From 71ba4ae796292423fa330a9baeaf6352c0b3fb81 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:46:26 -0400 Subject: [PATCH 01/13] docs(standards): add organization health defaults Adds organization-wide community health defaults, validation, ownership, contribution guidance, security policy, and hardened workflow checks. --- .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/bug_report.yml | 77 ++++++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.yml | 42 ++++ .github/ISSUE_TEMPLATE/question.yml | 36 +++ .github/PULL_REQUEST_TEMPLATE.md | 35 +++ .github/markdownlint-profile.yaml | 3 + .../tests/community_health_contract_test.py | 223 ++++++++++++++++++ .github/workflows/standards-validation.yml | 97 ++++++++ .github/workflows/update-stats.yml | 9 +- CONTRIBUTING.md | 56 +++++ SECURITY.md | 49 ++++ 12 files changed, 631 insertions(+), 2 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/ISSUE_TEMPLATE/question.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/markdownlint-profile.yaml create mode 100644 .github/tests/community_health_contract_test.py create mode 100644 .github/workflows/standards-validation.yml create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..787cb14 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @scttbnsn @ALARGECOMPANY @biggest-littlest diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..0782aa8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,77 @@ +name: Bug report +description: Report a reproducible problem +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for reporting a bug. Do not include secrets or private data, and redact logs and configuration. Email security@codeswhat.com instead of filing a public issue for a suspected vulnerability. + + - type: input + id: version + attributes: + label: Version or commit + description: Provide the release version, image digest, or commit SHA. + placeholder: "e.g. 1.2.3 or abc1234" + validations: + required: true + + - type: textarea + id: description + attributes: + label: What happened? + description: Describe the observed behavior and its impact. + validations: + required: true + + - type: textarea + id: expected + attributes: + label: What did you expect? + description: Describe what should have happened instead. + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Provide the smallest complete reproduction you can. + placeholder: | + 1. Configure ... + 2. Run ... + 3. Observe ... + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: Include relevant OS, architecture, runtime, and install method. + placeholder: | + - OS and architecture: + - Runtime or toolchain: + - Install method: + validations: + required: true + + - type: textarea + id: configuration + attributes: + label: Redacted configuration + description: Include only configuration needed to reproduce the problem. + render: yaml + + - type: textarea + id: logs + attributes: + label: Redacted logs + description: Remove credentials, private paths, hostnames, and user data. + render: text + + - type: textarea + id: additional + attributes: + label: Additional context + description: Add screenshots, links, or other useful context. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..632633e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Report a security vulnerability + url: mailto:security@codeswhat.com + about: Report suspected vulnerabilities privately, not in a public issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..ff53175 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,42 @@ +name: Feature request +description: Propose a focused improvement +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Search existing issues and pull requests before proposing a feature. Describe the need before the implementation. + + - type: textarea + id: problem + attributes: + label: Problem + description: What use case or limitation should this change address? + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed change + description: Describe the behavior you want and who benefits from it. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Describe current workarounds or other approaches you considered. + + - type: textarea + id: compatibility + attributes: + label: Compatibility and security + description: Note migration, breaking-change, privacy, or security effects. + + - type: textarea + id: additional + attributes: + label: Additional context + description: Add examples, screenshots, or links that clarify the request. diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 0000000..2d697bf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,36 @@ +name: Question or help +description: Ask about setup, usage, or project behavior +labels: ["question"] +body: + - type: markdown + attributes: + value: | + Check the README, documentation, and existing issues first. Email security@codeswhat.com instead of asking publicly about a suspected vulnerability. + + - type: textarea + id: question + attributes: + label: Question + description: What are you trying to understand or accomplish? + validations: + required: true + + - type: textarea + id: attempted + attributes: + label: What have you tried? + description: Include the relevant documentation and commands you used. + validations: + required: true + + - type: input + id: version + attributes: + label: Version or commit + description: Provide the release version or commit SHA when relevant. + + - type: textarea + id: context + attributes: + label: Additional context + description: Add redacted configuration, logs, screenshots, or links. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..96f9712 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,35 @@ +# Pull request + +## Summary + + + +Related issue: + +## Changes + + + +## Verification + + + +## Security and compatibility + + + +## Checklist + +- [ ] This pull request targets the repository's documented target branch. +- [ ] The change is focused and follows the repository's contributor + instructions. +- [ ] Tests cover non-trivial behavior, or the verification section explains + why they do not apply. +- [ ] Relevant documentation and changelog entries are updated, or they do not + apply. +- [ ] Required local checks pass. +- [ ] The change contains no secrets, credentials, or private data. +- [ ] All required checks and reviews are complete before merge. diff --git a/.github/markdownlint-profile.yaml b/.github/markdownlint-profile.yaml new file mode 100644 index 0000000..b6a8dcf --- /dev/null +++ b/.github/markdownlint-profile.yaml @@ -0,0 +1,3 @@ +config: + MD033: false + MD041: false diff --git a/.github/tests/community_health_contract_test.py b/.github/tests/community_health_contract_test.py new file mode 100644 index 0000000..9439282 --- /dev/null +++ b/.github/tests/community_health_contract_test.py @@ -0,0 +1,223 @@ +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class CommunityHealthContractTest(unittest.TestCase): + def test_required_default_files_exist(self): + required = ( + "SECURITY.md", + "CONTRIBUTING.md", + ".github/CODEOWNERS", + ".github/markdownlint-profile.yaml", + ".github/ISSUE_TEMPLATE/bug_report.yml", + ".github/ISSUE_TEMPLATE/feature_request.yml", + ".github/ISSUE_TEMPLATE/question.yml", + ".github/ISSUE_TEMPLATE/config.yml", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/workflows/standards-validation.yml", + ) + + missing = [path for path in required if not (ROOT / path).is_file()] + + self.assertEqual([], missing, f"missing community health files: {missing}") + + def test_codeowners_has_org_catch_all(self): + codeowners = self.read_text(".github/CODEOWNERS") + + self.assertEqual( + "* @scttbnsn @ALARGECOMPANY @biggest-littlest\n", + codeowners, + ) + + def test_security_policy_has_org_baseline(self): + policy = self.read_text("SECURITY.md") + + for expected in ( + "## Supported versions", + "## Reporting a vulnerability", + "## Security scope", + "Do not open a public GitHub issue", + "security@codeswhat.com", + "private vulnerability reporting", + "48 hours", + "7 days", + ): + self.assertIn(expected, policy) + + def test_contributor_guide_matches_org_workflow(self): + guide = self.read_text("CONTRIBUTING.md") + + for expected in ( + "AGENTS.md", + "active development or integration branch", + "Conventional Commits", + "feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert", + "without emoji", + "tests", + "documentation", + "required checks and reviews", + "SECURITY.md", + ): + self.assertIn(expected, guide) + + def test_issue_forms_have_unique_ids_and_required_reproduction_fields(self): + bug_report = self.read_text(".github/ISSUE_TEMPLATE/bug_report.yml") + feature_request = self.read_text( + ".github/ISSUE_TEMPLATE/feature_request.yml" + ) + question = self.read_text(".github/ISSUE_TEMPLATE/question.yml") + + self.assertIn('labels: ["bug"]', bug_report) + self.assertIn('labels: ["enhancement"]', feature_request) + self.assertIn('labels: ["question"]', question) + self.assert_form_ids_are_unique(bug_report) + self.assert_form_ids_are_unique(feature_request) + self.assert_form_ids_are_unique(question) + + required_bug_fields = {"version", "description", "expected", "reproduce"} + actual_required = self.required_form_ids(bug_report) + self.assertTrue(required_bug_fields <= actual_required) + + self.assertIn("security@codeswhat.com", bug_report) + self.assertIn("redact", bug_report.lower()) + + def test_issue_template_config_routes_security_reports_privately(self): + config = self.read_text(".github/ISSUE_TEMPLATE/config.yml") + + self.assertIn("blank_issues_enabled: false", config) + self.assertIn("url: mailto:security@codeswhat.com", config) + self.assertIn("privately", config.lower()) + + def test_pull_request_template_requires_verification_and_safe_content(self): + template = self.read_text(".github/PULL_REQUEST_TEMPLATE.md") + + for expected in ( + "## Summary", + "## Changes", + "## Verification", + "## Security and compatibility", + "## Checklist", + "target branch", + "tests", + "documentation", + "required checks and reviews", + "secrets, credentials, or private data", + ): + self.assertIn(expected, template) + + def test_standards_workflow_has_unconditional_stable_jobs(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + + self.assertEqual(2, workflow.count("name: Standards Validation")) + self.assertIn(" name: CodeQL\n", workflow) + self.assertIn(" pull_request:\n", workflow) + self.assertIn(" - main\n", workflow) + self.assertIn(" - dev/repository-standards\n", workflow) + self.assertNotIn("\n push:", workflow) + self.assertIn("permissions: {}", workflow) + self.assertIn("runs-on: ubuntu-24.04", workflow) + self.assertNotIn("matrix:", workflow) + self.assertNotIn("continue-on-error:", workflow) + self.assertNotIn("\n if:", workflow) + + validation_job = workflow.split(" validation:\n", 1)[1].split( + "\n codeql:\n", 1 + )[0] + self.assertIn(" actions: read", validation_job) + self.assertIn(" contents: read", validation_job) + self.assertNotIn("write", validation_job) + + codeql_job = workflow.split("\n codeql:\n", 1)[1] + self.assertIn(" actions: read", codeql_job) + self.assertIn(" contents: read", codeql_job) + self.assertIn(" security-events: write", codeql_job) + + jobs = re.findall( + r"^ ([a-z][a-z0-9-]+):$", + workflow.split("\njobs:\n", 1)[1], + re.MULTILINE, + ) + self.assertEqual(["validation", "codeql"], jobs) + + def test_standards_workflow_pins_actions_and_runs_every_contract(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + for expected in ( + "persist-credentials: false", + "community_health_contract_test.py", + "python3 -m compileall -q .", + "YAML.parse_file", + "json.load", + "markdownlint-cli2@0.23.2", + "markdownlint-profile.yaml", + "actionlint", + "zizmorcore/zizmor-action@", + "github/codeql-action/init@", + "languages: python", + "github/codeql-action/analyze@", + ): + self.assertIn(expected, workflow) + + def test_zizmor_action_owns_its_supported_cli_version(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + zizmor_step = workflow.split( + " - name: Validate workflows with zizmor\n", 1 + )[1].split("\n codeql:\n", 1)[0] + + self.assertNotIn("\n version:", zizmor_step) + + def test_linter_configs_do_not_suppress_workflow_findings(self): + markdown_config = self.read_text(".github/markdownlint-profile.yaml") + + self.assertIn("MD033: false", markdown_config) + self.assertIn("MD041: false", markdown_config) + self.assertNotIn("ignores:", markdown_config) + self.assertEqual(2, markdown_config.count(": false")) + self.assertFalse((ROOT / ".github/zizmor.yml").exists()) + + def test_profile_workflow_pins_actions_and_pushes_without_persisted_credentials(self): + workflow = self.read_text(".github/workflows/update-stats.yml") + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertEqual(2, len(actions)) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + checkout_step = workflow.split(" - name: Checkout\n", 1)[1].split( + "\n - name:", 1 + )[0] + push_step = workflow.split(" - name: Commit and push\n", 1)[1] + + self.assertIn("persist-credentials: false", checkout_step) + self.assertIn("GH_TOKEN: ${{ github.token }}", push_step) + self.assertIn("gh auth setup-git", push_step) + + def read_text(self, relative_path): + path = ROOT / relative_path + self.assertTrue(path.is_file(), f"missing community health file: {relative_path}") + return path.read_text() + + def assert_form_ids_are_unique(self, form): + ids = re.findall(r"^ id: ([a-z][a-z0-9_-]*)$", form, re.MULTILINE) + self.assertEqual(len(ids), len(set(ids))) + + def required_form_ids(self, form): + required = set() + for field in form.split("\n - type:"): + field_id = re.search(r"^ id: ([a-z][a-z0-9_-]*)$", field, re.MULTILINE) + if field_id and " required: true" in field: + required.add(field_id.group(1)) + return required + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml new file mode 100644 index 0000000..edfbdfd --- /dev/null +++ b/.github/workflows/standards-validation.yml @@ -0,0 +1,97 @@ +name: Standards Validation + +on: + pull_request: + branches: + - main + - dev/repository-standards + +permissions: {} + +jobs: + validation: + name: Standards Validation + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + + - name: Validate community health contract + run: python3 .github/tests/community_health_contract_test.py + + - name: Compile Python + run: python3 -m compileall -q . + + - name: Parse YAML and JSON + run: | + set -euo pipefail + ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' + python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' + + - name: Lint Markdown + run: | + set -euo pipefail + npx --yes markdownlint-cli2@0.23.2 "**/*.md" "#profile/README.md" + npx --yes markdownlint-cli2@0.23.2 "profile/README.md" --config .github/markdownlint-profile.yaml + + - name: Validate workflows with actionlint + run: | + set -euo pipefail + bash <(curl -fsSL https://raw.githubusercontent.com/rhysd/actionlint/914e7df21a07ef503a81201c76d2b11c789d3fca/scripts/download-actionlint.bash) 1.7.12 "$RUNNER_TEMP" + "$RUNNER_TEMP/actionlint" -color + + - name: Validate workflows with zizmor + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + with: + advanced-security: false + annotations: true + collect: workflows + inputs: .github/workflows/ + online-audits: false + + codeql: + name: CodeQL + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + languages: python + + - name: Analyze + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + category: /language:python diff --git a/.github/workflows/update-stats.yml b/.github/workflows/update-stats.yml index ba295c7..cb79f7c 100644 --- a/.github/workflows/update-stats.yml +++ b/.github/workflows/update-stats.yml @@ -17,10 +17,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -31,7 +33,10 @@ jobs: python generate_svg.py - name: Commit and push + env: + GH_TOKEN: ${{ github.token }} run: | + gh auth setup-git git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git add profile/dark_mode.svg profile/light_mode.svg cache/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1d31cae --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contributing to CodesWhat projects + +Bug fixes, focused features, tests, and documentation improvements are welcome. +Repository-specific instructions take precedence over this organization +default. + +## Before starting + +1. Read the repository's `README.md`, `AGENTS.md`, and development or release + documentation. +2. Search existing issues and pull requests for the same problem. +3. Open an issue before starting a large feature, breaking change, architecture + change, or release-process change. +4. From the documented active development or integration branch, create a + focused branch in your fork. + +Do not commit credentials, private data, local planning files, editor state, or +generated artifacts that the repository does not track. + +## Making a change + +- Keep the change focused on one concern. +- Follow the repository's existing language, formatting, and architecture + conventions. +- Add regression tests for bug fixes and coverage for non-trivial behavior. +- Update documentation and changelog entries when public behavior changes. +- Run the exact formatter, lint, test, build, and local hook commands documented + by the repository. + +## Commits + +Use plain Conventional Commits without emoji: + +```text +(): +``` + +Allowed types are +`feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert`. Keep the subject +concise and describe the change in the imperative mood. + +## Pull requests + +- Select the repository's documented target branch. Do not assume the GitHub + default branch is the correct target. +- Explain the problem, the change, and the verification performed. +- Call out security, compatibility, migration, or release effects. +- Include tests and documentation, or explain why they do not apply. +- Wait for all required checks and reviews before merge. + +Maintainers may ask for changes or decline work that does not fit the project's +scope. Contributions are licensed under the license of the repository receiving +them. + +Report suspected vulnerabilities privately as described in +[`SECURITY.md`](SECURITY.md), never in a public issue or pull request. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5cf164d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,49 @@ +# Security policy + +## Supported versions + +CodesWhat ships security fixes for the latest maintained release or branch of +each project. A repository-specific policy, release notes, or README may define +a narrower support window and takes precedence over this organization default. + +| Release or branch | Supported | +| --- | --- | +| Latest maintained release or branch | Yes | +| Older or unmaintained releases and branches | No | + +For a project that does not publish versioned releases, support applies to the +latest commit on its documented maintained branch. + +## Reporting a vulnerability + +Do not open a public GitHub issue for a suspected security vulnerability. + +Open this repository's **Security** tab and select **Report a vulnerability** +to use GitHub private vulnerability reporting when it is enabled. Otherwise, +email ****. Include the affected version or commit, +minimal reproduction steps, observed and expected behavior, and your assessment +of the impact. Redact credentials, private data, and identifying environment +details. + +You can expect: + +- acknowledgement within 48 hours; +- a status update within 7 days; and +- a fix or mitigation as soon as feasible, depending on severity and release + safety. + +CodesWhat coordinates disclosure with the reporter and credits reporters in +release notes unless they prefer to remain anonymous. + +## Security scope + +Unless a repository-specific policy says otherwise, the following are in +scope: + +- source code and configuration maintained in the repository; +- artifacts published by CodesWhat from that repository; and +- repository-owned build, test, and release automation. + +Third-party services, dependencies, and deployments are out of scope unless the +repository introduces or amplifies the reported impact. If the boundary is +unclear, report the issue privately and let the maintainers triage it. From 22feaaa1cd49740a453e9cd611745edbb13c6e8d Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:15:14 -0400 Subject: [PATCH 02/13] ci(greptile): require manual review requests (#11) --- .github/tests/greptile_config_contract_test.py | 18 ++++++++++++++++++ .github/workflows/standards-validation.yml | 1 + greptile.json | 3 +++ 3 files changed, 22 insertions(+) create mode 100644 .github/tests/greptile_config_contract_test.py create mode 100644 greptile.json diff --git a/.github/tests/greptile_config_contract_test.py b/.github/tests/greptile_config_contract_test.py new file mode 100644 index 0000000..33f7fb8 --- /dev/null +++ b/.github/tests/greptile_config_contract_test.py @@ -0,0 +1,18 @@ +import json +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class GreptileConfigContractTest(unittest.TestCase): + def test_reviews_are_manual_only(self): + with (ROOT / "greptile.json").open() as config_file: + config = json.load(config_file) + + self.assertEqual({"skipReview": "AUTOMATIC"}, config) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index edfbdfd..d9079a1 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -44,6 +44,7 @@ jobs: set -euo pipefail ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' + python3 .github/tests/greptile_config_contract_test.py - name: Lint Markdown run: | diff --git a/greptile.json b/greptile.json new file mode 100644 index 0000000..34c493d --- /dev/null +++ b/greptile.json @@ -0,0 +1,3 @@ +{ + "skipReview": "AUTOMATIC" +} From bc5ab596a2c715cd8eb7b87230abb5e568c33021 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:54:26 -0400 Subject: [PATCH 03/13] ci(workflows): add reusable CI foundation (#13) * ci(workflows): add reusable CI foundation * fix(workflows): harden reusable release contracts --- .github/tests/reusable_ci_contract_test.py | 527 +++++++++++++++++++++ .github/workflows/go-ci.yml | 460 ++++++++++++++++++ .github/workflows/node-ci.yml | 186 ++++++++ .github/workflows/release-gate.yml | 172 +++++++ .github/workflows/standards-validation.yml | 1 + 5 files changed, 1346 insertions(+) create mode 100644 .github/tests/reusable_ci_contract_test.py create mode 100644 .github/workflows/go-ci.yml create mode 100644 .github/workflows/node-ci.yml create mode 100644 .github/workflows/release-gate.yml diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py new file mode 100644 index 0000000..a6ec3f5 --- /dev/null +++ b/.github/tests/reusable_ci_contract_test.py @@ -0,0 +1,527 @@ +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = { + "go": ROOT / ".github/workflows/go-ci.yml", + "node": ROOT / ".github/workflows/node-ci.yml", + "release": ROOT / ".github/workflows/release-gate.yml", +} +TARGET_SHA = "a" * 40 + + +class ReusableCIContractTest(unittest.TestCase): + def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self): + go_inputs = [ + "module-directory", + "go-version-file", + "go-cache-dependency-path", + "lint-check-name", + "test-check-name", + "fuzzers-json", + "run-govulncheck", + "run-qlty", + "run-goreleaser", + "run-workflow-security", + "run-commit-message", + "run-codeql", + ] + self.egress_inputs( + ( + "test", + "lint", + "govulncheck", + "workflow-security", + "commit-message", + "goreleaser", + "codeql", + "qlty", + "fuzz", + ) + ) + node_inputs = [ + "node-version", + "lockfile-path", + "lint-check-name", + "test-check-name", + "build-check-name", + "run-lint", + "run-test", + "run-build", + ] + self.egress_inputs(("lint", "test", "build")) + release_inputs = [ + "target-sha", + "workflow-files-json", + "max-attempts", + "sleep-seconds", + ] + + expected_inputs = { + "go": go_inputs, + "node": node_inputs, + "release": release_inputs, + } + expected_jobs = { + "go": { + "test", + "lint", + "govulncheck", + "workflow-security", + "commit-message", + "goreleaser", + "codeql", + "qlty", + "fuzz", + }, + "node": {"lint", "test", "build"}, + "release": {"gate"}, + } + + for name, path in WORKFLOWS.items(): + workflow = self.read_workflow(path) + header = workflow.split("\npermissions:", 1)[0] + inputs = re.findall(r"^ ([a-z][a-z0-9-]+):$", header, re.MULTILINE) + self.assertEqual(expected_inputs[name], inputs) + self.assertRegex(workflow, r"(?m)^on:\n workflow_call:\s+inputs:") + self.assertNotRegex( + workflow, + r"(?m)^ (push|pull_request|schedule|workflow_dispatch):", + ) + self.assertNotIn("concurrency:", workflow) + self.assertNotIn("outputs:", header) + self.assertNotIn("secrets:", header) + self.assertNotIn("secrets: inherit", workflow) + self.assertIn("permissions: {}", workflow) + + jobs = set( + re.findall( + r"^ ([a-z][a-z0-9-]+):$", + workflow.split("\njobs:\n", 1)[1], + re.MULTILINE, + ) + ) + self.assertEqual(expected_jobs[name], jobs) + + self.assert_input("go", "module-directory", "string", default=".") + self.assert_input("go", "go-version-file", "string", default="go.mod") + self.assert_input("go", "go-cache-dependency-path", "string", default="go.sum") + self.assert_input("go", "lint-check-name", "string", default="Go Lint") + self.assert_input("go", "test-check-name", "string", default="Go Test") + self.assert_input("go", "fuzzers-json", "string", default="[]") + for input_name in ( + "run-govulncheck", + "run-qlty", + "run-goreleaser", + "run-workflow-security", + "run-commit-message", + "run-codeql", + ): + self.assert_input("go", input_name, "boolean", default="false") + + self.assert_input("node", "node-version", "string", default="24") + self.assert_input("node", "lockfile-path", "string", default="package-lock.json") + self.assert_input("node", "lint-check-name", "string", default="Node Lint") + self.assert_input("node", "test-check-name", "string", default="Node Test") + self.assert_input("node", "build-check-name", "string", default="Node Build") + for input_name in ("run-lint", "run-test", "run-build"): + self.assert_input("node", input_name, "boolean", default="false") + + for workflow_name, job_names in ( + ("go", ("test", "lint", "govulncheck", "workflow-security", "commit-message", "goreleaser", "codeql", "qlty", "fuzz")), + ("node", ("lint", "test", "build")), + ): + for job_name in job_names: + self.assert_input(workflow_name, f"{job_name}-egress-policy", "string", default="audit") + self.assert_input(workflow_name, f"{job_name}-allowed-endpoints", "string", default="") + + self.assert_input("release", "target-sha", "string", required=True) + self.assert_input("release", "workflow-files-json", "string", required=True) + self.assert_input("release", "max-attempts", "number", default="12") + self.assert_input("release", "sleep-seconds", "number", default="300") + + def test_central_jobs_have_exact_names_fixed_commands_and_per_job_egress(self): + go = self.read_workflow(WORKFLOWS["go"]) + expected_go_names = { + "test": "${{ inputs.test-check-name }}", + "lint": "${{ inputs.lint-check-name }}", + "govulncheck": "Govulncheck", + "workflow-security": "Workflow Security", + "commit-message": "Commit Message", + "goreleaser": "GoReleaser Config", + "codeql": "CodeQL Analysis", + "qlty": "Qlty Check", + "fuzz": 'Go Fuzz (${{ matrix.fuzzer.name }})', + } + for job, display_name in expected_go_names.items(): + self.assertEqual(display_name, self.job_name(go, job)) + self.assert_job_egress(go, job) + + expected_go_scripts = [ + "./scripts/ci/go-test.sh", + "./scripts/ci/go-lint.sh", + "./scripts/ci/go-govulncheck.sh", + "./scripts/ci/commit-message.sh", + "./scripts/ci/go-release-check.sh", + "./scripts/ci/go-codeql-build.sh", + "./scripts/ci/go-qlty.sh", + "./scripts/ci/go-fuzz.sh", + ] + self.assertEqual(expected_go_scripts, self.fixed_scripts(go)) + self.assertIn("fuzzer: ${{ fromJSON(inputs.fuzzers-json) }}", go) + self.assertIn( + "if: inputs.fuzzers-json != '' && fromJSON(inputs.fuzzers-json)[0] != null", + go, + ) + self.assertIn("go-version-file: ${{ inputs.go-version-file }}", go) + self.assertIn("cache-dependency-path: ${{ inputs.go-cache-dependency-path }}", go) + self.assertIn("MODULE_DIRECTORY: ${{ inputs.module-directory }}", go) + + node = self.read_workflow(WORKFLOWS["node"]) + expected_node_names = { + "lint": "${{ inputs.lint-check-name }}", + "test": "${{ inputs.test-check-name }}", + "build": "${{ inputs.build-check-name }}", + } + for job, display_name in expected_node_names.items(): + self.assertEqual(display_name, self.job_name(node, job)) + self.assert_job_egress(node, job) + self.assertEqual( + [ + "./scripts/ci/node-lint.sh", + "./scripts/ci/node-test.sh", + "./scripts/ci/node-build.sh", + ], + self.fixed_scripts(node), + ) + self.assertIn("node-version: ${{ inputs.node-version }}", node) + self.assertIn("cache-dependency-path: ${{ inputs.lockfile-path }}", node) + + for workflow in (go, node, self.read_workflow(WORKFLOWS["release"])): + self.assertIn("runs-on: ubuntu-24.04", workflow) + for action in re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE): + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + for run_block in re.findall(r"run: \|\n((?: {10}.*\n|\n)+)", workflow): + self.assertNotIn("${{ inputs.", run_block) + + def test_artifact_uploads_are_central_and_fixed(self): + go = self.read_workflow(WORKFLOWS["go"]) + node = self.read_workflow(WORKFLOWS["node"]) + + self.assertEqual( + ["artifacts/go-test/", "artifacts/go-fuzz/"], + re.findall(r"^\s+path: (artifacts/[^\s]+)$", go, re.MULTILINE), + ) + self.assertEqual( + ["artifacts/node-test/", "artifacts/node-build/"], + re.findall(r"^\s+path: (artifacts/[^\s]+)$", node, re.MULTILINE), + ) + for workflow in (go, node): + upload_count = workflow.count("actions/upload-artifact@") + self.assertEqual(upload_count, workflow.count("if-no-files-found: ignore")) + self.assertEqual(upload_count, workflow.count("retention-days: 14")) + self.assertEqual(upload_count, workflow.count("if: always()")) + + def test_release_gate_checks_every_workflow_by_exact_sha_push_and_nonempty_branch(self): + exact_success = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "dev/v1.7", + "status": "completed", + "conclusion": "success", + } + wrong_runs = [ + {**exact_success, "event": "pull_request"}, + {**exact_success, "head_branch": ""}, + {**exact_success, "head_sha": "b" * 40}, + ] + result = self.run_release_gate( + { + "ci-verify.yml": wrong_runs + [exact_success], + "e2e-playwright.yml": [ + {**exact_success, "head_branch": "main"}, + ], + } + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn(f"Found successful CI Verify push run for {TARGET_SHA}.", result.stdout) + self.assertIn(f"Found successful E2E Playwright push run for {TARGET_SHA}.", result.stdout) + + def test_release_gate_keeps_authoritative_client_side_run_filtering(self): + release = self.read_workflow(WORKFLOWS["release"]) + + self.assertIn("/runs?per_page=100\"", release) + self.assertNotRegex(release, r"runs\?[^\"\n]*(?:event|head_sha)=") + self.assertIn("spuriously returning 0 results since 2026-04-27", release) + + def test_release_gate_uses_one_shared_poll_budget_for_every_workflow(self): + in_progress = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "dev/v1.7", + "status": "in_progress", + "conclusion": None, + } + success = {**in_progress, "status": "completed", "conclusion": "success"} + result = self.run_release_gate( + {}, + runs_sequences_by_workflow={ + "ci-verify.yml": [[in_progress], [success]], + "e2e-playwright.yml": [[in_progress], [success]], + }, + max_attempts="2", + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(1, result.stdout.count("FAKE_SLEEP"), result.stdout) + + def test_release_gate_waits_when_an_exact_run_is_still_in_progress(self): + exact_run = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + } + result = self.run_release_gate( + { + "ci-verify.yml": [ + {**exact_run, "status": "completed", "conclusion": "failure"}, + {**exact_run, "status": "in_progress", "conclusion": None}, + ], + } + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("Attempt 1/1: waiting for CI Verify", result.stdout) + self.assertIn( + f"Timed out waiting for successful CI Verify push run for {TARGET_SHA}.", + result.stdout, + ) + + def test_release_gate_fails_closed_when_any_workflow_lacks_exact_success(self): + result = self.run_release_gate( + { + "ci-verify.yml": [ + { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ], + "e2e-playwright.yml": [ + { + "head_sha": "b" * 40, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ], + } + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn( + f"Timed out waiting for successful E2E Playwright push run for {TARGET_SHA}.", + result.stdout, + ) + + def test_release_gate_rejects_unsafe_inputs_before_api_calls(self): + bad_sha = self.run_release_gate({"ci-verify.yml": []}, target_sha="main") + bad_files = self.run_release_gate( + {"ci-verify.yml": []}, + workflow_files_json='["../../release.yml"]', + ) + bad_attempts = self.run_release_gate( + {"ci-verify.yml": []}, + max_attempts="0", + ) + + self.assertNotEqual(0, bad_sha.returncode) + self.assertIn("target-sha must be a full lowercase commit SHA", bad_sha.stdout) + self.assertNotEqual(0, bad_files.returncode) + self.assertIn("workflow-files-json must be a nonempty JSON array", bad_files.stdout) + self.assertNotEqual(0, bad_attempts.returncode) + self.assertIn("max-attempts must be a positive integer", bad_attempts.stdout) + + def test_release_gate_rejects_non_github_api_hosts_before_api_calls(self): + result = self.run_release_gate( + { + "ci-verify.yml": [ + { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ] + }, + api_url="https://github.example/api/v3", + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("release-gate supports api.github.com only", result.stdout) + + def test_standards_validation_runs_this_contract(self): + workflow = self.read_workflow(ROOT / ".github/workflows/standards-validation.yml") + self.assertEqual(1, workflow.count("python3 .github/tests/reusable_ci_contract_test.py")) + + def egress_inputs(self, job_names): + return [item for job in job_names for item in (f"{job}-egress-policy", f"{job}-allowed-endpoints")] + + def read_workflow(self, path): + self.assertTrue(path.is_file(), f"missing reusable workflow: {path.name}") + return path.read_text() + + def input_header(self, workflow_name): + workflow = self.read_workflow(WORKFLOWS[workflow_name]) + return workflow.split("\npermissions:", 1)[0] + + def assert_input(self, workflow_name, input_name, input_type, default=None, required=False): + header = self.input_header(workflow_name) + block = header.split(f" {input_name}:\n", 1)[1] + next_input = re.search(r"^ [a-z][a-z0-9-]+:$", block, re.MULTILINE) + if next_input: + block = block[: next_input.start()] + self.assertIn(f" type: {input_type}\n", block) + if default is not None: + expected = ( + f' default: "{default}"\n' + if default in {"", "[]"} + else f" default: {default}\n" + ) + self.assertIn(expected, block) + if required: + self.assertIn(" required: true\n", block) + + def job_section(self, workflow, job_name): + jobs = workflow.split("\njobs:\n", 1)[1] + section = jobs.split(f" {job_name}:\n", 1)[1] + next_job = re.search(r"^ [a-z][a-z0-9-]+:$", section, re.MULTILINE) + return section[: next_job.start()] if next_job else section + + def job_name(self, workflow, job_name): + section = self.job_section(workflow, job_name) + return re.search(r"^ name: (.+)$", section, re.MULTILINE).group(1).strip('"') + + def assert_job_egress(self, workflow, job_name): + section = self.job_section(workflow, job_name) + self.assertIn(f"egress-policy: ${{{{ inputs.{job_name}-egress-policy }}}}", section) + self.assertIn(f"allowed-endpoints: ${{{{ inputs.{job_name}-allowed-endpoints }}}}", section) + + def fixed_scripts(self, workflow): + return re.findall(r"^\s+run: (\./scripts/ci/[^\s]+)$", workflow, re.MULTILINE) + + def release_gate_script(self): + workflow = self.read_workflow(WORKFLOWS["release"]) + marker = " - name: Verify required CI workflows\n" + self.assertIn(marker, workflow) + step = workflow.split(marker, 1)[1] + block = step.split(" run: |\n", 1)[1] + lines = [] + for line in block.splitlines(): + if line.startswith(" "): + lines.append(line[10:]) + elif not line: + lines.append("") + else: + break + self.assertTrue(lines, "release gate shell step is empty") + return "\n".join(lines) + + def run_release_gate( + self, + runs_by_workflow, + target_sha=TARGET_SHA, + workflow_files_json=None, + max_attempts="1", + api_url="https://api.github.com", + runs_sequences_by_workflow=None, + ): + with tempfile.TemporaryDirectory() as temp_dir: + temp = Path(temp_dir) + fake_bin = temp / "bin" + fake_bin.mkdir() + fixtures = temp / "fixtures" + fixtures.mkdir() + + metadata = { + "ci-verify.yml": (123, "CI Verify"), + "e2e-playwright.yml": (456, "E2E Playwright"), + } + sequences = { + workflow_file: [runs] + for workflow_file, runs in runs_by_workflow.items() + } + sequences.update(runs_sequences_by_workflow or {}) + for workflow_file, runs_sequence in sequences.items(): + workflow_id = metadata[workflow_file][0] + for index, runs in enumerate(runs_sequence, start=1): + (fixtures / f"{workflow_id}.{index}.json").write_text( + json.dumps({"workflow_runs": runs}) + ) + (fixtures / f"{workflow_id}.last").write_text(str(len(runs_sequence))) + + curl = fake_bin / "curl" + curl.write_text( + "#!/usr/bin/env bash\n" + "for argument in \"$@\"; do url=\"$argument\"; done\n" + "case \"$url\" in\n" + " */actions/workflows/ci-verify.yml) printf '%s' '{\"id\":123,\"name\":\"CI Verify\",\"path\":\".github/workflows/ci-verify.yml\"}' ;;\n" + " */actions/workflows/e2e-playwright.yml) printf '%s' '{\"id\":456,\"name\":\"E2E Playwright\",\"path\":\".github/workflows/e2e-playwright.yml\"}' ;;\n" + " */actions/workflows/123/runs?per_page=100) workflow_id=123 ;;\n" + " */actions/workflows/456/runs?per_page=100) workflow_id=456 ;;\n" + " *) printf 'unexpected URL: %s\\n' \"$url\" >&2; exit 22 ;;\n" + "esac\n" + "if [ -n \"${workflow_id:-}\" ]; then\n" + " count_file=\"$RUNS_FIXTURES/$workflow_id.count\"\n" + " count=0\n" + " [ ! -f \"$count_file\" ] || count=$(cat \"$count_file\")\n" + " count=$((count + 1))\n" + " printf '%s' \"$count\" > \"$count_file\"\n" + " last=$(cat \"$RUNS_FIXTURES/$workflow_id.last\")\n" + " [ \"$count\" -le \"$last\" ] || count=\"$last\"\n" + " cat \"$RUNS_FIXTURES/$workflow_id.$count.json\"\n" + "fi\n" + ) + curl.chmod(0o755) + + sleep = fake_bin / "sleep" + sleep.write_text("#!/usr/bin/env bash\nprintf 'FAKE_SLEEP %s\\n' \"$1\"\n") + sleep.chmod(0o755) + + if workflow_files_json is None: + workflow_files_json = json.dumps(list(sequences)) + env = os.environ.copy() + env.update( + { + "GH_TOKEN": "test-token", + "GITHUB_API_URL": api_url, + "GITHUB_REPOSITORY": "CodesWhat/example", + "MAX_ATTEMPTS": max_attempts, + "PATH": f"{fake_bin}:{env['PATH']}", + "RUNS_FIXTURES": str(fixtures), + "SLEEP_SECONDS": "0", + "TARGET_SHA": target_sha, + "WORKFLOW_FILES_JSON": workflow_files_json, + } + ) + return subprocess.run( + ["bash", "-c", self.release_gate_script()], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml new file mode 100644 index 0000000..46601d4 --- /dev/null +++ b/.github/workflows/go-ci.yml @@ -0,0 +1,460 @@ +name: Reusable Go CI + +on: + workflow_call: + inputs: + module-directory: + description: Repository-relative Go module directory + required: false + type: string + default: . + go-version-file: + description: Repository-relative Go version file + required: false + type: string + default: go.mod + go-cache-dependency-path: + description: Repository-relative Go cache dependency path + required: false + type: string + default: go.sum + lint-check-name: + description: Exact display name for the Go lint job + required: false + type: string + default: Go Lint + test-check-name: + description: Exact display name for the Go test job + required: false + type: string + default: Go Test + fuzzers-json: + description: JSON array of caller-owned fuzzer name and pkg objects + required: false + type: string + default: "[]" + run-govulncheck: + description: Run the fixed repository govulncheck gate + required: false + type: boolean + default: false + run-qlty: + description: Run the fixed repository Qlty gate + required: false + type: boolean + default: false + run-goreleaser: + description: Run the fixed repository GoReleaser gate + required: false + type: boolean + default: false + run-workflow-security: + description: Run the central workflow security gate + required: false + type: boolean + default: false + run-commit-message: + description: Run the fixed repository commit-message gate on pull requests + required: false + type: boolean + default: false + run-codeql: + description: Run CodeQL for Actions and Go + required: false + type: boolean + default: false + test-egress-policy: + description: Harden Runner egress policy for Go test + required: false + type: string + default: audit + test-allowed-endpoints: + description: Harden Runner allowlist for Go test + required: false + type: string + default: "" + lint-egress-policy: + description: Harden Runner egress policy for Go lint + required: false + type: string + default: audit + lint-allowed-endpoints: + description: Harden Runner allowlist for Go lint + required: false + type: string + default: "" + govulncheck-egress-policy: + description: Harden Runner egress policy for govulncheck + required: false + type: string + default: audit + govulncheck-allowed-endpoints: + description: Harden Runner allowlist for govulncheck + required: false + type: string + default: "" + workflow-security-egress-policy: + description: Harden Runner egress policy for workflow security + required: false + type: string + default: audit + workflow-security-allowed-endpoints: + description: Harden Runner allowlist for workflow security + required: false + type: string + default: "" + commit-message-egress-policy: + description: Harden Runner egress policy for commit-message validation + required: false + type: string + default: audit + commit-message-allowed-endpoints: + description: Harden Runner allowlist for commit-message validation + required: false + type: string + default: "" + goreleaser-egress-policy: + description: Harden Runner egress policy for GoReleaser + required: false + type: string + default: audit + goreleaser-allowed-endpoints: + description: Harden Runner allowlist for GoReleaser + required: false + type: string + default: "" + codeql-egress-policy: + description: Harden Runner egress policy for CodeQL + required: false + type: string + default: audit + codeql-allowed-endpoints: + description: Harden Runner allowlist for CodeQL + required: false + type: string + default: "" + qlty-egress-policy: + description: Harden Runner egress policy for Qlty + required: false + type: string + default: audit + qlty-allowed-endpoints: + description: Harden Runner allowlist for Qlty + required: false + type: string + default: "" + fuzz-egress-policy: + description: Harden Runner egress policy for Go fuzzing + required: false + type: string + default: audit + fuzz-allowed-endpoints: + description: Harden Runner allowlist for Go fuzzing + required: false + type: string + default: "" + +permissions: {} + +jobs: + test: + name: ${{ inputs.test-check-name }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.test-egress-policy }} + allowed-endpoints: ${{ inputs.test-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go test gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-test.sh + + - name: Upload Go test artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: go-test-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/go-test/ + if-no-files-found: ignore + retention-days: 14 + + lint: + name: ${{ inputs.lint-check-name }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.lint-egress-policy }} + allowed-endpoints: ${{ inputs.lint-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go lint gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-lint.sh + + govulncheck: + name: Govulncheck + if: inputs.run-govulncheck + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.govulncheck-egress-policy }} + allowed-endpoints: ${{ inputs.govulncheck-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository govulncheck gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-govulncheck.sh + + workflow-security: + name: Workflow Security + if: inputs.run-workflow-security + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.workflow-security-egress-policy }} + allowed-endpoints: ${{ inputs.workflow-security-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + with: + token: ${{ github.token }} + advanced-security: false + online-audits: false + + commit-message: + name: Commit Message + if: inputs.run-commit-message && github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.commit-message-egress-policy }} + allowed-endpoints: ${{ inputs.commit-message-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Run repository commit-message gate + env: + BASE_REF: ${{ github.base_ref }} + HEAD_REF: ${{ github.head_ref }} + run: ./scripts/ci/commit-message.sh + + goreleaser: + name: GoReleaser Config + if: inputs.run-goreleaser + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.goreleaser-egress-policy }} + allowed-endpoints: ${{ inputs.goreleaser-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository GoReleaser gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-release-check.sh + + codeql: + name: CodeQL Analysis + if: inputs.run-codeql + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.codeql-egress-policy }} + allowed-endpoints: ${{ inputs.codeql-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + languages: actions, go + queries: security-and-quality + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Build for CodeQL + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-codeql-build.sh + + - name: Analyze with CodeQL + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + + qlty: + name: Qlty Check + if: inputs.run-qlty + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.qlty-egress-policy }} + allowed-endpoints: ${{ inputs.qlty-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Qlty + uses: qltysh/qlty-action/install@08a0a862c159eae9b9003081da6663d96efef637 # v2.3.0 + + - name: Run repository Qlty gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-qlty.sh + + fuzz: + name: "Go Fuzz (${{ matrix.fuzzer.name }})" + if: inputs.fuzzers-json != '' && fromJSON(inputs.fuzzers-json)[0] != null + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + fuzzer: ${{ fromJSON(inputs.fuzzers-json) }} + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.fuzz-egress-policy }} + allowed-endpoints: ${{ inputs.fuzz-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go fuzzer + env: + FUZZER: ${{ matrix.fuzzer.name }} + MODULE_DIRECTORY: ${{ inputs.module-directory }} + PKG: ${{ matrix.fuzzer.pkg }} + run: ./scripts/ci/go-fuzz.sh + + - name: Upload Go fuzz artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: go-fuzz-${{ matrix.fuzzer.name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/go-fuzz/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml new file mode 100644 index 0000000..e699a9b --- /dev/null +++ b/.github/workflows/node-ci.yml @@ -0,0 +1,186 @@ +name: Reusable Node CI + +on: + workflow_call: + inputs: + node-version: + description: Node.js version used by enabled jobs + required: false + type: string + default: 24 + lockfile-path: + description: Repository-relative npm lockfile path or multiline paths + required: false + type: string + default: package-lock.json + lint-check-name: + description: Exact display name for the Node lint job + required: false + type: string + default: Node Lint + test-check-name: + description: Exact display name for the Node test job + required: false + type: string + default: Node Test + build-check-name: + description: Exact display name for the Node build job + required: false + type: string + default: Node Build + run-lint: + description: Run the fixed repository Node lint gate + required: false + type: boolean + default: false + run-test: + description: Run the fixed repository Node test gate + required: false + type: boolean + default: false + run-build: + description: Run the fixed repository Node build gate + required: false + type: boolean + default: false + lint-egress-policy: + description: Harden Runner egress policy for Node lint + required: false + type: string + default: audit + lint-allowed-endpoints: + description: Harden Runner allowlist for Node lint + required: false + type: string + default: "" + test-egress-policy: + description: Harden Runner egress policy for Node test + required: false + type: string + default: audit + test-allowed-endpoints: + description: Harden Runner allowlist for Node test + required: false + type: string + default: "" + build-egress-policy: + description: Harden Runner egress policy for Node build + required: false + type: string + default: audit + build-allowed-endpoints: + description: Harden Runner allowlist for Node build + required: false + type: string + default: "" + +permissions: {} + +jobs: + lint: + name: ${{ inputs.lint-check-name }} + if: inputs.run-lint + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.lint-egress-policy }} + allowed-endpoints: ${{ inputs.lint-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node lint gate + run: ./scripts/ci/node-lint.sh + + test: + name: ${{ inputs.test-check-name }} + if: inputs.run-test + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.test-egress-policy }} + allowed-endpoints: ${{ inputs.test-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node test gate + run: ./scripts/ci/node-test.sh + + - name: Upload Node test artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-test-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/node-test/ + if-no-files-found: ignore + retention-days: 14 + + build: + name: ${{ inputs.build-check-name }} + if: inputs.run-build + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.build-egress-policy }} + allowed-endpoints: ${{ inputs.build-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node build gate + run: ./scripts/ci/node-build.sh + + - name: Upload Node build artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-build-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/node-build/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 0000000..f86ff64 --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,172 @@ +name: Reusable Release Gate + +on: + workflow_call: + inputs: + target-sha: + description: Full commit SHA whose branch push runs must have succeeded + required: true + type: string + workflow-files-json: + description: Nonempty JSON array of workflow filenames to verify + required: true + type: string + max-attempts: + description: Maximum polls for each required workflow + required: false + type: number + default: 12 + sleep-seconds: + description: Seconds between polls + required: false + type: number + default: 300 + +permissions: {} + +jobs: + gate: + name: Release Gate + runs-on: ubuntu-24.04 + timeout-minutes: 360 + permissions: + actions: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: api.github.com:443 + + - name: Verify required CI workflows + env: + GH_TOKEN: ${{ github.token }} + MAX_ATTEMPTS: ${{ inputs.max-attempts }} + SLEEP_SECONDS: ${{ inputs.sleep-seconds }} + TARGET_SHA: ${{ inputs.target-sha }} + WORKFLOW_FILES_JSON: ${{ inputs.workflow-files-json }} + run: | + set -euo pipefail + + if [ "${GITHUB_API_URL}" != "https://api.github.com" ]; then + echo "::error::release-gate supports api.github.com only so its blocked egress allowlist matches the API host." + exit 1 + fi + if ! [[ "${TARGET_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::target-sha must be a full lowercase commit SHA." + exit 1 + fi + if ! [[ "${MAX_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::max-attempts must be a positive integer." + exit 1 + fi + if ! [[ "${SLEEP_SECONDS}" =~ ^[0-9]+$ ]]; then + echo "::error::sleep-seconds must be a nonnegative integer." + exit 1 + fi + if ! jq -e ' + type == "array" + and length > 0 + and all(.[]; + type == "string" + and test("^[A-Za-z0-9][A-Za-z0-9._-]*[.]ya?ml$") + ) + ' <<< "${WORKFLOW_FILES_JSON}" >/dev/null; then + echo "::error::workflow-files-json must be a nonempty JSON array of workflow filenames." + exit 1 + fi + + workflow_files=() + while IFS= read -r workflow_file; do + workflow_files+=("${workflow_file}") + done < <(jq -r '.[]' <<< "${WORKFLOW_FILES_JSON}") + + workflow_ids=() + workflow_names=() + for workflow_file in "${workflow_files[@]}"; do + workflow_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow_file}" + workflow_json="$(curl --proto '=https' --tlsv1.2 --retry 3 --fail --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${workflow_url}")" + workflow_id="$(echo "${workflow_json}" | jq -r '.id // empty')" + workflow_name="$(echo "${workflow_json}" | jq -r '.name // empty')" + workflow_path="$(echo "${workflow_json}" | jq -r '.path // empty')" + expected_path=".github/workflows/${workflow_file}" + if [ -z "${workflow_id}" ] || [ -z "${workflow_name}" ] || [ "${workflow_path}" != "${expected_path}" ]; then + echo "::error::Failed to resolve workflow metadata for ${workflow_file}." + exit 1 + fi + workflow_ids+=("${workflow_id}") + workflow_names+=("${workflow_name}") + done + + workflow_succeeded=() + for index in "${!workflow_ids[@]}"; do + workflow_succeeded["${index}"]=false + done + + for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do + waiting=false + for index in "${!workflow_ids[@]}"; do + if [ "${workflow_succeeded[${index}]}" = "true" ]; then + continue + fi + + workflow_id="${workflow_ids[${index}]}" + workflow_name="${workflow_names[${index}]}" + # GitHub's `?head_sha=` and `?event=` query filters have been spuriously returning 0 results since 2026-04-27. + # Filter the latest 100 runs client-side instead. + runs_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow_id}/runs?per_page=100" + runs_json="$(curl --proto '=https' --tlsv1.2 --retry 3 --fail --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${runs_url}")" + filtered="$(echo "${runs_json}" | jq --arg sha "${TARGET_SHA}" '{ + workflow_runs: [.workflow_runs[] + | select( + .head_sha == $sha + and .event == "push" + and ((.head_branch // "") | length > 0) + )] + }')" + + success_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.conclusion == "success")] | length')" + in_progress_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.status != "completed")] | length')" + completed_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.status == "completed")] | length')" + + if [ "${success_count}" -gt 0 ]; then + echo "Found successful ${workflow_name} push run for ${TARGET_SHA}." + workflow_succeeded["${index}"]=true + continue + fi + + if [ "${completed_count}" -gt 0 ] && [ "${in_progress_count}" -eq 0 ]; then + echo "::error::${workflow_name} push runs for ${TARGET_SHA} completed without success." + echo "${filtered}" | jq '.workflow_runs[] | {id, status, conclusion, head_branch, html_url}' + exit 1 + fi + + echo "Attempt ${attempt}/${MAX_ATTEMPTS}: waiting for ${workflow_name} push run for ${TARGET_SHA}." + waiting=true + done + + if [ "${waiting}" != "true" ]; then + break + fi + if [ "${attempt}" -lt "${MAX_ATTEMPTS}" ]; then + sleep "${SLEEP_SECONDS}" + fi + done + + timed_out=false + for index in "${!workflow_ids[@]}"; do + if [ "${workflow_succeeded[${index}]}" != "true" ]; then + echo "::error::Timed out waiting for successful ${workflow_names[${index}]} push run for ${TARGET_SHA}." + timed_out=true + fi + done + if [ "${timed_out}" = "true" ]; then + exit 1 + fi diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index d9079a1..192e5bd 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -45,6 +45,7 @@ jobs: ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' python3 .github/tests/greptile_config_contract_test.py + python3 .github/tests/reusable_ci_contract_test.py - name: Lint Markdown run: | From 16a6680ee0bdeb32f5a768557413503adc4c36c8 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:35:36 -0400 Subject: [PATCH 04/13] feat(quality): standardize long-run reporting (#15) * feat(quality): add normalized reporting foundation * test(quality): run reporting contracts in standards validation * fix(quality): align report validator with schema * test(quality): verify GitHub integration outputs --- .../quality-result-config/target-result.json | 22 + .../quality-result-filter/target-result.json | 22 + .../quality-result-stats/target-result.json | 16 + .../quality-result-verify/target-result.json | 16 + .../quality-result-stats/target-result.json | 15 + .../quality-result-verify/target-result.json | 16 + .../quality-result-ops-1/target-result.json | 22 + .../quality-result-ops-1/target-result.json | 22 + .../quality-result-ops-2/target-result.json | 22 + .../quality-result-config/target-result.json | 22 + .../target-result.invalid | 21 + .github/tests/quality_report_contract_test.py | 447 ++++++++++ .../workflows/quality-report-aggregate.yml | 157 ++++ .github/workflows/standards-validation.yml | 1 + decisions/quality-reporting.md | 162 ++++ quality-report/v1/quality_report.py | 765 ++++++++++++++++++ quality-report/v1/schema.json | 496 ++++++++++++ 17 files changed, 2244 insertions(+) create mode 100644 .github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid create mode 100644 .github/tests/quality_report_contract_test.py create mode 100644 .github/workflows/quality-report-aggregate.yml create mode 100644 decisions/quality-reporting.md create mode 100644 quality-report/v1/quality_report.py create mode 100644 quality-report/v1/schema.json diff --git a/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json new file mode 100644 index 0000000..d4f7466 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/config", + "outcome": "passed", + "metrics": { + "killed": 40, + "timeout": 10, + "survived": 5, + "no_coverage": 5, + "invalid": 2, + "ignored": 3, + "tool_score_numerator": 40, + "tool_score_denominator": 45, + "tool_score_pct": 88.89, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json new file mode 100644 index 0000000..a4b3164 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/filter", + "outcome": "passed", + "metrics": { + "killed": 30, + "timeout": 0, + "survived": 0, + "no_coverage": 10, + "invalid": 1, + "ignored": 2, + "tool_score_numerator": 30, + "tool_score_denominator": 30, + "tool_score_pct": 100.0, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json new file mode 100644 index 0000000..627a14e --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/docker/FuzzDecodeStats", + "outcome": "passed", + "metrics": { + "budget_seconds": 120, + "elapsed_seconds": 111.25, + "executions": 650000, + "new_interesting_inputs": 2 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json new file mode 100644 index 0000000..472e276 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 288.25, + "executions": 1250000, + "new_interesting_inputs": 3 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json new file mode 100644 index 0000000..0db4a95 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json @@ -0,0 +1,15 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/docker/FuzzDecodeStats", + "outcome": "crashed", + "metrics": null, + "diagnostic": "tool process exited 2", + "reproduction": { + "seed": "-test.fuzzcachedir=/tmp/fuzz-cache" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json new file mode 100644 index 0000000..c2f7e89 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 300.0, + "executions": 990000, + "new_interesting_inputs": 0 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json b/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json new file mode 100644 index 0000000..4a5d472 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-1", + "outcome": "passed", + "metrics": { + "killed": 20, + "timeout": 2, + "survived": 3, + "no_coverage": 5, + "invalid": 1, + "ignored": 2, + "tool_score_numerator": 22, + "tool_score_denominator": 25, + "tool_score_pct": 88.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json new file mode 100644 index 0000000..f79465e --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-1", + "outcome": "passed", + "metrics": { + "killed": 3, + "timeout": 1, + "survived": 1, + "no_coverage": 1, + "invalid": 0, + "ignored": 0, + "tool_score_numerator": 4, + "tool_score_denominator": 5, + "tool_score_pct": 80.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json new file mode 100644 index 0000000..403a77f --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-2", + "outcome": "passed", + "metrics": { + "killed": 2, + "timeout": 0, + "survived": 2, + "no_coverage": 0, + "invalid": 1, + "ignored": 1, + "tool_score_numerator": 2, + "tool_score_denominator": 4, + "tool_score_pct": 50.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json new file mode 100644 index 0000000..d1f3655 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/config", + "outcome": "passed", + "metrics": { + "killed": 12, + "timeout": 1, + "survived": 2, + "no_coverage": 1, + "invalid": 0, + "ignored": 1, + "tool_score_numerator": 12, + "tool_score_denominator": 14, + "tool_score_pct": 85.71, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid new file mode 100644 index 0000000..783dac7 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid @@ -0,0 +1,21 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/filter", + "outcome": "passed", + "metrics": { + "killed": 31, + "timeout": 4, + "survived": 2, + "no_coverage": 3, + "invalid": 1, + "ignored": 0, + "tool_score_numerator": 31, + "tool_score_denominator": 33, + "tool_score_pct": 93.94, + "tool_score_definition": "killed / (killed + survived)" + } + } diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py new file mode 100644 index 0000000..b456ce7 --- /dev/null +++ b/.github/tests/quality_report_contract_test.py @@ -0,0 +1,447 @@ +import json +from pathlib import Path +import runpy +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +CONTRACT_ROOT = ROOT / "quality-report" / "v1" +FIXTURES = ROOT / ".github" / "tests" / "fixtures" / "quality-report" / "v1" +SCRIPT = CONTRACT_ROOT / "quality_report.py" +SCHEMA = CONTRACT_ROOT / "schema.json" +WORKFLOW = ROOT / ".github" / "workflows" / "quality-report-aggregate.yml" +DECISION = ROOT / "decisions" / "quality-reporting.md" +SHA = "0123456789abcdef0123456789abcdef01234567" +QUALITY_REPORT = runpy.run_path(str(SCRIPT)) + + +class QualityReportContractTest(unittest.TestCase): + def test_fixture_inventory_covers_failure_and_score_boundaries(self): + self.assertEqual( + { + "canonical-score", + "complete", + "crash", + "incomplete", + "native-score", + "parse-error", + }, + {path.name for path in FIXTURES.iterdir() if path.is_dir()}, + ) + + def test_complete_fuzz_fixture_emits_a_valid_report_and_summary(self): + result, report, summary = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("quality-report/v1", report["schema_version"]) + self.assertEqual("report", report["kind"]) + self.assertEqual( + {"expected": 2, "reported": 2, "complete": True}, + report["completeness"], + ) + self.assertEqual("passed", report["outcome"]) + self.assertEqual(420, report["metrics"]["declared_budget_seconds"]) + self.assertEqual(399.5, report["metrics"]["elapsed_seconds"]) + self.assertEqual(1900000, report["metrics"]["executions"]) + self.assertEqual(5, report["metrics"]["new_interesting_inputs"]) + self.assertNotIn("canonical_score_pct", report["metrics"]) + self.assertIn("complete 2/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_incomplete_fixture_fails_closed_without_a_subset_score(self): + result, report, summary = self.aggregate( + "incomplete", + track="mutation", + tool="stryker", + expected=["apps-api-ops-1", "apps-api-ops-2"], + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual( + {"expected": 2, "reported": 1, "complete": False}, + report["completeness"], + ) + self.assertEqual("error", report["outcome"]) + self.assertIsNone(report["metrics"]) + self.assertIn("missing target: apps-api-ops-2", report["errors"]) + self.assertIn("partial 1/2", summary) + self.assertNotIn("Canonical score", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_crash_fixture_fails_even_when_every_target_reports(self): + result, report, summary = self.aggregate( + "crash", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + + self.assertNotEqual(0, result.returncode) + self.assertTrue(report["completeness"]["complete"]) + self.assertEqual("crashed", report["outcome"]) + self.assertIsNone(report["metrics"]) + self.assertIn("tool process exited 2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_parse_error_fixture_still_emits_an_error_report(self): + result, report, summary = self.aggregate( + "parse-error", + track="mutation", + tool="gremlins", + expected=["./internal/config", "./internal/filter"], + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual("error", report["outcome"]) + self.assertFalse(report["completeness"]["complete"]) + self.assertIsNone(report["metrics"]) + self.assertTrue( + any("invalid JSON" in error for error in report["errors"]), + report["errors"], + ) + self.assertIn("partial 1/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_canonical_score_counts_timeout_and_no_coverage(self): + result, report, summary = self.aggregate( + "canonical-score", + track="mutation", + tool="gremlins", + expected=["./internal/config", "./internal/filter"], + ) + + self.assertEqual(0, result.returncode, result.stderr) + metrics = report["metrics"] + self.assertEqual(70, metrics["killed"]) + self.assertEqual(10, metrics["timeout"]) + self.assertEqual(5, metrics["survived"]) + self.assertEqual(15, metrics["no_coverage"]) + self.assertEqual(80, metrics["detected"]) + self.assertEqual(20, metrics["missed"]) + self.assertEqual(80.0, metrics["canonical_score_pct"]) + self.assertEqual(93.33, metrics["tool_score_pct"]) + self.assertIn("Canonical score: **80.00%**", summary) + + def test_decision_defines_canonical_score_as_a_percentage(self): + decision = DECISION.read_text() + + self.assertIn( + "canonical_score_pct = 100 * detected / (detected + missed)", + decision, + ) + + def test_decision_documents_track_and_tool_pairings(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn("`mutation` permits `gremlins` and `stryker`", decision) + self.assertIn("`fuzz` permits `fast-check` and `go-fuzz`", decision) + + def test_native_score_is_weighted_and_keeps_its_definition(self): + result, report, summary = self.aggregate( + "native-score", + track="mutation", + tool="stryker", + expected=["apps-api-ops-1", "apps-api-ops-2"], + ) + + self.assertEqual(0, result.returncode, result.stderr) + metrics = report["metrics"] + self.assertEqual(60.0, metrics["canonical_score_pct"]) + self.assertEqual(66.67, metrics["tool_score_pct"]) + self.assertEqual(6, metrics["tool_score_numerator"]) + self.assertEqual(9, metrics["tool_score_denominator"]) + self.assertEqual( + "detected / mutants with test coverage", + metrics["tool_score_definition"], + ) + self.assertIn("Native score: **66.67%**", summary) + + def test_validator_rejects_unknown_report_fields(self): + result, report, _ = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + self.assertEqual(0, result.returncode, result.stderr) + report["unreviewed"] = True + + validation = self.validate_report(report) + + self.assertNotEqual(0, validation.returncode) + self.assertIn("unknown field", validation.stderr) + + def test_validator_rejects_missing_start_time_from_a_success_report(self): + result, report, _ = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + self.assertEqual(0, result.returncode, result.stderr) + report["run"]["started_at"] = None + + validation = self.validate_report(report) + + self.assertNotEqual(0, validation.returncode) + self.assertIn("started_at", validation.stderr) + + def test_validator_rejects_target_names_beyond_the_schema_limit(self): + target = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + target["target"]["name"] = "x" * 257 + + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], "256"): + QUALITY_REPORT["validate_target_document"](target) + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], "256"): + QUALITY_REPORT["parse_expected_targets"]( + json.dumps([target["target"]["name"]]) + ) + + def test_validator_requires_a_positive_declared_fuzz_budget(self): + metrics = { + "declared_budget_seconds": 0, + "elapsed_seconds": 0, + "executions": 0, + "new_interesting_inputs": 0, + } + + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], ">= 1"): + QUALITY_REPORT["validate_fuzz_report_metrics"](metrics, [], "$.metrics") + + def test_validator_requires_time_and_offset_in_timestamps(self): + for timestamp in ("2026-08-14", "2026-08-14T15:00:00"): + with self.subTest(timestamp=timestamp): + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], "RFC 3339" + ): + QUALITY_REPORT["require_timestamp"](timestamp, "$.run.started_at") + + def test_schema_is_versioned_and_fail_closed(self): + with SCHEMA.open() as schema_file: + schema = json.load(schema_file) + + self.assertEqual( + "https://json-schema.org/draft/2020-12/schema", + schema["$schema"], + ) + self.assertEqual( + "https://codeswhat.com/schemas/quality-report/v1/schema.json", + schema["$id"], + ) + self.assertEqual("quality-report/v1", schema["$defs"]["version"]["const"]) + self.assertNotIn("additionalProperties", schema) + self.assertTrue(self.all_objects_are_closed(schema)) + + def test_reusable_workflow_matches_the_public_contract(self): + workflow = WORKFLOW.read_text() + + self.assertIn(" workflow_call:", workflow) + for input_name in ( + "track", + "tool", + "policy", + "expected_targets", + "result_artifact_pattern", + "report_name", + "retention_days", + "fail_on_incomplete", + ): + self.assertIn(f" {input_name}:\n", workflow) + for output_name in ( + "report_artifact_name", + "completeness", + "expected_targets", + "reported_targets", + "outcome", + "canonical_score_pct", + ): + self.assertIn(f" {output_name}:\n", workflow) + + self.assertIn(" default: quality-result-*", workflow) + self.assertIn(" default: 90", workflow) + self.assertIn(" default: true", workflow) + self.assertIn(" if: always()", workflow) + self.assertIn( + "repository: ${{ fromJSON(toJSON(job)).workflow_repository }}", workflow + ) + self.assertIn("ref: ${{ fromJSON(toJSON(job)).workflow_sha }}", workflow) + self.assertIn( + "name: quality-report-${{ inputs.track }}-${{ github.run_id }}-${{ github.run_attempt }}", + workflow, + ) + self.assertIn("retention-days: ${{ inputs.retention_days }}", workflow) + self.assertIn("if: always()", workflow) + + actions = [ + line.split("uses: ", 1)[1].split()[0] + for line in workflow.splitlines() + if "uses: " in line + ] + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + for forbidden in ("git add", "git commit", "git push", "badges/"): + self.assertNotIn(forbidden, workflow) + + def test_public_decision_records_fail_closed_and_no_source_badges(self): + decision = DECISION.read_text() + + for expected in ( + "quality-report/v1", + "target-result.json", + "if: always()", + "quality-report---", + "90 days", + "partial N/M", + "Do not commit badge JSON to `main`", + "Native percentages are not comparable", + ): + self.assertIn(expected, decision) + + def test_standards_validation_runs_this_contract(self): + workflow = (ROOT / ".github/workflows/standards-validation.yml").read_text() + self.assertEqual( + 1, + workflow.count("python3 .github/tests/quality_report_contract_test.py"), + ) + + def aggregate(self, fixture, *, track, tool, expected): + self.assertTrue(SCRIPT.is_file(), f"missing aggregator: {SCRIPT}") + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) / "input" + shutil.copytree(FIXTURES / fixture, input_dir) + for invalid_fixture in input_dir.rglob("target-result.invalid"): + invalid_fixture.rename(invalid_fixture.with_name("target-result.json")) + output_dir = Path(temp_dir) / "output" + github_output = Path(temp_dir) / "github-output" + summary_path = Path(temp_dir) / "step-summary" + command = [ + "python3", + str(SCRIPT), + "aggregate", + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + "--repository", + "CodesWhat/example", + "--ref", + "refs/heads/dev/example", + "--sha", + SHA, + "--run-id", + "1234567890", + "--run-attempt", + "2", + "--event", + "workflow_dispatch", + "--run-url", + "https://github.com/CodesWhat/example/actions/runs/1234567890", + "--started-at", + "2026-08-13T12:34:56Z", + "--track", + track, + "--tool", + tool, + "--policy", + "advisory", + "--expected-targets", + json.dumps(expected), + "--report-name", + "Fixture quality report", + "--fail-on-incomplete", + "true", + "--github-output", + str(github_output), + "--github-step-summary", + str(summary_path), + ] + result = subprocess.run(command, capture_output=True, text=True) + report_path = output_dir / "report.json" + summary_file = output_dir / "summary.md" + self.assertTrue(report_path.is_file(), result.stderr) + self.assertTrue(summary_file.is_file(), result.stderr) + report = json.loads(report_path.read_text()) + summary = summary_file.read_text() + self.assertTrue(github_output.is_file(), result.stderr) + self.assertTrue(summary_path.is_file(), result.stderr) + github_outputs = dict( + line.split("=", 1) for line in github_output.read_text().splitlines() + ) + canonical_score = "" + if ( + report["track"] == "mutation" + and report["metrics"] is not None + and report["metrics"]["canonical_score_pct"] is not None + ): + canonical_score = "{0:.2f}".format( + report["metrics"]["canonical_score_pct"] + ) + self.assertEqual( + { + "report_artifact_name": "quality-report-{0}-1234567890-2".format( + track + ), + "completeness": str(report["completeness"]["complete"]).lower(), + "expected_targets": str(report["completeness"]["expected"]), + "reported_targets": str(report["completeness"]["reported"]), + "outcome": report["outcome"], + "canonical_score_pct": canonical_score, + }, + github_outputs, + ) + self.assertEqual(summary, summary_path.read_text()) + return result, report, summary + + def validate_report(self, report): + with tempfile.TemporaryDirectory() as temp_dir: + report_path = Path(temp_dir) / "report.json" + report_path.write_text(json.dumps(report)) + return subprocess.run( + ["python3", str(SCRIPT), "validate", str(report_path)], + capture_output=True, + text=True, + ) + + def all_objects_are_closed(self, value): + if isinstance(value, dict): + if ( + value.get("type") == "object" + and value.get("additionalProperties") is not False + ): + return False + return all(self.all_objects_are_closed(item) for item in value.values()) + if isinstance(value, list): + return all(self.all_objects_are_closed(item) for item in value) + return True + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/quality-report-aggregate.yml b/.github/workflows/quality-report-aggregate.yml new file mode 100644 index 0000000..86ef420 --- /dev/null +++ b/.github/workflows/quality-report-aggregate.yml @@ -0,0 +1,157 @@ +name: Quality Report Aggregate + +on: + workflow_call: + inputs: + track: + description: Quality track, mutation or fuzz + required: true + type: string + tool: + description: Native tool that produced the normalized target results + required: true + type: string + policy: + description: Reporting policy, advisory or signal + required: true + type: string + expected_targets: + description: Non-empty JSON array of unique target names + required: true + type: string + result_artifact_pattern: + description: Artifact-name pattern containing target-result.json files + required: false + type: string + default: quality-result-* + report_name: + description: Human-readable report name + required: true + type: string + retention_days: + description: Aggregate artifact retention in days + required: false + type: number + default: 90 + fail_on_incomplete: + description: Must remain true for CodesWhat telemetry + required: false + type: boolean + default: true + outputs: + report_artifact_name: + description: Fixed aggregate artifact name + value: ${{ jobs.aggregate.outputs.report_artifact_name }} + completeness: + description: Whether every expected target reported valid JSON + value: ${{ jobs.aggregate.outputs.completeness }} + expected_targets: + description: Number of expected targets + value: ${{ jobs.aggregate.outputs.expected_targets }} + reported_targets: + description: Number of valid expected target results + value: ${{ jobs.aggregate.outputs.reported_targets }} + outcome: + description: Aggregate outcome + value: ${{ jobs.aggregate.outputs.outcome }} + canonical_score_pct: + description: Canonical mutation score, empty for fuzz or partial reports + value: ${{ jobs.aggregate.outputs.canonical_score_pct }} + +permissions: {} + +jobs: + aggregate: + name: Quality Report Aggregate + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + outputs: + report_artifact_name: ${{ steps.aggregate.outputs.report_artifact_name }} + completeness: ${{ steps.aggregate.outputs.completeness }} + expected_targets: ${{ steps.aggregate.outputs.expected_targets }} + reported_targets: ${{ steps.aggregate.outputs.reported_targets }} + outcome: ${{ steps.aggregate.outputs.outcome }} + canonical_score_pct: ${{ steps.aggregate.outputs.canonical_score_pct }} + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out the called workflow contract + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # actionlint 1.7.12 does not model these current reusable-workflow fields. + repository: ${{ fromJSON(toJSON(job)).workflow_repository }} + ref: ${{ fromJSON(toJSON(job)).workflow_sha }} + path: quality-report-contract + persist-credentials: false + + - name: Read workflow run metadata + id: metadata + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + started_at="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq .run_started_at)" + test -n "$started_at" + echo "started_at=${started_at}" >> "$GITHUB_OUTPUT" + + - name: Download normalized target results + id: download + if: always() + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ inputs.result_artifact_pattern }} + path: quality-result-artifacts + + - name: Validate and aggregate target results + id: aggregate + if: always() + env: + EXPECTED_TARGETS: ${{ inputs.expected_targets }} + FAIL_ON_INCOMPLETE: ${{ inputs.fail_on_incomplete }} + POLICY: ${{ inputs.policy }} + REPORT_NAME: ${{ inputs.report_name }} + RUN_STARTED_AT: ${{ steps.metadata.outputs.started_at }} + TOOL: ${{ inputs.tool }} + TRACK: ${{ inputs.track }} + run: | + set -euo pipefail + python3 quality-report-contract/quality-report/v1/quality_report.py aggregate \ + --input-dir quality-result-artifacts \ + --output-dir quality-report-output \ + --repository "$GITHUB_REPOSITORY" \ + --ref "$GITHUB_REF" \ + --sha "$GITHUB_SHA" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --event "$GITHUB_EVENT_NAME" \ + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --started-at "$RUN_STARTED_AT" \ + --track "$TRACK" \ + --tool "$TOOL" \ + --policy "$POLICY" \ + --expected-targets "$EXPECTED_TARGETS" \ + --report-name "$REPORT_NAME" \ + --fail-on-incomplete "$FAIL_ON_INCOMPLETE" \ + --github-output "$GITHUB_OUTPUT" \ + --github-step-summary "$GITHUB_STEP_SUMMARY" + + - name: Upload normalized quality report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: quality-report-${{ inputs.track }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + quality-report-output/report.json + quality-report-output/summary.md + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index 192e5bd..ceb4606 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -45,6 +45,7 @@ jobs: ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' python3 .github/tests/greptile_config_contract_test.py + python3 .github/tests/quality_report_contract_test.py python3 .github/tests/reusable_ci_contract_test.py - name: Lint Markdown diff --git a/decisions/quality-reporting.md b/decisions/quality-reporting.md new file mode 100644 index 0000000..6182c97 --- /dev/null +++ b/decisions/quality-reporting.md @@ -0,0 +1,162 @@ +# Quality Reporting + +Last updated: 2026-08-13 + +## Decision + +Keep each repository's native test tool. Normalize the result envelope, not the +runner: + +- Drydock keeps Stryker and fast-check. +- Sockguard keeps Gremlins and Go fuzzing. +- Portwing keeps Gremlins and Go fuzzing. + +Mutation stays advisory and absent from required PR checks. A scheduled or +manual mutation workflow must still fail when its telemetry is incomplete or a +tool crashes. Non-blocking means it does not gate a PR. It does not mean the +workflow may report success after losing results. + +Long fuzzing succeeds only when every declared target completes its budget. A +crash, repeated boundary flake, setup failure, or missing target result makes +the workflow red. + +## Evidence behind the decision + +The 2026-08-01 Drydock mutation run completed at the workflow level while 13 of +27 Stryker shards failed. Its aggregate used `--allow-missing` and published an +81.79% score from the remaining 14 shards. The workflow status and numeric +badge therefore described a partial result as success. + +Sockguard's existing 97.8% badge uses `killed / (killed + lived)`. Counting +timeout as detected and no-coverage as missed gives 93.91% for the same run. +Portwing has no aggregate badge; applying the same canonical definition to its +latest complete Gremlins run gives 77.31%, versus 81.98% under Gremlins' +efficacy definition. Native percentages are not comparable until their +denominators are explicit. + +Sockguard inventories all 48 in-tree fuzzers and covers every target in at +least one tier. Portwing's `FuzzVerifyRequest` runs in primary CI and scheduled +long tiers. Drydock's five fast-check fuzz tests run as ordinary Vitest tests +with default run counts and have no scheduled long-fuzz workflow. + +## Normalized envelope + +Every matrix leg writes one `target-result.json`. An always-running aggregate +job validates the set and publishes `quality-report/v1` with: + +- `repository`, `ref`, `sha` +- `run`: `id`, `attempt`, `event`, `url`, `started_at` +- `track`: `mutation` or `fuzz` +- `tool`: `mutation` permits `gremlins` and `stryker`; `fuzz` permits + `fast-check` and `go-fuzz` +- `policy`: `advisory` or `signal` +- `completeness`: `expected`, `reported`, `complete` +- `outcome`: `passed`, `failed`, `crashed`, `flaked`, `cancelled`, `error` +- `targets`: the declared target names and their individual results +- track-specific `metrics` + +The schema and dependency-free validator live under [`quality-report/v1`](../quality-report/v1). +Both target-result fragments and aggregate reports carry the exact +`quality-report/v1` version. Objects reject unknown fields, JSON parsing rejects +duplicate fields and non-finite numbers, and semantic validation recomputes +completeness, outcomes, and scores. Callers pass `expected_targets` as a +non-empty JSON array of unique target names. + +Mutation metrics are: + +- `killed`, `timeout`, `survived`, `no_coverage`, `invalid`, `ignored` +- `detected = killed + timeout` +- `missed = survived + no_coverage` +- `canonical_score_pct = 100 * detected / (detected + missed)`, rounded + half-up to two decimals, or `null` when the denominator is zero +- `tool_score_pct` and `tool_score_definition` + +Each target also supplies the native numerator and denominator. The aggregate +weights those counts instead of averaging percentages, and rejects mixed native +score definitions. The native score remains available with its definition. +Only the canonical score is comparable across mutation tools. + +Fuzz metrics include the declared budget, elapsed time, and executions or new +interesting inputs when the runner exposes them. Do not compare execution +counts between runners or tools. A failed, crashed, or flaked fuzz result must +retain a seed, path, or corpus reference for reproduction. + +## Reusable workflow contract + +Inputs: + +- `track` +- `tool` +- `policy` +- `expected_targets` +- `result_artifact_pattern`, default `quality-result-*` +- `report_name` +- `retention_days`, default `90` +- `fail_on_incomplete`, always `true` for CodesWhat telemetry + +Outputs: + +- `report_artifact_name` +- `completeness` +- `expected_targets` +- `reported_targets` +- `outcome` +- `canonical_score_pct` for mutation reports + +The aggregate artifact is named +`quality-report---` and contains `report.json` plus +`summary.md`. Native reports and crash corpora remain separate artifacts. +Target result JSON is always uploaded; crash forensics remain failure-only. +Normalized aggregate artifacts use at least 90 days of retention. + +The reusable workflow checks out its own contract at the exact called-workflow +commit via `job.workflow_repository` and `job.workflow_sha`. Product workflows +remain responsible for native tool adapters and uploading their target-result +artifacts. + +## Status and badge semantics + +- The aggregate job runs under `if: always()` and fails on an incomplete set, + tool crash, parse error, invalid contract, or policy failure. +- Numeric mutation badges publish only a complete canonical score. An + incomplete run displays `partial N/M`, never a subset score or stale score. +- Do not commit badge JSON to `main`. Bot commits bypass the dev-to-main flow + and make telemetry part of source history. +- Use the native workflow-status badge as the stable shared badge. Put numeric + scores and native dashboard links in the run summary and retained artifact + until there is a non-source telemetry endpoint. +- Long-fuzz failures must retain the runner seed, path, or corpus needed to + reproduce the failure. + +## Migration order + +1. Add and test the schema, validator, and reusable aggregate in + `CodesWhat/.github`. +2. Canary Portwing. It has the smallest Gremlins and fuzz matrices. Keep + `FuzzVerifyRequest` in a long tier and preserve its source-to-tier inventory + contract. +3. Migrate Sockguard's Gremlins aggregation, retire its direct badge commit, + and preserve its existing fuzz inventory contract. +4. Migrate Drydock's Stryker aggregate, remove partial score publication, then + add a scheduled fast-check workflow with explicit run count and seed/path + reproduction. +5. Manually dispatch each workflow on the exact active dev head before syncing + it to `main`. Scheduled workflows exercise `main`, not the active dev branch. + +Native runner adapters and product callers are intentionally outside this +foundation change. Each migration supplies and tests its own mapping from +native output into the target-result contract. + +## Risks and controls + +- The three first-of-month mutation runs compete for organization runner + capacity. Stagger their schedules. +- Gremlins text parsing is version-sensitive. Pin the tool and test fixtures + from real output. +- Stryker reports compile/runtime-invalid mutants and may lose shards. Preserve + native categories and fail closed on a missing shard. +- Fast-check failures are not reproducible without seed and path. Make both + mandatory failure fields. +- Artifacts expire. Ninety days is the minimum retention for normalized + reports; durable trend storage is a separate decision. +- `continue-on-error` on a matrix leg must not make the final aggregate green. diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py new file mode 100644 index 0000000..8b8c93a --- /dev/null +++ b/quality-report/v1/quality_report.py @@ -0,0 +1,765 @@ +#!/usr/bin/env python3 +"""Validate and aggregate CodesWhat quality-report/v1 result envelopes.""" + +import argparse +from datetime import datetime +from decimal import Decimal, ROUND_HALF_UP +import json +import math +from pathlib import Path +import re +import sys +from urllib.parse import urlparse + + +VERSION = "quality-report/v1" +KINDS = {"target-result", "report"} +TRACK_TOOLS = { + "mutation": {"gremlins", "stryker"}, + "fuzz": {"fast-check", "go-fuzz"}, +} +POLICIES = {"advisory", "signal"} +OUTCOMES = {"passed", "failed", "crashed", "flaked", "cancelled", "error"} +OUTCOME_PRIORITY = ("error", "crashed", "cancelled", "flaked", "failed") +MUTATION_COUNT_FIELDS = ( + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", +) +TARGET_MUTATION_FIELDS = set(MUTATION_COUNT_FIELDS) | { + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition", +} +REPORT_MUTATION_FIELDS = TARGET_MUTATION_FIELDS | { + "detected", + "missed", + "canonical_score_pct", +} +TARGET_FUZZ_REQUIRED = {"budget_seconds", "elapsed_seconds"} +TARGET_FUZZ_OPTIONAL = {"executions", "new_interesting_inputs"} +REPORT_FUZZ_FIELDS = { + "declared_budget_seconds", + "elapsed_seconds", + "executions", + "new_interesting_inputs", +} +SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +RFC3339_PATTERN = re.compile( + r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})$" +) + + +class ContractError(ValueError): + pass + + +def reject_constant(value): + raise ContractError("non-finite JSON number: {0}".format(value)) + + +def reject_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ContractError("duplicate JSON field: {0}".format(key)) + result[key] = value + return result + + +def load_json_text(text, source): + try: + return json.loads( + text, + object_pairs_hook=reject_duplicate_keys, + parse_constant=reject_constant, + ) + except (json.JSONDecodeError, ContractError) as error: + raise ContractError("{0}: invalid JSON: {1}".format(source, error)) + + +def load_json_file(path): + try: + return load_json_text(path.read_text(), str(path)) + except OSError as error: + raise ContractError("{0}: cannot read JSON: {1}".format(path, error)) + + +def require_object(value, path, required, optional=()): + if not isinstance(value, dict): + raise ContractError("{0}: expected object".format(path)) + required = set(required) + allowed = required | set(optional) + missing = sorted(required - set(value)) + unknown = sorted(set(value) - allowed) + if missing: + raise ContractError( + "{0}: missing field(s): {1}".format(path, ", ".join(missing)) + ) + if unknown: + raise ContractError( + "{0}: unknown field(s): {1}".format(path, ", ".join(unknown)) + ) + return value + + +def require_string(value, path, *, pattern=None, max_length=None): + if not isinstance(value, str) or not value or "\n" in value or "\r" in value: + raise ContractError("{0}: expected non-empty single-line string".format(path)) + if max_length is not None and len(value) > max_length: + raise ContractError( + "{0}: expected at most {1} characters".format(path, max_length) + ) + if pattern is not None and pattern.fullmatch(value) is None: + raise ContractError("{0}: invalid value".format(path)) + return value + + +def require_enum(value, path, choices): + require_string(value, path) + if value not in choices: + raise ContractError( + "{0}: expected one of {1}".format(path, ", ".join(sorted(choices))) + ) + return value + + +def require_integer(value, path, *, minimum=0): + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ContractError("{0}: expected integer >= {1}".format(path, minimum)) + return value + + +def require_number(value, path, *, minimum=0): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ContractError("{0}: expected number >= {1}".format(path, minimum)) + if not math.isfinite(value) or value < minimum: + raise ContractError("{0}: expected finite number >= {1}".format(path, minimum)) + return value + + +def require_nullable_integer(value, path): + if value is not None: + require_integer(value, path) + + +def require_percentage(value, path, *, nullable=False): + if value is None and nullable: + return + require_number(value, path) + if value > 100: + raise ContractError("{0}: expected percentage <= 100".format(path)) + + +def require_url(value, path): + require_string(value, path) + parsed = urlparse(value) + if parsed.scheme != "https" or not parsed.netloc: + raise ContractError("{0}: expected HTTPS URL".format(path)) + + +def require_timestamp(value, path): + require_string(value, path) + try: + if RFC3339_PATTERN.fullmatch(value) is None: + raise ValueError + datetime.fromisoformat(value.replace("Z", "+00:00").replace("z", "+00:00")) + except ValueError: + raise ContractError("{0}: expected RFC 3339 timestamp".format(path)) + + +def percentage(numerator, denominator): + if denominator == 0: + return None + value = Decimal(numerator) * Decimal(100) / Decimal(denominator) + return float(value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)) + + +def validate_mutation_target_metrics(metrics, path): + require_object(metrics, path, TARGET_MUTATION_FIELDS) + for field in MUTATION_COUNT_FIELDS: + require_integer(metrics[field], "{0}.{1}".format(path, field)) + numerator = require_integer( + metrics["tool_score_numerator"], path + ".tool_score_numerator" + ) + denominator = require_integer( + metrics["tool_score_denominator"], path + ".tool_score_denominator" + ) + require_percentage( + metrics["tool_score_pct"], path + ".tool_score_pct", nullable=True + ) + require_string(metrics["tool_score_definition"], path + ".tool_score_definition") + expected = percentage(numerator, denominator) + if metrics["tool_score_pct"] != expected: + raise ContractError( + "{0}.tool_score_pct: expected {1} from numerator/denominator".format( + path, expected + ) + ) + + +def validate_fuzz_target_metrics(metrics, path): + require_object(metrics, path, TARGET_FUZZ_REQUIRED, TARGET_FUZZ_OPTIONAL) + require_integer(metrics["budget_seconds"], path + ".budget_seconds", minimum=1) + require_number(metrics["elapsed_seconds"], path + ".elapsed_seconds") + for field in TARGET_FUZZ_OPTIONAL: + if field in metrics: + require_integer(metrics[field], "{0}.{1}".format(path, field)) + + +def validate_reproduction(value, path): + require_object(value, path, (), {"seed", "path", "corpus"}) + if not value: + raise ContractError("{0}: expected seed, path, or corpus".format(path)) + for field, item in value.items(): + require_string(item, "{0}.{1}".format(path, field)) + + +def validate_target(target, track, path): + require_object( + target, + path, + {"name", "outcome", "metrics"}, + {"diagnostic", "reproduction"}, + ) + require_string(target["name"], path + ".name", max_length=256) + outcome = require_enum(target["outcome"], path + ".outcome", OUTCOMES) + metrics = target["metrics"] + if metrics is None: + if outcome == "passed": + raise ContractError( + "{0}.metrics: passed target requires metrics".format(path) + ) + elif track == "mutation": + validate_mutation_target_metrics(metrics, path + ".metrics") + else: + validate_fuzz_target_metrics(metrics, path + ".metrics") + + if outcome == "passed": + if "diagnostic" in target or "reproduction" in target: + raise ContractError( + "{0}: passed target cannot contain diagnostic or reproduction".format( + path + ) + ) + else: + if "diagnostic" not in target: + raise ContractError( + "{0}: non-passed target requires diagnostic".format(path) + ) + require_string(target["diagnostic"], path + ".diagnostic") + if track == "fuzz" and outcome in {"failed", "crashed", "flaked"}: + if "reproduction" not in target: + raise ContractError( + "{0}: failed fuzz target requires reproduction".format(path) + ) + if "reproduction" in target: + validate_reproduction(target["reproduction"], path + ".reproduction") + + +def validate_target_document(document, path="$"): + require_object( + document, + path, + {"schema_version", "kind", "track", "tool", "target"}, + ) + if document["schema_version"] != VERSION: + raise ContractError("{0}.schema_version: expected {1}".format(path, VERSION)) + if document["kind"] != "target-result": + raise ContractError("{0}.kind: expected target-result".format(path)) + track = require_enum(document["track"], path + ".track", set(TRACK_TOOLS)) + tool = require_enum( + document["tool"], path + ".tool", set().union(*TRACK_TOOLS.values()) + ) + if tool not in TRACK_TOOLS[track]: + raise ContractError("{0}.tool: {1} is not a {2} tool".format(path, tool, track)) + validate_target(document["target"], track, path + ".target") + return document + + +def aggregate_mutation_metrics(targets): + metrics = {field: 0 for field in MUTATION_COUNT_FIELDS} + definitions = set() + for target in targets: + target_metrics = target["metrics"] + for field in MUTATION_COUNT_FIELDS: + metrics[field] += target_metrics[field] + metrics.setdefault("tool_score_numerator", 0) + metrics.setdefault("tool_score_denominator", 0) + metrics["tool_score_numerator"] += target_metrics["tool_score_numerator"] + metrics["tool_score_denominator"] += target_metrics["tool_score_denominator"] + definitions.add(target_metrics["tool_score_definition"]) + if len(definitions) != 1: + raise ContractError("target results use inconsistent native score definitions") + metrics["detected"] = metrics["killed"] + metrics["timeout"] + metrics["missed"] = metrics["survived"] + metrics["no_coverage"] + metrics["canonical_score_pct"] = percentage( + metrics["detected"], metrics["detected"] + metrics["missed"] + ) + metrics["tool_score_pct"] = percentage( + metrics["tool_score_numerator"], metrics["tool_score_denominator"] + ) + metrics["tool_score_definition"] = definitions.pop() + return metrics + + +def aggregate_fuzz_metrics(targets): + metrics = { + "declared_budget_seconds": sum( + target["metrics"]["budget_seconds"] for target in targets + ), + "elapsed_seconds": sum( + target["metrics"]["elapsed_seconds"] for target in targets + ), + } + for field in ("executions", "new_interesting_inputs"): + if all(field in target["metrics"] for target in targets): + metrics[field] = sum(target["metrics"][field] for target in targets) + else: + metrics[field] = None + return metrics + + +def derive_outcome(targets, errors): + if errors: + return "error" + outcomes = {target["outcome"] for target in targets} + for outcome in OUTCOME_PRIORITY: + if outcome in outcomes: + return outcome + return "passed" + + +def validate_mutation_report_metrics(metrics, targets, path): + require_object(metrics, path, REPORT_MUTATION_FIELDS) + for field in MUTATION_COUNT_FIELDS + ( + "detected", + "missed", + "tool_score_numerator", + "tool_score_denominator", + ): + require_integer(metrics[field], "{0}.{1}".format(path, field)) + require_percentage( + metrics["canonical_score_pct"], path + ".canonical_score_pct", nullable=True + ) + require_percentage( + metrics["tool_score_pct"], path + ".tool_score_pct", nullable=True + ) + require_string(metrics["tool_score_definition"], path + ".tool_score_definition") + expected = aggregate_mutation_metrics(targets) + if metrics != expected: + raise ContractError( + "{0}: aggregate mutation metrics do not match targets".format(path) + ) + + +def validate_fuzz_report_metrics(metrics, targets, path): + require_object(metrics, path, REPORT_FUZZ_FIELDS) + require_integer( + metrics["declared_budget_seconds"], + path + ".declared_budget_seconds", + minimum=1, + ) + require_number(metrics["elapsed_seconds"], path + ".elapsed_seconds") + require_nullable_integer(metrics["executions"], path + ".executions") + require_nullable_integer( + metrics["new_interesting_inputs"], path + ".new_interesting_inputs" + ) + expected = aggregate_fuzz_metrics(targets) + if metrics != expected: + raise ContractError( + "{0}: aggregate fuzz metrics do not match targets".format(path) + ) + + +def validate_report(document, path="$"): + require_object( + document, + path, + { + "schema_version", + "kind", + "repository", + "ref", + "sha", + "run", + "report_name", + "track", + "tool", + "policy", + "completeness", + "outcome", + "targets", + "metrics", + "errors", + }, + ) + if document["schema_version"] != VERSION: + raise ContractError("{0}.schema_version: expected {1}".format(path, VERSION)) + if document["kind"] != "report": + raise ContractError("{0}.kind: expected report".format(path)) + require_string(document["repository"], path + ".repository") + require_string(document["ref"], path + ".ref") + require_string(document["sha"], path + ".sha", pattern=SHA_PATTERN) + require_string(document["report_name"], path + ".report_name") + track = require_enum(document["track"], path + ".track", set(TRACK_TOOLS)) + tool = require_enum( + document["tool"], path + ".tool", set().union(*TRACK_TOOLS.values()) + ) + if tool not in TRACK_TOOLS[track]: + raise ContractError("{0}.tool: {1} is not a {2} tool".format(path, tool, track)) + require_enum(document["policy"], path + ".policy", POLICIES) + + run = require_object( + document["run"], + path + ".run", + {"id", "attempt", "event", "url", "started_at"}, + ) + require_string(run["id"], path + ".run.id") + require_integer(run["attempt"], path + ".run.attempt", minimum=1) + require_string(run["event"], path + ".run.event") + require_url(run["url"], path + ".run.url") + if run["started_at"] is not None: + require_timestamp(run["started_at"], path + ".run.started_at") + + completeness = require_object( + document["completeness"], + path + ".completeness", + {"expected", "reported", "complete"}, + ) + require_integer( + completeness["expected"], path + ".completeness.expected", minimum=1 + ) + require_integer(completeness["reported"], path + ".completeness.reported") + if not isinstance(completeness["complete"], bool): + raise ContractError(path + ".completeness.complete: expected boolean") + + targets = document["targets"] + if not isinstance(targets, list): + raise ContractError(path + ".targets: expected array") + names = [] + for index, target in enumerate(targets): + validate_target(target, track, "{0}.targets[{1}]".format(path, index)) + names.append(target["name"]) + if len(names) != len(set(names)): + raise ContractError(path + ".targets: duplicate target name") + if names != sorted(names): + raise ContractError(path + ".targets: targets must be sorted by name") + if completeness["reported"] != len(targets): + raise ContractError(path + ".completeness.reported: does not match targets") + + errors = document["errors"] + if not isinstance(errors, list) or any( + not isinstance(error, str) or not error for error in errors + ): + raise ContractError(path + ".errors: expected non-empty strings") + if errors != sorted(set(errors)): + raise ContractError(path + ".errors: errors must be sorted and unique") + if run["started_at"] is None and not errors: + raise ContractError(path + ".run.started_at: required for an error-free report") + expected_complete = ( + completeness["expected"] == completeness["reported"] and not errors + ) + if completeness["complete"] != expected_complete: + raise ContractError(path + ".completeness.complete: inconsistent value") + + outcome = require_enum(document["outcome"], path + ".outcome", OUTCOMES) + if outcome != derive_outcome(targets, errors): + raise ContractError(path + ".outcome: does not match target results") + + metrics = document["metrics"] + metrics_available = ( + completeness["complete"] + and bool(targets) + and all(target["metrics"] is not None for target in targets) + ) + if not metrics_available: + if metrics is not None: + raise ContractError(path + ".metrics: must be null for partial results") + elif track == "mutation": + validate_mutation_report_metrics(metrics, targets, path + ".metrics") + else: + validate_fuzz_report_metrics(metrics, targets, path + ".metrics") + return document + + +def validate_document(document): + if not isinstance(document, dict): + raise ContractError("$: expected object") + kind = document.get("kind") + if kind not in KINDS: + raise ContractError("$.kind: expected one of report, target-result") + if kind == "target-result": + return validate_target_document(document) + return validate_report(document) + + +def parse_expected_targets(raw): + value = load_json_text(raw, "expected_targets") + if not isinstance(value, list) or not value: + raise ContractError("expected_targets: expected a non-empty JSON array") + targets = [] + for index, target in enumerate(value): + targets.append( + require_string( + target, + "expected_targets[{0}]".format(index), + max_length=256, + ) + ) + if len(targets) != len(set(targets)): + raise ContractError("expected_targets: duplicate target") + return targets + + +def relative_label(path, root): + try: + return str(path.relative_to(root)) + except ValueError: + return path.name + + +def collect_targets(input_dir, expected_targets, track, tool): + errors = [] + reported = {} + paths = sorted(input_dir.rglob("target-result.json")) if input_dir.is_dir() else [] + if not paths: + errors.append("no target-result.json files found") + for path in paths: + label = relative_label(path, input_dir) + try: + document = load_json_file(path) + validate_target_document(document, label) + if document["track"] != track: + raise ContractError( + "{0}.track: expected {1}, got {2}".format( + label, track, document["track"] + ) + ) + if document["tool"] != tool: + raise ContractError( + "{0}.tool: expected {1}, got {2}".format( + label, tool, document["tool"] + ) + ) + target = document["target"] + name = target["name"] + if name not in expected_targets: + raise ContractError("{0}: unexpected target: {1}".format(label, name)) + if name in reported: + raise ContractError("{0}: duplicate target: {1}".format(label, name)) + reported[name] = target + except ContractError as error: + errors.append(str(error)) + for target in expected_targets: + if target not in reported: + errors.append("missing target: {0}".format(target)) + return [reported[name] for name in sorted(reported)], sorted(set(errors)) + + +def build_summary(report): + completeness = report["completeness"] + if completeness["complete"]: + completeness_text = "complete {0}/{1}".format( + completeness["reported"], completeness["expected"] + ) + else: + completeness_text = "partial {0}/{1}".format( + completeness["reported"], completeness["expected"] + ) + lines = [ + "# {0}".format(report["report_name"]), + "", + "- Track: `{0}` via `{1}`".format(report["track"], report["tool"]), + "- Policy: `{0}`".format(report["policy"]), + "- Completeness: **{0}**".format(completeness_text), + "- Outcome: **{0}**".format(report["outcome"]), + ] + metrics = report["metrics"] + if report["track"] == "mutation" and metrics is not None: + canonical = metrics["canonical_score_pct"] + native = metrics["tool_score_pct"] + if canonical is not None: + lines.append("- Canonical score: **{0:.2f}%**".format(canonical)) + if native is not None: + lines.append( + "- Native score: **{0:.2f}%** ({1})".format( + native, metrics["tool_score_definition"] + ) + ) + lines.extend(("", "## Targets", "")) + for target in report["targets"]: + detail = "" + if "diagnostic" in target: + detail = ": {0}".format(target["diagnostic"]) + lines.append( + "- `{0}`: {1}{2}".format(target["name"], target["outcome"], detail) + ) + if report["errors"]: + lines.extend(("", "## Contract errors", "")) + lines.extend("- {0}".format(error) for error in report["errors"]) + return "\n".join(lines) + "\n" + + +def append_lines(path, values): + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as output: + for key, value in values.items(): + output.write("{0}={1}\n".format(key, value)) + + +def command_aggregate(args): + errors = [] + try: + expected_targets = parse_expected_targets(args.expected_targets) + except ContractError as error: + expected_targets = ["invalid-expected-targets"] + errors.append(str(error)) + if args.fail_on_incomplete != "true": + errors.append("fail_on_incomplete must be true for CodesWhat telemetry") + try: + require_timestamp(args.started_at, "started_at") + started_at = args.started_at + except ContractError as error: + started_at = None + errors.append(str(error)) + + targets, collection_errors = collect_targets( + args.input_dir, expected_targets, args.track, args.tool + ) + errors.extend(collection_errors) + errors = sorted(set(errors)) + completeness = { + "expected": len(expected_targets), + "reported": len(targets), + "complete": len(targets) == len(expected_targets) and not errors, + } + metrics = None + if completeness["complete"] and all( + target["metrics"] is not None for target in targets + ): + try: + if args.track == "mutation": + metrics = aggregate_mutation_metrics(targets) + else: + metrics = aggregate_fuzz_metrics(targets) + except ContractError as error: + errors.append(str(error)) + errors = sorted(set(errors)) + completeness["complete"] = False + + report = { + "schema_version": VERSION, + "kind": "report", + "repository": args.repository, + "ref": args.ref, + "sha": args.sha, + "run": { + "id": args.run_id, + "attempt": args.run_attempt, + "event": args.event, + "url": args.run_url, + "started_at": started_at, + }, + "report_name": args.report_name, + "track": args.track, + "tool": args.tool, + "policy": args.policy, + "completeness": completeness, + "outcome": derive_outcome(targets, errors), + "targets": targets, + "metrics": metrics, + "errors": errors, + } + validate_report(report) + + args.output_dir.mkdir(parents=True, exist_ok=True) + summary = build_summary(report) + (args.output_dir / "report.json").write_text( + json.dumps(report, indent=2, allow_nan=False) + "\n" + ) + (args.output_dir / "summary.md").write_text(summary) + append_lines( + args.github_output, + { + "report_artifact_name": "quality-report-{0}-{1}-{2}".format( + args.track, args.run_id, args.run_attempt + ), + "completeness": str(completeness["complete"]).lower(), + "expected_targets": completeness["expected"], + "reported_targets": completeness["reported"], + "outcome": report["outcome"], + "canonical_score_pct": ( + "" + if metrics is None + or args.track != "mutation" + or metrics["canonical_score_pct"] is None + else "{0:.2f}".format(metrics["canonical_score_pct"]) + ), + }, + ) + if args.github_step_summary is not None: + args.github_step_summary.parent.mkdir(parents=True, exist_ok=True) + with args.github_step_summary.open("a") as step_summary: + step_summary.write(summary) + return 0 if completeness["complete"] and report["outcome"] == "passed" else 1 + + +def command_validate(args): + document = load_json_file(args.document) + validate_document(document) + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate_parser = subparsers.add_parser("validate") + validate_parser.add_argument("document", type=Path) + validate_parser.set_defaults(handler=command_validate) + + aggregate_parser = subparsers.add_parser("aggregate") + aggregate_parser.add_argument("--input-dir", type=Path, required=True) + aggregate_parser.add_argument("--output-dir", type=Path, required=True) + aggregate_parser.add_argument("--repository", required=True) + aggregate_parser.add_argument("--ref", required=True) + aggregate_parser.add_argument("--sha", required=True) + aggregate_parser.add_argument("--run-id", required=True) + aggregate_parser.add_argument("--run-attempt", type=int, required=True) + aggregate_parser.add_argument("--event", required=True) + aggregate_parser.add_argument("--run-url", required=True) + aggregate_parser.add_argument("--started-at", required=True) + aggregate_parser.add_argument("--track", choices=sorted(TRACK_TOOLS), required=True) + aggregate_parser.add_argument( + "--tool", choices=sorted(set().union(*TRACK_TOOLS.values())), required=True + ) + aggregate_parser.add_argument("--policy", choices=sorted(POLICIES), required=True) + aggregate_parser.add_argument("--expected-targets", required=True) + aggregate_parser.add_argument("--report-name", required=True) + aggregate_parser.add_argument( + "--fail-on-incomplete", choices=("true", "false"), required=True + ) + aggregate_parser.add_argument("--github-output", type=Path) + aggregate_parser.add_argument("--github-step-summary", type=Path) + aggregate_parser.set_defaults(handler=command_aggregate) + return parser + + +def main(): + args = build_parser().parse_args() + try: + return args.handler(args) + except ContractError as error: + print("quality-report: {0}".format(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/quality-report/v1/schema.json b/quality-report/v1/schema.json new file mode 100644 index 0000000..46c38f1 --- /dev/null +++ b/quality-report/v1/schema.json @@ -0,0 +1,496 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeswhat.com/schemas/quality-report/v1/schema.json", + "title": "CodesWhat quality-report/v1", + "description": "Normalized target results and aggregate quality reports.", + "oneOf": [ + { + "$ref": "#/$defs/targetResult" + }, + { + "$ref": "#/$defs/report" + } + ], + "$defs": { + "version": { + "type": "string", + "const": "quality-report/v1" + }, + "outcome": { + "type": "string", + "enum": [ + "passed", + "failed", + "crashed", + "flaked", + "cancelled", + "error" + ] + }, + "reproduction": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "seed": { + "type": "string", + "minLength": 1 + }, + "path": { + "type": "string", + "minLength": 1 + }, + "corpus": { + "type": "string", + "minLength": 1 + } + } + }, + "mutationTargetMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition" + ], + "properties": { + "killed": { + "type": "integer", + "minimum": 0 + }, + "timeout": { + "type": "integer", + "minimum": 0 + }, + "survived": { + "type": "integer", + "minimum": 0 + }, + "no_coverage": { + "type": "integer", + "minimum": 0 + }, + "invalid": { + "type": "integer", + "minimum": 0 + }, + "ignored": { + "type": "integer", + "minimum": 0 + }, + "tool_score_numerator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_denominator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_definition": { + "type": "string", + "minLength": 1 + } + } + }, + "fuzzTargetMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "budget_seconds", + "elapsed_seconds" + ], + "properties": { + "budget_seconds": { + "type": "integer", + "minimum": 1 + }, + "elapsed_seconds": { + "type": "number", + "minimum": 0 + }, + "executions": { + "type": "integer", + "minimum": 0 + }, + "new_interesting_inputs": { + "type": "integer", + "minimum": 0 + } + } + }, + "target": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "outcome", + "metrics" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\r\\n]+$" + }, + "outcome": { + "$ref": "#/$defs/outcome" + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationTargetMetrics" + }, + { + "$ref": "#/$defs/fuzzTargetMetrics" + }, + { + "type": "null" + } + ] + }, + "diagnostic": { + "type": "string", + "minLength": 1 + }, + "reproduction": { + "$ref": "#/$defs/reproduction" + } + } + }, + "targetResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "track", + "tool", + "target" + ], + "properties": { + "schema_version": { + "$ref": "#/$defs/version" + }, + "kind": { + "type": "string", + "const": "target-result" + }, + "track": { + "type": "string", + "enum": [ + "mutation", + "fuzz" + ] + }, + "tool": { + "type": "string", + "enum": [ + "stryker", + "gremlins", + "go-fuzz", + "fast-check" + ] + }, + "target": { + "$ref": "#/$defs/target" + } + } + }, + "mutationReportMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", + "detected", + "missed", + "canonical_score_pct", + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition" + ], + "properties": { + "killed": { + "type": "integer", + "minimum": 0 + }, + "timeout": { + "type": "integer", + "minimum": 0 + }, + "survived": { + "type": "integer", + "minimum": 0 + }, + "no_coverage": { + "type": "integer", + "minimum": 0 + }, + "invalid": { + "type": "integer", + "minimum": 0 + }, + "ignored": { + "type": "integer", + "minimum": 0 + }, + "detected": { + "type": "integer", + "minimum": 0 + }, + "missed": { + "type": "integer", + "minimum": 0 + }, + "canonical_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_numerator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_denominator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_definition": { + "type": "string", + "minLength": 1 + } + } + }, + "fuzzReportMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "declared_budget_seconds", + "elapsed_seconds", + "executions", + "new_interesting_inputs" + ], + "properties": { + "declared_budget_seconds": { + "type": "integer", + "minimum": 1 + }, + "elapsed_seconds": { + "type": "number", + "minimum": 0 + }, + "executions": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "new_interesting_inputs": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + } + } + }, + "run": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "attempt", + "event", + "url", + "started_at" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "attempt": { + "type": "integer", + "minimum": 1 + }, + "event": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https://" + }, + "started_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } + }, + "completeness": { + "type": "object", + "additionalProperties": false, + "required": [ + "expected", + "reported", + "complete" + ], + "properties": { + "expected": { + "type": "integer", + "minimum": 1 + }, + "reported": { + "type": "integer", + "minimum": 0 + }, + "complete": { + "type": "boolean" + } + } + }, + "report": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "repository", + "ref", + "sha", + "run", + "report_name", + "track", + "tool", + "policy", + "completeness", + "outcome", + "targets", + "metrics", + "errors" + ], + "properties": { + "schema_version": { + "$ref": "#/$defs/version" + }, + "kind": { + "type": "string", + "const": "report" + }, + "repository": { + "type": "string", + "minLength": 1 + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "run": { + "$ref": "#/$defs/run" + }, + "report_name": { + "type": "string", + "minLength": 1 + }, + "track": { + "type": "string", + "enum": [ + "mutation", + "fuzz" + ] + }, + "tool": { + "type": "string", + "enum": [ + "stryker", + "gremlins", + "go-fuzz", + "fast-check" + ] + }, + "policy": { + "type": "string", + "enum": [ + "advisory", + "signal" + ] + }, + "completeness": { + "$ref": "#/$defs/completeness" + }, + "outcome": { + "$ref": "#/$defs/outcome" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/$defs/target" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationReportMetrics" + }, + { + "$ref": "#/$defs/fuzzReportMetrics" + }, + { + "type": "null" + } + ] + }, + "errors": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + } + } + } + } +} From 99a7a997425bec33f51328d10a784f4612a4d3d3 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:50:56 -0400 Subject: [PATCH 05/13] fix(quality): enforce report contract boundaries --- .../quality-result-stats/target-result.json | 2 +- .../quality-result-verify/target-result.json | 2 +- .../quality-result-stats/target-result.json | 2 +- .github/tests/quality_report_contract_test.py | 259 +++++++++++++++++- decisions/quality-reporting.md | 7 + quality-report/v1/quality_report.py | 23 +- quality-report/v1/schema.json | 216 ++++++++++++++- 7 files changed, 495 insertions(+), 16 deletions(-) diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json index 627a14e..1f79ab0 100644 --- a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json @@ -8,7 +8,7 @@ "outcome": "passed", "metrics": { "budget_seconds": 120, - "elapsed_seconds": 111.25, + "elapsed_seconds": 120.0, "executions": 650000, "new_interesting_inputs": 2 } diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json index 472e276..3e0c633 100644 --- a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json @@ -8,7 +8,7 @@ "outcome": "passed", "metrics": { "budget_seconds": 300, - "elapsed_seconds": 288.25, + "elapsed_seconds": 300.0, "executions": 1250000, "new_interesting_inputs": 3 } diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json index 0db4a95..4530c33 100644 --- a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json @@ -9,7 +9,7 @@ "metrics": null, "diagnostic": "tool process exited 2", "reproduction": { - "seed": "-test.fuzzcachedir=/tmp/fuzz-cache" + "path": "internal/docker/testdata/fuzz/FuzzDecodeStats/6f2c1b8d9a4e5c37" } } } diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index b456ce7..6c65a24 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -3,6 +3,7 @@ import runpy import shutil import subprocess +import sys import tempfile import unittest @@ -52,7 +53,7 @@ def test_complete_fuzz_fixture_emits_a_valid_report_and_summary(self): ) self.assertEqual("passed", report["outcome"]) self.assertEqual(420, report["metrics"]["declared_budget_seconds"]) - self.assertEqual(399.5, report["metrics"]["elapsed_seconds"]) + self.assertEqual(420.0, report["metrics"]["elapsed_seconds"]) self.assertEqual(1900000, report["metrics"]["executions"]) self.assertEqual(5, report["metrics"]["new_interesting_inputs"]) self.assertNotIn("canonical_score_pct", report["metrics"]) @@ -97,6 +98,23 @@ def test_crash_fixture_fails_even_when_every_target_reports(self): self.assertIn("tool process exited 2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_crash_fixture_uses_a_native_go_crasher_path(self): + result = json.loads( + ( + FIXTURES + / "crash" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + + self.assertEqual( + { + "path": "internal/docker/testdata/fuzz/FuzzDecodeStats/6f2c1b8d9a4e5c37" + }, + result["target"]["reproduction"], + ) + def test_parse_error_fixture_still_emits_an_error_report(self): result, report, summary = self.aggregate( "parse-error", @@ -116,6 +134,33 @@ def test_parse_error_fixture_still_emits_an_error_report(self): self.assertIn("partial 1/2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_non_utf8_target_still_emits_an_error_report(self): + def corrupt_target(input_dir): + target = next(input_dir.rglob("target-result.json")) + target.write_bytes(b"\xff") + + result, report, summary = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + input_mutator=corrupt_target, + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual("error", report["outcome"]) + self.assertFalse(report["completeness"]["complete"]) + self.assertIsNone(report["metrics"]) + self.assertTrue( + any("cannot read JSON" in error for error in report["errors"]), + report["errors"], + ) + self.assertIn("partial 1/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + def test_canonical_score_counts_timeout_and_no_coverage(self): result, report, summary = self.aggregate( "canonical-score", @@ -150,6 +195,14 @@ def test_decision_documents_track_and_tool_pairings(self): self.assertIn("`mutation` permits `gremlins` and `stryker`", decision) self.assertIn("`fuzz` permits `fast-check` and `go-fuzz`", decision) + def test_decision_defines_go_fuzz_as_the_native_go_engine(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn( + "`go-fuzz` identifies Go's native `go test -fuzz` engine, not the legacy third-party go-fuzz tool.", + decision, + ) + def test_native_score_is_weighted_and_keeps_its_definition(self): result, report, summary = self.aggregate( "native-score", @@ -235,6 +288,21 @@ def test_validator_requires_a_positive_declared_fuzz_budget(self): with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], ">= 1"): QUALITY_REPORT["validate_fuzz_report_metrics"](metrics, [], "$.metrics") + def test_passed_fuzz_target_requires_its_full_declared_budget(self): + target = { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 299.99, + }, + } + + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], "completed fuzz budget" + ): + QUALITY_REPORT["validate_target"](target, "fuzz", "$.target") + def test_validator_requires_time_and_offset_in_timestamps(self): for timestamp in ("2026-08-14", "2026-08-14T15:00:00"): with self.subTest(timestamp=timestamp): @@ -243,6 +311,91 @@ def test_validator_requires_time_and_offset_in_timestamps(self): ): QUALITY_REPORT["require_timestamp"](timestamp, "$.run.started_at") + def test_invalid_json_contract_error_retains_its_cause(self): + with self.assertRaises(QUALITY_REPORT["ContractError"]) as raised: + QUALITY_REPORT["load_json_text"]("{", "target-result.json") + + self.assertIsInstance(raised.exception.__cause__, json.JSONDecodeError) + + def test_invalid_timestamp_contract_error_retains_its_cause(self): + with self.assertRaises(QUALITY_REPORT["ContractError"]) as raised: + QUALITY_REPORT["require_timestamp"]("2026-08-14", "$.run.started_at") + + self.assertIsInstance(raised.exception.__cause__, ValueError) + + def test_json_parser_rejects_duplicate_fields_and_non_finite_numbers(self): + cases = ( + ('{"value": 1, "value": 2}', "duplicate JSON field"), + ('{"value": NaN}', "non-finite JSON number"), + ('{"value": Infinity}', "non-finite JSON number"), + ) + + for document, message in cases: + with self.subTest(document=document): + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], message): + QUALITY_REPORT["load_json_text"](document, "target-result.json") + + def test_mutation_aggregate_rejects_mixed_native_score_definitions(self): + targets = [] + for definition in ("detected / covered", "killed / lived"): + targets.append( + { + "metrics": { + "killed": 1, + "timeout": 0, + "survived": 0, + "no_coverage": 0, + "invalid": 0, + "ignored": 0, + "tool_score_numerator": 1, + "tool_score_denominator": 1, + "tool_score_pct": 100.0, + "tool_score_definition": definition, + } + } + ) + + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], + "inconsistent native score definitions", + ): + QUALITY_REPORT["aggregate_mutation_metrics"](targets) + + def test_target_collection_rejects_unexpected_and_duplicate_names(self): + document = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + target_name = document["target"]["name"] + + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) + (input_dir / "unexpected").mkdir() + (input_dir / "unexpected" / "target-result.json").write_text( + json.dumps(document) + ) + targets, errors = QUALITY_REPORT["collect_targets"]( + input_dir, ["expected-target"], "fuzz", "go-fuzz" + ) + self.assertEqual([], targets) + self.assertTrue(any("unexpected target" in error for error in errors)) + + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) + for directory in ("first", "second"): + path = input_dir / directory + path.mkdir() + (path / "target-result.json").write_text(json.dumps(document)) + targets, errors = QUALITY_REPORT["collect_targets"]( + input_dir, [target_name], "fuzz", "go-fuzz" + ) + self.assertEqual([target_name], [target["name"] for target in targets]) + self.assertTrue(any("duplicate target" in error for error in errors)) + def test_schema_is_versioned_and_fail_closed(self): with SCHEMA.open() as schema_file: schema = json.load(schema_file) @@ -259,6 +412,97 @@ def test_schema_is_versioned_and_fail_closed(self): self.assertNotIn("additionalProperties", schema) self.assertTrue(self.all_objects_are_closed(schema)) + def test_schema_encodes_python_target_and_track_guards(self): + schema = json.loads(SCHEMA.read_text()) + definitions = schema["$defs"] + + self.assertEqual( + [ + { + "if": { + "properties": {"outcome": {"const": "passed"}}, + "required": ["outcome"], + }, + "then": { + "properties": {"metrics": {"not": {"type": "null"}}}, + "not": { + "anyOf": [ + {"required": ["diagnostic"]}, + {"required": ["reproduction"]}, + ] + }, + }, + "else": {"required": ["diagnostic"]}, + } + ], + definitions["target"]["allOf"], + ) + self.assertEqual( + {"gremlins", "stryker"}, + set( + definitions["targetResult"]["allOf"][0]["then"]["properties"][ + "tool" + ]["enum"] + ), + ) + self.assertEqual( + {"fast-check", "go-fuzz"}, + set( + definitions["targetResult"]["allOf"][0]["else"]["properties"][ + "tool" + ]["enum"] + ), + ) + self.assertEqual( + "#/$defs/mutationTargetMetrics", + definitions["mutationTarget"]["allOf"][1]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzTargetMetrics", + definitions["fuzzTarget"]["allOf"][1]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + ["reproduction"], + definitions["fuzzTarget"]["allOf"][2]["then"]["required"], + ) + self.assertEqual( + "#/$defs/mutationTarget", + definitions["report"]["allOf"][0]["then"]["properties"]["targets"][ + "items" + ]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzTarget", + definitions["report"]["allOf"][0]["else"]["properties"]["targets"][ + "items" + ]["$ref"], + ) + self.assertEqual( + "#/$defs/mutationReportMetrics", + definitions["report"]["allOf"][0]["then"]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzReportMetrics", + definitions["report"]["allOf"][0]["else"]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertTrue(definitions["report"]["properties"]["targets"]["uniqueItems"]) + + def test_decision_names_python_as_the_cross_field_semantic_authority(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn( + "The Python validator is authoritative for cross-field semantics that JSON Schema cannot express, including unique and sorted target names, recomputed completeness, outcomes, and aggregate metrics.", + decision, + ) + def test_reusable_workflow_matches_the_public_contract(self): workflow = WORKFLOW.read_text() @@ -332,18 +576,20 @@ def test_standards_validation_runs_this_contract(self): workflow.count("python3 .github/tests/quality_report_contract_test.py"), ) - def aggregate(self, fixture, *, track, tool, expected): + def aggregate(self, fixture, *, track, tool, expected, input_mutator=None): self.assertTrue(SCRIPT.is_file(), f"missing aggregator: {SCRIPT}") with tempfile.TemporaryDirectory() as temp_dir: input_dir = Path(temp_dir) / "input" shutil.copytree(FIXTURES / fixture, input_dir) + if input_mutator is not None: + input_mutator(input_dir) for invalid_fixture in input_dir.rglob("target-result.invalid"): invalid_fixture.rename(invalid_fixture.with_name("target-result.json")) output_dir = Path(temp_dir) / "output" github_output = Path(temp_dir) / "github-output" summary_path = Path(temp_dir) / "step-summary" command = [ - "python3", + sys.executable, str(SCRIPT), "aggregate", "--input-dir", @@ -383,7 +629,9 @@ def aggregate(self, fixture, *, track, tool, expected): "--github-step-summary", str(summary_path), ] - result = subprocess.run(command, capture_output=True, text=True) + result = subprocess.run( + command, capture_output=True, text=True, timeout=30 + ) report_path = output_dir / "report.json" summary_file = output_dir / "summary.md" self.assertTrue(report_path.is_file(), result.stderr) @@ -425,9 +673,10 @@ def validate_report(self, report): report_path = Path(temp_dir) / "report.json" report_path.write_text(json.dumps(report)) return subprocess.run( - ["python3", str(SCRIPT), "validate", str(report_path)], + [sys.executable, str(SCRIPT), "validate", str(report_path)], capture_output=True, text=True, + timeout=30, ) def all_objects_are_closed(self, value): diff --git a/decisions/quality-reporting.md b/decisions/quality-reporting.md index 6182c97..b826c22 100644 --- a/decisions/quality-reporting.md +++ b/decisions/quality-reporting.md @@ -55,6 +55,9 @@ job validates the set and publishes `quality-report/v1` with: - `targets`: the declared target names and their individual results - track-specific `metrics` +`go-fuzz` identifies Go's native `go test -fuzz` engine, not the legacy +third-party go-fuzz tool. + The schema and dependency-free validator live under [`quality-report/v1`](../quality-report/v1). Both target-result fragments and aggregate reports carry the exact `quality-report/v1` version. Objects reject unknown fields, JSON parsing rejects @@ -62,6 +65,10 @@ duplicate fields and non-finite numbers, and semantic validation recomputes completeness, outcomes, and scores. Callers pass `expected_targets` as a non-empty JSON array of unique target names. +The Python validator is authoritative for cross-field semantics that JSON +Schema cannot express, including unique and sorted target names, recomputed +completeness, outcomes, and aggregate metrics. + Mutation metrics are: - `killed`, `timeout`, `survived`, `no_coverage`, `invalid`, `ignored` diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py index 8b8c93a..274308f 100644 --- a/quality-report/v1/quality_report.py +++ b/quality-report/v1/quality_report.py @@ -79,14 +79,16 @@ def load_json_text(text, source): parse_constant=reject_constant, ) except (json.JSONDecodeError, ContractError) as error: - raise ContractError("{0}: invalid JSON: {1}".format(source, error)) + raise ContractError("{0}: invalid JSON: {1}".format(source, error)) from error def load_json_file(path): try: return load_json_text(path.read_text(), str(path)) - except OSError as error: - raise ContractError("{0}: cannot read JSON: {1}".format(path, error)) + except (OSError, UnicodeDecodeError) as error: + raise ContractError( + "{0}: cannot read JSON: {1}".format(path, error) + ) from error def require_object(value, path, required, optional=()): @@ -168,8 +170,10 @@ def require_timestamp(value, path): if RFC3339_PATTERN.fullmatch(value) is None: raise ValueError datetime.fromisoformat(value.replace("Z", "+00:00").replace("z", "+00:00")) - except ValueError: - raise ContractError("{0}: expected RFC 3339 timestamp".format(path)) + except ValueError as error: + raise ContractError( + "{0}: expected RFC 3339 timestamp".format(path) + ) from error def percentage(numerator, denominator): @@ -238,6 +242,15 @@ def validate_target(target, track, path): validate_mutation_target_metrics(metrics, path + ".metrics") else: validate_fuzz_target_metrics(metrics, path + ".metrics") + if ( + outcome == "passed" + and metrics["elapsed_seconds"] < metrics["budget_seconds"] + ): + raise ContractError( + "{0}.metrics.elapsed_seconds: passed target has not completed fuzz budget".format( + path + ) + ) if outcome == "passed": if "diagnostic" in target or "reproduction" in target: diff --git a/quality-report/v1/schema.json b/quality-report/v1/schema.json index 46c38f1..2ce2acc 100644 --- a/quality-report/v1/schema.json +++ b/quality-report/v1/schema.json @@ -172,7 +172,112 @@ "reproduction": { "$ref": "#/$defs/reproduction" } - } + }, + "allOf": [ + { + "if": { + "properties": { + "outcome": { + "const": "passed" + } + }, + "required": [ + "outcome" + ] + }, + "then": { + "properties": { + "metrics": { + "not": { + "type": "null" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "diagnostic" + ] + }, + { + "required": [ + "reproduction" + ] + } + ] + } + }, + "else": { + "required": [ + "diagnostic" + ] + } + } + ] + }, + "mutationTarget": { + "allOf": [ + { + "$ref": "#/$defs/target" + }, + { + "properties": { + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationTargetMetrics" + }, + { + "type": "null" + } + ] + } + } + } + ] + }, + "fuzzTarget": { + "allOf": [ + { + "$ref": "#/$defs/target" + }, + { + "properties": { + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/fuzzTargetMetrics" + }, + { + "type": "null" + } + ] + } + } + }, + { + "if": { + "properties": { + "outcome": { + "enum": [ + "failed", + "crashed", + "flaked" + ] + } + }, + "required": [ + "outcome" + ] + }, + "then": { + "required": [ + "reproduction" + ] + } + } + ] }, "targetResult": { "type": "object", @@ -211,7 +316,47 @@ "target": { "$ref": "#/$defs/target" } - } + }, + "allOf": [ + { + "if": { + "properties": { + "track": { + "const": "mutation" + } + }, + "required": [ + "track" + ] + }, + "then": { + "properties": { + "tool": { + "enum": [ + "gremlins", + "stryker" + ] + }, + "target": { + "$ref": "#/$defs/mutationTarget" + } + } + }, + "else": { + "properties": { + "tool": { + "enum": [ + "fast-check", + "go-fuzz" + ] + }, + "target": { + "$ref": "#/$defs/fuzzTarget" + } + } + } + } + ] }, "mutationReportMetrics": { "type": "object", @@ -465,6 +610,7 @@ }, "targets": { "type": "array", + "uniqueItems": true, "items": { "$ref": "#/$defs/target" } @@ -490,7 +636,71 @@ "minLength": 1 } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "track": { + "const": "mutation" + } + }, + "required": [ + "track" + ] + }, + "then": { + "properties": { + "tool": { + "enum": [ + "gremlins", + "stryker" + ] + }, + "targets": { + "items": { + "$ref": "#/$defs/mutationTarget" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationReportMetrics" + }, + { + "type": "null" + } + ] + } + } + }, + "else": { + "properties": { + "tool": { + "enum": [ + "fast-check", + "go-fuzz" + ] + }, + "targets": { + "items": { + "$ref": "#/$defs/fuzzTarget" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/fuzzReportMetrics" + }, + { + "type": "null" + } + ] + } + } + } + } + ] } } } From 9a5362a6666d6ee42a3c5e47bf78b88eb21021f6 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:08:46 -0400 Subject: [PATCH 06/13] fix(quality): decode reports as utf-8 --- .github/tests/quality_report_contract_test.py | 30 +++++++++++++++++++ quality-report/v1/quality_report.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index 6c65a24..858507c 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -6,6 +6,7 @@ import sys import tempfile import unittest +from unittest import mock ROOT = Path(__file__).resolve().parents[2] @@ -161,6 +162,35 @@ def corrupt_target(input_dir): self.assertIn("partial 1/2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_utf8_target_load_does_not_depend_on_the_locale_encoding(self): + document = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + document["target"]["name"] = "internal/docker/FuzzDécodage" + + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "target-result.json" + path.write_bytes(json.dumps(document, ensure_ascii=False).encode("utf-8")) + + def read_with_ascii_default(*, encoding=None, errors=None): + return path.read_bytes().decode( + encoding or "ascii", errors or "strict" + ) + + with mock.patch.object( + Path, "read_text", side_effect=read_with_ascii_default + ): + loaded = QUALITY_REPORT["load_json_file"](path) + + self.assertEqual( + "internal/docker/FuzzDécodage", loaded["target"]["name"] + ) + def test_canonical_score_counts_timeout_and_no_coverage(self): result, report, summary = self.aggregate( "canonical-score", diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py index 274308f..1c4a4c2 100644 --- a/quality-report/v1/quality_report.py +++ b/quality-report/v1/quality_report.py @@ -84,7 +84,7 @@ def load_json_text(text, source): def load_json_file(path): try: - return load_json_text(path.read_text(), str(path)) + return load_json_text(path.read_text(encoding="utf-8"), str(path)) except (OSError, UnicodeDecodeError) as error: raise ContractError( "{0}: cannot read JSON: {1}".format(path, error) From a39c5bd4697a7e513694c4a227dc8b15099f12a8 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:38:16 -0400 Subject: [PATCH 07/13] test(quality): pin fixture encoding --- .github/tests/quality_report_contract_test.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index 858507c..6fabc70 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -163,13 +163,19 @@ def corrupt_target(input_dir): self.assertEqual(0, self.validate_report(report).returncode) def test_utf8_target_load_does_not_depend_on_the_locale_encoding(self): - document = json.loads( - ( - FIXTURES - / "complete" - / "quality-result-stats" - / "target-result.json" - ).read_text() + fixture = ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ) + original_read_text = Path.read_text + with mock.patch.object( + Path, "read_text", autospec=True, side_effect=original_read_text + ) as fixture_read: + document = json.loads(fixture.read_text(encoding="utf-8")) + self.assertEqual( + mock.call(fixture, encoding="utf-8"), fixture_read.call_args ) document["target"]["name"] = "internal/docker/FuzzDécodage" From 7be30f375a4303edd1f135e919d02b9ac3cfe875 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:23:49 -0400 Subject: [PATCH 08/13] ci(profile): make asset generation read-only (#10) * ci(profile): make asset generation read-only * fix(profile): restrict asset validation egress --- .../tests/community_health_contract_test.py | 78 ++++++++++++++++--- .github/workflows/profile-assets.yml | 60 ++++++++++++++ .github/workflows/standards-validation.yml | 14 ++-- .github/workflows/update-stats.yml | 44 ----------- .gitignore | 1 + cache/.gitignore | 2 - cache/stats.json | 22 ------ .../generate_profile_svg.py | 6 +- 8 files changed, 140 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/profile-assets.yml delete mode 100644 .github/workflows/update-stats.yml delete mode 100644 cache/.gitignore delete mode 100644 cache/stats.json rename generate_svg.py => scripts/generate_profile_svg.py (98%) diff --git a/.github/tests/community_health_contract_test.py b/.github/tests/community_health_contract_test.py index 9439282..7d36a25 100644 --- a/.github/tests/community_health_contract_test.py +++ b/.github/tests/community_health_contract_test.py @@ -1,5 +1,7 @@ from pathlib import Path import re +import subprocess +import sys import unittest @@ -119,6 +121,9 @@ def test_standards_workflow_has_unconditional_stable_jobs(self): self.assertIn(" - dev/repository-standards\n", workflow) self.assertNotIn("\n push:", workflow) self.assertIn("permissions: {}", workflow) + self.assertIn("concurrency:", workflow) + self.assertIn("group: standards-validation-", workflow) + self.assertIn("cancel-in-progress: true", workflow) self.assertIn("runs-on: ubuntu-24.04", workflow) self.assertNotIn("matrix:", workflow) self.assertNotIn("continue-on-error:", workflow) @@ -127,14 +132,14 @@ def test_standards_workflow_has_unconditional_stable_jobs(self): validation_job = workflow.split(" validation:\n", 1)[1].split( "\n codeql:\n", 1 )[0] - self.assertIn(" actions: read", validation_job) - self.assertIn(" contents: read", validation_job) + self.assertIn(" actions: read # Read workflow metadata", validation_job) + self.assertIn(" contents: read # Check out and validate repository files", validation_job) self.assertNotIn("write", validation_job) codeql_job = workflow.split("\n codeql:\n", 1)[1] - self.assertIn(" actions: read", codeql_job) - self.assertIn(" contents: read", codeql_job) - self.assertIn(" security-events: write", codeql_job) + self.assertIn(" actions: read # Read workflow metadata", codeql_job) + self.assertIn(" contents: read # Check out Python sources", codeql_job) + self.assertIn(" security-events: write # Upload CodeQL analysis", codeql_job) jobs = re.findall( r"^ ([a-z][a-z0-9-]+):$", @@ -184,22 +189,73 @@ def test_linter_configs_do_not_suppress_workflow_findings(self): self.assertEqual(2, markdown_config.count(": false")) self.assertFalse((ROOT / ".github/zizmor.yml").exists()) - def test_profile_workflow_pins_actions_and_pushes_without_persisted_credentials(self): - workflow = self.read_text(".github/workflows/update-stats.yml") + def test_profile_generator_has_one_canonical_location(self): + self.assertFalse((ROOT / "generate_svg.py").exists()) + self.assertTrue((ROOT / "scripts/generate_profile_svg.py").is_file()) + + def test_dead_profile_cache_is_removed(self): + self.assertFalse((ROOT / "cache").exists()) + + def test_font_reference_is_local_only(self): + gitignore = self.read_text(".gitignore") + self.assertIn("profile/font_reference.svg", gitignore.splitlines()) + + def test_profile_generator_reproduces_committed_assets(self): + generator = ROOT / "scripts/generate_profile_svg.py" + self.assertTrue(generator.is_file(), f"missing generator: {generator}") + outputs = ( + ROOT / "profile/dark_mode.svg", + ROOT / "profile/light_mode.svg", + ) + before = {path: path.read_bytes() for path in outputs} + + subprocess.run( + [sys.executable, generator], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + after = {path: path.read_bytes() for path in outputs} + self.assertEqual(before, after) + + def test_profile_workflow_is_read_only_bounded_and_pinned(self): + workflow = self.read_text(".github/workflows/profile-assets.yml") actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) self.assertEqual(2, len(actions)) for action in actions: self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") - checkout_step = workflow.split(" - name: Checkout\n", 1)[1].split( + harden_marker = " - name: Harden runner\n" + self.assertIn(harden_marker, workflow) + harden_step = workflow.split(harden_marker, 1)[1].split("\n - name:", 1)[0] + + self.assertIn("egress-policy: block", harden_step) + self.assertIn("allowed-endpoints: github.com:443", harden_step) + + checkout_marker = " - name: Check out repository\n" + self.assertIn(checkout_marker, workflow) + checkout_step = workflow.split(checkout_marker, 1)[1].split( "\n - name:", 1 )[0] - push_step = workflow.split(" - name: Commit and push\n", 1)[1] self.assertIn("persist-credentials: false", checkout_step) - self.assertIn("GH_TOKEN: ${{ github.token }}", push_step) - self.assertIn("gh auth setup-git", push_step) + self.assertIn("permissions: {}", workflow) + self.assertIn("name: Profile Assets", workflow) + self.assertIn("contents: read", workflow) + self.assertIn("timeout-minutes:", workflow) + self.assertIn("concurrency:", workflow) + self.assertIn("step-security/harden-runner@", workflow) + self.assertIn("python3 scripts/generate_profile_svg.py", workflow) + self.assertIn("git diff --exit-code -- profile/", workflow) + self.assertIn("git status --porcelain --untracked-files=all -- profile/", workflow) + self.assertNotIn("contents: write", workflow) + self.assertNotIn("git push", workflow) + self.assertNotIn("GH_TOKEN", workflow) + self.assertNotIn("schedule:", workflow) + self.assertFalse((ROOT / ".github/workflows/update-stats.yml").exists()) def read_text(self, relative_path): path = ROOT / relative_path diff --git a/.github/workflows/profile-assets.yml b/.github/workflows/profile-assets.yml new file mode 100644 index 0000000..0fd6154 --- /dev/null +++ b/.github/workflows/profile-assets.yml @@ -0,0 +1,60 @@ +name: Profile Assets + +on: + pull_request: + branches: + - main + - dev/repository-standards + paths: + - .github/tests/community_health_contract_test.py + - .github/workflows/profile-assets.yml + - .gitignore + - profile/*.svg + - scripts/generate_profile_svg.py + push: + branches: + - main + paths: + - .github/tests/community_health_contract_test.py + - .github/workflows/profile-assets.yml + - .gitignore + - profile/*.svg + - scripts/generate_profile_svg.py + workflow_dispatch: + +permissions: {} + +concurrency: + group: profile-assets-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + validate: + name: Profile Assets + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: github.com:443 + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Verify generated profile assets + run: | + set -euo pipefail + python3 scripts/generate_profile_svg.py + git diff --exit-code -- profile/ + changes="$(git status --porcelain --untracked-files=all -- profile/)" + if [[ -n "$changes" ]]; then + printf '%s\n' "$changes" + exit 1 + fi diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index ceb4606..30ea7f4 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -8,14 +8,18 @@ on: permissions: {} +concurrency: + group: standards-validation-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: validation: name: Standards Validation runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: - actions: read - contents: read + actions: read # Read workflow metadata + contents: read # Check out and validate repository files steps: - name: Harden runner @@ -74,9 +78,9 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: - actions: read - contents: read - security-events: write + actions: read # Read workflow metadata + contents: read # Check out Python sources + security-events: write # Upload CodeQL analysis steps: - name: Harden runner diff --git a/.github/workflows/update-stats.yml b/.github/workflows/update-stats.yml deleted file mode 100644 index cb79f7c..0000000 --- a/.github/workflows/update-stats.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Update Org Profile Stats - -on: - push: - branches: - - main - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -permissions: - contents: write - -jobs: - update: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.11' - - - name: Generate SVGs - env: - GH_TOKEN: ${{ secrets.ACCESS_TOKEN }} - run: | - python generate_svg.py - - - name: Commit and push - env: - GH_TOKEN: ${{ github.token }} - run: | - gh auth setup-git - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - git add profile/dark_mode.svg profile/light_mode.svg cache/ - git diff --staged --quiet || git commit -m "Update stats" - git push diff --git a/.gitignore b/.gitignore index c54ade8..d66d040 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ # Local planning/scratch workspace — not for commit .planning/ +profile/font_reference.svg diff --git a/cache/.gitignore b/cache/.gitignore deleted file mode 100644 index 688f1fa..0000000 --- a/cache/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Track cache files (needed for CI) -!*.json diff --git a/cache/stats.json b/cache/stats.json deleted file mode 100644 index 42f10e4..0000000 --- a/cache/stats.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "timestamp": "2026-02-09T23:00:00.795596", - "org": { - "public_repos": 2, - "private_repos": 3, - "total_repos": 5, - "total_stars": 0, - "total_forks": 0, - "total_commits": 398, - "total_prs": 11, - "total_issues": 1, - "members": 0, - "loc_added": 354732, - "loc_deleted": 283673, - "loc_total": 71059, - "languages": [ - "Python", - "Swift", - "TypeScript" - ] - } -} \ No newline at end of file diff --git a/generate_svg.py b/scripts/generate_profile_svg.py similarity index 98% rename from generate_svg.py rename to scripts/generate_profile_svg.py index 02f2b1b..436b7c6 100644 --- a/generate_svg.py +++ b/scripts/generate_profile_svg.py @@ -211,16 +211,16 @@ def art_color(ch): def main(): """Generate both dark and light mode SVGs.""" - script_dir = Path(__file__).parent + repo_root = Path(__file__).resolve().parents[1] dark_svg = generate_svg("dark") - dark_path = script_dir / "profile" / "dark_mode.svg" + dark_path = repo_root / "profile" / "dark_mode.svg" with open(dark_path, 'w', encoding='utf-8') as f: f.write(dark_svg) print(f"Generated: {dark_path}") light_svg = generate_svg("light") - light_path = script_dir / "profile" / "light_mode.svg" + light_path = repo_root / "profile" / "light_mode.svg" with open(light_path, 'w', encoding='utf-8') as f: f.write(light_svg) print(f"Generated: {light_path}") From 55c58e856c5f2a680b8e616158b76328831fad76 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:31:52 -0400 Subject: [PATCH 09/13] ci(review): add deduplicated Greptile summon (#9) * ci(review): add deduplicated Greptile summon * fix(review): serialize exact-head Greptile summons * test(review): lock Greptile security controls --- .../tests/greptile_summon_contract_test.py | 92 ++++++++++++++ .github/workflows/greptile-summon.yml | 120 ++++++++++++++++++ .github/workflows/standards-validation.yml | 4 +- 3 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 .github/tests/greptile_summon_contract_test.py create mode 100644 .github/workflows/greptile-summon.yml diff --git a/.github/tests/greptile_summon_contract_test.py b/.github/tests/greptile_summon_contract_test.py new file mode 100644 index 0000000..7584077 --- /dev/null +++ b/.github/tests/greptile_summon_contract_test.py @@ -0,0 +1,92 @@ +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/greptile-summon.yml" + + +class GreptileSummonContractTest(unittest.TestCase): + def test_reusable_workflow_has_exact_head_inputs_and_narrow_permissions(self): + workflow = self.read_workflow() + + for expected in ( + " workflow_call:\n", + " pr_number:\n", + " type: number\n", + " head_sha:\n", + " type: string\n", + " focus:\n", + "permissions: {}", + " runs-on: ubuntu-24.04", + " timeout-minutes: 5", + " pull-requests: write", + "uses: step-security/harden-runner@", + "egress-policy: block", + "api.github.com:443", + "github-token: ${{ github.token }}", + ): + self.assertIn(expected, workflow) + + self.assertNotIn("actions/checkout", workflow) + + def test_workflow_verifies_head_and_deduplicates_before_commenting(self): + workflow = self.read_workflow() + + for expected in ( + "github.rest.pulls.get", + 'pull.state !== "open"', + "pull.head.sha !== headSha", + "github.paginate", + "github.rest.issues.listComments", + "greptile-summon:${headSha}", + 'comment.user?.login === "github-actions[bot]"', + "github.rest.issues.createComment", + "@greptileai", + ): + self.assertIn(expected, workflow) + + self.assertIn( + "greptile-summon-${{ github.repository }}-${{ inputs.pr_number }}-${{ inputs.head_sha }}", + workflow, + ) + self.assertIn(" cancel-in-progress: false", workflow) + self.assertGreaterEqual(workflow.count("github.rest.pulls.get"), 2) + self.assertIn("currentPull.head.sha !== headSha", workflow) + self.assertIn("HEAD_SHA: ${{ inputs.head_sha }}", workflow) + self.assertIn("REVIEW_FOCUS: ${{ inputs.focus }}", workflow) + self.assertNotIn("const headSha = '${{ inputs.head_sha }}'", workflow) + self.assertNotIn("const focus = '${{ inputs.focus }}'", workflow) + + def test_workflow_rejects_partial_pull_request_numbers(self): + workflow = self.read_workflow() + + self.assertIn("const prNumberText = process.env.PR_NUMBER", workflow) + self.assertIn("const prNumber = Number(prNumberText)", workflow) + self.assertIn("!/^[1-9]\\d*$/.test(prNumberText)", workflow) + self.assertIn("PR_NUMBER: ${{ inputs.pr_number }}", workflow) + self.assertIn("if (!/^[0-9a-f]{40}$/.test(headSha))", workflow) + self.assertIn("if (focus.length === 0 || focus.length > 500)", workflow) + self.assertNotIn("Number.parseInt(process.env.PR_NUMBER", workflow) + + def test_workflow_pins_actions_and_is_run_by_standards_validation(self): + workflow = self.read_workflow() + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + validation = ( + ROOT / ".github/workflows/standards-validation.yml" + ).read_text() + self.assertIn("greptile_summon_contract_test.py", validation) + + def read_workflow(self): + self.assertTrue(WORKFLOW.is_file(), f"missing workflow: {WORKFLOW}") + return WORKFLOW.read_text() + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/greptile-summon.yml b/.github/workflows/greptile-summon.yml new file mode 100644 index 0000000..3484f9b --- /dev/null +++ b/.github/workflows/greptile-summon.yml @@ -0,0 +1,120 @@ +name: Summon Greptile + +on: + workflow_call: + inputs: + pr_number: + description: Pull request number in the caller repository + required: true + type: number + head_sha: + description: Exact pull request head to review + required: true + type: string + focus: + description: Short review focus based on the risk in the diff + required: false + default: Review for correctness, security issues, and cross-file regressions. + type: string + +permissions: {} + +jobs: + summon: + name: Summon Greptile + runs-on: ubuntu-24.04 + timeout-minutes: 5 + concurrency: + group: greptile-summon-${{ github.repository }}-${{ inputs.pr_number }}-${{ inputs.head_sha }} + cancel-in-progress: false + permissions: + pull-requests: write # Create the intentional Greptile summon comment. + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + + - name: Comment once on the exact pull request head + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PR_NUMBER: ${{ inputs.pr_number }} + HEAD_SHA: ${{ inputs.head_sha }} + REVIEW_FOCUS: ${{ inputs.focus }} + with: + github-token: ${{ github.token }} + script: | + const prNumberText = process.env.PR_NUMBER; + const prNumber = Number(prNumberText); + const headSha = process.env.HEAD_SHA; + const focus = process.env.REVIEW_FOCUS.trim(); + const { owner, repo } = context.repo; + + if (!/^[1-9]\d*$/.test(prNumberText) || !Number.isSafeInteger(prNumber)) { + core.setFailed(`Invalid pull request number: ${prNumberText}`); + return; + } + if (!/^[0-9a-f]{40}$/.test(headSha)) { + core.setFailed(`Invalid full head SHA: ${headSha}`); + return; + } + if (focus.length === 0 || focus.length > 500) { + core.setFailed("Review focus must contain 1 to 500 characters"); + return; + } + + const { data: pull } = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber, + }); + if (pull.state !== "open") { + core.setFailed(`Pull request #${prNumber} is ${pull.state}`); + return; + } + if (pull.head.sha !== headSha) { + core.setFailed( + `Pull request head changed: expected ${headSha}, got ${pull.head.sha}`, + ); + return; + } + + const marker = ``; + const comments = await github.paginate( + github.rest.issues.listComments, + { owner, repo, issue_number: prNumber, per_page: 100 }, + ); + const alreadySummoned = comments.some( + (comment) => comment.user?.login === "github-actions[bot]" + && comment.body?.includes(marker), + ); + if (alreadySummoned) { + core.notice(`Greptile already summoned for ${headSha}`); + return; + } + + const { data: currentPull } = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber, + }); + if (currentPull.state !== "open") { + core.setFailed(`Pull request #${prNumber} is ${currentPull.state}`); + return; + } + if (currentPull.head.sha !== headSha) { + core.setFailed( + `Pull request head changed: expected ${headSha}, got ${currentPull.head.sha}`, + ); + return; + } + + await github.rest.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body: `@greptileai Review exact head \`${headSha}\`. ${focus}\n\n${marker}`, + }); diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index 30ea7f4..0ea5ad0 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -38,7 +38,9 @@ jobs: node-version: 24 - name: Validate community health contract - run: python3 .github/tests/community_health_contract_test.py + run: | + python3 .github/tests/community_health_contract_test.py + python3 .github/tests/greptile_summon_contract_test.py - name: Compile Python run: python3 -m compileall -q . From 6e7a78dc13d099cb27669c0a0f8a75836de97f03 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 08:24:16 -0400 Subject: [PATCH 10/13] ci(workflows): add run-test and run-lint toggles to go-ci (#19) go-ci.yml's test and lint jobs ran unconditionally, so a Go-less repo that only wants the language-agnostic workflow-security (zizmor) job couldn't call it. Add run-test/run-lint boolean inputs, mirroring the existing run-govulncheck/run-workflow-security/etc. toggle pattern, defaulting to true so existing callers see no behavior change. Fixes: #18 --- .github/tests/reusable_ci_contract_test.py | 4 ++++ .github/workflows/go-ci.yml | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py index a6ec3f5..b958eb6 100644 --- a/.github/tests/reusable_ci_contract_test.py +++ b/.github/tests/reusable_ci_contract_test.py @@ -25,6 +25,8 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) "lint-check-name", "test-check-name", "fuzzers-json", + "run-test", + "run-lint", "run-govulncheck", "run-qlty", "run-goreleaser", @@ -113,6 +115,8 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) self.assert_input("go", "lint-check-name", "string", default="Go Lint") self.assert_input("go", "test-check-name", "string", default="Go Test") self.assert_input("go", "fuzzers-json", "string", default="[]") + for input_name in ("run-test", "run-lint"): + self.assert_input("go", input_name, "boolean", default="true") for input_name in ( "run-govulncheck", "run-qlty", diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 46601d4..8e14b1c 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -33,6 +33,16 @@ on: required: false type: string default: "[]" + run-test: + description: Run the fixed repository Go test gate + required: false + type: boolean + default: true + run-lint: + description: Run the fixed repository Go lint gate + required: false + type: boolean + default: true run-govulncheck: description: Run the fixed repository govulncheck gate required: false @@ -159,6 +169,7 @@ permissions: {} jobs: test: name: ${{ inputs.test-check-name }} + if: inputs.run-test runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: @@ -198,6 +209,7 @@ jobs: lint: name: ${{ inputs.lint-check-name }} + if: inputs.run-lint runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: From 04ad251abff43346cd46b836f323421a0f99bd00 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:05:50 -0400 Subject: [PATCH 11/13] ci(workflows): add module-directory input to node-ci (#22) * ci(workflows): add module-directory input to node-ci Mirrors go-ci's module-directory idiom: a string input defaulting to "." threaded into each fixed script's env as MODULE_DIRECTORY, so a repo with several independently-gated Node projects can call node-ci once per project. The default preserves current behavior for existing callers. Extends the reusable CI contract test to assert the new input and its threading, matching how run-test/run-lint were added for go-ci in #19. * test(workflows): assert module-directory threads into all three node jobs --- .github/tests/reusable_ci_contract_test.py | 5 +++++ .github/workflows/node-ci.yml | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py index b958eb6..b4fc0fa 100644 --- a/.github/tests/reusable_ci_contract_test.py +++ b/.github/tests/reusable_ci_contract_test.py @@ -47,6 +47,7 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) ) ) node_inputs = [ + "module-directory", "node-version", "lockfile-path", "lint-check-name", @@ -127,6 +128,7 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) ): self.assert_input("go", input_name, "boolean", default="false") + self.assert_input("node", "module-directory", "string", default=".") self.assert_input("node", "node-version", "string", default="24") self.assert_input("node", "lockfile-path", "string", default="package-lock.json") self.assert_input("node", "lint-check-name", "string", default="Node Lint") @@ -204,6 +206,9 @@ def test_central_jobs_have_exact_names_fixed_commands_and_per_job_egress(self): ) self.assertIn("node-version: ${{ inputs.node-version }}", node) self.assertIn("cache-dependency-path: ${{ inputs.lockfile-path }}", node) + self.assertEqual( + 3, node.count("MODULE_DIRECTORY: ${{ inputs.module-directory }}") + ) for workflow in (go, node, self.read_workflow(WORKFLOWS["release"])): self.assertIn("runs-on: ubuntu-24.04", workflow) diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index e699a9b..c953151 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -3,6 +3,11 @@ name: Reusable Node CI on: workflow_call: inputs: + module-directory: + description: Repository-relative Node module directory + required: false + type: string + default: . node-version: description: Node.js version used by enabled jobs required: false @@ -105,6 +110,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node lint gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-lint.sh test: @@ -135,6 +142,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node test gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-test.sh - name: Upload Node test artifacts @@ -174,6 +183,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node build gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-build.sh - name: Upload Node build artifacts From 7f9be8b854dd345814537fd0d7f24b2509760f96 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:30:27 -0400 Subject: [PATCH 12/13] docs(onboarding): record the qlty alignment baseline (#24) --- REPOSITORY_ONBOARDING.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index a06bb7a..7510744 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -115,10 +115,8 @@ repositories. Language-specific tests do not. - [ ] Add language formatters to `pre-commit` only when they can safely operate on staged files. Verify the hook does not rewrite unrelated work. -Qlty Cloud enrollment and usage data are not an organization-wide prerequisite. -Do not make a Qlty upload or check required until the repository is enrolled and -the check has proved stable. Local Qlty checks may still be stricter than this -baseline. +Qlty spans local and CI gates; the full baseline, including why Qlty Cloud +checks stay non-required, is in the Qlty subsection of section 4. ## 3. Protect `main` @@ -194,6 +192,37 @@ Add these only when the behavior exists: - [ ] Translation synchronization only for a repository with a translation source of truth and configured provider credentials. +### Qlty + +Qlty appears in three places. The committed configuration and the repository-run +gates are the alignment surface; the Qlty Cloud GitHub App is not. + +- [ ] Commit `.qlty/qlty.toml` (`config_version = "0"`) with the organization + exclude baseline for generated, minified, and vendored paths. Drydock's file + is the reference posture and Portwing's is its Go-repo mirror; start from the + closer of the two. The same file drives the local CLI and Qlty Cloud, so + changes to it are quality-gate changes, not formatting. +- [ ] Add the repository's Qlty gate script (`scripts/qlty-check-gate.sh all`) + to `pre-push`, fail-fast like every other local gate. An advisory smells + gate may run alongside it (Drydock pattern) but must not mask the gating + script's failure. +- [ ] Gate Qlty in CI with the repository-run check, which needs no Qlty Cloud + account. Go repositories call `go-ci.yml` with `run-qlty` enabled, + `qlty-egress-policy: block`, and the proven endpoint allowlist — copy the + allowlist from Portwing's `ci-verify.yml` rather than re-deriving it, and do + not widen it. Node repositories run the SHA-pinned + `qltysh/qlty-action/install` plus the same gate script inside their own + workflow (Drydock pattern). + +The Qlty Cloud GitHub App's statuses (`qlty check`, `qlty coverage`, +`qlty coverage diff`) currently error organization-wide with out-of-minutes +billing failures. Treat them as non-gating: never add them to required status +checks, and do not chase these failures on PRs — the repository-run gate above +is the enforced check. Qlty Cloud enrollment and usage data are not an +organization-wide prerequisite; do not make a Cloud upload or check required +until the repository is enrolled and the check has proved stable. Local Qlty +checks may still be stricter than this baseline. + ## 5. Add delivery controls only for shipped artifacts - [ ] Keep release construction separate from ordinary CI. A release-cut From f4c1d509fe98551d6e6d6c819d2bb618711e460f Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 18:34:58 -0400 Subject: [PATCH 13/13] docs(onboarding): align with the codified standards registry (#26) * docs(onboarding): align with the codified standards registry - docs(onboarding): name Codecov as the coverage cloud; Qlty Cloud App and maintainability badge stay, checks stay non-required - docs(onboarding): trivy deprecated in favor of Grype, including the qlty plugin blocks in the two reference configs (drydock#753, portwing#135) - docs(onboarding): CodeRabbit free Pro is public-only; private repos use cross-account human review - docs(onboarding): add the greptile.json contract and the label-gated second-opinion caller * docs(onboarding): reword the CodeRabbit private-repo claim as org policy - docs(onboarding): free-plan private-repo reviews exist but are rate-limited and never fired here; the skip is policy, not a plan fact - docs(onboarding): pair the Greptile caller with auto-applied CodeRabbit labeling so the second-opinion label is criteria-driven --- REPOSITORY_ONBOARDING.md | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index 7510744..8929480 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -100,6 +100,20 @@ repositories. Language-specific tests do not. request an explicit CodeRabbit review on the first PR to prove it works. - [ ] Confirm CodeRabbit can access the repository and posts an actual review comment. A green check without review output is not enough for this test. + Automated Pro-level reviews are free on public repositories only; on + private repositories the free plan rate-limits automated reviews, and they + have not fired for this organization in practice. Organization policy: on a + private repository skip both CodeRabbit items, use cross-account human + review, and add the config when the repository goes public. +- [ ] Keep `greptile.json` at exactly `{"skipReview": "AUTOMATIC"}` + (contract-tested in this repository) so Greptile never reviews unbidden. A + repository that wants opt-in second opinions adds a label-gated caller + workflow (`.github/workflows/greptile.yml`, firing on the `second-opinion` + label) that calls this repository's `greptile-summon.yml` at a pinned full + commit SHA. Pair the caller with a CodeRabbit `labeling_instructions` entry + for `second-opinion` and `auto_apply_labels` enabled, so applying the label + is criteria-driven rather than left to memory. The label is Greptile's only + trigger; never wire it to review every PR directly. ### Local gates @@ -156,7 +170,8 @@ must push, and use `step-security/harden-runner` with the smallest practical egress allowlist. Every repository with executable code needs the applicable format or lint and -test gates. Require coverage when the language and test tooling measure it, a +test gates. Require coverage when the language and test tooling measure it +(uploaded to Codecov, not Qlty Cloud), a production build when the repository ships a buildable artifact, dependency review when it has dependencies, workflow validation when it has GitHub Actions, and CodeQL when its language is supported. Add the matching language or artifact @@ -174,8 +189,9 @@ checks below: - **GitHub Actions:** actionlint and zizmor on every PR, plus tests for non-trivial embedded shell and workflow contracts. - **Container image:** build the real Dockerfile, smoke-test the image, and scan - the built image and locked dependencies with Grype or the current organization - scanner. + the built image and locked dependencies with Grype. Trivy is deprecated + organization-wide in favor of Grype; do not adopt it, including as a Qlty + plugin. - **Public repository:** OpenSSF Scorecard on its supported triggers and dependency review on PRs. @@ -200,8 +216,11 @@ gates are the alignment surface; the Qlty Cloud GitHub App is not. - [ ] Commit `.qlty/qlty.toml` (`config_version = "0"`) with the organization exclude baseline for generated, minified, and vendored paths. Drydock's file is the reference posture and Portwing's is its Go-repo mirror; start from the - closer of the two. The same file drives the local CLI and Qlty Cloud, so - changes to it are quality-gate changes, not formatting. + closer of the two. Do not copy the trivy plugin block from either reference: + trivy is deprecated in favor of Grype and its removal from both files is + tracked (drydock#753, portwing#135). The trufflehog plugin stays. The same + file drives the local CLI and Qlty Cloud, so changes to it are quality-gate + changes, not formatting. - [ ] Add the repository's Qlty gate script (`scripts/qlty-check-gate.sh all`) to `pre-push`, fail-fast like every other local gate. An advisory smells gate may run alongside it (Drydock pattern) but must not mask the gating @@ -218,8 +237,12 @@ The Qlty Cloud GitHub App's statuses (`qlty check`, `qlty coverage`, `qlty coverage diff`) currently error organization-wide with out-of-minutes billing failures. Treat them as non-gating: never add them to required status checks, and do not chase these failures on PRs — the repository-run gate above -is the enforced check. Qlty Cloud enrollment and usage data are not an -organization-wide prerequisite; do not make a Cloud upload or check required +is the enforced check. Coverage reporting is Codecov's job (decided +2026-08-16): upload coverage to Codecov and carry its badge in the README. +The Qlty Cloud App and its maintainability badge stay installed, but its +coverage upload is not the coverage system and its checks stay non-required. +Qlty Cloud enrollment and usage data are not an organization-wide +prerequisite; do not make a Cloud upload or check required until the repository is enrolled and the check has proved stable. Local Qlty checks may still be stricter than this baseline.