From 633dc88e6b4446b912b714754ed47ec5167627f1 Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 11:34:23 -0500 Subject: [PATCH 1/9] CHORE: Add workflow for coverage report. --- .../pyright_typecompleteness_summary.py | 142 ++++++++++++++++++ .../workflows/pyright-typecompleteness.yml | 50 ++++++ 2 files changed, 192 insertions(+) create mode 100644 .github/scripts/pyright_typecompleteness_summary.py create mode 100644 .github/workflows/pyright-typecompleteness.yml diff --git a/.github/scripts/pyright_typecompleteness_summary.py b/.github/scripts/pyright_typecompleteness_summary.py new file mode 100644 index 00000000..a5d44563 --- /dev/null +++ b/.github/scripts/pyright_typecompleteness_summary.py @@ -0,0 +1,142 @@ +""" +Builds a Markdown summary comparing pyright `--verifytypes` reports for a +PR's base and head commits, for posting as a PR comment. + +"Project" completeness is read directly from the head report's +typeCompleteness.completenessScore. "Patch" completeness is computed by +matching exported symbols between the base and head reports by their +dotted name: a symbol counts toward the patch if it is new in head, or if +its known/ambiguous/unknown status changed between base and head. This +avoids needing to map symbols to source line ranges, since pyright only +reports a file/line for symbols that already have a type problem. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +STATUS_ICON = {"known": "✅", "ambiguous": "⚠️", "unknown": "❌"} + + +def status_of(symbol: dict[str, Any]) -> str: + if symbol["isTypeKnown"]: + return "known" + if symbol["isTypeAmbiguous"]: + return "ambiguous" + return "unknown" + + +def exported_symbols(report_path: Path) -> dict[str, dict[str, Any]]: + with report_path.open() as f: + data = json.load(f) + symbols = data["typeCompleteness"]["symbols"] + return {s["name"]: s for s in symbols if s["isExported"]} + + +def counts_by_status(symbols: list[dict[str, Any]]) -> dict[str, int]: + counts = {"known": 0, "ambiguous": 0, "unknown": 0} + for s in symbols: + counts[status_of(s)] += 1 + return counts + + +def completeness_pct(counts: dict[str, int]) -> float: + total = sum(counts.values()) + return 100.0 * counts["known"] / total if total else 0.0 + + +def render_counts_table(rows: list[tuple[str, dict[str, int]]]) -> str: + lines = ["| | Known | Ambiguous | Unknown | Total |", "|---|---|---|---|---|"] + for label, counts in rows: + total = sum(counts.values()) + lines.append( + f"| {label} | {counts['known']} | {counts['ambiguous']} | " + f"{counts['unknown']} | {total} |" + ) + return "\n".join(lines) + + +def render_patch_detail( + patch_names: list[str], + base: dict[str, dict[str, Any]], + head: dict[str, dict[str, Any]], +) -> str: + lines = ["| Symbol | Status | Change |", "|---|---|---|"] + for name in sorted(patch_names): + head_status = status_of(head[name]) + icon = STATUS_ICON[head_status] + if name not in base: + change = "new" + else: + base_status = status_of(base[name]) + change = f"changed (was {STATUS_ICON[base_status]} {base_status})" + lines.append(f"| `{name}` | {icon} {head_status} | {change} |") + return "\n".join(lines) + + +def build_summary(base_path: Path, head_path: Path) -> str: + base = exported_symbols(base_path) + head = exported_symbols(head_path) + + project_counts = counts_by_status(list(head.values())) + project_pct = completeness_pct(project_counts) + + patch_names = [ + name + for name, symbol in head.items() + if name not in base or status_of(base[name]) != status_of(symbol) + ] + + sections = [ + "## Pyright Type Completeness", + "", + f"**Project (full `chainladder` package, at this PR's head):** " + f"{project_pct:.1f}% of exported symbols fully typed " + f"({project_counts['known']} / {sum(project_counts.values())})", + "", + render_counts_table([("Project (head)", project_counts)]), + "", + ] + + if patch_names: + patch_counts = counts_by_status([head[n] for n in patch_names]) + patch_pct = completeness_pct(patch_counts) + sections += [ + f"**Patch (exported symbols added or changed by this PR):** " + f"{patch_pct:.1f}% fully typed " + f"({patch_counts['known']} / {sum(patch_counts.values())})", + "", + render_counts_table([("Patch", patch_counts)]), + "", + "
", + "Patch symbol details", + "", + render_patch_detail(patch_names, base, head), + "", + "
", + ] + else: + sections += [ + "**Patch (exported symbols added or changed by this PR):** " + "no exported symbol type-completeness changes detected.", + ] + + return "\n".join(sections) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True, type=Path) + parser.add_argument("--head", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + + summary = build_summary(args.base, args.head) + args.output.write_text(summary) + print(summary) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/pyright-typecompleteness.yml b/.github/workflows/pyright-typecompleteness.yml new file mode 100644 index 00000000..3fbebb59 --- /dev/null +++ b/.github/workflows/pyright-typecompleteness.yml @@ -0,0 +1,50 @@ +name: Pyright Type Completeness + +on: + pull_request: + +permissions: + contents: read + pull-requests: write + +jobs: + typecompleteness: + name: pyright --verifytypes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "latest" + python-version: "3.14" + + - name: Install dependencies + run: uv sync --extra dev + + - name: Run pyright --verifytypes on PR head + run: uv run pyright --outputjson --verifytypes chainladder > head.json || true + + - name: Run pyright --verifytypes on PR base + run: | + git fetch origin ${{ github.event.pull_request.base.sha }} + git checkout ${{ github.event.pull_request.base.sha }} -- chainladder + uv run pyright --outputjson --verifytypes chainladder > base.json || true + git checkout ${{ github.event.pull_request.head.sha }} -- chainladder + + - name: Build summary + run: | + uv run python .github/scripts/pyright_typecompleteness_summary.py \ + --base base.json \ + --head head.json \ + --output summary.md + + - name: Comment on PR + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: pyright-typecompleteness + path: summary.md From 50df01cbb0db6ee111f31c2913af11232261814a Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 11:56:49 -0500 Subject: [PATCH 2/9] CHORE: Add additional report items. --- .../pyright_typecompleteness_summary.py | 35 ++++++++++++++++--- .../workflows/pyright-typecompleteness.yml | 7 +++- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/scripts/pyright_typecompleteness_summary.py b/.github/scripts/pyright_typecompleteness_summary.py index a5d44563..7c87f20b 100644 --- a/.github/scripts/pyright_typecompleteness_summary.py +++ b/.github/scripts/pyright_typecompleteness_summary.py @@ -28,11 +28,23 @@ def status_of(symbol: dict[str, Any]) -> str: return "unknown" -def exported_symbols(report_path: Path) -> dict[str, dict[str, Any]]: +def load_type_completeness(report_path: Path) -> dict[str, Any]: with report_path.open() as f: data = json.load(f) - symbols = data["typeCompleteness"]["symbols"] - return {s["name"]: s for s in symbols if s["isExported"]} + return data["typeCompleteness"] + + +def exported_symbols(type_completeness: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {s["name"]: s for s in type_completeness["symbols"] if s["isExported"]} + + +def other_symbol_counts(type_completeness: dict[str, Any]) -> dict[str, int]: + other = type_completeness["otherSymbolCounts"] + return { + "known": other["withKnownType"], + "ambiguous": other["withAmbiguousType"], + "unknown": other["withUnknownType"], + } def counts_by_status(symbols: list[dict[str, Any]]) -> dict[str, int]: @@ -77,11 +89,14 @@ def render_patch_detail( def build_summary(base_path: Path, head_path: Path) -> str: - base = exported_symbols(base_path) - head = exported_symbols(head_path) + base_tc = load_type_completeness(base_path) + head_tc = load_type_completeness(head_path) + base = exported_symbols(base_tc) + head = exported_symbols(head_tc) project_counts = counts_by_status(list(head.values())) project_pct = completeness_pct(project_counts) + other_counts = other_symbol_counts(head_tc) patch_names = [ name @@ -98,6 +113,16 @@ def build_summary(base_path: Path, head_path: Path) -> str: "", render_counts_table([("Project (head)", project_counts)]), "", + f"Other symbols referenced but not exported by `chainladder`: " + f"{sum(other_counts.values())}", + "", + render_counts_table([("Other (head)", other_counts)]), + "", + "Symbols without documentation:", + f"- Functions without docstring: {head_tc['missingFunctionDocStringCount']}", + f"- Functions without default param: {head_tc['missingDefaultParamCount']}", + f"- Classes without docstring: {head_tc['missingClassDocStringCount']}", + "", ] if patch_names: diff --git a/.github/workflows/pyright-typecompleteness.yml b/.github/workflows/pyright-typecompleteness.yml index 3fbebb59..23ba675c 100644 --- a/.github/workflows/pyright-typecompleteness.yml +++ b/.github/workflows/pyright-typecompleteness.yml @@ -27,14 +27,19 @@ jobs: run: uv sync --extra dev - name: Run pyright --verifytypes on PR head - run: uv run pyright --outputjson --verifytypes chainladder > head.json || true + run: | + uv run pyright --outputjson --verifytypes chainladder > head.json || true + uv run pyright --verifytypes chainladder > head.txt || true + .github/scripts/append_verifytypes_step_summary.sh head.txt "PR head" - name: Run pyright --verifytypes on PR base run: | git fetch origin ${{ github.event.pull_request.base.sha }} git checkout ${{ github.event.pull_request.base.sha }} -- chainladder uv run pyright --outputjson --verifytypes chainladder > base.json || true + uv run pyright --verifytypes chainladder > base.txt || true git checkout ${{ github.event.pull_request.head.sha }} -- chainladder + .github/scripts/append_verifytypes_step_summary.sh base.txt "PR base" - name: Build summary run: | From 87f97d51838716ee324ba69152ce928eadd751a8 Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 11:58:00 -0500 Subject: [PATCH 3/9] CHORE: Add step summary. --- .../append_verifytypes_step_summary.sh | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 .github/scripts/append_verifytypes_step_summary.sh diff --git a/.github/scripts/append_verifytypes_step_summary.sh b/.github/scripts/append_verifytypes_step_summary.sh new file mode 100755 index 00000000..47748fd4 --- /dev/null +++ b/.github/scripts/append_verifytypes_step_summary.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Appends a pyright --verifytypes text report to $GITHUB_STEP_SUMMARY, +# wrapped in a collapsible
block. GitHub caps each step's summary +# at 1MiB and silently drops the whole upload if exceeded, so the report is +# truncated to a safe byte budget with a note when that happens, rather than +# risking the entire block vanishing. +set -euo pipefail + +report_file="$1" +label="$2" +max_bytes=900000 + +total_bytes="$(wc -c < "$report_file")" + +{ + echo "
Full pyright --verifytypes output (${label})" + echo "" + echo '```text' + head -c "$max_bytes" "$report_file" + if [ "$total_bytes" -gt "$max_bytes" ]; then + echo "" + echo "... (truncated: ${total_bytes} bytes total, GITHUB_STEP_SUMMARY caps each step at 1MiB)" + fi + echo '```' + echo "
" +} >> "$GITHUB_STEP_SUMMARY" From 1d0721cda3db8503173bfacd45bf21b88eb6ec57 Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 12:51:20 -0500 Subject: [PATCH 4/9] FIX: Fix bugs and tidy up filenames. --- ...pleteness_summary.py => type_completeness.py} | 16 ++++++++++++++-- ...ep_summary.sh => type_completeness_output.sh} | 0 ...ypecompleteness.yml => type_completeness.yml} | 9 ++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) rename .github/scripts/{pyright_typecompleteness_summary.py => type_completeness.py} (92%) rename .github/scripts/{append_verifytypes_step_summary.sh => type_completeness_output.sh} (100%) rename .github/workflows/{pyright-typecompleteness.yml => type_completeness.yml} (80%) diff --git a/.github/scripts/pyright_typecompleteness_summary.py b/.github/scripts/type_completeness.py similarity index 92% rename from .github/scripts/pyright_typecompleteness_summary.py rename to .github/scripts/type_completeness.py index 7c87f20b..08c952e3 100644 --- a/.github/scripts/pyright_typecompleteness_summary.py +++ b/.github/scripts/type_completeness.py @@ -88,7 +88,7 @@ def render_patch_detail( return "\n".join(lines) -def build_summary(base_path: Path, head_path: Path) -> str: +def build_summary(base_path: Path, head_path: Path, run_url: str | None = None) -> str: base_tc = load_type_completeness(base_path) head_tc = load_type_completeness(head_path) base = exported_symbols(base_tc) @@ -107,6 +107,13 @@ def build_summary(base_path: Path, head_path: Path) -> str: sections = [ "## Pyright Type Completeness", "", + ] + if run_url: + sections += [ + f"[View the full `pyright --verifytypes` output for this commit]({run_url})", + "", + ] + sections += [ f"**Project (full `chainladder` package, at this PR's head):** " f"{project_pct:.1f}% of exported symbols fully typed " f"({project_counts['known']} / {sum(project_counts.values())})", @@ -156,9 +163,14 @@ def main() -> None: parser.add_argument("--base", required=True, type=Path) parser.add_argument("--head", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) + parser.add_argument( + "--run-url", + default=None, + help="Link to the workflow run, e.g. for its step summary.", + ) args = parser.parse_args() - summary = build_summary(args.base, args.head) + summary = build_summary(args.base, args.head, run_url=args.run_url) args.output.write_text(summary) print(summary) diff --git a/.github/scripts/append_verifytypes_step_summary.sh b/.github/scripts/type_completeness_output.sh similarity index 100% rename from .github/scripts/append_verifytypes_step_summary.sh rename to .github/scripts/type_completeness_output.sh diff --git a/.github/workflows/pyright-typecompleteness.yml b/.github/workflows/type_completeness.yml similarity index 80% rename from .github/workflows/pyright-typecompleteness.yml rename to .github/workflows/type_completeness.yml index 23ba675c..897eb676 100644 --- a/.github/workflows/pyright-typecompleteness.yml +++ b/.github/workflows/type_completeness.yml @@ -30,7 +30,7 @@ jobs: run: | uv run pyright --outputjson --verifytypes chainladder > head.json || true uv run pyright --verifytypes chainladder > head.txt || true - .github/scripts/append_verifytypes_step_summary.sh head.txt "PR head" + bash .github/scripts/type_completeness_output.sh head.txt "PR head" - name: Run pyright --verifytypes on PR base run: | @@ -39,13 +39,16 @@ jobs: uv run pyright --outputjson --verifytypes chainladder > base.json || true uv run pyright --verifytypes chainladder > base.txt || true git checkout ${{ github.event.pull_request.head.sha }} -- chainladder - .github/scripts/append_verifytypes_step_summary.sh base.txt "PR base" + bash .github/scripts/type_completeness_output.sh base.txt "PR base" - name: Build summary + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | - uv run python .github/scripts/pyright_typecompleteness_summary.py \ + uv run python .github/scripts/type_completeness.py \ --base base.json \ --head head.json \ + --run-url "$RUN_URL" \ --output summary.md - name: Comment on PR From baf44ceb094d5fe955b145be02786d8b3c1e4946 Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 13:23:37 -0500 Subject: [PATCH 5/9] FIX: Apply bugbot fixes, remove external dependencies from report. --- .github/workflows/type_completeness.yml | 38 +++++++++++++------ .../workflows/type_completeness_comment.yml | 38 +++++++++++++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/type_completeness_comment.yml diff --git a/.github/workflows/type_completeness.yml b/.github/workflows/type_completeness.yml index 897eb676..eec7e154 100644 --- a/.github/workflows/type_completeness.yml +++ b/.github/workflows/type_completeness.yml @@ -5,7 +5,6 @@ on: permissions: contents: read - pull-requests: write jobs: typecompleteness: @@ -28,17 +27,21 @@ jobs: - name: Run pyright --verifytypes on PR head run: | - uv run pyright --outputjson --verifytypes chainladder > head.json || true - uv run pyright --verifytypes chainladder > head.txt || true + uv run pyright --outputjson --verifytypes chainladder --ignoreexternal > head.json || true + uv run pyright --verifytypes chainladder --ignoreexternal > head.txt || true bash .github/scripts/type_completeness_output.sh head.txt "PR head" - name: Run pyright --verifytypes on PR base + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - git fetch origin ${{ github.event.pull_request.base.sha }} - git checkout ${{ github.event.pull_request.base.sha }} -- chainladder - uv run pyright --outputjson --verifytypes chainladder > base.json || true - uv run pyright --verifytypes chainladder > base.txt || true - git checkout ${{ github.event.pull_request.head.sha }} -- chainladder + git fetch origin "$BASE_SHA" + MERGE_BASE="$(git merge-base "$BASE_SHA" "$HEAD_SHA")" + git checkout "$MERGE_BASE" -- chainladder + uv run pyright --outputjson --verifytypes chainladder --ignoreexternal > base.json || true + uv run pyright --verifytypes chainladder --ignoreexternal > base.txt || true + git checkout "$HEAD_SHA" -- chainladder bash .github/scripts/type_completeness_output.sh base.txt "PR base" - name: Build summary @@ -51,8 +54,19 @@ jobs: --run-url "$RUN_URL" \ --output summary.md - - name: Comment on PR - uses: marocchino/sticky-pull-request-comment@v2 + - name: Save PR number + run: echo "${{ github.event.pull_request.number }}" > pr_number.txt + + # This workflow runs on pull_request, so on PRs from forks it only ever + # gets a read-only GITHUB_TOKEN and can't post a comment directly. The + # artifact is picked up by type_completeness_comment.yml, which runs on + # workflow_run (always with the base repo's write-capable token, without + # ever checking out or executing the fork's code) and posts the comment. + - name: Upload summary artifact + uses: actions/upload-artifact@v4 with: - header: pyright-typecompleteness - path: summary.md + name: type-completeness-summary + path: | + summary.md + pr_number.txt + retention-days: 1 diff --git a/.github/workflows/type_completeness_comment.yml b/.github/workflows/type_completeness_comment.yml new file mode 100644 index 00000000..81189e8e --- /dev/null +++ b/.github/workflows/type_completeness_comment.yml @@ -0,0 +1,38 @@ +name: Pyright Type Completeness Comment + +# Split out from type_completeness.yml so the comment can be posted with a +# write-capable token even for PRs from forks. workflow_run always executes +# using the workflow file from the default branch, with the base repo's own +# token, regardless of whether the triggering run came from a fork - and +# this workflow never checks out or executes the fork's code, only the +# already-computed summary.md/pr_number.txt artifact it produced. +on: + workflow_run: + workflows: ["Pyright Type Completeness"] + types: [completed] + +permissions: + pull-requests: write + +jobs: + comment: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + - name: Download summary artifact + uses: actions/download-artifact@v4 + with: + name: type-completeness-summary + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read PR number + id: pr + run: echo "number=$(cat pr_number.txt)" >> "$GITHUB_OUTPUT" + + - name: Comment on PR + uses: marocchino/sticky-pull-request-comment@v3 + with: + header: pyright-typecompleteness + number_force: ${{ steps.pr.outputs.number }} + path: summary.md From 78fa10d93469e4dcada39f35dca146caf483bbf6 Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 13:36:41 -0500 Subject: [PATCH 6/9] FIX: Apply bugbot fixes. --- .github/workflows/type_completeness.yml | 1 + .github/workflows/type_completeness_comment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/type_completeness.yml b/.github/workflows/type_completeness.yml index eec7e154..1bc8eda8 100644 --- a/.github/workflows/type_completeness.yml +++ b/.github/workflows/type_completeness.yml @@ -5,6 +5,7 @@ on: permissions: contents: read + actions: write jobs: typecompleteness: diff --git a/.github/workflows/type_completeness_comment.yml b/.github/workflows/type_completeness_comment.yml index 81189e8e..3833b263 100644 --- a/.github/workflows/type_completeness_comment.yml +++ b/.github/workflows/type_completeness_comment.yml @@ -13,6 +13,7 @@ on: permissions: pull-requests: write + actions: read jobs: comment: From 233c544b04b0f0a7cb04c65e7656659976f94f08 Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 18:43:22 -0500 Subject: [PATCH 7/9] DOCS: Add annotations. --- .github/scripts/type_completeness.py | 181 +++++++++++++++++- .github/workflows/type_completeness.yml | 27 ++- .../workflows/type_completeness_comment.yml | 8 +- 3 files changed, 194 insertions(+), 22 deletions(-) diff --git a/.github/scripts/type_completeness.py b/.github/scripts/type_completeness.py index 08c952e3..336979a8 100644 --- a/.github/scripts/type_completeness.py +++ b/.github/scripts/type_completeness.py @@ -1,26 +1,37 @@ """ Builds a Markdown summary comparing pyright `--verifytypes` reports for a PR's base and head commits, for posting as a PR comment. - -"Project" completeness is read directly from the head report's -typeCompleteness.completenessScore. "Patch" completeness is computed by -matching exported symbols between the base and head reports by their -dotted name: a symbol counts toward the patch if it is new in head, or if -its known/ambiguous/unknown status changed between base and head. This -avoids needing to map symbols to source line ranges, since pyright only -reports a file/line for symbols that already have a type problem. """ from __future__ import annotations import argparse import json from pathlib import Path -from typing import Any +from typing import ( + Any, + Literal +) +# Decorates the patch coverage table. STATUS_ICON = {"known": "✅", "ambiguous": "⚠️", "unknown": "❌"} -def status_of(symbol: dict[str, Any]) -> str: +def status_of(symbol: dict[str, Any]) -> Literal['known', 'ambiguous', 'unknown']: + """ + Maps the --verifytypes JSON boolean flags, isTypeKnown and isTypeAmbiguous, to internal + representation in script: known, ambiguous, and unknown. + + Parameters + ---------- + symbol: dict[str, Any] + A single entry for a symbol from typeCompleteless.symbols in the JSON output. + + Returns + ------- + Literal['known', 'ambiguous', 'unknown'] + The mapped symbol type. + + """ if symbol["isTypeKnown"]: return "known" if symbol["isTypeAmbiguous"]: @@ -29,16 +40,60 @@ def status_of(symbol: dict[str, Any]) -> str: def load_type_completeness(report_path: Path) -> dict[str, Any]: + """ + Loads the --verifytypes JSON output and extracts the typeCompleteness section. + Discards the rest. + + Parameters + ---------- + report_path: Path + The path to the --verifytypes JSON report to parse. + + Returns + ------- + dict[str, Any] + A dictionary representation of the typeCompleteness section of the --verifytypes JSON report. + + """ with report_path.open() as f: data = json.load(f) return data["typeCompleteness"] def exported_symbols(type_completeness: dict[str, Any]) -> dict[str, dict[str, Any]]: + """ + Parses the dict generated by load_type_completeness() and filters it on exported symbols. + + Parameters + ---------- + type_completeness: dict[str, Any] + The dict generated by load_type_completeness(). + + Returns + ------- + dict[str, dict[str, Any]] + A dict of exported symbols, indexed by symbol name. + + """ return {s["name"]: s for s in type_completeness["symbols"] if s["isExported"]} def other_symbol_counts(type_completeness: dict[str, Any]) -> dict[str, int]: + """ + Parses the dict generated by load_type_completeness(). Extract the section on otherSymbolCounts + and return as a summary dict. + + Parameters + ---------- + type_completeness: dict[str, Any] + The dict generated by load_type_completeness(). + + Returns + ------- + dict[str, int] + Summary dict containing counts of other symbols. + + """ other = type_completeness["otherSymbolCounts"] return { "known": other["withKnownType"], @@ -48,6 +103,21 @@ def other_symbol_counts(type_completeness: dict[str, Any]) -> dict[str, int]: def counts_by_status(symbols: list[dict[str, Any]]) -> dict[str, int]: + """ + Parses the dict (transformed to list) produced by exported_symbols(). Loops through the symbols and counts + them by type. Return a dict summary of the counts. + + Parameters + ---------- + symbols: list[dict[str, Any]] + The output dict produced by exported_symbols(), converted to a list. + + Returns + ------- + dict[str, int] + A summary count of exported symbols by type. + + """ counts = {"known": 0, "ambiguous": 0, "unknown": 0} for s in symbols: counts[status_of(s)] += 1 @@ -55,11 +125,41 @@ def counts_by_status(symbols: list[dict[str, Any]]) -> dict[str, int]: def completeness_pct(counts: dict[str, int]) -> float: + """ + Calculates the type completeness score. Equals the percentage of known symbols + relative to total symbols. + + Parameters + ---------- + counts: dict[str, int] + A summary dictionary of symbol counts by type (known, ambiguous, unknown). + + Returns + ------- + float + A type completeness score. + + """ total = sum(counts.values()) return 100.0 * counts["known"] / total if total else 0.0 def render_counts_table(rows: list[tuple[str, dict[str, int]]]) -> str: + """ + Generate a Markdown type completeness summary table, for the project base, used for PR comment. + + Parameters + ---------- + rows: list[tuple[str, dict[str, int]]] + List of tuples in the form of [("Section Name", symbol_counts)]. e.g., [("Patch", patch_counts)] + + + Returns + ------- + str + A Markdown table of the type completeness summary, used for PR comment. + + """ lines = ["| | Known | Ambiguous | Unknown | Total |", "|---|---|---|---|---|"] for label, counts in rows: total = sum(counts.values()) @@ -75,6 +175,23 @@ def render_patch_detail( base: dict[str, dict[str, Any]], head: dict[str, dict[str, Any]], ) -> str: + """ + Generate a Markdown type completeness summary table, for the PR patch, used for PR comment. + + Parameters + ---------- + patch_names: list[str] + A list of symbol names whose status changed in the PR, or are newly exported. + base: dict[str, dict[str, Any]] + dict generated by exported_symbols(), base case. + head: dict[str, dict[str, Any]] + dict generated by exported_symbols(), head case. + + Returns + ------- + str + A Markdown type completeness summary table, for the PR patch, used for PR comment. + """ lines = ["| Symbol | Status | Change |", "|---|---|---|"] for name in sorted(patch_names): head_status = status_of(head[name]) @@ -89,21 +206,49 @@ def render_patch_detail( def build_summary(base_path: Path, head_path: Path, run_url: str | None = None) -> str: + """ + Builds the full Markdown summary for the PR comment, combining the project-level + type completeness report with the patch-level diff between base and head. + + Parameters + ---------- + base_path: Path + The path to the --verifytypes JSON report for the PR's base (or merge-base) + commit. + head_path: Path + The path to the --verifytypes JSON report for the PR's head commit. + run_url: str | None + Optional link to the workflow run, e.g. its step summary. When provided, a + link to it is included near the top of the summary. Omitted entirely when + None. + + Returns + ------- + str + The rendered Markdown summary, ready to be written to a file or posted as + a PR comment. + + """ + # Extract the type completeness section of --verifytypes JSON, and filter on exported symbols + # for both the head and base case. base_tc = load_type_completeness(base_path) head_tc = load_type_completeness(head_path) base = exported_symbols(base_tc) head = exported_symbols(head_tc) + # Count up the symbol types. project_counts = counts_by_status(list(head.values())) project_pct = completeness_pct(project_counts) other_counts = other_symbol_counts(head_tc) + # Gather the patch symbols, i.e., those whose status changed in the PR, or newly exported ones. patch_names = [ name for name, symbol in head.items() if name not in base or status_of(base[name]) != status_of(symbol) ] + # Generate the Markdown report. sections = [ "## Pyright Type Completeness", "", @@ -159,6 +304,22 @@ def build_summary(base_path: Path, head_path: Path, run_url: str | None = None) def main() -> None: + """ + Parse -verifytypes JSON output and produce a Markdown summary report for PR comment. + + Accepts command line arguments: + + --base: Path to the --verifytypes JSON report for the PR's base (or merge-base) commit. + --head: Path to the --verifytypes JSON report for the PR's head commit. + --output: Path to write the rendered Markdown summary to. + --run-url: Optional link to the workflow run, e.g. its step summary. Omitted from the + summary entirely when not provided. + + Returns + ------- + None + + """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", required=True, type=Path) parser.add_argument("--head", required=True, type=Path) diff --git a/.github/workflows/type_completeness.yml b/.github/workflows/type_completeness.yml index 1bc8eda8..5efdebfa 100644 --- a/.github/workflows/type_completeness.yml +++ b/.github/workflows/type_completeness.yml @@ -1,5 +1,18 @@ name: Pyright Type Completeness +# Main workflow for generating type completeness report. Calculates type completeness scores on both a +# project basis and a patch basis. +# +# Results are fed into a subsequent workflow, +# defined in type_completeness_comment.yml. This workflow must be split into 2 files in order to work +# securely on PRs from external forks of the repo by splitting read/write permissions. +# +# Auxiliary Files +# --------------- +# type_completeness_comment.yml - Writes a comment on the PR summarizing the type completeness. +# ../scripts/type_completeness.py - Parses the resulting JSON to extract coverage statistics. +# ../scripts/type_completeness_output.sh - Appends the Pyright terminal output to the GitHub action for user review. + on: pull_request: @@ -27,12 +40,18 @@ jobs: run: uv sync --extra dev - name: Run pyright --verifytypes on PR head + # Get type coverage on the PR head. + # head.json -> Machine-readable report + # head.txt -> human-readable report run: | uv run pyright --outputjson --verifytypes chainladder --ignoreexternal > head.json || true uv run pyright --verifytypes chainladder --ignoreexternal > head.txt || true bash .github/scripts/type_completeness_output.sh head.txt "PR head" - name: Run pyright --verifytypes on PR base + # Get type coverage on main, append terminal output to GitHub actions. + # base.json -> Machine-readable report + # base.txt -> Human-readable report env: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} @@ -46,6 +65,7 @@ jobs: bash .github/scripts/type_completeness_output.sh base.txt "PR base" - name: Build summary + # Construct summary by feeding JSON files into type_completeness.py. env: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | @@ -57,12 +77,7 @@ jobs: - name: Save PR number run: echo "${{ github.event.pull_request.number }}" > pr_number.txt - - # This workflow runs on pull_request, so on PRs from forks it only ever - # gets a read-only GITHUB_TOKEN and can't post a comment directly. The - # artifact is picked up by type_completeness_comment.yml, which runs on - # workflow_run (always with the base repo's write-capable token, without - # ever checking out or executing the fork's code) and posts the comment. + # Generate summary artifact to be read in by subsequent workflow type_completeness_comment.yml. - name: Upload summary artifact uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/type_completeness_comment.yml b/.github/workflows/type_completeness_comment.yml index 3833b263..53a0ae58 100644 --- a/.github/workflows/type_completeness_comment.yml +++ b/.github/workflows/type_completeness_comment.yml @@ -1,11 +1,7 @@ name: Pyright Type Completeness Comment -# Split out from type_completeness.yml so the comment can be posted with a -# write-capable token even for PRs from forks. workflow_run always executes -# using the workflow file from the default branch, with the base repo's own -# token, regardless of whether the triggering run came from a fork - and -# this workflow never checks out or executes the fork's code, only the -# already-computed summary.md/pr_number.txt artifact it produced. +# Generate a comment containing type completeness summary. Triggered after completion of +# preceding workflow Pyright Type Completeness. on: workflow_run: workflows: ["Pyright Type Completeness"] From b8cebddb79b311a346302c47888fdf9d5dab7807 Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 19:16:42 -0500 Subject: [PATCH 8/9] FIX: Apply bugbot fixes. --- .github/scripts/type_completeness.py | 37 ++++++++++++++++++++----- .github/workflows/type_completeness.yml | 2 ++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.github/scripts/type_completeness.py b/.github/scripts/type_completeness.py index 336979a8..4234c683 100644 --- a/.github/scripts/type_completeness.py +++ b/.github/scripts/type_completeness.py @@ -172,6 +172,7 @@ def render_counts_table(rows: list[tuple[str, dict[str, int]]]) -> str: def render_patch_detail( patch_names: list[str], + removed_names: list[str], base: dict[str, dict[str, Any]], head: dict[str, dict[str, Any]], ) -> str: @@ -182,6 +183,9 @@ def render_patch_detail( ---------- patch_names: list[str] A list of symbol names whose status changed in the PR, or are newly exported. + removed_names: list[str] + A list of symbol names that were exported at base but are no longer exported at + head (e.g. de-exported, renamed, or deleted). base: dict[str, dict[str, Any]] dict generated by exported_symbols(), base case. head: dict[str, dict[str, Any]] @@ -202,6 +206,11 @@ def render_patch_detail( base_status = status_of(base[name]) change = f"changed (was {STATUS_ICON[base_status]} {base_status})" lines.append(f"| `{name}` | {icon} {head_status} | {change} |") + for name in sorted(removed_names): + base_status = status_of(base[name]) + lines.append( + f"| `{name}` | — | no longer exported (was {STATUS_ICON[base_status]} {base_status}) |" + ) return "\n".join(lines) @@ -247,6 +256,10 @@ def build_summary(base_path: Path, head_path: Path, run_url: str | None = None) for name, symbol in head.items() if name not in base or status_of(base[name]) != status_of(symbol) ] + # Symbols exported at base but no longer exported at head (de-exported, renamed, + # or deleted). These can't be found by iterating head, since they're absent from + # it entirely - iterate base instead and check for absence from head. + removed_names = [name for name in base if name not in head] # Generate the Markdown report. sections = [ @@ -277,20 +290,30 @@ def build_summary(base_path: Path, head_path: Path, run_url: str | None = None) "", ] - if patch_names: - patch_counts = counts_by_status([head[n] for n in patch_names]) - patch_pct = completeness_pct(patch_counts) + if patch_names or removed_names: + patch_counts = ( + counts_by_status([head[n] for n in patch_names]) + if patch_names + else {"known": 0, "ambiguous": 0, "unknown": 0} + ) + parts = [] + if patch_names: + patch_pct = completeness_pct(patch_counts) + parts.append( + f"{patch_pct:.1f}% fully typed " + f"({patch_counts['known']} / {sum(patch_counts.values())})" + ) + if removed_names: + parts.append(f"{len(removed_names)} no longer exported") sections += [ - f"**Patch (exported symbols added or changed by this PR):** " - f"{patch_pct:.1f}% fully typed " - f"({patch_counts['known']} / {sum(patch_counts.values())})", + "**Patch (exported symbols added or changed by this PR):** " + "; ".join(parts), "", render_counts_table([("Patch", patch_counts)]), "", "
", "Patch symbol details", "", - render_patch_detail(patch_names, base, head), + render_patch_detail(patch_names, removed_names, base, head), "", "
", ] diff --git a/.github/workflows/type_completeness.yml b/.github/workflows/type_completeness.yml index 5efdebfa..70a3e10f 100644 --- a/.github/workflows/type_completeness.yml +++ b/.github/workflows/type_completeness.yml @@ -58,9 +58,11 @@ jobs: run: | git fetch origin "$BASE_SHA" MERGE_BASE="$(git merge-base "$BASE_SHA" "$HEAD_SHA")" + rm -rf chainladder git checkout "$MERGE_BASE" -- chainladder uv run pyright --outputjson --verifytypes chainladder --ignoreexternal > base.json || true uv run pyright --verifytypes chainladder --ignoreexternal > base.txt || true + rm -rf chainladder git checkout "$HEAD_SHA" -- chainladder bash .github/scripts/type_completeness_output.sh base.txt "PR base" From f4abd3379df1ef7a41d91ae937e92ecd4a62b48d Mon Sep 17 00:00:00 2001 From: Gene Dan Date: Mon, 22 Jun 2026 19:30:17 -0500 Subject: [PATCH 9/9] FIX: Apply bugbot fixes. --- .github/scripts/type_completeness_output.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/scripts/type_completeness_output.sh b/.github/scripts/type_completeness_output.sh index 47748fd4..fc2970b1 100755 --- a/.github/scripts/type_completeness_output.sh +++ b/.github/scripts/type_completeness_output.sh @@ -4,11 +4,17 @@ # at 1MiB and silently drops the whole upload if exceeded, so the report is # truncated to a safe byte budget with a note when that happens, rather than # risking the entire block vanishing. +# +# This script is called twice per workflow run (PR head and PR base). The +# budget below is half of the 1MiB cap, not the full cap, so the two calls +# stay safely under the limit even in the worst case where they end up +# sharing one combined budget instead of each getting their own per-step +# allotment (true today, per GitHub's docs, but cheap to hedge against). set -euo pipefail report_file="$1" label="$2" -max_bytes=900000 +max_bytes=450000 total_bytes="$(wc -c < "$report_file")"