From 266293ef122269bc95290e570748ed9e83244e1d Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:00:21 +0100 Subject: [PATCH 1/7] feat(ci): CLAUDE_AUTH switch for the Claude steps, cost lines, pair-mode daily regen - CLAUDE_AUTH (repository variable): unset or `subscription` keeps every claude-code-action step on CLAUDE_CODE_OAUTH_TOKEN; `api` moves it to ANTHROPIC_API_KEY. Each of the 13 steps passes exactly one non-empty credential. A "Check Claude credentials" step opens every Claude job and stops on an unknown value or on `api` without the key, never falling back to the subscription. - "Claude cost to the job summary" after every Claude step: total_cost_usd, model and credential from the execution file, never fatal. collect_week.py --costs reads it, counts daily-regen too and splits out the API part. - daily-regen pair mode: REGEN_PAIRS_PER_DAY on the schedule, or the new `pairs` input, picks the N oldest (spec, library) pairs (automation/scripts/regen_pick_pairs.py), skips the spec polish and dispatches each pair straight to impl-generate with bulk-generate's arguments and pace. The watchdog's cron rescue forwards the variable. Unset, the schedule runs spec mode as before. - tests/unit/workflows/test_claude_auth.py pins the wiring and runs the check, cost, mode and dispatch scripts; docs and changelog fragment. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H7HSkGyea1k2v293XiAMo8 --- .claude/skills/review-regen-week/SKILL.md | 9 +- .../skills/review-regen-week/collect_week.py | 52 ++- .github/workflows/daily-regen.yml | 203 ++++++++- .github/workflows/impl-generate.yml | 64 ++- .github/workflows/impl-repair.yml | 62 ++- .github/workflows/impl-review.yml | 44 +- .github/workflows/report-validate.yml | 45 +- .github/workflows/review-retest.yml | 44 +- .github/workflows/spec-create.yml | 63 ++- .github/workflows/util-claude.yml | 62 ++- .github/workflows/watchdog-stuck-jobs.yml | 6 +- agentic/docs/project-guide.md | 3 +- automation/scripts/README.md | 15 + automation/scripts/regen_pick_pairs.py | 287 +++++++++++++ changelog.d/ci-claude-auth-switch.md | 28 ++ docs/workflows/overview.md | 168 +++++++- prompts/README.md | 4 +- prompts/workflow-prompts/README.md | 4 +- .../scripts/test_regen_pick_pairs.py | 281 ++++++++++++ tests/unit/skills/test_collect_week.py | 25 ++ tests/unit/workflows/test_claude_auth.py | 404 ++++++++++++++++++ 21 files changed, 1835 insertions(+), 38 deletions(-) create mode 100644 automation/scripts/regen_pick_pairs.py create mode 100644 changelog.d/ci-claude-auth-switch.md create mode 100644 tests/unit/automation/scripts/test_regen_pick_pairs.py create mode 100644 tests/unit/workflows/test_claude_auth.py diff --git a/.claude/skills/review-regen-week/SKILL.md b/.claude/skills/review-regen-week/SKILL.md index 6ae55be0ccf..bab0826b50e 100644 --- a/.claude/skills/review-regen-week/SKILL.md +++ b/.claude/skills/review-regen-week/SKILL.md @@ -5,7 +5,8 @@ description: Weekly review of what daily-regen produced — collect every regene # Weekly regen review -`daily-regen.yml` regenerates one spec per night (02:17 UTC): 15 pairs, +`daily-regen.yml` regenerates one spec per night (02:17 UTC): 15 pairs +(in pair mode, `REGEN_PAIRS_PER_DAY` single pairs across the catalogue), each reviewed once on Opus, and the regen gate either merges the new version (`regen:improved`) or keeps the live one (`regen:kept`, with a write-back of the re-score). Once a week you look at what that did to @@ -65,8 +66,10 @@ uv run python -m automation.scripts.review_retest first-reviews \ no gate record; it is listed so you can check why it was forced. The collector also counts the window's pipeline runs by conclusion, prints the failed ones under the metric table, and with `--costs` - sums `total_cost_usd` from every generate, review and repair job log - (about one minute per 50 runs; leave it off for a quick look). Its + sums `total_cost_usd` from every daily-regen pre-flight, generate, + review and repair job log, and names the part that ran on API credits + (`CLAUDE_AUTH=api`, read from each step's `claude-cost` line; about + one minute per 50 runs; leave it off for a quick look). Its stdout is the metric table and the PR table the report starts from; `◆` marks the render sample. - `gate-report` adds the aggregate view and its soft alarms, among diff --git a/.claude/skills/review-regen-week/collect_week.py b/.claude/skills/review-regen-week/collect_week.py index c5926d66776..cdb91dcb0f3 100644 --- a/.claude/skills/review-regen-week/collect_week.py +++ b/.claude/skills/review-regen-week/collect_week.py @@ -52,6 +52,13 @@ WRITEBACK_BRANCH_RE = re.compile(r"^review-writeback/(?P[^/]+)/(?P[^/]+)/(?P\d+)$") RUN_URL_RE = re.compile(r"/actions/runs/(\d+)") COST_RE = re.compile(r'"total_cost_usd":\s*([0-9.]+)') +# The line every Claude step's "Claude cost to the job summary" step prints, +# with the credential it ran on (vars.CLAUDE_AUTH). A job from before that +# step carries only Claude Code's own `"total_cost_usd": n` and ran on the +# subscription. +COST_LINE_RE = re.compile( + r"claude-cost step=\S+ auth=(?Papi|subscription) model=\S+ total_cost_usd=(?P[0-9.]+)" +) GCS_PUBLIC = "https://storage.googleapis.com/anyplot-images/plots" VERDICT_LABELS = ("regen:improved", "regen:kept", "ai-approved", "ai-rejected") PIPELINE_WORKFLOWS = ( @@ -62,7 +69,7 @@ "impl-repair.yml", "impl-merge.yml", ) -CLAUDE_WORKFLOWS = ("impl-generate.yml", "impl-review.yml", "impl-repair.yml") +CLAUDE_WORKFLOWS = ("daily-regen.yml", "impl-generate.yml", "impl-review.yml", "impl-repair.yml") # A keep whose re-score sits this far below the stored score is a render look. # Under the same judge (both reviews from one model family) a few points is @@ -309,7 +316,9 @@ def previously_reported(rows: Sequence[dict[str, Any]]) -> set[int]: return {int(r["pr"]) for r in rows if r.get("state") != "OPEN"} -def summarize(rows: Sequence[dict[str, Any]], runs: dict[str, Counter], cost: float | None) -> dict[str, Any]: +def summarize( + rows: Sequence[dict[str, Any]], runs: dict[str, Counter], cost: float | None, api_cost: float | None = None +) -> dict[str, Any]: decided = [r for r in rows if r["verdict"]] merges = [r for r in decided if r["verdict"] == "merge"] keeps = [r for r in decided if r["verdict"] == "keep"] @@ -337,6 +346,7 @@ def summarize(rows: Sequence[dict[str, Any]], runs: dict[str, Counter], cost: fl "identical_vectors": identical_vectors(rows), "runs": {wf: dict(counter) for wf, counter in runs.items()}, "cost_usd": round(cost, 2) if cost is not None else None, + "cost_usd_api": round(api_cost, 2) if api_cost is not None else None, } @@ -364,10 +374,18 @@ def summarize(rows: Sequence[dict[str, Any]], runs: dict[str, Counter], cost: fl ("Missing pair artifact", "0", lambda s: s["flags"].get("no_pair_artifact", 0)), ("daily-regen runs (success/all)", "7/7", lambda s: _success(s["runs"].get("daily-regen.yml", {}))), ("Failed pipeline runs", "few", lambda s: sum(c.get("failure", 0) for c in s["runs"].values())), - ("Session cost (USD)", "", lambda s: "not collected" if s["cost_usd"] is None else f"${s['cost_usd']:.0f}"), + ("Session cost (USD)", "", lambda s: _cost(s)), ) +def _cost(summary: dict[str, Any]) -> str: + """The week's session cost, and the part that ran on API credits when there is one.""" + if summary["cost_usd"] is None: + return "not collected" + api = summary.get("cost_usd_api") + return f"${summary['cost_usd']:.0f}" + (f" (API ${api:.0f})" if api else "") + + def _cell(value: Any) -> str: """A record key that older records lack (pre-P3.1 kinds, pre-P8 code) reads as a dash.""" return "–" if value is None else str(value) @@ -544,9 +562,20 @@ def pipeline_runs(since: str) -> dict[str, list[dict[str, Any]]]: return runs -def session_cost(runs: dict[str, list[dict[str, Any]]]) -> float: - """Sum of ``total_cost_usd`` over every job log of the Claude-running workflows.""" - total = 0.0 +def log_cost(log: str) -> dict[str, float]: + """One job log's session cost by credential: the cost lines, else Claude Code's own lines.""" + lines = COST_LINE_RE.findall(log) + if not lines: + return {"subscription": sum(float(m) for m in COST_RE.findall(log))} + by_auth: dict[str, float] = {} + for auth, usd in lines: + by_auth[auth] = by_auth.get(auth, 0.0) + float(usd) + return by_auth + + +def session_cost(runs: dict[str, list[dict[str, Any]]]) -> dict[str, float]: + """``total_cost_usd`` over every job log of the Claude-running workflows, by credential.""" + total: dict[str, float] = {} for workflow in CLAUDE_WORKFLOWS: for run in runs.get(workflow, []): jobs = _gh_json(["api", f"repos/{{owner}}/{{repo}}/actions/runs/{run['databaseId']}/jobs"]) or {} @@ -555,7 +584,8 @@ def session_cost(runs: dict[str, list[dict[str, Any]]]) -> float: log = _gh(["api", f"repos/{{owner}}/{{repo}}/actions/jobs/{job['id']}/logs"]) except subprocess.CalledProcessError: continue # an expired or missing log costs nothing to skip - total += sum(float(m) for m in COST_RE.findall(log)) + for auth, usd in log_cost(log).items(): + total[auth] = total.get(auth, 0.0) + usd return total @@ -602,7 +632,13 @@ def main(argv: Sequence[str] | None = None) -> int: ] for wf, listed in runs.items() } - summary = summarize(rows, run_counts, session_cost(runs) if args.costs else None) + costs = session_cost(runs) if args.costs else None + summary = summarize( + rows, + run_counts, + sum(costs.values()) if costs is not None else None, + costs.get("api", 0.0) if costs is not None else None, + ) sample, not_opened = render_sample(rows, seed=args.since) previous = previous_payload.get("summary") payload = { diff --git a/.github/workflows/daily-regen.yml b/.github/workflows/daily-regen.yml index 03b1a7bc10a..cffd91543b9 100644 --- a/.github/workflows/daily-regen.yml +++ b/.github/workflows/daily-regen.yml @@ -1,9 +1,19 @@ name: "Scheduled: Regen oldest specs" -run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.event.inputs.count || '1' }} / ${{ github.event.inputs.model || 'auto' }})" +run-name: "Scheduled regen (${{ github.event.inputs.specification_id || (github.event.inputs.pairs && format('{0} pairs', github.event.inputs.pairs)) || github.event.inputs.count || '1' }} / ${{ github.event.inputs.model || 'auto' }})" # Picks the N oldest specs (by most-recent implementation `updated` timestamp) # and re-dispatches `bulk-generate.yml` for each. Default N=1 per cron tick. # +# Pair mode (off unless configured): with the repository variable +# REGEN_PAIRS_PER_DAY set to N >= 1, the scheduled run instead picks the N +# oldest implementations — single (spec, library) pairs across the catalogue, +# aged by their own metadata and regeneration attempts — +# (automation/scripts/regen_pick_pairs.py) and dispatches each pair straight +# to impl-generate.yml, skipping the spec polish. A manual run opts in with the +# `pairs` input; the variable never changes a manual run, except the +# watchdog's cron rescue, which forwards it as `pairs` because it replaces the +# missed scheduled run. Unset, the schedule runs spec mode exactly as before. +# # Schedule: once per day at 02:17 UTC (04:17 Berlin under CEST, 03:17 under # CET — Actions cron is always UTC). Until 2026-09 it ticked every 2 hours # (10× per day); the owner cut it to one tick when re-enabling the workflow @@ -29,8 +39,9 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.e # # Triggers: # - schedule: once daily at 02:17 UTC → auto (Sonnet regenerations, Opus reviews) -# - workflow_dispatch: manual, with inputs for spec id, model, count, dry-run -# (also used by the watchdog's cron-liveness rescue, with all defaults) +# - workflow_dispatch: manual, with inputs for spec id, model, count, dry-run, +# pairs (also used by the watchdog's cron-liveness rescue, with all +# defaults — plus `pairs` when REGEN_PAIRS_PER_DAY is set) on: schedule: @@ -63,6 +74,10 @@ on: description: "Just print picks, do not trigger bulk-generate" type: boolean default: false + pairs: + description: "Pair mode: regen this many of the oldest implementations (spec × library) instead of whole specs, without spec polish (empty = spec mode; ignored when specification_id is set)" + required: false + default: '' permissions: contents: read @@ -77,11 +92,15 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - issues: read # spec issue updatedAt = last regen attempt per spec + issues: read # spec issue updatedAt = last regen attempt per spec + pull-requests: read # pair mode: implementation PRs = last attempt per pair + actions: read # pair mode: impl-generate runs = last attempt per pair outputs: - specs: ${{ steps.pick.outputs.specs }} - specs_json: ${{ steps.pick.outputs.specs_json }} - count: ${{ steps.pick.outputs.count }} + mode: ${{ steps.mode.outputs.mode }} + specs: ${{ steps.pick.outputs.specs || steps.pick_pairs.outputs.specs }} + specs_json: ${{ steps.pick.outputs.specs_json || steps.pick_pairs.outputs.specs_json }} + count: ${{ steps.pick.outputs.count || steps.pick_pairs.outputs.count }} + pairs_json: ${{ steps.pick_pairs.outputs.pairs_json }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -93,8 +112,32 @@ jobs: - name: Install PyYAML run: pip install pyyaml + # Spec mode unless a pair count is given: the `pairs` input on a manual + # run (the watchdog's cron rescue forwards REGEN_PAIRS_PER_DAY there), + # the REGEN_PAIRS_PER_DAY variable on the schedule. A spec override + # always wins. Anything but a positive integer stops the run instead of + # falling back to a whole spec. + - name: Resolve regen mode + id: mode + env: + PAIRS: ${{ github.event_name == 'schedule' && vars.REGEN_PAIRS_PER_DAY || inputs.pairs }} + SPEC_OVERRIDE: ${{ inputs.specification_id }} + run: | + if [ -n "$SPEC_OVERRIDE" ] || [ -z "$PAIRS" ]; then + echo "mode=spec" >> "$GITHUB_OUTPUT" + echo "Regen mode: spec" + elif [[ "$PAIRS" =~ ^[1-9][0-9]*$ ]]; then + echo "mode=pairs" >> "$GITHUB_OUTPUT" + echo "pairs=$PAIRS" >> "$GITHUB_OUTPUT" + echo "::notice::Regen mode: pairs (${PAIRS} per run)" + else + echo "::error::The pair count (REGEN_PAIRS_PER_DAY or the pairs input) must be a positive integer, got '${PAIRS//[^A-Za-z0-9 ._-]/?}'" + exit 1 + fi + - name: Pick oldest spec(s) (or use override) id: pick + if: steps.mode.outputs.mode == 'spec' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} @@ -235,6 +278,22 @@ jobs: f.write(f"count={len(picks)}\n") PY + # Pair mode: the N oldest (spec, library) pairs, with the spec mode's + # age rule applied per pair (see the script's docstring). The picks go + # to the log and the run summary, also on a dry run. + - name: Pick oldest pairs (pair mode) + id: pick_pairs + if: steps.mode.outputs.mode == 'pairs' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COUNT: ${{ steps.mode.outputs.pairs }} + MIN_AGE_HOURS: ${{ inputs.min_age_hours || '20' }} + run: | + LIBRARIES=$(python3 -c "from core.constants import SUPPORTED_LIBRARIES; print(' '.join(sorted(SUPPORTED_LIBRARIES)))") + python3 automation/scripts/regen_pick_pairs.py --count "$COUNT" --min-age-hours "$MIN_AGE_HOURS" \ + --repo "$GH_REPO" --libraries "$LIBRARIES" + # ============================================================================ # Pre-flight: per spec, run autonomous spec polish + cross-library similarity # audit, then dispatch bulk-generate with the resulting change_requests. @@ -248,6 +307,15 @@ jobs: # model ("auto" unless a manual run pins one) is passed through to # bulk-generate (and from there to impl-generate and repair) unchanged; # reviews always run on Opus. + # + # Pair mode: one matrix entry per spec among the picked pairs. The spec + # polish is skipped (a spec rewritten for one library would leave its other + # implementations written against the old text), the similarity audit runs + # as in spec mode, and each picked pair of the spec is dispatched straight + # to impl-generate.yml with bulk-generate's arguments and pacing — not via + # bulk-generate.yml, whose single concurrency group keeps at most one + # pending run and cancels the older one (see the watchdog's + # dispatch_generate). # ============================================================================ preflight-dispatch: needs: pick @@ -268,6 +336,31 @@ jobs: # Side-effect steps (polish, dispatch) are individually gated on # `!inputs.dry_run` below. steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one per + # run and each Claude step passes exactly that one, the other input + # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; + # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, + # stops the job here, never a silent fallback to the subscription. + # Identical in every job that runs Claude (tests/unit/workflows/ + # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + - name: Check Claude credentials + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + case "${CLAUDE_AUTH,,}" in + ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; + api) + if [ "$API_KEY_SET" != "true" ]; then + echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" + exit 1 + fi + echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; + *) + echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" + exit 1 ;; + esac + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -294,14 +387,16 @@ jobs: fi - name: Spec polish (autonomous, opens PR with auto-merge) - if: ${{ steps.gate.outputs.skip_polish == '0' && !inputs.dry_run }} + id: polish + if: ${{ steps.gate.outputs.skip_polish == '0' && !inputs.dry_run && needs.pick.outputs.mode != 'pairs' }} # Optional quality pass: a transient action failure here must not # block the main regeneration pipeline. Skip cleanly and continue. continue-on-error: true timeout-minutes: 15 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: '--model sonnet --settings ${{ github.workspace }}/.claude/settings.json' allowed_bots: '*' prompt: | @@ -310,14 +405,32 @@ jobs: Variables for this run: - SPEC_ID: ${{ matrix.spec_id }} + # The session's cost (Claude Code's own estimate at API list price) and + # the credential it ran on, for the weekly regen review. Never fatal. + - name: Claude cost to the job summary (spec polish) + if: ${{ !cancelled() && (steps.polish.outcome == 'success' || steps.polish.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.polish.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: daily-regen/polish + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Cross-library similarity audit + id: similarity # Read-only audit; if it fails, fall back to empty change_requests # rather than aborting the dispatch. continue-on-error: true timeout-minutes: 15 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: '--model sonnet --settings ${{ github.workspace }}/.claude/settings.json' allowed_bots: '*' prompt: | @@ -326,6 +439,20 @@ jobs: Variables for this run: - SPEC_ID: ${{ matrix.spec_id }} + - name: Claude cost to the job summary (similarity audit) + if: ${{ !cancelled() && (steps.similarity.outcome == 'success' || steps.similarity.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.similarity.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: daily-regen/similarity + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Collect change_requests id: collect run: | @@ -348,7 +475,7 @@ jobs: echo "::notice::change_requests for ${{ matrix.spec_id }}: ${flagged} lib(s) flagged — ${CR_COMPACT}" - name: Dispatch bulk-generate with change_requests - if: ${{ !inputs.dry_run }} + if: ${{ !inputs.dry_run && needs.pick.outputs.mode != 'pairs' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SPEC_ID: ${{ matrix.spec_id }} @@ -366,3 +493,57 @@ jobs: # the next matrix entry's dispatch (matrix is serialized via # max-parallel: 1, so this is between specs). sleep 5 + + # Pair mode: bulk-generate's per-pair dispatch for each picked library of + # this spec — the issue from specification.yaml, the library's + # divergence hint, the regen gate on, three dispatch attempts — with + # bulk-generate's default pace (180 s, 120 s for a pinned sonnet or + # haiku) before every dispatch but the run's first. impl-generate sets + # `impl::pending` itself, so this job needs no issue permission. + - name: Dispatch impl-generate for each picked pair (pair mode) + if: ${{ !inputs.dry_run && needs.pick.outputs.mode == 'pairs' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SPEC_ID: ${{ matrix.spec_id }} + MODEL: ${{ inputs.model || 'auto' }} + CHANGE_REQUESTS: ${{ steps.collect.outputs.change_requests }} + PAIRS_JSON: ${{ needs.pick.outputs.pairs_json }} + PACE_SECONDS: ${{ (inputs.model == 'sonnet' || inputs.model == 'haiku') && '120' || '180' }} + run: | + set -u + ISSUE="" + if [ -f "plots/${SPEC_ID}/specification.yaml" ]; then + ISSUE=$(yq '.issue' "plots/${SPEC_ID}/specification.yaml" 2>/dev/null || echo "") + [ "$ISSUE" = "null" ] && ISSUE="" + fi + FIRST=$(echo "$PAIRS_JSON" | jq -r '.[0] | "\(.spec)/\(.library)"') + failed=0 + for LIBRARY in $(echo "$PAIRS_JSON" | jq -r --arg spec "$SPEC_ID" '.[] | select(.spec == $spec) | .library'); do + if [ "${SPEC_ID}/${LIBRARY}" != "$FIRST" ]; then + echo "Sleeping ${PACE_SECONDS}s before the next dispatch…" + sleep "$PACE_SECONDS" + fi + HINT=$(echo "$CHANGE_REQUESTS" | jq -r --arg lib "$LIBRARY" '.[$lib] // ""') + args=(-f specification_id="${SPEC_ID}" -f library="${LIBRARY}" -f model="${MODEL}" + -f change_request="${HINT}" -f regen_gate=true -f source_ref= -f source_model=) + if [ -n "$ISSUE" ]; then + args+=(-f issue_number="${ISSUE}") + fi + echo "::notice::$(date -u +%H:%M:%SZ) dispatching impl-generate for ${SPEC_ID}/${LIBRARY} (issue: ${ISSUE:-none}, model=${MODEL}, change_request: ${HINT:-none})" + dispatched=0 + for attempt in 1 2 3; do + if gh workflow run impl-generate.yml --repo "${{ github.repository }}" "${args[@]}"; then + dispatched=1 + break + fi + echo "::warning::Dispatch attempt $attempt failed for ${SPEC_ID}/${LIBRARY}, retrying in 10s" + sleep 10 + done + if [ "$dispatched" = "0" ]; then + echo "::error::Failed to dispatch ${SPEC_ID}/${LIBRARY} after 3 attempts — continuing" + failed=$((failed + 1)) + fi + done + if [ "$failed" -gt 0 ]; then + exit 1 + fi diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml index b17dce9091c..d02377ce669 100644 --- a/.github/workflows/impl-generate.yml +++ b/.github/workflows/impl-generate.yml @@ -98,6 +98,33 @@ jobs: pr_number: ${{ steps.pr.outputs.pr_number }} steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one per + # run and each Claude step passes exactly that one, the other input + # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; + # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, + # stops the job here, never a silent fallback to the subscription. + # Identical in every job that runs Claude (tests/unit/workflows/ + # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + # First step: a stop here precedes "Set final issue number", so "Handle + # generation failure" never counts it against the pair's attempts. + - name: Check Claude credentials + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + case "${CLAUDE_AUTH,,}" in + ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; + api) + if [ "$API_KEY_SET" != "true" ]; then + echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" + exit 1 + fi + echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; + *) + echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" + exit 1 ;; + esac + # ======================================================================== # Setup: Extract inputs and validate # ======================================================================== @@ -651,7 +678,8 @@ jobs: timeout-minutes: 60 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model ${{ steps.inputs.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -665,13 +693,31 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} + # The session's cost (Claude Code's own estimate at API list price) and + # the credential it ran on, for the weekly regen review. Read right after + # the step: the retry writes the same execution file. Never fatal. + - name: Claude cost to the job summary + if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: impl-generate + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Retry Claude (on failure) if: steps.claude.outcome == 'failure' && steps.inputs.outputs.source_ref == '' id: claude_retry timeout-minutes: 60 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model ${{ steps.inputs.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -685,6 +731,20 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} + - name: Claude cost to the job summary (retry) + if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: impl-generate/retry + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + # ======================================================================== # Create metadata file (before PR) # ======================================================================== diff --git a/.github/workflows/impl-repair.yml b/.github/workflows/impl-repair.yml index c91dc9810a0..6c746f27011 100644 --- a/.github/workflows/impl-repair.yml +++ b/.github/workflows/impl-repair.yml @@ -52,6 +52,31 @@ jobs: id-token: write steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one per + # run and each Claude step passes exactly that one, the other input + # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; + # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, + # stops the job here, never a silent fallback to the subscription. + # Identical in every job that runs Claude (tests/unit/workflows/ + # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + - name: Check Claude credentials + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + case "${CLAUDE_AUTH,,}" in + ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; + api) + if [ "$API_KEY_SET" != "true" ]; then + echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" + exit 1 + fi + echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; + *) + echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" + exit 1 ;; + esac + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -229,7 +254,8 @@ jobs: timeout-minutes: 45 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model ${{ steps.model.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -243,13 +269,31 @@ jobs: - ATTEMPT: ${{ inputs.attempt }} - BRANCH: ${{ env.branch }} + # The session's cost (Claude Code's own estimate at API list price) and + # the credential it ran on, for the weekly regen review. Read right after + # the step: the retry writes the same execution file. Never fatal. + - name: Claude cost to the job summary + if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: impl-repair + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Retry Claude (on failure) if: steps.claude.outcome == 'failure' id: claude_retry timeout-minutes: 45 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model ${{ steps.model.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -263,6 +307,20 @@ jobs: - ATTEMPT: ${{ inputs.attempt }} - BRANCH: ${{ env.branch }} + - name: Claude cost to the job summary (retry) + if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: impl-repair/retry + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Process repaired image if: success() env: diff --git a/.github/workflows/impl-review.yml b/.github/workflows/impl-review.yml index 2f402e32f66..42198256950 100644 --- a/.github/workflows/impl-review.yml +++ b/.github/workflows/impl-review.yml @@ -48,6 +48,31 @@ jobs: actions: write steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one per + # run and each Claude step passes exactly that one, the other input + # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; + # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, + # stops the job here, never a silent fallback to the subscription. + # Identical in every job that runs Claude (tests/unit/workflows/ + # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + - name: Check Claude credentials + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + case "${CLAUDE_AUTH,,}" in + ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; + api) + if [ "$API_KEY_SET" != "true" ]; then + echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" + exit 1 + fi + echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; + *) + echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" + exit 1 ;; + esac + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -890,7 +915,8 @@ jobs: PR_NUMBER: ${{ steps.pr.outputs.pr_number }} ATTEMPT: ${{ steps.attempts.outputs.display }} with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model ${{ steps.pr.outputs.review_model_alias }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -925,6 +951,22 @@ jobs: run: | python3 "$RUNNER_TEMP/regen-tools/review_provenance.py" model --execution-file "${EXECUTION_FILE:-}" + # The session's cost (Claude Code's own estimate at API list price) and + # the credential it ran on, for the weekly regen review. Never fatal. + - name: Claude cost to the job summary + if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: impl-review + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + # Undo "Hide the previous review", whatever the review step did: the # metadata step below reads the metadata file from disk and rewrites # the header. The implementation file goes back to the branch's copy diff --git a/.github/workflows/report-validate.yml b/.github/workflows/report-validate.yml index 9e0d9512ca9..07983083e83 100644 --- a/.github/workflows/report-validate.yml +++ b/.github/workflows/report-validate.yml @@ -41,6 +41,32 @@ jobs: fi echo "should_run=true" >> $GITHUB_OUTPUT + # Claude credentials: the repository variable CLAUDE_AUTH picks one per + # run and each Claude step passes exactly that one, the other input + # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; + # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, + # stops the job here, never a silent fallback to the subscription. + # Identical in every job that runs Claude (tests/unit/workflows/ + # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + - name: Check Claude credentials + if: steps.check.outputs.should_run == 'true' + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + case "${CLAUDE_AUTH,,}" in + ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; + api) + if [ "$API_KEY_SET" != "true" ]; then + echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" + exit 1 + fi + echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; + *) + echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" + exit 1 ;; + esac + - name: Checkout repository if: steps.check.outputs.should_run == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -65,7 +91,8 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Validate Issue Report @@ -157,6 +184,22 @@ jobs: - Do NOT trigger any fixes - Keep the analysis comment concise but informative + # The session's cost (Claude Code's own estimate at API list price) and + # the credential it ran on. Never fatal. + - name: Claude cost to the job summary + if: ${{ !cancelled() && (steps.validate.outcome == 'success' || steps.validate.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.validate.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: report-validate + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Add reaction on success if: steps.check.outputs.should_run == 'true' && steps.validate.outcome == 'success' env: diff --git a/.github/workflows/review-retest.yml b/.github/workflows/review-retest.yml index 8ce9efac753..ba314c0c696 100644 --- a/.github/workflows/review-retest.yml +++ b/.github/workflows/review-retest.yml @@ -224,6 +224,31 @@ jobs: CELL_JSON: ${{ toJSON(matrix.cell) }} RULES_SHA: ${{ needs.prep.outputs.rules_sha }} steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one per + # run and each Claude step passes exactly that one, the other input + # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; + # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, + # stops the job here, never a silent fallback to the subscription. + # Identical in every job that runs Claude (tests/unit/workflows/ + # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + - name: Check Claude credentials + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + case "${CLAUDE_AUTH,,}" in + ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; + api) + if [ "$API_KEY_SET" != "true" ]; then + echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" + exit 1 + fi + echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; + *) + echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" + exit 1 ;; + esac + - name: Checkout harness uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -309,7 +334,8 @@ jobs: PR_NUMBER: '0' ATTEMPT: '1' with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} github_token: ${{ github.token }} claude_args: "--model ${{ matrix.cell.model }} --settings ${{ github.workspace }}/automation/retest/claude-settings.json" prompt: | @@ -333,6 +359,22 @@ jobs: to the PR, write the exact comment body to `review_comment.md` in the repository root instead. Do not run `gh` or `git push`; every other instruction applies unchanged. + # The session's cost (Claude Code's own estimate at API list price) and + # the credential it ran on. Never fatal. + - name: Claude cost to the job summary + if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: review-retest + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Collect the cell if: always() env: diff --git a/.github/workflows/spec-create.yml b/.github/workflows/spec-create.yml index c99c68987b4..ed362f647ca 100644 --- a/.github/workflows/spec-create.yml +++ b/.github/workflows/spec-create.yml @@ -69,6 +69,32 @@ jobs: fi echo "should_run=true" >> "$GITHUB_OUTPUT" + # Claude credentials: the repository variable CLAUDE_AUTH picks one per + # run and each Claude step passes exactly that one, the other input + # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; + # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, + # stops the job here, never a silent fallback to the subscription. + # Identical in every job that runs Claude (tests/unit/workflows/ + # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + - name: Check Claude credentials + if: steps.check.outputs.should_run == 'true' + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + case "${CLAUDE_AUTH,,}" in + ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; + api) + if [ "$API_KEY_SET" != "true" ]; then + echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" + exit 1 + fi + echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; + *) + echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" + exit 1 ;; + esac + - name: Checkout repository if: steps.check.outputs.should_run == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -99,7 +125,8 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Create New Specification @@ -248,6 +275,23 @@ jobs: - Do NOT close the issue - STOP after step 11 + # The session's cost (Claude Code's own estimate at API list price) and + # the credential it ran on. Read right after the step: the retry writes + # the same execution file. Never fatal. + - name: Claude cost to the job summary + if: ${{ !cancelled() && (steps.process.outcome == 'success' || steps.process.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.process.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: spec-create + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Retry Claude (on failure) if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' id: process_retry @@ -259,7 +303,8 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Create New Specification @@ -417,6 +462,20 @@ jobs: - Do NOT close the issue - STOP after step 11 + - name: Claude cost to the job summary (retry) + if: ${{ !cancelled() && (steps.process_retry.outcome == 'success' || steps.process_retry.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.process_retry.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: spec-create/retry + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Extract outputs if: steps.check.outputs.should_run == 'true' id: extract diff --git a/.github/workflows/util-claude.yml b/.github/workflows/util-claude.yml index a4dd6fb9a7f..9aaf8c3042d 100644 --- a/.github/workflows/util-claude.yml +++ b/.github/workflows/util-claude.yml @@ -33,6 +33,31 @@ jobs: id-token: write actions: read # Required for Claude to read CI results on PRs steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one per + # run and each Claude step passes exactly that one, the other input + # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; + # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, + # stops the job here, never a silent fallback to the subscription. + # Identical in every job that runs Claude (tests/unit/workflows/ + # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + - name: Check Claude credentials + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + run: | + case "${CLAUDE_AUTH,,}" in + ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; + api) + if [ "$API_KEY_SET" != "true" ]; then + echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" + exit 1 + fi + echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; + *) + echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" + exit 1 ;; + esac + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -43,21 +68,54 @@ jobs: continue-on-error: true uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" # This is an optional setting that allows Claude to read CI results on PRs additional_permissions: | actions: read + # The session's cost (Claude Code's own estimate at API list price) and + # the credential it ran on. Read right after the step: the retry writes + # the same execution file. Never fatal. + - name: Claude cost to the job summary + if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: util-claude + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Retry Claude (on failure) if: steps.claude.outcome == 'failure' id: claude_retry uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" additional_permissions: | actions: read + - name: Claude cost to the job summary (retry) + if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_STEP: util-claude/retry + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index 541f8c74108..923a3718f93 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -69,6 +69,10 @@ jobs: GH_REPO: ${{ github.repository }} STALE_HOURS: ${{ inputs.stale_hours || '4' }} DRY_RUN: ${{ inputs.dry_run || 'false' }} + # daily-regen's pair mode: the cron rescue replaces a missed + # scheduled run, so it forwards the pair count the schedule would + # have used. Empty = spec mode, the dispatch as before. + REGEN_PAIRS: ${{ vars.REGEN_PAIRS_PER_DAY }} run: | set -euo pipefail STALE_SEC=$(( STALE_HOURS * 3600 )) @@ -568,7 +572,7 @@ jobs: gap_hm=$(printf '%dh%02dm' $(( gap / 3600 )) $(( (gap % 3600) / 60 ))) if (( gap > LIVENESS_HOURS * 3600 )); then echo "::warning::daily-regen: no new run on main for ${gap_hm} (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" - dispatch "daily-regen: cron starved (${gap_hm} silent)" daily-regen.yml || true + dispatch "daily-regen: cron starved (${gap_hm} silent)" daily-regen.yml ${REGEN_PAIRS:+-f "pairs=$REGEN_PAIRS"} || true else echo "daily-regen liveness: ${ref_label} ${gap_hm} ago — healthy" fi diff --git a/agentic/docs/project-guide.md b/agentic/docs/project-guide.md index bed00e977f3..eee5027e5e2 100644 --- a/agentic/docs/project-guide.md +++ b/agentic/docs/project-guide.md @@ -768,7 +768,7 @@ Issue ready for maintainer review | **sync-labels.yml** | Auto-syncs labels after manual PR merges (spec-ready, impl:*:done) | | **notify-deployment.yml** | Deployment notifications | | **util-claude.yml** | Claude utility workflow | -| **daily-regen.yml** | Scheduled: picks the oldest specs, runs spec polish + cross-library similarity audit, then dispatches bulk-generate | +| **daily-regen.yml** | Scheduled: picks the oldest specs, runs spec polish + cross-library similarity audit, then dispatches bulk-generate. With the repository variable `REGEN_PAIRS_PER_DAY` (or the manual `pairs` input) it picks the N oldest (spec, library) pairs instead (`automation/scripts/regen_pick_pairs.py`), skips the polish and dispatches each pair straight to impl-generate; see `docs/workflows/overview.md#daily-regeneration-modes` | | **watchdog-stuck-jobs.yml** | Periodic safety net: detects and unsticks stalled impl-pipeline PRs | | **bot-serving-check.yml** | Daily synthetic monitor for the nginx bot -> seo-proxy path; routes and titles derived from `api/routers/seo.py` and the spec files, and a failure opens/comments the fixed-title issue "Bot serving check is red" (closed again by the next green run) | @@ -799,6 +799,7 @@ The new architecture separates specification and implementation processes: - **Adopt mode.** The `source_ref` input (default empty) on `impl-generate.yml` and `bulk-generate.yml` takes each implementation file from that branch, tag or SHA instead of running Claude, for code written outside the pipeline (a `gemini/*` branch). The run renders the file with the generator's commands, checks the canvas with `core.canvas`, and is always a forced regeneration (`regen:forced`); the routed model only runs repair, and the PR body adds a `**Generation:** adopted from …` line above `**Model:**`. The optional `source_model` input (a model name such as `gemini-4-argon`, only with `source_ref`) names the model that wrote the code and goes on that line. The metadata records `generated_by: {source_model}`, or `generated_by: adopted:{source_ref}` without one, followed by `adopted_from: {source_ref} ({sha})` for every adopted file. Both workflows reject a malformed `source_ref` or `source_model` without echoing it. Retries keep both inputs; the watchdog forwards neither. Python and JavaScript only. A missing file or a failed render is never retried and never counts toward the 3-attempt cap. See `docs/workflows/overview.md#adopt-implementations-from-a-branch`. - **Review model.** Every quality review runs on Opus, whatever model generated the implementation: first implementations, reviews after a repair, and the regen gate's session (the predecessor's re-score and the new render's review). Only `impl-review.yml` chooses it; its `review_model` input (default `opus`) pins another model for one manual review and that run's own auto-retry, and no other workflow forwards one. The retest harness's `models=production` runs every cell on Opus to match (`review_retest.PRODUCTION_MODELS`, kept equal by `tests/unit/workflows/test_model_routing.py`). - Fixed models: `spec-create.yml` runs on Opus; the spec polish and cross-library similarity audit in `daily-regen.yml` run on Sonnet. See `docs/workflows/overview.md#pipeline-models`. +- **Credentials.** The repository variable `CLAUDE_AUTH` chooses what every Claude step runs on: unset or `subscription` = the `CLAUDE_CODE_OAUTH_TOKEN` secret (the owner's subscription), `api` = the `ANTHROPIC_API_KEY` secret (API credits). Each step passes exactly one of the two; the step "Check Claude credentials" at the start of every Claude job stops on any other value or on `api` without the key, never falling back to the subscription. After each Claude step, "Claude cost to the job summary" records `total_cost_usd` and the credential. A new Claude step copies both patterns, which `tests/unit/workflows/test_claude_auth.py` enforces. See `docs/workflows/overview.md#claude-credentials`. ## GitHub Issue Labels diff --git a/automation/scripts/README.md b/automation/scripts/README.md index 71d18cd31a4..48b38b5a813 100644 --- a/automation/scripts/README.md +++ b/automation/scripts/README.md @@ -256,6 +256,21 @@ Closes a spec issue once every supported library is `impl::done` or `:failed`. Shared by `impl-merge.yml` and the kept-regen branch of `impl-review.yml`. +### `regen_pick_pairs.py` +The pick of `daily-regen.yml`'s pair mode (`REGEN_PAIRS_PER_DAY`, or the +`pairs` input): the N oldest (spec, library) pairs across the catalogue, aged by +the newer of their metadata `updated` and their last attempt (implementation PR, +`regen:kept` PR, `impl-generate.yml` run). It skips pairs younger than +`--min-age-hours`, pairs with an open implementation PR, and every pair of a +spec whose issue saw activity in that window. PyYAML plus `gh`; a failed listing +degrades with a warning. The library list comes from the caller. + +```bash +python3 automation/scripts/regen_pick_pairs.py --count 5 --min-age-hours 20 \ + --repo OWNER/NAME --libraries "altair bokeh ..." +# writes pairs_json, specs, specs_json and count to $GITHUB_OUTPUT +``` + ### `spec_pr_guard.py` Fail-closed file allowlist for the spec PR that the merge job in `spec-create.yml` merges: the diff of the PR head against its merge base with `main` may only add or diff --git a/automation/scripts/regen_pick_pairs.py b/automation/scripts/regen_pick_pairs.py new file mode 100644 index 00000000000..6fabc5011a2 --- /dev/null +++ b/automation/scripts/regen_pick_pairs.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Pick the oldest implementations for daily-regen's pair mode. + +Spec mode, the default, regenerates the N oldest specs with all their +libraries. Pair mode picks the N oldest implementations, one (spec, library) +pair each, across the catalogue, and applies the spec mode's rules per pair: + +- A pair's age counts from the newer of its metadata ``updated`` (``created`` + when there is none) and its last regeneration attempt, so a regeneration the + gate kept, which leaves ``updated`` alone, is not picked again the next + night. The attempt record is the newest of the pair's implementation PR + (head ``implementation//``, any state), its ``regen:kept`` PR + and its ``impl-generate.yml`` run (``Generate: for ``, which + also covers a generation that failed before it opened a PR). +- A pair younger than ``--min-age-hours`` is skipped, and so is every pair of + a spec whose issue saw activity within that window: a comment, a label or a + report postpones the spec as in spec mode, and a night does not stack a + spec whose last batch is still in flight. +- A pair with an open implementation PR is skipped: its last attempt is still + running or stuck, and the watchdog owns it. +- An unparseable metadata timestamp skips the pair, as in spec mode. Only + pairs with a metadata file and a ``specification.md`` are candidates, so + pair mode never writes a first implementation; gap backfill stays with + ``bulk-generate.yml``. + +Each listing is one ``gh`` call of at most 1000 entries, most recent first. A +listing that fails degrades to the other records with a warning, like the +spec mode's issue listing. + +Usage, from the repository root:: + + python3 automation/scripts/regen_pick_pairs.py --count 5 --min-age-hours 20 \\ + --repo OWNER/NAME --libraries "altair bokeh ..." + +The library list comes from the caller (``core.constants.SUPPORTED_LIBRARIES``), +so this file carries no copy that could drift. Writes ``pairs_json``, +``specs``, ``specs_json`` and ``count`` to ``$GITHUB_OUTPUT`` and a table of +the picks to ``$GITHUB_STEP_SUMMARY``. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import yaml + + +ANCIENT = datetime.min.replace(tzinfo=timezone.utc) +LIMIT = "1000" +BRANCH_RE = re.compile(r"^implementation/(?P[a-z0-9][a-z0-9-]*)/(?P[a-z0-9]+)$") +RUN_TITLE_RE = re.compile(r"^Generate: (?P[a-z0-9]+) for (?P[a-z0-9][a-z0-9-]*)$") + +Pair = tuple[str, str] + + +@dataclass(frozen=True) +class Candidate: + spec: str + library: str + metadata: datetime | None # None: the metadata file names no timestamp + + +@dataclass(frozen=True) +class Pick: + spec: str + library: str + touched: datetime + basis: str + + +@dataclass +class Records: + """What GitHub says about past attempts, per pair and per spec issue.""" + + attempts: dict[Pair, tuple[datetime, str]] + open_pairs: set[Pair] + issue_activity: dict[int, datetime] + + +def parse_ts(raw: str) -> datetime | None: + try: + dt = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + return None + return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc) + + +def load_candidates(plots: Path, libraries: Iterable[str]) -> tuple[list[Candidate], dict[str, int]]: + """Every pair with a metadata file, plus each spec's issue number.""" + supported = set(libraries) + candidates: list[Candidate] = [] + issues: dict[str, int] = {} + for spec_dir in sorted(plots.iterdir()): + if not spec_dir.is_dir() or spec_dir.name.startswith(".") or not (spec_dir / "specification.md").is_file(): + continue + try: + issue = (yaml.safe_load((spec_dir / "specification.yaml").read_text(encoding="utf-8")) or {}).get("issue") + except OSError, yaml.YAMLError, AttributeError: + issue = None + if isinstance(issue, int): + issues[spec_dir.name] = issue + for meta in sorted(spec_dir.glob("metadata/*/*.yaml")): + if meta.stem not in supported: + continue + try: + data = yaml.safe_load(meta.read_text(encoding="utf-8")) or {} + except OSError, yaml.YAMLError: + continue + if not isinstance(data, dict): + continue + raw = data.get("updated") or data.get("created") + if not raw: + candidates.append(Candidate(spec_dir.name, meta.stem, None)) + continue + dt = parse_ts(str(raw)) + if dt is None: + continue # unparseable timestamp: skipped, as in spec mode + candidates.append(Candidate(spec_dir.name, meta.stem, dt)) + return candidates, issues + + +def _newer(records: dict[Pair, tuple[datetime, str]], pair: Pair, when: datetime | None, basis: str) -> None: + if when is not None and (pair not in records or when > records[pair][0]): + records[pair] = (when, basis) + + +def fold_pulls(pulls: Iterable[dict[str, Any]], basis: str, records: Records) -> None: + for pull in pulls: + match = BRANCH_RE.match(str(pull.get("headRefName") or "")) + if not match: + continue + pair = (match["spec"], match["lib"]) + _newer(records.attempts, pair, parse_ts(str(pull.get("updatedAt") or "")), basis) + if pull.get("state") == "OPEN": + records.open_pairs.add(pair) + + +def fold_runs(runs: Iterable[dict[str, Any]], records: Records) -> None: + for run in runs: + match = RUN_TITLE_RE.match(str(run.get("displayTitle") or "")) + if match: + _newer(records.attempts, (match["spec"], match["lib"]), parse_ts(str(run.get("createdAt") or "")), "run") + + +def fold_issues(issues: Iterable[dict[str, Any]], records: Records) -> None: + for issue in issues: + ts = parse_ts(str(issue.get("updatedAt") or "")) + if ts and isinstance(issue.get("number"), int): + records.issue_activity[issue["number"]] = ts + + +def pick( + candidates: Iterable[Candidate], + records: Records, + spec_issues: dict[str, int], + now: datetime, + min_age: timedelta, + count: int, +) -> tuple[list[Pick], int]: + """The ``count`` oldest eligible pairs, oldest first, and how many were eligible.""" + eligible: list[Pick] = [] + for cand in candidates: + pair = (cand.spec, cand.library) + if pair in records.open_pairs: + continue + issue_ts = records.issue_activity.get(spec_issues.get(cand.spec, -1)) + if issue_ts is not None and now - issue_ts < min_age: + continue + touched, basis = (cand.metadata, "metadata") if cand.metadata else (None, "none") + attempt = records.attempts.get(pair) + if attempt and (touched is None or attempt[0] > touched): + touched, basis = attempt + if touched is None: + eligible.append(Pick(cand.spec, cand.library, ANCIENT, basis)) + continue + if now - touched < min_age: + continue # too fresh to re-roll + eligible.append(Pick(cand.spec, cand.library, touched, basis)) + eligible.sort(key=lambda p: (p.touched, p.spec, p.library)) + return eligible[:count], len(eligible) + + +def _gh_json(args: list[str], what: str) -> list[dict[str, Any]] | None: + try: + raw = subprocess.run(["gh", *args], check=True, capture_output=True, text=True).stdout + data = json.loads(raw) + except (subprocess.CalledProcessError, ValueError, OSError) as exc: + print(f"::warning::Could not list {what} ({' '.join(str(exc).split())[:200]}) — picking without them") + return None + return data if isinstance(data, list) else None + + +def read_records(repo: str) -> Records: + records = Records(attempts={}, open_pairs=set(), issue_activity={}) + pr_list = ["pr", "list", "--repo", repo, "--limit", LIMIT, "--json", "headRefName,state,updatedAt"] + pulls = _gh_json( + [*pr_list, "--state", "all", "--search", "head:implementation/ sort:updated-desc"], "implementation PRs" + ) + fold_pulls(pulls or [], "implementation PR", records) + # Kept regenerations again on their own: they are the attempts that leave + # `updated` alone, and this listing reaches much further back. + kept = _gh_json( + [*pr_list, "--state", "closed", "--label", "regen:kept", "--search", "sort:updated-desc"], "kept regenerations" + ) + fold_pulls(kept or [], "kept PR", records) + runs = _gh_json( + ["run", "list", "--repo", repo, "--workflow", "impl-generate.yml", "--limit", LIMIT] + + ["--json", "displayTitle,createdAt"], + "impl-generate runs", + ) + fold_runs(runs or [], records) + issues = _gh_json( + ["issue", "list", "--repo", repo, "--state", "all", "--search", "sort:updated-desc", "--limit", LIMIT] + + ["--json", "number,updatedAt"], + "issues", + ) + fold_issues(issues or [], records) + print( + f"::notice::Attempt records for {len(records.attempts)} pair(s), {len(records.open_pairs)} open PR(s), " + f"activity for {len(records.issue_activity)} issue(s)" + ) + return records + + +def summary_table(picks: list[Pick], eligible: int, count: int) -> str: + lines = [ + f"### Pair mode: {len(picks)} of {count} requested ({eligible} eligible)", + "", + "| # | Spec | Library | Last touched | From |", + "|---|------|---------|--------------|------|", + ] + for i, p in enumerate(picks, 1): + touched = "never" if p.touched == ANCIENT else p.touched.strftime("%Y-%m-%dT%H:%M:%SZ") + lines.append(f"| {i} | `{p.spec}` | `{p.library}` | {touched} | {p.basis} |") + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) + parser.add_argument("--count", type=int, required=True) + parser.add_argument("--min-age-hours", type=int, required=True) + parser.add_argument("--repo", required=True) + parser.add_argument("--libraries", required=True, help="space-separated supported libraries") + parser.add_argument("--plots", default="plots") + args = parser.parse_args(argv) + + libraries = args.libraries.split() + if args.count < 1 or not libraries: + print("::error::--count must be at least 1 and --libraries must name at least one library") + return 1 + + candidates, spec_issues = load_candidates(Path(args.plots), libraries) + records = read_records(args.repo) + picks, eligible = pick( + candidates, records, spec_issues, datetime.now(timezone.utc), timedelta(hours=args.min_age_hours), args.count + ) + + print(f"::notice::Eligible pairs: {eligible} picked: {[f'{p.spec}/{p.library}' for p in picks]}") + for p in picks: + print(f" - {p.spec + '/' + p.library:52s} last_touched={p.touched.isoformat()} ({p.basis})") + + specs = list(dict.fromkeys(p.spec for p in picks)) + pairs = [{"spec": p.spec, "library": p.library} for p in picks] + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"pairs_json={json.dumps(pairs, separators=(',', ':'))}\n") + f.write(f"specs={' '.join(specs)}\n") + f.write(f"specs_json={json.dumps(specs)}\n") + f.write(f"count={len(picks)}\n") + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as f: + f.write(summary_table(picks, eligible, args.count)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/changelog.d/ci-claude-auth-switch.md b/changelog.d/ci-claude-auth-switch.md new file mode 100644 index 00000000000..505b2ee2297 --- /dev/null +++ b/changelog.d/ci-claude-auth-switch.md @@ -0,0 +1,28 @@ +### Added + +- **The pipeline's Claude steps can run on API credits: the repository + variable `CLAUDE_AUTH`.** Unset or `subscription`, all 13 Claude steps in the + 8 workflows that run Claude use the `CLAUDE_CODE_OAUTH_TOKEN` secret as + before; `api` switches them to the `ANTHROPIC_API_KEY` secret. Each step + passes exactly one credential, because the action does not document which + one wins when both are set. A new first step, "Check Claude credentials", + stops the job on any other value and on `api` without the key, instead of + falling back to the subscription and the owner's interactive quota. + `docs/workflows/overview.md` has the switch, the spend limit and what the + pipeline does when the limit is reached. +- **Every Claude session's cost in the job summary.** A step after each + Claude step writes `total_cost_usd` (Claude Code's estimate at API list + price), the model and the credential to the job summary and the log, and + never fails the job. The weekly regen review's `collect_week.py --costs` + reads it, now counts the `daily-regen.yml` pre-flight too, and names the + part that ran on API credits. +- **Daily regeneration by implementation instead of by spec: the repository + variable `REGEN_PAIRS_PER_DAY`.** Set to N, the scheduled `daily-regen.yml` + run picks the N oldest (spec, library) pairs across the catalogue + (`automation/scripts/regen_pick_pairs.py`), aged by their own metadata and + their last regeneration attempt so a kept regeneration is not picked again, + skips the spec polish, and dispatches each pair straight to + `impl-generate.yml` with `bulk-generate.yml`'s arguments and pace, since + that workflow's single concurrency group would cancel queued runs. Unset, + the schedule regenerates whole specs as before; a manual run opts in with the + new `pairs` input, and the watchdog's cron rescue forwards the variable. diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index cfc745291a5..f45ad8d91d8 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -398,7 +398,7 @@ Located in `.github/workflows/`: | `impl-merge.yml` | Merges approved PRs, and the review write-back PRs a kept regeneration opens (job `writeback`) | | `bulk-generate.yml` | Batch implementation generation | | `review-retest.yml` | Dispatch-only measurement: re-runs the AI quality review on the frozen retest set (several fresh sessions per item, regen pairs in both orders) for one rules version, and reports score spread, verdict flips, weakness agreement, and gate order bias against a baseline arm. Read-only tokens; writes nothing outside its own artifacts. See [Review retest](review-retest.md) | -| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (once a day at 02:17 UTC, off the top of the hour to dodge GitHub's scheduler overload). A spec's age counts from the newer of its last merged update and the last activity on its spec issue (every regen touches the issue), so a kept regeneration is not re-picked the next night. Any activity on the issue — a comment, a label, a report — postpones that spec's regen the same way | +| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (once a day at 02:17 UTC, off the top of the hour to dodge GitHub's scheduler overload). A spec's age counts from the newer of its last merged update and the last activity on its spec issue (every regen touches the issue), so a kept regeneration is not re-picked the next night. Any activity on the issue — a comment, a label, a report — postpones that spec's regen the same way. With the repository variable `REGEN_PAIRS_PER_DAY`, the scheduled run regenerates single (spec, library) pairs instead of whole specs; see [Daily regeneration modes](#daily-regeneration-modes) | | `watchdog-stuck-jobs.yml` | 6-hourly safety net: re-dispatches stuck reviews, repairs (including a repair that crashed after a rejection), merges and generations (straight to `impl-generate.yml`, marked only once the run exists), re-closes open `regen:kept` PRs, never rescues a regeneration into repair or re-dispatches its failed review (it flags it for manual attention instead; "regeneration" = the `regen` label or the implementation file on main, unless `regen:forced`), and rescues daily-regen when its cron is silently starved by GitHub (>26 h without a run) | | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main, and runs `alembic upgrade head` against Cloud SQL first; a push that only adds a revision under `alembic/versions/` triggers it too, and so does one that changes a pin the library versions are read from (`uv.lock`, `package-lock.json`, `Manifest.toml`, `.python-version`, a setup action) | @@ -493,6 +493,54 @@ on the source ref gets an "Adoption Failed" comment. The watchdog's generation rescues don't carry `source_ref` or `source_model`: a pair the watchdog re-dispatches is generated by Claude. +### Daily regeneration modes + +`daily-regen.yml` has two modes. The repository variable +`REGEN_PAIRS_PER_DAY` chooses the mode of the scheduled run: + +| `REGEN_PAIRS_PER_DAY` | Scheduled run | +|-----------------------|---------------| +| Unset or empty | Spec mode, as before: the oldest spec (`count`, default 1), spec polish, the cross-library similarity audit, then `bulk-generate.yml` for all 15 libraries | +| A positive integer N | Pair mode: the N oldest implementations across the catalogue, the similarity audit for each spec among them, then one `impl-generate.yml` dispatch per pair | +| Anything else | The run stops with an error before it picks anything | + +The variable changes only the scheduled run. A manual run keeps its inputs and +their meaning, and switches to pair mode only with the `pairs` input. The +watchdog's cron rescue replaces a missed scheduled run, so it forwards the +variable as `pairs`. A `specification_id` always wins over a pair count. + +Pair mode picks pairs the way spec mode picks specs, per pair: + +- A pair's age is the newer of its metadata `updated` timestamp (`created` + when there's none) and its last regeneration attempt: its newest + `implementation/{spec-id}/{library}` PR in any state, its newest + `regen:kept` PR, and its newest `impl-generate.yml` run. A kept regeneration + leaves `updated` alone, so without the attempt record the same pair would + come back the next night. +- A pair younger than `min_age_hours` (default 20) is skipped. So is every + pair of a spec whose issue saw activity within that window. +- A pair with an open implementation PR is skipped. +- Only pairs with a metadata file are candidates, so pair mode never writes a + first implementation. Gap backfill stays with `bulk-generate.yml`. + +Pair mode skips the spec polish, because a spec rewritten for one library +would leave its other implementations written against the old text. It +dispatches each pair straight to `impl-generate.yml` with the arguments and +pace of `bulk-generate.yml` (180 seconds between dispatches, 120 seconds for a +pinned `sonnet` or `haiku`), and never through `bulk-generate.yml`: that +workflow's single concurrency group keeps one pending run and cancels the +older one. The regen gate, the review, and the merge are unchanged. + +The picks go to the log and the run summary, so a dry run shows them: + +```bash +gh workflow run daily-regen.yml -f pairs=5 -f dry_run=true +``` + +The dry run still runs the similarity audit, as in spec mode. To turn pair mode +on for the schedule, run `gh variable set REGEN_PAIRS_PER_DAY --body 5`; to +turn it off, run `gh variable delete REGEN_PAIRS_PER_DAY`. + --- ## Pipeline models @@ -594,6 +642,124 @@ gate record and notice line show `model=n/a`. --- +## Claude credentials + +Every Claude step of the pipeline runs on one of two credentials. The +repository variable `CLAUDE_AUTH` chooses it for every run: + +| `CLAUDE_AUTH` | Credential | Billed to | +|---------------|------------|-----------| +| Unset, empty, or `subscription` | The `CLAUDE_CODE_OAUTH_TOKEN` secret | The owner's Claude subscription, as before the switch | +| `api` | The `ANTHROPIC_API_KEY` secret | The Claude API credits of the key's Console workspace | +| Anything else | None: the job stops | Nothing | + +The value ignores case. The switch covers all 13 Claude steps in the 8 +workflows that run Claude: `daily-regen.yml`, `impl-generate.yml`, +`impl-repair.yml`, `impl-review.yml`, `report-validate.yml`, +`review-retest.yml`, `spec-create.yml`, and `util-claude.yml`. Each step +passes exactly one credential and leaves the other input empty, because the +action doesn't document which one wins when both are set. The models don't +change: the aliases `sonnet` and `opus` resolve to the same models on the API. + +Every job that runs Claude starts with the step "Check Claude credentials". +It stops the job with an error when `CLAUDE_AUTH` has an unknown value, or +when it's `api` and the `ANTHROPIC_API_KEY` secret is missing. It never falls +back to the subscription, so a broken switch can't spend the owner's +interactive quota. In `spec-create.yml` and `report-validate.yml`, the check +runs only for an issue that reaches Claude. In `impl-generate.yml`, it runs +before the issue number is known, so a stop never counts against the pair's +three attempts. + +With an API key, the action also classifies Claude's unconfirmed inline PR +comments with Haiku on that key and drops the ones it reads as test probes. +Without a key, it posts all of them. Only `util-claude.yml` on a pull request +writes inline comments. + +### Session cost + +After every Claude step, the step "Claude cost to the job summary" writes one +line to the job summary and the log: + +```text +claude-cost step=impl-generate auth=api model=claude-sonnet-5-5 total_cost_usd=0.2931 +``` + +`total_cost_usd` is Claude Code's own estimate at API list price. On the +subscription it's what the session would have cost on the API. It reads +`n/a` when the session wrote no result, and the step never fails the job. The +weekly regen review sums these lines (`collect_week.py --costs`) and shows the +part that ran on API credits. Job logs measured from 2026-10-28 to 2026-10-31 +put a generation on Sonnet at about $0.29 per implementation, a review on +Opus at about $1.10, and a repair with its re-review at about $2.70 per +implementation in all. + +### Switch to API credits + +1. In the Claude Console, create a workspace for the pipeline, for example + `anyplot-ci`, and set a monthly spend limit on it. +2. Create an API key in that workspace. +3. Store the key as a repository secret. The command prompts for the value, + so the key stays out of your shell history: + + ```bash + gh secret set ANTHROPIC_API_KEY + ``` + +4. Switch the pipeline to the key: + + ```bash + gh variable set CLAUDE_AUTH --body api + ``` + +5. Check the next Claude run: the step "Check Claude credentials" logs + `Claude credentials: api (ANTHROPIC_API_KEY)`, and the cost line in the job + summary reads `auth=api`. + +To switch back to the subscription, run +`gh variable set CLAUDE_AUTH --body subscription`, or delete the variable. The +key can stay in place. + +### At the spend limit + +When the workspace reaches its spend limit, the API refuses every request +until the limit resets or you raise it. Each Claude step then fails, and its +retry fails too. What happens next depends on the error text in the step's +execution log: + +- **The text names a provider-side error**, such as a `429` with + `rate_limit_error`, `too many requests`, `overloaded`, or a 5xx status. + `impl-generate.yml` counts the failure as an infrastructure failure: it + doesn't spend the pair's three attempts, and it retries the pair until the + fifth infrastructure failure in 12 hours, which parks the pair without + `impl:{library}:failed` and with a comment that holds the command to + re-dispatch it. +- **The text names anything else**, such as a low credit balance. + `impl-generate.yml` counts the failure as the pair's own. A regeneration is + never marked `impl:{library}:failed`, because its implementation is live on + main. A first implementation is marked failed after its third attempt. + +A failed review gets one automatic re-review and then `ai-review-failed`, a +failed repair gets one automatic retry, and the watchdog rescues each once. +The spec polish +and the similarity audit in `daily-regen.yml` fail without stopping the +dispatch. To recover, raise the limit or switch back to the subscription, then +re-dispatch the parked pairs. + +### Exposure + +The key reaches the same steps that the OAuth token reaches, and nothing +else: `tests/unit/workflows/test_claude_auth.py` checks that the secret +appears only in the action input and in the check step's `!= ''` test. None of +the eight workflows runs on `pull_request` or `pull_request_target`, so a pull +request from a fork never gets either credential. `util-claude.yml` runs only +for comments by owners, members, and collaborators. `spec-create.yml` and +`report-validate.yml` run on a label, which only maintainers can add, but +they read issue text that anyone can write. A key is billed per token, so keep +it in its own workspace: the workspace's spend limit bounds what a leaked key +can cost. + +--- + ## CLI model tiers The agentic workflows use abstract model tiers (`small`, `medium`, `large`) instead of CLI-specific model names. This allows the same command to work across different AI tools. diff --git a/prompts/README.md b/prompts/README.md index 52cbc7a6855..1f572bcfcc3 100644 --- a/prompts/README.md +++ b/prompts/README.md @@ -47,7 +47,9 @@ See `workflow-prompts/README.md` for the wiring pattern + per-variable reference # Example: from .github/workflows/impl-review.yml - uses: anthropics/claude-code-action@ # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Exactly one credential, chosen by vars.CLAUDE_AUTH (docs/workflows/overview.md#claude-credentials) + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model sonnet" prompt: | Read `prompts/workflow-prompts/ai-quality-review.md` and follow those instructions. diff --git a/prompts/workflow-prompts/README.md b/prompts/workflow-prompts/README.md index b5a3a4f73df..7a225673ec2 100644 --- a/prompts/workflow-prompts/README.md +++ b/prompts/workflow-prompts/README.md @@ -20,7 +20,9 @@ Example (from `.github/workflows/impl-review.yml`): ```yaml - uses: anthropics/claude-code-action@ # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Exactly one credential, chosen by vars.CLAUDE_AUTH (docs/workflows/overview.md#claude-credentials) + anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} claude_args: "--model sonnet" prompt: | Read `prompts/workflow-prompts/ai-quality-review.md` and follow those instructions. diff --git a/tests/unit/automation/scripts/test_regen_pick_pairs.py b/tests/unit/automation/scripts/test_regen_pick_pairs.py new file mode 100644 index 00000000000..d85307447f8 --- /dev/null +++ b/tests/unit/automation/scripts/test_regen_pick_pairs.py @@ -0,0 +1,281 @@ +"""Tests for automation.scripts.regen_pick_pairs — daily-regen's pair mode. + +The picker mirrors the spec mode's age rule per (spec, library) pair: the +newer of the metadata ``updated`` and the pair's last attempt, a minimum age, +the spec issue's activity as a postponement. Every exclusion is pinned here, +and ``main`` runs once against a fake ``gh`` the way the workflow calls it. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +import yaml + +from automation.scripts import regen_pick_pairs as rp + + +NOW = datetime(2026, 10, 31, 2, 17, tzinfo=timezone.utc) +MIN_AGE = timedelta(hours=20) +LIBS = ["altair", "matplotlib", "plotly"] +SCRIPT = Path(rp.__file__) + + +def ts(days: float) -> str: + return (NOW - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def at(days: float) -> datetime: + return NOW - timedelta(days=days) + + +def write_spec(root: Path, spec: str, metadata: dict[str, dict | str | None], issue: int | None = None) -> None: + spec_dir = root / spec + spec_dir.mkdir(parents=True) + (spec_dir / "specification.md").write_text("# s\n", encoding="utf-8") + (spec_dir / "specification.yaml").write_text(yaml.safe_dump({"issue": issue} if issue else {}), encoding="utf-8") + for lib, data in metadata.items(): + lang = "javascript" if lib == "chartjs" else "python" + meta = spec_dir / "metadata" / lang + meta.mkdir(parents=True, exist_ok=True) + text = data if isinstance(data, str) else yaml.safe_dump(data or {}) + (meta / f"{lib}.yaml").write_text(text, encoding="utf-8") + + +def records(**kw) -> rp.Records: + return rp.Records( + attempts=kw.get("attempts", {}), open_pairs=kw.get("open_pairs", set()), issue_activity=kw.get("issues", {}) + ) + + +def picked(picks: list[rp.Pick]) -> list[str]: + return [f"{p.spec}/{p.library}" for p in picks] + + +class TestLoadCandidates: + def test_reads_every_supported_pair_with_its_timestamp(self, tmp_path): + write_spec(tmp_path, "a", {"altair": {"updated": ts(10)}, "plotly": {"created": ts(30)}}, issue=7) + cands, issues = rp.load_candidates(tmp_path, LIBS) + assert {(c.spec, c.library, c.metadata) for c in cands} == {("a", "altair", at(10)), ("a", "plotly", at(30))} + assert issues == {"a": 7} + + def test_skips_unsupported_libraries_and_specs_without_specification(self, tmp_path): + write_spec(tmp_path, "a", {"chartjs": {"updated": ts(10)}, "legacy": {"updated": ts(10)}}) + (tmp_path / "no-spec" / "metadata" / "python").mkdir(parents=True) + (tmp_path / "no-spec" / "metadata" / "python" / "altair.yaml").write_text("updated: x\n", encoding="utf-8") + assert rp.load_candidates(tmp_path, LIBS)[0] == [] + assert [c.library for c in rp.load_candidates(tmp_path, [*LIBS, "chartjs"])[0]] == ["chartjs"] + + def test_unparseable_timestamp_skips_the_pair_and_missing_one_keeps_it(self, tmp_path): + write_spec(tmp_path, "a", {"altair": {"updated": "not a date"}, "plotly": {}, "matplotlib": "- a list\n"}) + cands, _ = rp.load_candidates(tmp_path, LIBS) + assert [(c.library, c.metadata) for c in cands] == [("plotly", None)] + + +class TestPick: + def test_oldest_first_across_specs(self): + cands = [ + rp.Candidate("a", "altair", at(5)), + rp.Candidate("b", "plotly", at(50)), + rp.Candidate("a", "plotly", at(20)), + ] + picks, eligible = rp.pick(cands, records(), {}, NOW, MIN_AGE, 2) + assert picked(picks) == ["b/plotly", "a/plotly"] + assert eligible == 3 + + def test_a_kept_regeneration_ages_from_its_attempt(self): + # The gate kept a/altair two days ago: `updated` is still 90 days old, + # but the pair is now younger than b/plotly. + cands = [rp.Candidate("a", "altair", at(90)), rp.Candidate("b", "plotly", at(30))] + recs = records(attempts={("a", "altair"): (at(2), "kept PR")}) + picks, _ = rp.pick(cands, recs, {}, NOW, MIN_AGE, 2) + assert picked(picks) == ["b/plotly", "a/altair"] + assert picks[1].basis == "kept PR" and picks[1].touched == at(2) + + def test_an_attempt_older_than_the_metadata_changes_nothing(self): + cands = [rp.Candidate("a", "altair", at(5))] + picks, _ = rp.pick(cands, records(attempts={("a", "altair"): (at(9), "run")}), {}, NOW, MIN_AGE, 1) + assert picks[0].basis == "metadata" and picks[0].touched == at(5) + + def test_too_fresh_pairs_are_skipped(self): + cands = [rp.Candidate("a", "altair", at(0.5)), rp.Candidate("a", "plotly", at(30))] + recs = records(attempts={("a", "plotly"): (at(0.2), "run")}) # a generation started 5 h ago + assert rp.pick(cands, recs, {}, NOW, MIN_AGE, 5) == ([], 0) + + def test_open_pr_skips_only_its_pair(self): + cands = [rp.Candidate("a", "altair", at(30)), rp.Candidate("a", "plotly", at(40))] + picks, _ = rp.pick(cands, records(open_pairs={("a", "plotly")}), {}, NOW, MIN_AGE, 5) + assert picked(picks) == ["a/altair"] + + def test_recent_issue_activity_postpones_every_pair_of_the_spec(self): + cands = [rp.Candidate("a", "altair", at(30)), rp.Candidate("b", "altair", at(10))] + recs = records(issues={1: at(0.3), 2: at(3)}) + picks, _ = rp.pick(cands, recs, {"a": 1, "b": 2}, NOW, MIN_AGE, 5) + assert picked(picks) == ["b/altair"] # b's issue is three days old: no postponement + + def test_issue_activity_is_a_gate_not_an_age(self): + # Old activity on a's issue does not re-date a's pairs. + cands = [rp.Candidate("a", "altair", at(30)), rp.Candidate("b", "altair", at(20))] + picks, _ = rp.pick(cands, records(issues={1: at(2)}), {"a": 1}, NOW, MIN_AGE, 1) + assert picked(picks) == ["a/altair"] and picks[0].touched == at(30) + + def test_no_timestamp_at_all_is_ancient(self): + cands = [rp.Candidate("a", "altair", at(300)), rp.Candidate("b", "altair", None)] + picks, _ = rp.pick(cands, records(), {}, NOW, MIN_AGE, 2) + assert picked(picks) == ["b/altair", "a/altair"] + assert picks[0].touched == rp.ANCIENT and picks[0].basis == "none" + + def test_no_metadata_timestamp_ages_from_the_attempt(self): + cands = [rp.Candidate("b", "altair", None)] + assert rp.pick(cands, records(attempts={("b", "altair"): (at(0.1), "run")}), {}, NOW, MIN_AGE, 1)[0] == [] + + def test_ties_break_by_spec_then_library(self): + cands = [ + rp.Candidate("b", "altair", at(9)), + rp.Candidate("a", "plotly", at(9)), + rp.Candidate("a", "altair", at(9)), + ] + assert picked(rp.pick(cands, records(), {}, NOW, MIN_AGE, 3)[0]) == ["a/altair", "a/plotly", "b/altair"] + + +class TestFold: + def test_pulls_keep_the_newest_attempt_and_mark_open_ones(self): + recs = records() + rp.fold_pulls( + [ + {"headRefName": "implementation/a/altair", "state": "MERGED", "updatedAt": ts(9)}, + {"headRefName": "implementation/a/altair", "state": "CLOSED", "updatedAt": ts(3)}, + {"headRefName": "implementation/b/plotly", "state": "OPEN", "updatedAt": ts(1)}, + {"headRefName": "review-writeback/a/altair/12", "state": "MERGED", "updatedAt": ts(0)}, + {"headRefName": "implementation/a/altair/extra", "state": "OPEN", "updatedAt": ts(0)}, + ], + "implementation PR", + recs, + ) + assert recs.attempts == { + ("a", "altair"): (at(3), "implementation PR"), + ("b", "plotly"): (at(1), "implementation PR"), + } + assert recs.open_pairs == {("b", "plotly")} + + def test_runs_parse_the_impl_generate_run_name(self): + recs = records() + rp.fold_runs( + [ + {"displayTitle": "Generate: plotly for line-basic", "createdAt": ts(4)}, + {"displayTitle": "Generate: generate:plotly for issue", "createdAt": ts(1)}, + {"displayTitle": "Review: PR #12", "createdAt": ts(1)}, + ], + recs, + ) + assert recs.attempts == {("line-basic", "plotly"): (at(4), "run")} + + def test_issues(self): + recs = records() + rp.fold_issues([{"number": 3, "updatedAt": ts(2)}, {"number": "x", "updatedAt": ts(1)}], recs) + assert recs.issue_activity == {3: at(2)} + + +# Answers each listing the picker makes from a JSON file named after it; a +# listing whose file is missing fails like an API error. Logs every call. +FAKE_GH = """#!/bin/sh +echo "$*" >> "$GH_LOG" +case "$*" in + *"head:implementation/"*) f=prs ;; + *"regen:kept"*) f=kept ;; + "run list"*) f=runs ;; + "issue list"*) f=issues ;; + *) exit 1 ;; +esac +[ -f "$GH_DATA/$f.json" ] || { echo "HTTP 502" >&2; exit 1; } +cat "$GH_DATA/$f.json" +""" + + +def run_main(tmp_path: Path, data: dict[str, list], count: int = 3) -> tuple[subprocess.CompletedProcess, dict, str]: + bin_dir, data_dir = tmp_path / "bin", tmp_path / "data" + bin_dir.mkdir() + data_dir.mkdir() + (bin_dir / "gh").write_text(FAKE_GH, encoding="utf-8") + (bin_dir / "gh").chmod(0o755) + for name, rows in data.items(): + (data_dir / f"{name}.json").write_text(json.dumps(rows), encoding="utf-8") + out, summary = tmp_path / "out", tmp_path / "summary" + env = { + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "GH_LOG": str(tmp_path / "gh.log"), + "GH_DATA": str(data_dir), + "GITHUB_OUTPUT": str(out), + "GITHUB_STEP_SUMMARY": str(summary), + } + args = ["--count", str(count), "--min-age-hours", "20", "--repo", "o/r", "--libraries", " ".join(LIBS)] + result = subprocess.run( + [sys.executable, str(SCRIPT), *args, "--plots", str(tmp_path / "plots")], + env=env, + capture_output=True, + text=True, + ) + outputs = dict(line.split("=", 1) for line in out.read_text().splitlines()) if out.exists() else {} + return result, outputs, summary.read_text() if summary.exists() else "" + + +class TestMain: + def _plots(self, tmp_path: Path) -> None: + plots = tmp_path / "plots" + now = datetime.now(timezone.utc) + + def old(days: float) -> str: + return (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ") + + write_spec(plots, "a-spec", {"altair": {"updated": old(90)}, "plotly": {"updated": old(60)}}, issue=11) + write_spec(plots, "b-spec", {"altair": {"updated": old(80)}, "matplotlib": {"updated": old(70)}}, issue=12) + self.recent = old(0.1) + self.week = old(7) + + def test_writes_pairs_specs_and_summary(self, tmp_path): + self._plots(tmp_path) + result, outputs, summary = run_main( + tmp_path, + { + "prs": [{"headRefName": "implementation/b-spec/matplotlib", "state": "OPEN", "updatedAt": self.recent}], + "kept": [{"headRefName": "implementation/a-spec/altair", "state": "CLOSED", "updatedAt": self.week}], + "runs": [], + "issues": [{"number": 11, "updatedAt": self.week}], + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + # a-spec/altair is aged by its keep (7 d), b-spec/matplotlib has an open PR. + assert json.loads(outputs["pairs_json"]) == [ + {"spec": "b-spec", "library": "altair"}, + {"spec": "a-spec", "library": "plotly"}, + {"spec": "a-spec", "library": "altair"}, + ] + assert json.loads(outputs["specs_json"]) == ["b-spec", "a-spec"] + assert outputs["specs"] == "b-spec a-spec" and outputs["count"] == "3" + assert "| 3 | `a-spec` | `altair` |" in summary and "kept PR" in summary + + def test_a_failed_listing_degrades_with_a_warning(self, tmp_path): + self._plots(tmp_path) + result, outputs, _ = run_main(tmp_path, {"issues": []}, count=1) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.count("::warning::Could not list") == 3 + assert json.loads(outputs["pairs_json"]) == [{"spec": "a-spec", "library": "altair"}] + + def test_nothing_eligible_writes_an_empty_pick(self, tmp_path): + (tmp_path / "plots").mkdir() + result, outputs, _ = run_main(tmp_path, {"prs": [], "kept": [], "runs": [], "issues": []}) + assert result.returncode == 0, result.stdout + result.stderr + assert outputs == {"pairs_json": "[]", "specs": "", "specs_json": "[]", "count": "0"} + + @pytest.mark.parametrize("count", [0, -1]) + def test_refuses_a_count_below_one(self, tmp_path, count): + (tmp_path / "plots").mkdir() + result, _, _ = run_main(tmp_path, {}, count=count) + assert result.returncode == 1 and "::error::" in result.stdout diff --git a/tests/unit/skills/test_collect_week.py b/tests/unit/skills/test_collect_week.py index 9fd1d9aa2f8..4f2a01d2b94 100644 --- a/tests/unit/skills/test_collect_week.py +++ b/tests/unit/skills/test_collect_week.py @@ -278,3 +278,28 @@ def test_previous_summary_with_missing_keys_still_renders(cw, tmp_path): text = cw.render_markdown(summary, rows, [], previous={"regen_prs": 3}) assert "| Regen PRs (specs) |" in text json.dumps(summary) # the summary is what --out stores + + +class TestSessionCost: + """A job log's cost: the workflows' `claude-cost` lines, else Claude Code's own lines.""" + + # What a job log holds since the cost step: Claude Code's own result line, + # the step's script as the runner echoes it, and the line it printed. + LOG = """\ +2026-10-31T02:30:00.0Z "total_cost_usd": 0.2931, +2026-10-31T02:30:01.0Z line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" +2026-10-31T02:30:01.1Z claude-cost step=impl-generate auth=api model=claude-sonnet-5-5 total_cost_usd=0.2931 +2026-10-31T02:40:00.0Z claude-cost step=impl-generate/retry auth=api model=n/a total_cost_usd=n/a +""" + + def test_cost_lines_win_and_carry_the_credential(self, cw): + assert cw.log_cost(self.LOG) == {"api": 0.2931} + + def test_an_older_log_counts_as_subscription(self, cw): + assert cw.log_cost(' "total_cost_usd": 1.1,\n "total_cost_usd": 0.4\n') == {"subscription": 1.5} + + def test_the_digest_names_the_api_part(self, cw): + assert "| Session cost (USD) | – | $42 (API $30) |" in cw.render_markdown( + cw.summarize([], {}, 42.0, 30.0), [], [], None + ) + assert "| Session cost (USD) | – | $42 |" in cw.render_markdown(cw.summarize([], {}, 42.0, 0.0), [], [], None) diff --git a/tests/unit/workflows/test_claude_auth.py b/tests/unit/workflows/test_claude_auth.py new file mode 100644 index 00000000000..bad768a86ec --- /dev/null +++ b/tests/unit/workflows/test_claude_auth.py @@ -0,0 +1,404 @@ +"""Claude credential switch (vars.CLAUDE_AUTH) and the per-session cost line. + +Every claude-code-action step passes exactly one non-empty credential: the +subscription's CLAUDE_CODE_OAUTH_TOKEN unless CLAUDE_AUTH is `api`, then +ANTHROPIC_API_KEY. The action's precedence with both set is undocumented, so +the other input is always the empty string. Every job that runs Claude starts +with the same check step, which stops on an unknown value or on `api` without +the key instead of falling back to the subscription, and every Claude step is +followed by the same cost step. There is no pre-merge smoke for these +workflows (claude-code-action refuses to run on a PR that changes them), so +the wiring is pinned here and the check step runs for real. + +Also here: daily-regen's mode switch (REGEN_PAIRS_PER_DAY / the `pairs` +input), its pair dispatch and the watchdog's rescue, which forwards the pair +count. The pick itself is tested in tests/unit/automation/scripts/ +test_regen_pick_pairs.py. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from typing import Any + +import pytest +import yaml + + +WORKFLOWS_DIR = Path(__file__).parent.parent.parent.parent / ".github" / "workflows" +ACTION = "anthropics/claude-code-action@" +API_KEY = "${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }}" +OAUTH = "${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }}" +AUTH_MODE = "${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }}" +CHECK = "Check Claude credentials" +COST = "Claude cost to the job summary" +# Inputs that would select another credential or provider. +OTHER_CREDENTIALS = { + "anthropic_federation_rule_id", + "anthropic_organization_id", + "anthropic_service_account_id", + "anthropic_workspace_id", + "use_bedrock", + "use_vertex", + "use_foundry", + "claude_env", +} +# The eight workflows (13 steps) that ran Claude on the OAuth token before the switch. +CLAUDE_WORKFLOWS = { + "daily-regen.yml": 2, + "impl-generate.yml": 2, + "impl-repair.yml": 2, + "spec-create.yml": 2, + "util-claude.yml": 2, + "impl-review.yml": 1, + "report-validate.yml": 1, + "review-retest.yml": 1, +} + + +def _load(filename: str) -> dict[str, Any]: + return yaml.safe_load((WORKFLOWS_DIR / filename).read_text(encoding="utf-8")) + + +def _claude_jobs() -> list[tuple[str, str, list[dict[str, Any]]]]: + jobs = [] + for path in sorted(WORKFLOWS_DIR.glob("*.yml")): + for name, job in (_load(path.name).get("jobs") or {}).items(): + steps = job.get("steps") or [] + if any(str(s.get("uses", "")).startswith(ACTION) for s in steps): + jobs.append((path.name, name, steps)) + return jobs + + +def _is_claude(step: dict[str, Any]) -> bool: + return str(step.get("uses", "")).startswith(ACTION) + + +def _check_steps() -> list[dict[str, Any]]: + return [s for _, _, steps in _claude_jobs() for s in steps if s.get("name") == CHECK] + + +def test_every_claude_workflow_is_covered(): + found: dict[str, int] = {} + for filename, _, steps in _claude_jobs(): + found[filename] = found.get(filename, 0) + sum(1 for s in steps if _is_claude(s)) + assert found == CLAUDE_WORKFLOWS + + +@pytest.mark.parametrize(("filename", "job", "steps"), _claude_jobs(), ids=lambda v: v if isinstance(v, str) else "") +def test_each_claude_step_passes_exactly_the_switched_credentials(filename, job, steps): + for step in filter(_is_claude, steps): + given = step.get("with") or {} + assert given.get("anthropic_api_key") == API_KEY, (filename, step.get("name")) + assert given.get("claude_code_oauth_token") == OAUTH, (filename, step.get("name")) + assert not OTHER_CREDENTIALS & set(given), (filename, step.get("name")) + env = step.get("env") or {} + assert not {"ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"} & set(env), (filename, step.get("name")) + + +@pytest.mark.parametrize(("filename", "job", "steps"), _claude_jobs(), ids=lambda v: v if isinstance(v, str) else "") +def test_the_check_runs_before_any_claude_step(filename, job, steps): + names = [s.get("name") for s in steps] + assert names.count(CHECK) == 1, (filename, job) + check = steps[names.index(CHECK)] + first_claude = next(i for i, s in enumerate(steps) if _is_claude(s)) + assert names.index(CHECK) < first_claude + # Unconditional, or gated exactly like the first Claude step (a label + # event that never reaches Claude must not fail on a misconfigured switch). + assert check.get("if") in (None, steps[first_claude].get("if")), (filename, job) + assert "continue-on-error" not in check + + +def test_every_check_step_is_identical(): + checks = _check_steps() + assert len(checks) == len(_claude_jobs()) == 8 + reference = {k: v for k, v in checks[0].items() if k != "if"} + for check in checks[1:]: + assert {k: v for k, v in check.items() if k != "if"} == reference + assert reference["env"] == { + "CLAUDE_AUTH": "${{ vars.CLAUDE_AUTH }}", + "API_KEY_SET": "${{ secrets.ANTHROPIC_API_KEY != '' }}", + } + + +@pytest.mark.parametrize(("filename", "job", "steps"), _claude_jobs(), ids=lambda v: v if isinstance(v, str) else "") +def test_each_claude_step_is_followed_by_its_cost_step(filename, job, steps): + """Read before the next Claude step: a retry writes the same execution file.""" + for i, step in enumerate(steps): + if not _is_claude(step): + continue + sid = step["id"] + later = steps[i + 1 :] + cost = next(s for s in later if str(s.get("name", "")).startswith(COST)) + next_claude = next((j for j, s in enumerate(later) if _is_claude(s)), len(later)) + assert later.index(cost) < next_claude, (filename, step.get("name")) + assert cost["if"] == ( + f"${{{{ !cancelled() && (steps.{sid}.outcome == 'success' || steps.{sid}.outcome == 'failure') }}}}" + ) + assert cost["continue-on-error"] is True + assert cost["env"]["EXECUTION_FILE"] == f"${{{{ steps.{sid}.outputs.execution_file }}}}" + assert cost["env"]["CLAUDE_AUTH_MODE"] == AUTH_MODE + + +def test_cost_steps_share_one_script_and_unique_labels(): + costs = [s for _, _, steps in _claude_jobs() for s in steps if s.get("name", "").startswith(COST)] + labels = [s["env"]["CLAUDE_STEP"] for s in costs] + assert len(labels) == len(set(labels)) == sum(CLAUDE_WORKFLOWS.values()) + assert all(" " not in label for label in labels) + assert len({s["run"] for s in costs}) == 1 + # The line must not look like Claude Code's own `"total_cost_usd": n`, or + # collect_week.py's fallback regex would count a session twice. + assert '"total_cost_usd":' not in costs[0]["run"] + + +def _run_cost(tmp_path: Path, execution: list[dict[str, Any]] | None, mode: str = "subscription") -> tuple[int, str]: + script = next(s for _, _, steps in _claude_jobs() for s in steps if s.get("name", "").startswith(COST))["run"] + exec_file = tmp_path / "claude-execution-output.json" + if execution is not None: + exec_file.write_text(json.dumps(execution), encoding="utf-8") + summary = tmp_path / "summary" + env = { + **os.environ, + "EXECUTION_FILE": str(exec_file) if execution is not None else "", + "CLAUDE_AUTH_MODE": mode, + "CLAUDE_STEP": "impl-generate", + "GITHUB_STEP_SUMMARY": str(summary), + } + result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True, text=True) + return result.returncode, summary.read_text() if summary.exists() else "" + + +def test_cost_step_reads_the_result_message(tmp_path): + execution = [ + {"type": "system", "subtype": "init", "model": "claude-sonnet-5-5"}, + {"type": "assistant"}, + {"type": "result", "subtype": "success", "total_cost_usd": 0.2931}, + ] + rc, summary = _run_cost(tmp_path, execution, mode="api") + assert rc == 0 + assert summary == "- `claude-cost step=impl-generate auth=api model=claude-sonnet-5-5 total_cost_usd=0.2931`\n" + + +@pytest.mark.parametrize("execution", [None, [], [{"type": "assistant"}], [{"type": "result"}]]) +def test_cost_step_never_fails_without_a_cost(tmp_path, execution): + rc, summary = _run_cost(tmp_path, execution) + assert rc == 0 + assert "auth=subscription model=n/a total_cost_usd=n/a" in summary + + +def test_the_api_key_reaches_nothing_but_the_action_input(): + """The key is never put into an env block, a script or another action.""" + for path in WORKFLOWS_DIR.glob("*.yml"): + for line in path.read_text(encoding="utf-8").splitlines(): + if "ANTHROPIC_API_KEY" not in line or line.lstrip().startswith("#"): + continue + stripped = line.strip() + assert ( + stripped in {f"anthropic_api_key: {API_KEY}", "API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }}"} + or "secrets.ANTHROPIC_API_KEY" not in stripped + ), (path.name, stripped) + + +def test_no_pull_request_target_workflow_runs_claude(): + """A fork PR never reaches either credential (no secrets on `pull_request`).""" + for filename in CLAUDE_WORKFLOWS: + triggers = _load(filename).get(True) or _load(filename).get("on") + assert "pull_request_target" not in (triggers or {}), filename + assert "pull_request" not in (triggers or {}), filename + + +# --- the check step, run for real ------------------------------------------ + + +def _run_check(tmp_path: Path, auth: str | None, key_set: bool) -> subprocess.CompletedProcess: + script = _check_steps()[0]["run"] + env = {k: v for k, v in os.environ.items() if k not in {"CLAUDE_AUTH", "API_KEY_SET"}} + env["API_KEY_SET"] = "true" if key_set else "false" + if auth is not None: + env["CLAUDE_AUTH"] = auth + return subprocess.run(["bash", "-e", "-c", script], cwd=tmp_path, env=env, capture_output=True, text=True) + + +def _expression(auth: str, key: str, token: str) -> tuple[str, str]: + """The two inputs as GitHub evaluates them: `==` ignores case, `a && b || ''` yields b or ''.""" + is_api = auth.casefold() == "api" + return (key if is_api and key else "", token if not is_api and token else "") + + +@pytest.mark.parametrize( + ("auth", "key_set", "ok", "mode"), + [ + (None, False, True, "subscription"), + ("", False, True, "subscription"), + ("subscription", False, True, "subscription"), + ("Subscription", True, True, "subscription"), + ("api", True, True, "api"), + ("API", True, True, "api"), + ("api", False, False, None), + ("apikey", True, False, None), + (" api", True, False, None), + ("api\n::warning::injected", True, False, None), + ], +) +def test_check_step(tmp_path, auth, key_set, ok, mode): + result = _run_check(tmp_path, auth, key_set) + assert (result.returncode == 0) is ok, result.stdout + result.stderr + if ok: + assert f"Claude credentials: {mode}" in result.stdout + # Whatever the check lets through, the step gets exactly one credential, the right one. + key, token = _expression(auth or "", "sk-ant-key" if key_set else "", "oauth-token") + assert bool(key) != bool(token) + assert bool(key) is (mode == "api") + else: + assert result.stdout.startswith("::error::") + assert "\n::warning::" not in result.stdout + + +# --- daily-regen modes ------------------------------------------------------ + + +def _daily_step(name: str) -> dict[str, Any]: + steps = [s for job in _load("daily-regen.yml")["jobs"].values() for s in job.get("steps", [])] + return next(s for s in steps if s.get("name") == name) + + +@pytest.mark.parametrize( + ("pairs", "override", "rc", "mode"), + [ + ("", "", 0, "spec"), + ("", "scatter-basic", 0, "spec"), + ("5", "scatter-basic", 0, "spec"), + ("5", "", 0, "pairs"), + ("12", "", 0, "pairs"), + ("0", "", 1, None), + ("-3", "", 1, None), + ("five", "", 1, None), + (" 5", "", 1, None), + ], +) +def test_daily_regen_mode(tmp_path, pairs, override, rc, mode): + out = tmp_path / "out" + env = {**os.environ, "PAIRS": pairs, "SPEC_OVERRIDE": override, "GITHUB_OUTPUT": str(out)} + result = subprocess.run( + ["bash", "-e", "-c", _daily_step("Resolve regen mode")["run"]], env=env, capture_output=True, text=True + ) + assert result.returncode == rc, result.stdout + result.stderr + if mode: + assert f"mode={mode}" in out.read_text() + assert (f"pairs={pairs}" in out.read_text()) is (mode == "pairs") + + +def test_daily_regen_mode_sources(): + """The variable drives only the schedule; a manual run keeps its inputs.""" + workflow = _load("daily-regen.yml") + env = _daily_step("Resolve regen mode")["env"] + assert env["PAIRS"] == "${{ github.event_name == 'schedule' && vars.REGEN_PAIRS_PER_DAY || inputs.pairs }}" + inputs = (workflow.get(True) or workflow["on"])["workflow_dispatch"]["inputs"] + assert inputs["pairs"]["default"] == "" + assert {"specification_id", "model", "count", "min_age_hours", "dry_run"} <= set(inputs) + assert _daily_step("Pick oldest spec(s) (or use override)")["if"] == "steps.mode.outputs.mode == 'spec'" + assert _daily_step("Pick oldest pairs (pair mode)")["if"] == "steps.mode.outputs.mode == 'pairs'" + outputs = workflow["jobs"]["pick"]["outputs"] + assert outputs["specs_json"] == "${{ steps.pick.outputs.specs_json || steps.pick_pairs.outputs.specs_json }}" + + +def test_spec_mode_steps_are_gated_off_in_pair_mode(): + assert ( + "needs.pick.outputs.mode != 'pairs'" in _daily_step("Spec polish (autonomous, opens PR with auto-merge)")["if"] + ) + assert _daily_step("Dispatch bulk-generate with change_requests")["if"] == ( + "${{ !inputs.dry_run && needs.pick.outputs.mode != 'pairs' }}" + ) + assert _daily_step("Dispatch impl-generate for each picked pair (pair mode)")["if"] == ( + "${{ !inputs.dry_run && needs.pick.outputs.mode == 'pairs' }}" + ) + # The similarity audit runs in both modes. + assert "if" not in _daily_step("Cross-library similarity audit") + + +FAKE_GH = """#!/bin/sh +printf '%s\\n' "$@" >> "$GH_LOG" +echo "--END--" >> "$GH_LOG" +""" +FAKE_YQ = """#!/bin/sh +sed -n 's/^issue: //p' "$2" +""" + + +def _calls(log: Path) -> list[list[str]]: + calls, current = [], [] + for line in log.read_text().splitlines() if log.exists() else []: + if line == "--END--": + calls.append(current) + current = [] + else: + current.append(line) + return calls + + +def test_pair_dispatch_mirrors_bulk_generate(tmp_path): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + for name, body in { + "gh": FAKE_GH, + "yq": FAKE_YQ, + "sleep": '#!/bin/sh\necho "slept $1" >> "$GH_LOG.sleep"\n', + }.items(): + (bin_dir / name).write_text(body, encoding="utf-8") + (bin_dir / name).chmod(0o755) + (tmp_path / "plots" / "a-spec").mkdir(parents=True) + (tmp_path / "plots" / "a-spec" / "specification.yaml").write_text("issue: 42\n", encoding="utf-8") + pairs = [ + {"spec": "b-spec", "library": "plotly"}, + {"spec": "a-spec", "library": "altair"}, + {"spec": "a-spec", "library": "plotly"}, + ] + step = _daily_step("Dispatch impl-generate for each picked pair (pair mode)") + script = step["run"].replace("${{ github.repository }}", "owner/repo") + env = { + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "GH_LOG": str(tmp_path / "gh.log"), + "SPEC_ID": "a-spec", + "MODEL": "auto", + "CHANGE_REQUESTS": json.dumps({"plotly": "Use a horizontal layout"}), + "PAIRS_JSON": json.dumps(pairs), + "PACE_SECONDS": "180", + } + result = subprocess.run(["bash", "-e", "-c", script], cwd=tmp_path, env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + + def dispatch(library: str, hint: str) -> list[str]: + fields = { + "specification_id": "a-spec", + "library": library, + "model": "auto", + "change_request": hint, + "regen_gate": "true", + "source_ref": "", + "source_model": "", + "issue_number": "42", + } + args = ["workflow", "run", "impl-generate.yml", "--repo", "owner/repo"] + for key, value in fields.items(): + args += ["-f", f"{key}={value}"] + return args + + # bulk-generate's arguments, pair by pair, with the library's own hint. + assert _calls(tmp_path / "gh.log") == [dispatch("altair", ""), dispatch("plotly", "Use a horizontal layout")] + # b-spec/plotly is the run's first dispatch (another matrix entry): both of + # this entry's dispatches wait the pace first. + assert (tmp_path / "gh.log.sleep").read_text().splitlines() == ["slept 180", "slept 180"] + assert ( + step["env"]["PACE_SECONDS"] == "${{ (inputs.model == 'sonnet' || inputs.model == 'haiku') && '120' || '180' }}" + ) + + +def test_watchdog_rescue_forwards_the_pair_count(): + steps = [s for job in _load("watchdog-stuck-jobs.yml")["jobs"].values() for s in job.get("steps", [])] + scan = next(s for s in steps if s.get("name") == "Scan and dispatch") + assert scan["env"]["REGEN_PAIRS"] == "${{ vars.REGEN_PAIRS_PER_DAY }}" + assert 'daily-regen.yml ${REGEN_PAIRS:+-f "pairs=$REGEN_PAIRS"} || true' in scan["run"] From 56641cee5a3d9dff18b08af0acae03ee227cc890 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:34:18 +0100 Subject: [PATCH 2/7] feat(ci): oidc mode, clean stop at the Claude spend limit, no pair re-pick loop - CLAUDE_AUTH=oidc: workload identity federation through the pinned action's native inputs, with the CLAUDE_WIF_RULE_ID, _ORG_ID, _SERVICE_ACCOUNT_ID and _WORKSPACE_ID variables. "Resolve Claude credentials" (id claude_auth) replaces the check step and hands the mode and the federation IDs to every Claude step as outputs, so a new mode changes one step; it stops on a missing variable, a non-ID value or a job without id-token. No job gains id-token; review-retest stays without it. - "Claude session summary" (id _cost) also reads, for api/oidc only, the API's spend-limit and credit messages and Claude Code's billing_error from the session's error fields. The retry is skipped, "Stop at the Claude spend limit" fails the job, impl-generate parks the pair (labels restored, own marker, no retry, no counted attempt, never :failed), impl-review and impl-repair hand the PR on without spending their auto-retry, and a daily-regen pre-flight at the limit skips that spec's dispatch. Subscription runs never take the new branches. - Pair mode: a pair's real attempts are its merged or closed implementation PRs and its failed generations (impl-fail comments without the infra tag, read lazily per spec issue); a pair touched within the attempt window (attempt_window_days, REGEN_ATTEMPT_WINDOW_DAYS, default 30) is skipped. Infrastructure failures and spend-limit stops are no attempt. - collect_week counts auth=oidc as API credits; docs, tests, changelog. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H7HSkGyea1k2v293XiAMo8 --- .claude/skills/review-regen-week/SKILL.md | 4 +- .../skills/review-regen-week/collect_week.py | 12 +- .github/workflows/daily-regen.yml | 159 +++-- .github/workflows/impl-generate.yml | 164 ++++- .github/workflows/impl-repair.yml | 152 ++++- .github/workflows/impl-review.yml | 119 +++- .github/workflows/report-validate.yml | 88 ++- .github/workflows/review-retest.yml | 90 ++- .github/workflows/spec-create.yml | 126 +++- .github/workflows/util-claude.yml | 126 +++- agentic/docs/project-guide.md | 4 +- automation/scripts/README.md | 14 +- automation/scripts/regen_pick_pairs.py | 207 ++++-- changelog.d/ci-claude-auth-switch.md | 56 +- docs/workflows/overview.md | 303 ++++++--- prompts/README.md | 11 +- prompts/workflow-prompts/README.md | 11 +- .../scripts/test_regen_pick_pairs.py | 265 +++++--- tests/unit/skills/test_collect_week.py | 5 + tests/unit/workflows/test_claude_auth.py | 621 +++++++++++++----- .../workflows/test_impl_generate_adopt.py | 1 + .../workflows/test_regen_gate_workflows.py | 5 +- 22 files changed, 1879 insertions(+), 664 deletions(-) diff --git a/.claude/skills/review-regen-week/SKILL.md b/.claude/skills/review-regen-week/SKILL.md index bab0826b50e..640e0c91fe2 100644 --- a/.claude/skills/review-regen-week/SKILL.md +++ b/.claude/skills/review-regen-week/SKILL.md @@ -68,8 +68,8 @@ uv run python -m automation.scripts.review_retest first-reviews \ prints the failed ones under the metric table, and with `--costs` sums `total_cost_usd` from every daily-regen pre-flight, generate, review and repair job log, and names the part that ran on API credits - (`CLAUDE_AUTH=api`, read from each step's `claude-cost` line; about - one minute per 50 runs; leave it off for a quick look). Its + (`CLAUDE_AUTH=oidc` or `api`, read from each step's `claude-cost` line; + about one minute per 50 runs; leave it off for a quick look). Its stdout is the metric table and the PR table the report starts from; `◆` marks the render sample. - `gate-report` adds the aggregate view and its soft alarms, among diff --git a/.claude/skills/review-regen-week/collect_week.py b/.claude/skills/review-regen-week/collect_week.py index cdb91dcb0f3..2ed1f212a94 100644 --- a/.claude/skills/review-regen-week/collect_week.py +++ b/.claude/skills/review-regen-week/collect_week.py @@ -52,13 +52,15 @@ WRITEBACK_BRANCH_RE = re.compile(r"^review-writeback/(?P[^/]+)/(?P[^/]+)/(?P\d+)$") RUN_URL_RE = re.compile(r"/actions/runs/(\d+)") COST_RE = re.compile(r'"total_cost_usd":\s*([0-9.]+)') -# The line every Claude step's "Claude cost to the job summary" step prints, -# with the credential it ran on (vars.CLAUDE_AUTH). A job from before that -# step carries only Claude Code's own `"total_cost_usd": n` and ran on the +# The line every Claude step's "Claude session summary" step prints, with the +# credential mode it ran on (vars.CLAUDE_AUTH). A job from before that step +# carries only Claude Code's own `"total_cost_usd": n` and ran on the # subscription. COST_LINE_RE = re.compile( - r"claude-cost step=\S+ auth=(?Papi|subscription) model=\S+ total_cost_usd=(?P[0-9.]+)" + r"claude-cost step=\S+ auth=(?Papi|oidc|subscription) model=\S+ total_cost_usd=(?P[0-9.]+)" ) +# The modes billed to Claude API credits; the subscription is not billed per session. +API_MODES = ("api", "oidc") GCS_PUBLIC = "https://storage.googleapis.com/anyplot-images/plots" VERDICT_LABELS = ("regen:improved", "regen:kept", "ai-approved", "ai-rejected") PIPELINE_WORKFLOWS = ( @@ -637,7 +639,7 @@ def main(argv: Sequence[str] | None = None) -> int: rows, run_counts, sum(costs.values()) if costs is not None else None, - costs.get("api", 0.0) if costs is not None else None, + sum(costs.get(mode, 0.0) for mode in API_MODES) if costs is not None else None, ) sample, not_opened = render_sample(rows, seed=args.since) previous = previous_payload.get("summary") diff --git a/.github/workflows/daily-regen.yml b/.github/workflows/daily-regen.yml index cffd91543b9..0e621180001 100644 --- a/.github/workflows/daily-regen.yml +++ b/.github/workflows/daily-regen.yml @@ -6,14 +6,29 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || (github. # # Pair mode (off unless configured): with the repository variable # REGEN_PAIRS_PER_DAY set to N >= 1, the scheduled run instead picks the N -# oldest implementations — single (spec, library) pairs across the catalogue, -# aged by their own metadata and regeneration attempts — -# (automation/scripts/regen_pick_pairs.py) and dispatches each pair straight +# oldest implementations, single (spec, library) pairs across the catalogue +# (automation/scripts/regen_pick_pairs.py), and dispatches each pair straight # to impl-generate.yml, skipping the spec polish. A manual run opts in with the # `pairs` input; the variable never changes a manual run, except the # watchdog's cron rescue, which forwards it as `pairs` because it replaces the # missed scheduled run. Unset, the schedule runs spec mode exactly as before. # +# No re-pick loop: a pair's age is its newest touch, the metadata `updated` +# or its last real attempt (a merged or closed implementation PR, `regen:kept` +# included, or a failed generation's `impl-fail` comment on the spec issue). +# A pair touched within the attempt window (`attempt_window_days`, else the +# REGEN_ATTEMPT_WINDOW_DAYS variable, else 30 days) is not picked, and after +# it queues behind every pair touched before it. Infrastructure failures and +# stops at the Claude spend limit are not attempts: the pair comes back on a +# later night. Spec mode has no window; a spec's age is the newer of its +# metadata and ANY activity on its issue, infrastructure failures included. +# +# Spend limit (CLAUDE_AUTH api/oidc): a pre-flight session that hits it skips +# the night's dispatch for its spec. Pairs dispatched before the limit was +# reached fail fast at their first refused request, and impl-generate, +# impl-review and impl-repair park them without a retry or a counted attempt +# (docs/workflows/overview.md#at-the-spend-limit). +# # Schedule: once per day at 02:17 UTC (04:17 Berlin under CEST, 03:17 under # CET — Actions cron is always UTC). Until 2026-09 it ticked every 2 hours # (10× per day); the owner cut it to one tick when re-enabling the workflow @@ -78,6 +93,10 @@ on: description: "Pair mode: regen this many of the oldest implementations (spec × library) instead of whole specs, without spec polish (empty = spec mode; ignored when specification_id is set)" required: false default: '' + attempt_window_days: + description: "Pair mode only: skip pairs merged or attempted within this many days (empty = the REGEN_ATTEMPT_WINDOW_DAYS variable, else 30)" + required: false + default: '' permissions: contents: read @@ -94,7 +113,6 @@ jobs: contents: read issues: read # spec issue updatedAt = last regen attempt per spec pull-requests: read # pair mode: implementation PRs = last attempt per pair - actions: read # pair mode: impl-generate runs = last attempt per pair outputs: mode: ${{ steps.mode.outputs.mode }} specs: ${{ steps.pick.outputs.specs || steps.pick_pairs.outputs.specs }} @@ -278,9 +296,9 @@ jobs: f.write(f"count={len(picks)}\n") PY - # Pair mode: the N oldest (spec, library) pairs, with the spec mode's - # age rule applied per pair (see the script's docstring). The picks go - # to the log and the run summary, also on a dry run. + # Pair mode: the N oldest (spec, library) pairs (see the script's + # docstring and the header above). The picks go to the log and the run + # summary, also on a dry run. - name: Pick oldest pairs (pair mode) id: pick_pairs if: steps.mode.outputs.mode == 'pairs' @@ -289,10 +307,11 @@ jobs: GH_REPO: ${{ github.repository }} COUNT: ${{ steps.mode.outputs.pairs }} MIN_AGE_HOURS: ${{ inputs.min_age_hours || '20' }} + ATTEMPT_WINDOW_DAYS: ${{ inputs.attempt_window_days || vars.REGEN_ATTEMPT_WINDOW_DAYS || '30' }} run: | LIBRARIES=$(python3 -c "from core.constants import SUPPORTED_LIBRARIES; print(' '.join(sorted(SUPPORTED_LIBRARIES)))") python3 automation/scripts/regen_pick_pairs.py --count "$COUNT" --min-age-hours "$MIN_AGE_HOURS" \ - --repo "$GH_REPO" --libraries "$LIBRARIES" + --attempt-window-days "$ATTEMPT_WINDOW_DAYS" --repo "$GH_REPO" --libraries "$LIBRARIES" # ============================================================================ # Pre-flight: per spec, run autonomous spec polish + cross-library similarity @@ -336,30 +355,47 @@ jobs: # Side-effect steps (polish, dispatch) are individually gated on # `!inputs.dry_run` below. steps: - # Claude credentials: the repository variable CLAUDE_AUTH picks one per - # run and each Claude step passes exactly that one, the other input - # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; - # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, - # stops the job here, never a silent fallback to the subscription. - # Identical in every job that runs Claude (tests/unit/workflows/ - # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. - - name: Check Claude credentials + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} run: | - case "${CLAUDE_AUTH,,}" in - ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; - api) - if [ "$API_KEY_SET" != "true" ]; then - echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" - exit 1 - fi - echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; - *) - echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" - exit 1 ;; + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -395,8 +431,12 @@ jobs: timeout-minutes: 15 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: '--model sonnet --settings ${{ github.workspace }}/.claude/settings.json' allowed_bots: '*' prompt: | @@ -405,21 +445,38 @@ jobs: Variables for this run: - SPEC_ID: ${{ matrix.spec_id }} - # The session's cost (Claude Code's own estimate at API list price) and - # the credential it ran on, for the weekly regen review. Never fatal. - - name: Claude cost to the job summary (spec polish) + # The session's cost (Claude Code's own estimate at API list price), + # model and credential, for the weekly regen review, and whether it + # stopped at the Claude spend limit (api/oidc only: the API's messages + # for a spend limit you set, the tier's spend cap and an empty credit + # balance, read only from the session's error fields, because its own + # text may quote them). Never fatal. + - name: Claude session summary (spec polish) + id: polish_cost if: ${{ !cancelled() && (steps.polish.outcome == 'success' || steps.polish.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.polish.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: daily-regen/polish + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" - name: Cross-library similarity audit id: similarity @@ -429,8 +486,12 @@ jobs: timeout-minutes: 15 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: '--model sonnet --settings ${{ github.workspace }}/.claude/settings.json' allowed_bots: '*' prompt: | @@ -439,19 +500,32 @@ jobs: Variables for this run: - SPEC_ID: ${{ matrix.spec_id }} - - name: Claude cost to the job summary (similarity audit) + - name: Claude session summary (similarity audit) + id: similarity_cost if: ${{ !cancelled() && (steps.similarity.outcome == 'success' || steps.similarity.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.similarity.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: daily-regen/similarity + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" - name: Collect change_requests id: collect @@ -474,6 +548,17 @@ jobs: flagged=$(echo "$CR_COMPACT" | jq 'length') echo "::notice::change_requests for ${{ matrix.spec_id }}: ${flagged} lib(s) flagged — ${CR_COMPACT}" + # A pre-flight session stopped at the Claude spend limit: every + # generation dispatched now would be refused the same way, so this + # spec's dispatch is skipped and the entry fails. A later entry's audit + # stops the same way (a refused request costs nothing), and the pairs + # stay in the queue for a later night. + - name: Stop at the Claude spend limit + if: ${{ steps.polish_cost.outputs.spend_limit == 'true' || steps.similarity_cost.outputs.spend_limit == 'true' }} + run: | + echo "::error::Claude spend limit reached in the pre-flight; nothing dispatched for ${{ matrix.spec_id }}" + exit 1 + - name: Dispatch bulk-generate with change_requests if: ${{ !inputs.dry_run && needs.pick.outputs.mode != 'pairs' }} env: diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml index d02377ce669..91960b98f75 100644 --- a/.github/workflows/impl-generate.yml +++ b/.github/workflows/impl-generate.yml @@ -98,32 +98,49 @@ jobs: pr_number: ${{ steps.pr.outputs.pr_number }} steps: - # Claude credentials: the repository variable CLAUDE_AUTH picks one per - # run and each Claude step passes exactly that one, the other input - # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; - # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, - # stops the job here, never a silent fallback to the subscription. - # Identical in every job that runs Claude (tests/unit/workflows/ - # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. # First step: a stop here precedes "Set final issue number", so "Handle # generation failure" never counts it against the pair's attempts. - - name: Check Claude credentials + - name: Resolve Claude credentials + id: claude_auth env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} run: | - case "${CLAUDE_AUTH,,}" in - ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; - api) - if [ "$API_KEY_SET" != "true" ]; then - echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" - exit 1 - fi - echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; - *) - echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" - exit 1 ;; + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" # ======================================================================== # Setup: Extract inputs and validate @@ -678,8 +695,12 @@ jobs: timeout-minutes: 60 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.inputs.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -693,31 +714,53 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} - # The session's cost (Claude Code's own estimate at API list price) and - # the credential it ran on, for the weekly regen review. Read right after - # the step: the retry writes the same execution file. Never fatal. - - name: Claude cost to the job summary + # The session's cost (Claude Code's own estimate at API list price), + # model and credential, for the weekly regen review, and whether it + # stopped at the Claude spend limit (api/oidc only: the API's messages + # for a spend limit you set, the tier's spend cap and an empty credit + # balance, read only from the session's error fields, because its own + # text may quote them). Read right after the step: the retry writes the + # same execution file. Never fatal. + - name: Claude session summary + id: claude_cost if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: impl-generate + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + # Not on a spend limit: the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' && steps.inputs.outputs.source_ref == '' + if: steps.claude.outcome == 'failure' && steps.inputs.outputs.source_ref == '' && steps.claude_cost.outputs.spend_limit != 'true' id: claude_retry timeout-minutes: 60 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.inputs.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -731,19 +774,41 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} - - name: Claude cost to the job summary (retry) + - name: Claude session summary (retry) + id: claude_retry_cost if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: impl-generate/retry + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + + # The first session stopped at the spend limit and its retry was skipped: + # fail here, so "Handle generation failure" parks the pair instead of + # the steps below running without an implementation. + - name: Stop at the Claude spend limit + if: steps.claude_cost.outputs.spend_limit == 'true' + run: | + echo "::error::Claude spend limit reached; the retry is skipped" + exit 1 # ======================================================================== # Create metadata file (before PR) @@ -1545,6 +1610,11 @@ jobs: # job by itself). CLAUDE_RETRY_OUTCOME: ${{ steps.claude_retry.outcome }} CLAUDE_EXEC_FILE: ${{ steps.claude_retry.outputs.execution_file }} + # Either session stopped at the Claude spend limit (api/oidc only; + # "Claude session summary" reads it): parked below, never retried, + # never counted. + SPEND_LIMIT: ${{ steps.claude_cost.outputs.spend_limit == 'true' || steps.claude_retry_cost.outputs.spend_limit == 'true' }} + CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} GCP_AUTH_OUTCOME: ${{ steps.gcp_auth.outcome }} GCLOUD_OUTCOME: ${{ steps.gcloud.outcome }} GCS_OUTCOME: ${{ steps.gcs.outcome }} @@ -1656,6 +1726,38 @@ jobs: exit 0 fi + # The Claude spend limit (api/oidc): an infrastructure stop that a + # retry cannot get past until the limit is raised, so the pair is + # parked at once — no auto-retry, no attempt counted (the comment + # carries its own marker, never the impl-fail one, so neither the + # 3-attempt cap nor daily-regen's attempt window sees it), never + # `impl::failed`. The pair stays in daily-regen's queue. + if [ "${SPEND_LIMIT}" = "true" ]; then + echo "::error::Claude spend limit reached (mode ${CLAUDE_MODE}) — ${LIBRARY}/${SPEC_ID} parked: no auto-retry, no attempt counted, not marked failed" + for stale in "generate:${LIBRARY}" "impl:${LIBRARY}:pending"; do + gh issue edit "$ISSUE" --remove-label "$stale" 2>/dev/null || true + done + if [ "$IS_LIVE" = "true" ]; then + gh issue edit "$ISSUE" --add-label "impl:${LIBRARY}:done" 2>/dev/null || true + fi + gh issue comment "$ISSUE" --body " + ## :pause_button: ${LIBRARY} Paused (Claude spend limit) + + The Claude API refused the **${LIBRARY}** generation for \`${SPEC_ID}\` at the spend limit (credentials: \`${CLAUDE_MODE}\`). This is **not** a capability verdict: nothing was retried, no attempt was counted, and the pair is not marked failed. + + Once the limit is raised, run: + \`\`\` + gh workflow run impl-generate.yml -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f issue_number=${ISSUE} -f model=${MODEL} + \`\`\` + + --- + :robot: *[impl-generate](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*" + if [ "$IS_LIVE" = "true" ]; then + close_issue_if_complete + fi + exit 0 + fi + # Classify the failure. The retry cap exists to stop re-running a pair # the model cannot solve; a provider incident, a cloud token refresh # or a GitHub 5xx says nothing about the pair. During the Claude diff --git a/.github/workflows/impl-repair.yml b/.github/workflows/impl-repair.yml index 6c746f27011..b42e0e5ab3c 100644 --- a/.github/workflows/impl-repair.yml +++ b/.github/workflows/impl-repair.yml @@ -52,30 +52,47 @@ jobs: id-token: write steps: - # Claude credentials: the repository variable CLAUDE_AUTH picks one per - # run and each Claude step passes exactly that one, the other input - # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; - # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, - # stops the job here, never a silent fallback to the subscription. - # Identical in every job that runs Claude (tests/unit/workflows/ - # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. - - name: Check Claude credentials + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} run: | - case "${CLAUDE_AUTH,,}" in - ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; - api) - if [ "$API_KEY_SET" != "true" ]; then - echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" - exit 1 - fi - echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; - *) - echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" - exit 1 ;; + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -254,8 +271,12 @@ jobs: timeout-minutes: 45 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.model.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -269,31 +290,53 @@ jobs: - ATTEMPT: ${{ inputs.attempt }} - BRANCH: ${{ env.branch }} - # The session's cost (Claude Code's own estimate at API list price) and - # the credential it ran on, for the weekly regen review. Read right after - # the step: the retry writes the same execution file. Never fatal. - - name: Claude cost to the job summary + # The session's cost (Claude Code's own estimate at API list price), + # model and credential, for the weekly regen review, and whether it + # stopped at the Claude spend limit (api/oidc only: the API's messages + # for a spend limit you set, the tier's spend cap and an empty credit + # balance, read only from the session's error fields, because its own + # text may quote them). Read right after the step: the retry writes the + # same execution file. Never fatal. + - name: Claude session summary + id: claude_cost if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: impl-repair + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + # Not on a spend limit: the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' + if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.spend_limit != 'true' id: claude_retry timeout-minutes: 45 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.model.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -307,19 +350,41 @@ jobs: - ATTEMPT: ${{ inputs.attempt }} - BRANCH: ${{ env.branch }} - - name: Claude cost to the job summary (retry) + - name: Claude session summary (retry) + id: claude_retry_cost if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: impl-repair/retry + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + + # The first session stopped at the spend limit and its retry was skipped: + # fail here, so "Handle repair failure" restores the PR without an + # auto-retry instead of the steps below running on an unrepaired file. + - name: Stop at the Claude spend limit + if: steps.claude_cost.outputs.spend_limit == 'true' + run: | + echo "::error::Claude spend limit reached; the retry is skipped" + exit 1 - name: Process repaired image if: success() @@ -476,11 +541,36 @@ jobs: MODEL: ${{ steps.model.outputs.model || inputs.model || 'auto' }} REPOSITORY: ${{ github.repository }} RUN_ID: ${{ github.run_id }} + # Either session stopped at the Claude spend limit (api/oidc only). + SPEND_LIMIT: ${{ steps.claude_cost.outputs.spend_limit == 'true' || steps.claude_retry_cost.outputs.spend_limit == 'true' }} + CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} run: | # Restore ai-rejected label that was removed at start of repair so the # PR state stays consistent (otherwise it looks "stuck approved"). gh pr edit "$PR_NUM" --add-label "ai-rejected" 2>/dev/null || true + # The Claude spend limit (api/oidc): an auto-retry now would be + # refused the same way. The comment carries its own marker, not the + # repair-retry one, so the attempt keeps its one crash retry for after + # the limit is raised; the watchdog's repair rescue (once per + # attempt) or a manual restart picks the PR up. + if [ "${SPEND_LIMIT}" = "true" ]; then + echo "::error::Claude spend limit reached (mode ${CLAUDE_MODE}) — repair attempt ${ATTEMPT} stopped, no auto-retry" + gh pr comment "$PR_NUM" --body " + ## :pause_button: Repair Paused (Claude spend limit, Attempt ${ATTEMPT}/4) + + The Claude API refused the repair at the spend limit (credentials: \`${CLAUDE_MODE}\`). Nothing was retried, and the attempt is not used up. + + **Restart once the limit is raised:** + \`\`\` + gh workflow run impl-repair.yml -f pr_number=${PR_NUM} -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f attempt=${ATTEMPT} -f model=${MODEL} + \`\`\` + + --- + :robot: *[impl-repair](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" || true + exit 0 + fi + MARKER="" # Paginate so the marker is found even on PRs with >30 comments. # `--paginate --jq` prints one count per page, so the pages are diff --git a/.github/workflows/impl-review.yml b/.github/workflows/impl-review.yml index 42198256950..a9850371335 100644 --- a/.github/workflows/impl-review.yml +++ b/.github/workflows/impl-review.yml @@ -48,30 +48,47 @@ jobs: actions: write steps: - # Claude credentials: the repository variable CLAUDE_AUTH picks one per - # run and each Claude step passes exactly that one, the other input - # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; - # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, - # stops the job here, never a silent fallback to the subscription. - # Identical in every job that runs Claude (tests/unit/workflows/ - # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. - - name: Check Claude credentials + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} run: | - case "${CLAUDE_AUTH,,}" in - ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; - api) - if [ "$API_KEY_SET" != "true" ]; then - echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" - exit 1 - fi - echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; - *) - echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" - exit 1 ;; + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -915,8 +932,12 @@ jobs: PR_NUMBER: ${{ steps.pr.outputs.pr_number }} ATTEMPT: ${{ steps.attempts.outputs.display }} with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.pr.outputs.review_model_alias }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -951,21 +972,38 @@ jobs: run: | python3 "$RUNNER_TEMP/regen-tools/review_provenance.py" model --execution-file "${EXECUTION_FILE:-}" - # The session's cost (Claude Code's own estimate at API list price) and - # the credential it ran on, for the weekly regen review. Never fatal. - - name: Claude cost to the job summary + # The session's cost (Claude Code's own estimate at API list price), + # model and credential, for the weekly regen review, and whether it + # stopped at the Claude spend limit (api/oidc only: the API's messages + # for a spend limit you set, the tier's spend cap and an empty credit + # balance, read only from the session's error fields, because its own + # text may quote them). "Validate review output" reads it. Never fatal. + - name: Claude session summary + id: review_cost if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: impl-review + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" # Undo "Hide the previous review", whatever the review step did: the # metadata step below reads the metadata file from disk and rewrites @@ -1070,9 +1108,40 @@ jobs: RUN_ID: ${{ github.run_id }} MODEL: ${{ steps.pr.outputs.model }} REVIEW_MODEL_ALIAS: ${{ steps.pr.outputs.review_model_alias }} + # The session stopped at the Claude spend limit (api/oidc only). + SPEND_LIMIT: ${{ steps.review_cost.outputs.spend_limit }} + CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} run: | echo "::error::AI Review did not produce valid output files (score=0)" + # The Claude spend limit (api/oidc): a re-review now would be refused + # the same way, and it says nothing about the implementation. Hand + # the PR to the watchdog with `ai-review-failed`, as an exhausted + # retry does, but without the review-retry marker, so the one + # auto-retry stays for after the limit is raised. No verdict, no + # score, nothing merged or closed. + if [ "${SPEND_LIMIT}" = "true" ]; then + echo "::error::Claude spend limit reached (mode ${CLAUDE_MODE}) — no auto-retry for PR #${PR_NUM}" + for attempt in 1 2 3; do + gh pr edit "$PR_NUM" --add-label "ai-review-failed" 2>/dev/null && break + [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) + done || echo "::error::could not add ai-review-failed to PR #${PR_NUM} — left to the watchdog's scan of stalled PRs" + gh pr comment "$PR_NUM" --body " + ## :pause_button: AI Review Paused (Claude spend limit) + + The Claude API refused the review at the spend limit (credentials: \`${CLAUDE_MODE}\`). No score and no verdict were recorded, and the one automatic re-review is still unused. + + **Rerun once the limit is raised:** + \`\`\` + gh workflow run impl-review.yml -f pr_number=$PR_NUM + \`\`\` + + --- + :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" \ + || echo "::error::spend-limit comment not posted on PR #${PR_NUM}" + exit 1 + fi + MARKER="" # Paginate so the marker is found even on PRs with >30 comments. # `--paginate --jq` prints one count per page, so the pages are diff --git a/.github/workflows/report-validate.yml b/.github/workflows/report-validate.yml index 07983083e83..bedd269fa1e 100644 --- a/.github/workflows/report-validate.yml +++ b/.github/workflows/report-validate.yml @@ -41,31 +41,48 @@ jobs: fi echo "should_run=true" >> $GITHUB_OUTPUT - # Claude credentials: the repository variable CLAUDE_AUTH picks one per - # run and each Claude step passes exactly that one, the other input - # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; - # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, - # stops the job here, never a silent fallback to the subscription. - # Identical in every job that runs Claude (tests/unit/workflows/ - # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. - - name: Check Claude credentials + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth if: steps.check.outputs.should_run == 'true' env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} run: | - case "${CLAUDE_AUTH,,}" in - ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; - api) - if [ "$API_KEY_SET" != "true" ]; then - echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" - exit 1 - fi - echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; - *) - echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" - exit 1 ;; + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" - name: Checkout repository if: steps.check.outputs.should_run == 'true' @@ -91,8 +108,12 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Validate Issue Report @@ -184,21 +205,38 @@ jobs: - Do NOT trigger any fixes - Keep the analysis comment concise but informative - # The session's cost (Claude Code's own estimate at API list price) and - # the credential it ran on. Never fatal. - - name: Claude cost to the job summary + # The session's cost (Claude Code's own estimate at API list price), + # model and credential, and whether it stopped at the Claude spend limit + # (api/oidc only: the API's messages for a spend limit you set, the + # tier's spend cap and an empty credit balance, read only from the + # session's error fields, because its own text may quote them). Never + # fatal; a spend-limit stop takes "Handle failure" like any other. + - name: Claude session summary + id: validate_cost if: ${{ !cancelled() && (steps.validate.outcome == 'success' || steps.validate.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.validate.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: report-validate + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" - name: Add reaction on success if: steps.check.outputs.should_run == 'true' && steps.validate.outcome == 'success' diff --git a/.github/workflows/review-retest.yml b/.github/workflows/review-retest.yml index ba314c0c696..febc9a32488 100644 --- a/.github/workflows/review-retest.yml +++ b/.github/workflows/review-retest.yml @@ -224,30 +224,49 @@ jobs: CELL_JSON: ${{ toJSON(matrix.cell) }} RULES_SHA: ${{ needs.prep.outputs.rules_sha }} steps: - # Claude credentials: the repository variable CLAUDE_AUTH picks one per - # run and each Claude step passes exactly that one, the other input - # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; - # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, - # stops the job here, never a silent fallback to the subscription. - # Identical in every job that runs Claude (tests/unit/workflows/ - # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. - - name: Check Claude credentials + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + # This job has no id-token on purpose (no Claude App token, so no + # session can write): under `oidc` it stops here. + - name: Resolve Claude credentials + id: claude_auth env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} run: | - case "${CLAUDE_AUTH,,}" in - ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; - api) - if [ "$API_KEY_SET" != "true" ]; then - echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" - exit 1 - fi - echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; - *) - echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" - exit 1 ;; + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" - name: Checkout harness uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -334,8 +353,12 @@ jobs: PR_NUMBER: '0' ATTEMPT: '1' with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} github_token: ${{ github.token }} claude_args: "--model ${{ matrix.cell.model }} --settings ${{ github.workspace }}/automation/retest/claude-settings.json" prompt: | @@ -359,21 +382,38 @@ jobs: to the PR, write the exact comment body to `review_comment.md` in the repository root instead. Do not run `gh` or `git push`; every other instruction applies unchanged. - # The session's cost (Claude Code's own estimate at API list price) and - # the credential it ran on. Never fatal. - - name: Claude cost to the job summary + # The session's cost (Claude Code's own estimate at API list price), + # model and credential, and whether it stopped at the Claude spend limit + # (api/oidc only: the API's messages for a spend limit you set, the + # tier's spend cap and an empty credit balance, read only from the + # session's error fields, because its own text may quote them). Never + # fatal; a stopped cell is collected as a failed one. + - name: Claude session summary + id: review_cost if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: review-retest + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" - name: Collect the cell if: always() diff --git a/.github/workflows/spec-create.yml b/.github/workflows/spec-create.yml index ed362f647ca..bb6e53edc5c 100644 --- a/.github/workflows/spec-create.yml +++ b/.github/workflows/spec-create.yml @@ -69,31 +69,48 @@ jobs: fi echo "should_run=true" >> "$GITHUB_OUTPUT" - # Claude credentials: the repository variable CLAUDE_AUTH picks one per - # run and each Claude step passes exactly that one, the other input - # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; - # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, - # stops the job here, never a silent fallback to the subscription. - # Identical in every job that runs Claude (tests/unit/workflows/ - # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. - - name: Check Claude credentials + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth if: steps.check.outputs.should_run == 'true' env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} run: | - case "${CLAUDE_AUTH,,}" in - ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; - api) - if [ "$API_KEY_SET" != "true" ]; then - echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" - exit 1 - fi - echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; - *) - echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" - exit 1 ;; + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" - name: Checkout repository if: steps.check.outputs.should_run == 'true' @@ -125,8 +142,12 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Create New Specification @@ -275,25 +296,43 @@ jobs: - Do NOT close the issue - STOP after step 11 - # The session's cost (Claude Code's own estimate at API list price) and - # the credential it ran on. Read right after the step: the retry writes - # the same execution file. Never fatal. - - name: Claude cost to the job summary + # The session's cost (Claude Code's own estimate at API list price), + # model and credential, and whether it stopped at the Claude spend limit + # (api/oidc only: the API's messages for a spend limit you set, the + # tier's spend cap and an empty credit balance, read only from the + # session's error fields, because its own text may quote them). Read + # right after the step: the retry writes the same execution file. Never + # fatal. + - name: Claude session summary + id: process_cost if: ${{ !cancelled() && (steps.process.outcome == 'success' || steps.process.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.process.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: spec-create + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + # Not on a spend limit: the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' + if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' && steps.process_cost.outputs.spend_limit != 'true' id: process_retry timeout-minutes: 30 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 @@ -303,8 +342,12 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Create New Specification @@ -462,19 +505,40 @@ jobs: - Do NOT close the issue - STOP after step 11 - - name: Claude cost to the job summary (retry) + - name: Claude session summary (retry) + id: process_retry_cost if: ${{ !cancelled() && (steps.process_retry.outcome == 'success' || steps.process_retry.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.process_retry.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: spec-create/retry + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + + # The first session stopped at the spend limit and its retry was skipped: + # fail here instead of reading outputs that were never written. + - name: Stop at the Claude spend limit + if: steps.process_cost.outputs.spend_limit == 'true' + run: | + echo "::error::Claude spend limit reached; the retry is skipped" + exit 1 - name: Extract outputs if: steps.check.outputs.should_run == 'true' diff --git a/.github/workflows/util-claude.yml b/.github/workflows/util-claude.yml index 9aaf8c3042d..d482202ad61 100644 --- a/.github/workflows/util-claude.yml +++ b/.github/workflows/util-claude.yml @@ -33,30 +33,47 @@ jobs: id-token: write actions: read # Required for Claude to read CI results on PRs steps: - # Claude credentials: the repository variable CLAUDE_AUTH picks one per - # run and each Claude step passes exactly that one, the other input - # empty. Unset or `subscription`: CLAUDE_CODE_OAUTH_TOKEN, as before; - # `api`: ANTHROPIC_API_KEY. Any other value, or `api` without the key, - # stops the job here, never a silent fallback to the subscription. - # Identical in every job that runs Claude (tests/unit/workflows/ - # test_claude_auth.py); docs/workflows/overview.md#claude-credentials. - - name: Check Claude credentials + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} run: | - case "${CLAUDE_AUTH,,}" in - ''|subscription) echo "Claude credentials: subscription (CLAUDE_CODE_OAUTH_TOKEN)" ;; - api) - if [ "$API_KEY_SET" != "true" ]; then - echo "::error::CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing; not falling back to the subscription" - exit 1 - fi - echo "Claude credentials: api (ANTHROPIC_API_KEY)" ;; - *) - echo "::error::CLAUDE_AUTH must be unset, 'subscription' or 'api', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" - exit 1 ;; + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -68,54 +85,101 @@ jobs: continue-on-error: true uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" # This is an optional setting that allows Claude to read CI results on PRs additional_permissions: | actions: read - # The session's cost (Claude Code's own estimate at API list price) and - # the credential it ran on. Read right after the step: the retry writes - # the same execution file. Never fatal. - - name: Claude cost to the job summary + # The session's cost (Claude Code's own estimate at API list price), + # model and credential, and whether it stopped at the Claude spend limit + # (api/oidc only: the API's messages for a spend limit you set, the + # tier's spend cap and an empty credit balance, read only from the + # session's error fields, because its own text may quote them). Read + # right after the step: the retry writes the same execution file. Never + # fatal. + - name: Claude session summary + id: claude_cost if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: util-claude + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + # Not on a spend limit: the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' + if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.spend_limit != 'true' id: claude_retry uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" additional_permissions: | actions: read - - name: Claude cost to the job summary (retry) + - name: Claude session summary (retry) + id: claude_retry_cost if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} continue-on-error: true env: EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} - CLAUDE_AUTH_MODE: ${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: util-claude/retry + SESSION_ERRORS: >- + .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) + SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + spend_limit=false + if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then + spend_limit=true + echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" + echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + fi + echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + + # The first session stopped at the spend limit and its retry was skipped: + # the run fails instead of reporting a success it did not have. + - name: Stop at the Claude spend limit + if: steps.claude_cost.outputs.spend_limit == 'true' + run: | + echo "::error::Claude spend limit reached; the retry is skipped" + exit 1 diff --git a/agentic/docs/project-guide.md b/agentic/docs/project-guide.md index eee5027e5e2..70e1b4b9c41 100644 --- a/agentic/docs/project-guide.md +++ b/agentic/docs/project-guide.md @@ -768,7 +768,7 @@ Issue ready for maintainer review | **sync-labels.yml** | Auto-syncs labels after manual PR merges (spec-ready, impl:*:done) | | **notify-deployment.yml** | Deployment notifications | | **util-claude.yml** | Claude utility workflow | -| **daily-regen.yml** | Scheduled: picks the oldest specs, runs spec polish + cross-library similarity audit, then dispatches bulk-generate. With the repository variable `REGEN_PAIRS_PER_DAY` (or the manual `pairs` input) it picks the N oldest (spec, library) pairs instead (`automation/scripts/regen_pick_pairs.py`), skips the polish and dispatches each pair straight to impl-generate; see `docs/workflows/overview.md#daily-regeneration-modes` | +| **daily-regen.yml** | Scheduled: picks the oldest specs, runs spec polish + cross-library similarity audit, then dispatches bulk-generate. With the repository variable `REGEN_PAIRS_PER_DAY` (or the manual `pairs` input) it picks the N oldest (spec, library) pairs instead (`automation/scripts/regen_pick_pairs.py`), skipping pairs merged or really attempted within the attempt window (30 days by default), skips the polish and dispatches each pair straight to impl-generate; see `docs/workflows/overview.md#daily-regeneration-modes` | | **watchdog-stuck-jobs.yml** | Periodic safety net: detects and unsticks stalled impl-pipeline PRs | | **bot-serving-check.yml** | Daily synthetic monitor for the nginx bot -> seo-proxy path; routes and titles derived from `api/routers/seo.py` and the spec files, and a failure opens/comments the fixed-title issue "Bot serving check is red" (closed again by the next green run) | @@ -799,7 +799,7 @@ The new architecture separates specification and implementation processes: - **Adopt mode.** The `source_ref` input (default empty) on `impl-generate.yml` and `bulk-generate.yml` takes each implementation file from that branch, tag or SHA instead of running Claude, for code written outside the pipeline (a `gemini/*` branch). The run renders the file with the generator's commands, checks the canvas with `core.canvas`, and is always a forced regeneration (`regen:forced`); the routed model only runs repair, and the PR body adds a `**Generation:** adopted from …` line above `**Model:**`. The optional `source_model` input (a model name such as `gemini-4-argon`, only with `source_ref`) names the model that wrote the code and goes on that line. The metadata records `generated_by: {source_model}`, or `generated_by: adopted:{source_ref}` without one, followed by `adopted_from: {source_ref} ({sha})` for every adopted file. Both workflows reject a malformed `source_ref` or `source_model` without echoing it. Retries keep both inputs; the watchdog forwards neither. Python and JavaScript only. A missing file or a failed render is never retried and never counts toward the 3-attempt cap. See `docs/workflows/overview.md#adopt-implementations-from-a-branch`. - **Review model.** Every quality review runs on Opus, whatever model generated the implementation: first implementations, reviews after a repair, and the regen gate's session (the predecessor's re-score and the new render's review). Only `impl-review.yml` chooses it; its `review_model` input (default `opus`) pins another model for one manual review and that run's own auto-retry, and no other workflow forwards one. The retest harness's `models=production` runs every cell on Opus to match (`review_retest.PRODUCTION_MODELS`, kept equal by `tests/unit/workflows/test_model_routing.py`). - Fixed models: `spec-create.yml` runs on Opus; the spec polish and cross-library similarity audit in `daily-regen.yml` run on Sonnet. See `docs/workflows/overview.md#pipeline-models`. -- **Credentials.** The repository variable `CLAUDE_AUTH` chooses what every Claude step runs on: unset or `subscription` = the `CLAUDE_CODE_OAUTH_TOKEN` secret (the owner's subscription), `api` = the `ANTHROPIC_API_KEY` secret (API credits). Each step passes exactly one of the two; the step "Check Claude credentials" at the start of every Claude job stops on any other value or on `api` without the key, never falling back to the subscription. After each Claude step, "Claude cost to the job summary" records `total_cost_usd` and the credential. A new Claude step copies both patterns, which `tests/unit/workflows/test_claude_auth.py` enforces. See `docs/workflows/overview.md#claude-credentials`. +- **Credentials.** The repository variable `CLAUDE_AUTH` chooses what every Claude step runs on: unset or `subscription` = the `CLAUDE_CODE_OAUTH_TOKEN` secret (the owner's subscription), `oidc` = workload identity federation with the `CLAUDE_WIF_*` variables (API credits of the `ci-regen` workspace; accepts only runs on `main`), `api` = the `ANTHROPIC_API_KEY` secret (API credits, the fallback). The step "Resolve Claude credentials" (id `claude_auth`) at the start of every Claude job stops on any other value or a missing credential, never falling back to the subscription, and every Claude step reads its outputs, so each passes exactly one credential. After each Claude step, "Claude session summary" (id `_cost`) records `total_cost_usd`, the model and the mode, and flags a stop at the Claude spend limit (api/oidc): the retry is skipped and impl-generate, impl-review and impl-repair park the work as an infrastructure stop, without a retry or a counted attempt. A new Claude step copies all three patterns, which `tests/unit/workflows/test_claude_auth.py` enforces. See `docs/workflows/overview.md#claude-credentials`. ## GitHub Issue Labels diff --git a/automation/scripts/README.md b/automation/scripts/README.md index 48b38b5a813..bae00b48d43 100644 --- a/automation/scripts/README.md +++ b/automation/scripts/README.md @@ -259,15 +259,17 @@ Closes a spec issue once every supported library is `impl::done` or ### `regen_pick_pairs.py` The pick of `daily-regen.yml`'s pair mode (`REGEN_PAIRS_PER_DAY`, or the `pairs` input): the N oldest (spec, library) pairs across the catalogue, aged by -the newer of their metadata `updated` and their last attempt (implementation PR, -`regen:kept` PR, `impl-generate.yml` run). It skips pairs younger than -`--min-age-hours`, pairs with an open implementation PR, and every pair of a -spec whose issue saw activity in that window. PyYAML plus `gh`; a failed listing -degrades with a warning. The library list comes from the caller. +the newer of their metadata `updated` and their last real attempt (a merged or +closed implementation PR, `regen:kept` included, or a failed generation's +`impl-fail` comment; infrastructure failures and spend-limit stops don't +count). It skips pairs touched within `--attempt-window-days` (default 30), +pairs with an open implementation PR, and every pair of a spec whose issue saw +activity within `--min-age-hours`. PyYAML plus `gh`; a failed read degrades +with a warning. The library list comes from the caller. ```bash python3 automation/scripts/regen_pick_pairs.py --count 5 --min-age-hours 20 \ - --repo OWNER/NAME --libraries "altair bokeh ..." + --attempt-window-days 30 --repo OWNER/NAME --libraries "altair bokeh ..." # writes pairs_json, specs, specs_json and count to $GITHUB_OUTPUT ``` diff --git a/automation/scripts/regen_pick_pairs.py b/automation/scripts/regen_pick_pairs.py index 6fabc5011a2..12446140923 100644 --- a/automation/scripts/regen_pick_pairs.py +++ b/automation/scripts/regen_pick_pairs.py @@ -3,34 +3,44 @@ Spec mode, the default, regenerates the N oldest specs with all their libraries. Pair mode picks the N oldest implementations, one (spec, library) -pair each, across the catalogue, and applies the spec mode's rules per pair: - -- A pair's age counts from the newer of its metadata ``updated`` (``created`` - when there is none) and its last regeneration attempt, so a regeneration the - gate kept, which leaves ``updated`` alone, is not picked again the next - night. The attempt record is the newest of the pair's implementation PR - (head ``implementation//``, any state), its ``regen:kept`` PR - and its ``impl-generate.yml`` run (``Generate: for ``, which - also covers a generation that failed before it opened a PR). -- A pair younger than ``--min-age-hours`` is skipped, and so is every pair of - a spec whose issue saw activity within that window: a comment, a label or a - report postpones the spec as in spec mode, and a night does not stack a - spec whose last batch is still in flight. -- A pair with an open implementation PR is skipped: its last attempt is still +pair each, across the catalogue: + +- A pair's age counts from its newest touch: its metadata ``updated`` + (``created`` when there is none), or its last real regeneration attempt, + whichever is newer. A pair touched within ``--attempt-window-days`` + (default 30) is not picked, so an attempted pair waits at least that long + and then queues behind every pair touched before it. +- A real attempt is one that reached a verdict or burned a generation attempt: + the pair's implementation PR (head ``implementation//``) + once it is merged or closed (a ``regen:kept`` PR included), and a failed + generation, which leaves an ```` comment + on the spec issue. A merge moves ``updated`` anyway; the other two leave it + alone, and without them the same pair would come back the next night. An + infrastructure failure (its comment carries ``impl-fail-cause:infra``) and + a stop at the Claude spend limit (its own pause comment) are not attempts: + the pair stays in the queue and comes back on a later night. +- A pair with an open implementation PR is skipped: its attempt is still running or stuck, and the watchdog owns it. +- Every pair of a spec whose issue saw activity within ``--min-age-hours`` is + skipped, like the spec mode's postponement: a comment, a label or a report + holds the spec back, and a night does not stack a spec whose last batch is + still in flight. - An unparseable metadata timestamp skips the pair, as in spec mode. Only pairs with a metadata file and a ``specification.md`` are candidates, so pair mode never writes a first implementation; gap backfill stays with ``bulk-generate.yml``. -Each listing is one ``gh`` call of at most 1000 entries, most recent first. A -listing that fails degrades to the other records with a warning, like the -spec mode's issue listing. +The PR and issue listings are one ``gh`` call each, at most 1000 entries, most +recent first (kept regenerations get a listing of their own, which reaches +much further back). The failed-generation comments are read per spec issue, +only for the specs at the front of the queue, at most ``--max-issue-reads`` +issues per run. A read that fails degrades to the other records with a +warning, like the spec mode's issue listing. Usage, from the repository root:: python3 automation/scripts/regen_pick_pairs.py --count 5 --min-age-hours 20 \\ - --repo OWNER/NAME --libraries "altair bokeh ..." + --attempt-window-days 30 --repo OWNER/NAME --libraries "altair bokeh ..." The library list comes from the caller (``core.constants.SUPPORTED_LIBRARIES``), so this file carries no copy that could drift. Writes ``pairs_json``, @@ -41,12 +51,13 @@ from __future__ import annotations import argparse +import heapq import json import os import re import subprocess import sys -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path @@ -58,7 +69,10 @@ ANCIENT = datetime.min.replace(tzinfo=timezone.utc) LIMIT = "1000" BRANCH_RE = re.compile(r"^implementation/(?P[a-z0-9][a-z0-9-]*)/(?P[a-z0-9]+)$") -RUN_TITLE_RE = re.compile(r"^Generate: (?P[a-z0-9]+) for (?P[a-z0-9][a-z0-9-]*)$") +# impl-generate.yml "Handle generation failure": every failure comment carries +# the marker; an infrastructure failure's comment also carries the infra tag. +FAIL_MARKER_RE = re.compile(r"") +INFRA_TAG = "" Pair = tuple[str, str] @@ -134,22 +148,23 @@ def _newer(records: dict[Pair, tuple[datetime, str]], pair: Pair, when: datetime records[pair] = (when, basis) -def fold_pulls(pulls: Iterable[dict[str, Any]], basis: str, records: Records) -> None: +def fold_pulls(pulls: Iterable[dict[str, Any]], records: Records) -> None: + """An open PR blocks its pair; a merged or closed one is the pair's attempt.""" for pull in pulls: match = BRANCH_RE.match(str(pull.get("headRefName") or "")) if not match: continue pair = (match["spec"], match["lib"]) - _newer(records.attempts, pair, parse_ts(str(pull.get("updatedAt") or "")), basis) - if pull.get("state") == "OPEN": + state = pull.get("state") + if state == "OPEN": records.open_pairs.add(pair) - - -def fold_runs(runs: Iterable[dict[str, Any]], records: Records) -> None: - for run in runs: - match = RUN_TITLE_RE.match(str(run.get("displayTitle") or "")) - if match: - _newer(records.attempts, (match["spec"], match["lib"]), parse_ts(str(run.get("createdAt") or "")), "run") + continue + if state not in ("MERGED", "CLOSED"): + continue + labels = {str(label.get("name")) for label in pull.get("labels") or [] if isinstance(label, dict)} + basis = "merged PR" if state == "MERGED" else "kept PR" if "regen:kept" in labels else "closed PR" + when = parse_ts(str(pull.get("closedAt") or pull.get("updatedAt") or "")) + _newer(records.attempts, pair, when, basis) def fold_issues(issues: Iterable[dict[str, Any]], records: Records) -> None: @@ -159,16 +174,48 @@ def fold_issues(issues: Iterable[dict[str, Any]], records: Records) -> None: records.issue_activity[issue["number"]] = ts +def failed_generations(comments: Iterable[dict[str, Any]], spec: str) -> dict[str, datetime]: + """Newest failed generation per library of ``spec``, from the spec issue's comments.""" + newest: dict[str, datetime] = {} + for comment in comments: + body = str(comment.get("body") or "") + if INFRA_TAG in body: + continue # an infrastructure failure, not an attempt + when = parse_ts(str(comment.get("created_at") or "")) + for match in FAIL_MARKER_RE.finditer(body): + if match["spec"] == spec and when and (match["lib"] not in newest or when > newest[match["lib"]]): + newest[match["lib"]] = when + return newest + + +def _touched(pair: Pair, metadata: datetime | None, records: Records) -> tuple[datetime | None, str]: + touched, basis = (metadata, "metadata") if metadata else (None, "none") + attempt = records.attempts.get(pair) + if attempt and (touched is None or attempt[0] > touched): + touched, basis = attempt + return touched, basis + + def pick( candidates: Iterable[Candidate], records: Records, spec_issues: dict[str, int], now: datetime, min_age: timedelta, + window: timedelta, count: int, + failures: Callable[[str], dict[str, datetime]] | None = None, + max_reads: int = 100, ) -> tuple[list[Pick], int]: - """The ``count`` oldest eligible pairs, oldest first, and how many were eligible.""" - eligible: list[Pick] = [] + """The ``count`` oldest eligible pairs, oldest first, and how many were eligible. + + ``failures(spec)`` returns the spec's failed generations per library. It is + asked lazily, once per spec, when one of the spec's pairs reaches the front + of the queue; a pair whose failure is newer than its age goes back into + the queue, or out of it while the failure is within the window. + """ + metadata: dict[Pair, datetime | None] = {} + queue: list[tuple[datetime, str, str, str]] = [] for cand in candidates: pair = (cand.spec, cand.library) if pair in records.open_pairs: @@ -176,18 +223,35 @@ def pick( issue_ts = records.issue_activity.get(spec_issues.get(cand.spec, -1)) if issue_ts is not None and now - issue_ts < min_age: continue - touched, basis = (cand.metadata, "metadata") if cand.metadata else (None, "none") - attempt = records.attempts.get(pair) - if attempt and (touched is None or attempt[0] > touched): - touched, basis = attempt - if touched is None: - eligible.append(Pick(cand.spec, cand.library, ANCIENT, basis)) + touched, basis = _touched(pair, cand.metadata, records) + if touched is not None and now - touched < window: + continue # touched within the window: not yet + metadata[pair] = cand.metadata + queue.append((touched or ANCIENT, cand.spec, cand.library, basis)) + heapq.heapify(queue) + eligible = len(queue) + + read: set[str] = set() + picks: list[Pick] = [] + while queue and len(picks) < count: + key, spec, lib, basis = heapq.heappop(queue) + if failures is not None and spec not in read: + if len(read) == max_reads: + print(f"::warning::Read the failed generations of {max_reads} specs; picking the rest without them") + failures = None + else: + read.add(spec) + for flib, when in failures(spec).items(): + _newer(records.attempts, (spec, flib), when, "failed generation") + touched, new_basis = _touched((spec, lib), metadata[(spec, lib)], records) + if (touched or ANCIENT) != key: + if touched is not None and now - touched < window: + eligible -= 1 + continue + heapq.heappush(queue, (touched or ANCIENT, spec, lib, new_basis)) continue - if now - touched < min_age: - continue # too fresh to re-roll - eligible.append(Pick(cand.spec, cand.library, touched, basis)) - eligible.sort(key=lambda p: (p.touched, p.spec, p.library)) - return eligible[:count], len(eligible) + picks.append(Pick(spec, lib, key, basis)) + return picks, eligible def _gh_json(args: list[str], what: str) -> list[dict[str, Any]] | None: @@ -202,23 +266,17 @@ def _gh_json(args: list[str], what: str) -> list[dict[str, Any]] | None: def read_records(repo: str) -> Records: records = Records(attempts={}, open_pairs=set(), issue_activity={}) - pr_list = ["pr", "list", "--repo", repo, "--limit", LIMIT, "--json", "headRefName,state,updatedAt"] + pr_list = ["pr", "list", "--repo", repo, "--limit", LIMIT, "--json", "headRefName,state,updatedAt,closedAt,labels"] pulls = _gh_json( [*pr_list, "--state", "all", "--search", "head:implementation/ sort:updated-desc"], "implementation PRs" ) - fold_pulls(pulls or [], "implementation PR", records) + fold_pulls(pulls or [], records) # Kept regenerations again on their own: they are the attempts that leave # `updated` alone, and this listing reaches much further back. kept = _gh_json( [*pr_list, "--state", "closed", "--label", "regen:kept", "--search", "sort:updated-desc"], "kept regenerations" ) - fold_pulls(kept or [], "kept PR", records) - runs = _gh_json( - ["run", "list", "--repo", repo, "--workflow", "impl-generate.yml", "--limit", LIMIT] - + ["--json", "displayTitle,createdAt"], - "impl-generate runs", - ) - fold_runs(runs or [], records) + fold_pulls(kept or [], records) issues = _gh_json( ["issue", "list", "--repo", repo, "--state", "all", "--search", "sort:updated-desc", "--limit", LIMIT] + ["--json", "number,updatedAt"], @@ -232,6 +290,30 @@ def read_records(repo: str) -> Records: return records +def issue_failures(repo: str, spec_issues: dict[str, int]) -> Callable[[str], dict[str, datetime]]: + """``failures(spec)`` backed by the spec issue's comments (one paginated read per spec).""" + + def failures(spec: str) -> dict[str, datetime]: + issue = spec_issues.get(spec) + if issue is None: + return {} + jq = '.[] | select(.body != null and (.body | contains("` comment on the spec issue. A + kept regeneration and a failed generation leave `updated` alone, so without + the attempt record the same pair would come back the next night. +- A pair touched within the attempt window is skipped. The window is the + `attempt_window_days` input, else the repository variable + `REGEN_ATTEMPT_WINDOW_DAYS`, else 30 days. After the window, the pair queues + behind every pair touched before it. +- An infrastructure failure, whose comment carries + ``, and a stop at the Claude spend limit are + not attempts: the pair stays at its place in the queue and comes back on a + later night. +- A pair with an open implementation PR is skipped: its attempt is still + running or stuck. +- Every pair of a spec whose issue saw activity within `min_age_hours` + (default 20) is skipped for that night, as in spec mode. - Only pairs with a metadata file are candidates, so pair mode never writes a first implementation. Gap backfill stays with `bulk-generate.yml`. +The picker reads the failed-generation comments only for the specs at the +front of the queue, at most 100 issues per run. Spec mode keeps its own rule: +a spec's age is the newer of its metadata and any activity on its issue, so a +kept or failed regeneration moves the spec to the back of the queue, and so +does an infrastructure failure or any comment. + Pair mode skips the spec polish, because a spec rewritten for one library would leave its other implementations written against the old text. It dispatches each pair straight to `impl-generate.yml` with the arguments and @@ -541,6 +558,14 @@ The dry run still runs the similarity audit, as in spec mode. To turn pair mode on for the schedule, run `gh variable set REGEN_PAIRS_PER_DAY --body 5`; to turn it off, run `gh variable delete REGEN_PAIRS_PER_DAY`. +When a pre-flight session (the spec polish or the similarity audit) stops at +the [Claude spend limit](#at-the-spend-limit), the night's dispatch for that +spec is skipped and the run fails. `daily-regen.yml` dispatches every pair +within a few minutes, before the first generation is reviewed, so it can't +see a limit that a later review of the same night reaches. The pairs +dispatched after the limit was reached fail at their first refused request, +which costs nothing, and are parked without a retry or a counted attempt. + --- ## Pipeline models @@ -644,41 +669,139 @@ gate record and notice line show `model=n/a`. ## Claude credentials -Every Claude step of the pipeline runs on one of two credentials. The +Every Claude step of the pipeline runs on one of three credentials. The repository variable `CLAUDE_AUTH` chooses it for every run: | `CLAUDE_AUTH` | Credential | Billed to | |---------------|------------|-----------| | Unset, empty, or `subscription` | The `CLAUDE_CODE_OAUTH_TOKEN` secret | The owner's Claude subscription, as before the switch | -| `api` | The `ANTHROPIC_API_KEY` secret | The Claude API credits of the key's Console workspace | +| `oidc` | Workload identity federation: the job's GitHub OIDC token, exchanged for a short-lived Claude API token. The repository variables `CLAUDE_WIF_RULE_ID`, `CLAUDE_WIF_ORG_ID`, `CLAUDE_WIF_SERVICE_ACCOUNT_ID`, and `CLAUDE_WIF_WORKSPACE_ID` name the federation. | The API credits of the federation's workspace | +| `api` | The `ANTHROPIC_API_KEY` secret, the fallback when federation isn't available | The API credits of the key's workspace | | Anything else | None: the job stops | Nothing | The value ignores case. The switch covers all 13 Claude steps in the 8 workflows that run Claude: `daily-regen.yml`, `impl-generate.yml`, `impl-repair.yml`, `impl-review.yml`, `report-validate.yml`, -`review-retest.yml`, `spec-create.yml`, and `util-claude.yml`. Each step -passes exactly one credential and leaves the other input empty, because the -action doesn't document which one wins when both are set. The models don't +`review-retest.yml`, `spec-create.yml`, and `util-claude.yml`. The models don't change: the aliases `sonnet` and `opus` resolve to the same models on the API. -Every job that runs Claude starts with the step "Check Claude credentials". -It stops the job with an error when `CLAUDE_AUTH` has an unknown value, or -when it's `api` and the `ANTHROPIC_API_KEY` secret is missing. It never falls -back to the subscription, so a broken switch can't spend the owner's -interactive quota. In `spec-create.yml` and `report-validate.yml`, the check -runs only for an issue that reaches Claude. In `impl-generate.yml`, it runs -before the issue number is known, so a stop never counts against the pair's -three attempts. +Every job that runs Claude starts with the step "Resolve Claude credentials". +It turns `CLAUDE_AUTH` into a mode, and every Claude step reads the step's +outputs, so a new mode changes only that step. Each Claude step gets exactly +one credential and leaves the others empty, because an API key or an OAuth +token silently shadows federation, and the action doesn't document which of +the two wins. The step stops the job with an error in these cases, and never +falls back to the subscription, so a broken switch can't spend the owner's +interactive quota: + +- `CLAUDE_AUTH` has an unknown value. +- `CLAUDE_AUTH` is `api` and the `ANTHROPIC_API_KEY` secret is missing. +- `CLAUDE_AUTH` is `oidc` and one of the four `CLAUDE_WIF_*` variables is + missing or isn't an ID, or the job has no `id-token: write` permission. + +In `spec-create.yml` and `report-validate.yml`, the step runs only for an +issue that reaches Claude. In `impl-generate.yml`, it runs before the issue +number is known, so a stop never counts against the pair's three attempts. With an API key, the action also classifies Claude's unconfirmed inline PR comments with Haiku on that key and drops the ones it reads as test probes. -Without a key, it posts all of them. Only `util-claude.yml` on a pull request -writes inline comments. +With the subscription or federation, it posts all of them. Only +`util-claude.yml` on a pull request writes inline comments. + +### Which runs federation accepts + +GitHub mints the OIDC token per job, and its `sub` claim names the trigger. +The federation rule accepts only the subject +`repo:MarkusNeusinger/anyplot:ref:refs/heads/main`, so with `CLAUDE_AUTH=oidc` +the Claude steps of these runs work: + +| Workflow | Triggers | Subject | +|----------|----------|---------| +| `daily-regen.yml` | `schedule`, `workflow_dispatch` on `main` | `…:ref:refs/heads/main` | +| `impl-generate.yml` | `issues` (a `generate:` label), `workflow_dispatch` on `main` from bulk-generate, daily-regen, the watchdog, and its own retries | `…:ref:refs/heads/main` | +| `impl-review.yml` | `repository_dispatch`, `workflow_dispatch` on `main` | `…:ref:refs/heads/main` | +| `impl-repair.yml` | `workflow_dispatch` on `main` | `…:ref:refs/heads/main` | +| `spec-create.yml`, `report-validate.yml` | `issues` (a label) | `…:ref:refs/heads/main` | +| `util-claude.yml` | `issue_comment`, also on a pull request | `…:ref:refs/heads/main` | + +Federation refuses these runs with `401 Authentication failed`, so they need +`subscription` or `api`: + +- `util-claude.yml` started by a `pull_request_review_comment` or a + `pull_request_review`: the subject is `repo:MarkusNeusinger/anyplot:pull_request`. +- Any workflow dispatched on another branch, such as a branch smoke with + `--ref feature/x` and the retries it dispatches on that branch: the subject + is `…:ref:refs/heads/feature/x`. `impl-generate.yml` counts such a refusal as + a failed attempt, like any other authentication error. +- `review-retest.yml`: its review job has no `id-token` permission on + purpose, because without it the action can't mint the Claude GitHub App + token, so no session can write to the repository. Under `oidc`, "Resolve + Claude credentials" stops the job. + +Every other job that runs Claude already has `id-token: write`, for the +Claude GitHub App token and, in some of them, Google Cloud. The federation +didn't add the permission to any job. + +### Set up workload identity federation + +1. In the Claude Console, open + [Workload identity](https://platform.claude.com/settings/workload-identity-federation), + click **Connect workload**, and select **GitHub Actions**. The wizard + registers GitHub's issuer, creates a service account, and creates the + federation rule. +2. Give the rule these values: + - Subject prefix: `repo:MarkusNeusinger/anyplot:ref:refs/heads/main`. + - Audience: `https://api.anthropic.com`, the action's default. + - Claims: `repository_owner=MarkusNeusinger`, `repository=MarkusNeusinger/anyplot`, + and `ref=refs/heads/main`. + - Workspace: `ci-regen` only, with the scope `workspace:developer` and a + token lifetime of 600 seconds. The action fetches a fresh GitHub token + every 4 minutes and Claude Code exchanges it again, so a session longer + than the lifetime keeps running. +3. Remove the `event_name=push` claim that the wizard fills in. With it, the + rule refuses every scheduled, dispatched, and issue-triggered run, which is + every run of the pipeline. +4. Set a spend limit on the `ci-regen` workspace. +5. Store the four IDs as repository variables. They're identifiers, not + secrets. The organization ID is on the + [Organization](https://platform.claude.com/settings/organization) page; the + rule ID (`fdrl_…`), the service account ID (`svac_…`), and the workspace ID + (`wrkspc_…`) belong to the resources the wizard created: + + ```bash + gh variable set CLAUDE_WIF_RULE_ID --body fdrl_... + gh variable set CLAUDE_WIF_ORG_ID --body 00000000-0000-0000-0000-000000000000 + gh variable set CLAUDE_WIF_SERVICE_ACCOUNT_ID --body svac_... + gh variable set CLAUDE_WIF_WORKSPACE_ID --body wrkspc_... + ``` + +6. Switch the pipeline to federation, as described in + [Switch modes](#switch-modes). + +A refused exchange always answers `401 Authentication failed`. The reason, such +as `match_subject_prefix`, is in the authentication history on the Workload +identity page. + +### Switch modes + +Switch with one command. Each takes effect with the next job that starts. + +- Federation: `gh variable set CLAUDE_AUTH --body oidc` +- API key: run `gh secret set ANTHROPIC_API_KEY` once, which prompts for the + value so the key stays out of your shell history, then + `gh variable set CLAUDE_AUTH --body api` +- Subscription: `gh variable set CLAUDE_AUTH --body subscription`, or + `gh variable delete CLAUDE_AUTH` + +To check a switch, open the next Claude run: the step "Resolve Claude +credentials" logs `Claude credentials: `, and the cost line in the job +summary reads `auth=`. For an API key, create it in a workspace of its +own with a spend limit. ### Session cost -After every Claude step, the step "Claude cost to the job summary" writes one -line to the job summary and the log: +After every Claude step, the step "Claude session summary" writes one line to +the job summary and the log: ```text claude-cost step=impl-generate auth=api model=claude-sonnet-5-5 total_cost_usd=0.2931 @@ -693,70 +816,92 @@ put a generation on Sonnet at about $0.29 per implementation, a review on Opus at about $1.10, and a repair with its re-review at about $2.70 per implementation in all. -### Switch to API credits +### At the spend limit -1. In the Claude Console, create a workspace for the pipeline, for example - `anyplot-ci`, and set a monthly spend limit on it. -2. Create an API key in that workspace. -3. Store the key as a repository secret. The command prompts for the value, - so the key stays out of your shell history: +With `oidc` or `api`, the workspace's spend limit stops the pipeline cleanly. +The API refuses every request once the limit is reached, and Claude Code's own +retries can't get past it: + +- A spend limit you set on the workspace or the organization answers HTTP 400 + `invalid_request_error` with a message that starts with + `You have reached your specified workspace API usage limits` (or + `… specified API usage limits` for the organization). +- The usage tier's monthly spend cap answers HTTP 429 `rate_limit_error` with + `error_code` `enforced_spend_limit_reached` and no `retry-after` header. +- An empty credit balance answers `Your credit balance is too low`. + +"Claude session summary" reads these messages, and Claude Code's +`billing_error`, from the session's error fields only: the error result, Claude +Code's error messages, and its API retries. It never reads the conversation, +which may quote the same text. When it finds one, it writes an `::error::` +and the line **Claude spend limit reached (mode oidc)** (or `api`) to the job +summary, and every path treats the stop as an infrastructure failure: + +| Workflow | What happens | +|----------|--------------| +| `impl-generate.yml` | The retry is skipped, and "Stop at the Claude spend limit" fails the job. "Handle generation failure" parks the pair: it removes `generate:{library}` and `impl:{library}:pending`, puts `impl:{library}:done` back for a live implementation, and comments with its own marker. Nothing is retried, no attempt counts toward the cap of three, and the pair is never marked `impl:{library}:failed`. No PR exists; the next generation recreates the branch. | +| `impl-review.yml` | "Validate review output" adds `ai-review-failed` and comments, without the auto re-review and without spending it. No score, no verdict: the PR stays open, nothing is merged or closed. The watchdog re-dispatches the review once at its next scan. | +| `impl-repair.yml` | The retry is skipped. "Handle repair failure" restores `ai-rejected` and comments, without the auto-retry and without spending it. The watchdog's repair rescue, once per attempt, or a manual restart picks the PR up. | +| `daily-regen.yml` | A spec polish or similarity audit at the limit skips that spec's dispatch, and the entry fails. | +| `spec-create.yml`, `util-claude.yml` | The retry is skipped, and the job fails. | +| `report-validate.yml`, `review-retest.yml` | The step fails as before: the issue gets the "Validation Failed" comment, and the cell is collected as failed. | + +In pair mode, a parked pair keeps its place in the queue: neither a parked +generation nor a review stopped on an open PR is a real attempt. A pair whose +review stopped keeps its open PR, which pair mode skips until the review +finishes. A session that reaches the limit after it wrote its review files is +scored on what it wrote, like any session that ends early. + +With the subscription nothing changes: the summary never reads a spend limit +there, and the subscription's own usage limit takes the paths it took before. + +### Test the spend limit + +Test the stop after the merge, with `CLAUDE_AUTH=oidc`: + +1. In the Claude Console, set the `ci-regen` workspace's spend limit to the + month's spend so far plus about 0.50 USD. +2. Dispatch one regeneration of a pair whose implementation is live on main: ```bash - gh secret set ANTHROPIC_API_KEY + gh workflow run impl-generate.yml -f specification_id=SPEC_ID -f library=LIBRARY ``` -4. Switch the pipeline to the key: + The generation on Sonnet costs about 0.29 USD and passes; the review on + Opus costs about 1.10 USD and reaches the limit while it runs. +3. Check the `impl-review.yml` run: its summary shows **Claude spend limit + reached (mode oidc)**, the PR has `ai-review-failed` and the "AI Review + Paused (Claude spend limit)" comment, no second review run started, and no + verdict or `quality:` label was added. +4. Check the spec issue: no `impl:{library}:failed` label, and no + `impl-fail` comment for the pair. +5. Restore the spend limit, then rerun the review, or wait for the watchdog: ```bash - gh variable set CLAUDE_AUTH --body api + gh workflow run impl-review.yml -f pr_number=PR_NUMBER ``` -5. Check the next Claude run: the step "Check Claude credentials" logs - `Claude credentials: api (ANTHROPIC_API_KEY)`, and the cost line in the job - summary reads `auth=api`. - -To switch back to the subscription, run -`gh variable set CLAUDE_AUTH --body subscription`, or delete the variable. The -key can stay in place. - -### At the spend limit +6. Check that the review finishes and the regen gate decides as usual. -When the workspace reaches its spend limit, the API refuses every request -until the limit resets or you raise it. Each Claude step then fails, and its -retry fails too. What happens next depends on the error text in the step's -execution log: - -- **The text names a provider-side error**, such as a `429` with - `rate_limit_error`, `too many requests`, `overloaded`, or a 5xx status. - `impl-generate.yml` counts the failure as an infrastructure failure: it - doesn't spend the pair's three attempts, and it retries the pair until the - fifth infrastructure failure in 12 hours, which parks the pair without - `impl:{library}:failed` and with a comment that holds the command to - re-dispatch it. -- **The text names anything else**, such as a low credit balance. - `impl-generate.yml` counts the failure as the pair's own. A regeneration is - never marked `impl:{library}:failed`, because its implementation is live on - main. A first implementation is marked failed after its third attempt. - -A failed review gets one automatic re-review and then `ai-review-failed`, a -failed repair gets one automatic retry, and the watchdog rescues each once. -The spec polish -and the similarity audit in `daily-regen.yml` fail without stopping the -dispatch. To recover, raise the limit or switch back to the subscription, then -re-dispatch the parked pairs. +To test the generation path as well, set the limit to the spend so far plus +about 0.05 USD: the generation stops, the pair is parked as described above, +and its retry is skipped. ### Exposure -The key reaches the same steps that the OAuth token reaches, and nothing -else: `tests/unit/workflows/test_claude_auth.py` checks that the secret -appears only in the action input and in the check step's `!= ''` test. None of -the eight workflows runs on `pull_request` or `pull_request_target`, so a pull -request from a fork never gets either credential. `util-claude.yml` runs only -for comments by owners, members, and collaborators. `spec-create.yml` and -`report-validate.yml` run on a label, which only maintainers can add, but -they read issue text that anyone can write. A key is billed per token, so keep -it in its own workspace: the workspace's spend limit bounds what a leaked key -can cost. +Each credential reaches only the Claude steps, and nothing else: +`tests/unit/workflows/test_claude_auth.py` checks that the API key appears only +in the action input and in the resolve step's `!= ''` test, and that no job +gained `id-token: write`. None of the eight workflows runs on `pull_request` +or `pull_request_target`, so a pull request from a fork never gets a +credential. `util-claude.yml` runs only for comments by owners, members, and +collaborators. `spec-create.yml` and `report-validate.yml` run on a label, +which only maintainers can add, but they read issue text that anyone can +write. Federation adds no stored secret: any step of a job with +`id-token: write` can mint an OIDC token, but the federation rule accepts only +`main`, so a token from a pull-request event or another branch is refused. A +leaked federated token expires after 600 seconds, and an API key is billed +until you revoke it; the workspace's spend limit bounds either. --- diff --git a/prompts/README.md b/prompts/README.md index 1f572bcfcc3..8c117708e40 100644 --- a/prompts/README.md +++ b/prompts/README.md @@ -47,9 +47,14 @@ See `workflow-prompts/README.md` for the wiring pattern + per-variable reference # Example: from .github/workflows/impl-review.yml - uses: anthropics/claude-code-action@ # v1 with: - # Exactly one credential, chosen by vars.CLAUDE_AUTH (docs/workflows/overview.md#claude-credentials) - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + # Exactly one credential, from the job's "Resolve Claude credentials" step + # (docs/workflows/overview.md#claude-credentials) + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model sonnet" prompt: | Read `prompts/workflow-prompts/ai-quality-review.md` and follow those instructions. diff --git a/prompts/workflow-prompts/README.md b/prompts/workflow-prompts/README.md index 7a225673ec2..6011cbb5c29 100644 --- a/prompts/workflow-prompts/README.md +++ b/prompts/workflow-prompts/README.md @@ -20,9 +20,14 @@ Example (from `.github/workflows/impl-review.yml`): ```yaml - uses: anthropics/claude-code-action@ # v1 with: - # Exactly one credential, chosen by vars.CLAUDE_AUTH (docs/workflows/overview.md#claude-credentials) - anthropic_api_key: ${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }} - claude_code_oauth_token: ${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + # Exactly one credential, from the job's "Resolve Claude credentials" step + # (docs/workflows/overview.md#claude-credentials) + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model sonnet" prompt: | Read `prompts/workflow-prompts/ai-quality-review.md` and follow those instructions. diff --git a/tests/unit/automation/scripts/test_regen_pick_pairs.py b/tests/unit/automation/scripts/test_regen_pick_pairs.py index d85307447f8..4ee40ad9f3b 100644 --- a/tests/unit/automation/scripts/test_regen_pick_pairs.py +++ b/tests/unit/automation/scripts/test_regen_pick_pairs.py @@ -1,9 +1,13 @@ """Tests for automation.scripts.regen_pick_pairs — daily-regen's pair mode. -The picker mirrors the spec mode's age rule per (spec, library) pair: the -newer of the metadata ``updated`` and the pair's last attempt, a minimum age, -the spec issue's activity as a postponement. Every exclusion is pinned here, -and ``main`` runs once against a fake ``gh`` the way the workflow calls it. +The picker ages each (spec, library) pair by its newest touch, the metadata +``updated`` or the last real attempt (a merged or closed implementation PR, a +failed generation), keeps every pair touched within the attempt window out +of the queue, and postpones a spec whose issue just saw activity. The +no-loop rule is pinned here: a pair the gate kept, or whose generation +failed, does not come back the next night, and an infrastructure failure or +a stop at the spend limit is no attempt. ``main`` runs once against a fake +``gh`` the way the workflow calls it. """ from __future__ import annotations @@ -23,12 +27,13 @@ NOW = datetime(2026, 10, 31, 2, 17, tzinfo=timezone.utc) MIN_AGE = timedelta(hours=20) +WINDOW = timedelta(days=30) LIBS = ["altair", "matplotlib", "plotly"] SCRIPT = Path(rp.__file__) -def ts(days: float) -> str: - return (NOW - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ") +def ts(days: float, now: datetime = NOW) -> str: + return (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ") def at(days: float) -> datetime: @@ -54,10 +59,19 @@ def records(**kw) -> rp.Records: ) +def run_pick(cands, recs=None, issues=None, count=5, window=WINDOW, failures=None, max_reads=100): + return rp.pick(cands, recs or records(), issues or {}, NOW, MIN_AGE, window, count, failures, max_reads) + + def picked(picks: list[rp.Pick]) -> list[str]: return [f"{p.spec}/{p.library}" for p in picks] +def fail_comment(spec: str, lib: str, days: float, infra: bool = False) -> dict[str, str]: + body = f"\n" + (f"{rp.INFRA_TAG}\n" if infra else "") + "## Failed" + return {"created_at": ts(days), "body": body} + + class TestLoadCandidates: def test_reads_every_supported_pair_with_its_timestamp(self, tmp_path): write_spec(tmp_path, "a", {"altair": {"updated": ts(10)}, "plotly": {"created": ts(30)}}, issue=7) @@ -81,131 +95,154 @@ def test_unparseable_timestamp_skips_the_pair_and_missing_one_keeps_it(self, tmp class TestPick: def test_oldest_first_across_specs(self): cands = [ - rp.Candidate("a", "altair", at(5)), - rp.Candidate("b", "plotly", at(50)), - rp.Candidate("a", "plotly", at(20)), + rp.Candidate("a", "altair", at(40)), + rp.Candidate("b", "plotly", at(90)), + rp.Candidate("a", "plotly", at(60)), ] - picks, eligible = rp.pick(cands, records(), {}, NOW, MIN_AGE, 2) + picks, eligible = run_pick(cands, count=2) assert picked(picks) == ["b/plotly", "a/plotly"] assert eligible == 3 - def test_a_kept_regeneration_ages_from_its_attempt(self): - # The gate kept a/altair two days ago: `updated` is still 90 days old, - # but the pair is now younger than b/plotly. - cands = [rp.Candidate("a", "altair", at(90)), rp.Candidate("b", "plotly", at(30))] - recs = records(attempts={("a", "altair"): (at(2), "kept PR")}) - picks, _ = rp.pick(cands, recs, {}, NOW, MIN_AGE, 2) + def test_a_kept_pair_waits_out_the_window(self): + # The gate kept a/altair 10 days ago: `updated` is still 200 days old. + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("b", "plotly", at(60))] + recs = records(attempts={("a", "altair"): (at(10), "kept PR")}) + assert picked(run_pick(cands, recs)[0]) == ["b/plotly"] + + def test_after_the_window_an_attempt_queues_behind_older_pairs(self): + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("b", "plotly", at(60))] + recs = records(attempts={("a", "altair"): (at(45), "kept PR")}) + picks, _ = run_pick(cands, recs) assert picked(picks) == ["b/plotly", "a/altair"] - assert picks[1].basis == "kept PR" and picks[1].touched == at(2) + assert picks[1].basis == "kept PR" and picks[1].touched == at(45) - def test_an_attempt_older_than_the_metadata_changes_nothing(self): - cands = [rp.Candidate("a", "altair", at(5))] - picks, _ = rp.pick(cands, records(attempts={("a", "altair"): (at(9), "run")}), {}, NOW, MIN_AGE, 1) - assert picks[0].basis == "metadata" and picks[0].touched == at(5) + def test_a_recent_merge_is_inside_the_window_too(self): + cands = [rp.Candidate("a", "altair", at(12)), rp.Candidate("a", "plotly", at(31))] + assert picked(run_pick(cands)[0]) == ["a/plotly"] - def test_too_fresh_pairs_are_skipped(self): - cands = [rp.Candidate("a", "altair", at(0.5)), rp.Candidate("a", "plotly", at(30))] - recs = records(attempts={("a", "plotly"): (at(0.2), "run")}) # a generation started 5 h ago - assert rp.pick(cands, recs, {}, NOW, MIN_AGE, 5) == ([], 0) + def test_window_zero_orders_only(self): + cands = [rp.Candidate("a", "altair", at(2)), rp.Candidate("a", "plotly", at(1))] + assert picked(run_pick(cands, window=timedelta(0))[0]) == ["a/altair", "a/plotly"] def test_open_pr_skips_only_its_pair(self): - cands = [rp.Candidate("a", "altair", at(30)), rp.Candidate("a", "plotly", at(40))] - picks, _ = rp.pick(cands, records(open_pairs={("a", "plotly")}), {}, NOW, MIN_AGE, 5) - assert picked(picks) == ["a/altair"] + cands = [rp.Candidate("a", "altair", at(60)), rp.Candidate("a", "plotly", at(70))] + assert picked(run_pick(cands, records(open_pairs={("a", "plotly")}))[0]) == ["a/altair"] def test_recent_issue_activity_postpones_every_pair_of_the_spec(self): - cands = [rp.Candidate("a", "altair", at(30)), rp.Candidate("b", "altair", at(10))] + cands = [rp.Candidate("a", "altair", at(60)), rp.Candidate("b", "altair", at(40))] recs = records(issues={1: at(0.3), 2: at(3)}) - picks, _ = rp.pick(cands, recs, {"a": 1, "b": 2}, NOW, MIN_AGE, 5) - assert picked(picks) == ["b/altair"] # b's issue is three days old: no postponement + assert picked(run_pick(cands, recs, {"a": 1, "b": 2})[0]) == ["b/altair"] def test_issue_activity_is_a_gate_not_an_age(self): - # Old activity on a's issue does not re-date a's pairs. - cands = [rp.Candidate("a", "altair", at(30)), rp.Candidate("b", "altair", at(20))] - picks, _ = rp.pick(cands, records(issues={1: at(2)}), {"a": 1}, NOW, MIN_AGE, 1) - assert picked(picks) == ["a/altair"] and picks[0].touched == at(30) + cands = [rp.Candidate("a", "altair", at(60)), rp.Candidate("b", "altair", at(50))] + picks, _ = run_pick(cands, records(issues={1: at(2)}), {"a": 1}, count=1) + assert picked(picks) == ["a/altair"] and picks[0].touched == at(60) def test_no_timestamp_at_all_is_ancient(self): cands = [rp.Candidate("a", "altair", at(300)), rp.Candidate("b", "altair", None)] - picks, _ = rp.pick(cands, records(), {}, NOW, MIN_AGE, 2) + picks, _ = run_pick(cands, count=2) assert picked(picks) == ["b/altair", "a/altair"] assert picks[0].touched == rp.ANCIENT and picks[0].basis == "none" - def test_no_metadata_timestamp_ages_from_the_attempt(self): - cands = [rp.Candidate("b", "altair", None)] - assert rp.pick(cands, records(attempts={("b", "altair"): (at(0.1), "run")}), {}, NOW, MIN_AGE, 1)[0] == [] - def test_ties_break_by_spec_then_library(self): - cands = [ - rp.Candidate("b", "altair", at(9)), - rp.Candidate("a", "plotly", at(9)), - rp.Candidate("a", "altair", at(9)), + cands = [rp.Candidate("b", "altair", at(90)), rp.Candidate("a", "plotly", at(90))] + cands.append(rp.Candidate("a", "altair", at(90))) + assert picked(run_pick(cands, count=3)[0]) == ["a/altair", "a/plotly", "b/altair"] + + +class TestFailedGenerations: + def test_a_failed_generation_is_an_attempt(self): + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("a", "plotly", at(150))] + cands.append(rp.Candidate("b", "altair", at(100))) + asked: list[str] = [] + + def failures(spec): + asked.append(spec) + return {"altair": at(3)} if spec == "a" else {} + + picks, eligible = run_pick(cands, count=2, failures=failures) + # a/altair failed three days ago: out for the window; its sibling is not. + assert picked(picks) == ["a/plotly", "b/altair"] + assert eligible == 2 + assert asked == ["a", "b"] # once per spec, only for the front of the queue + + def test_an_old_failure_re_ages_the_pair(self): + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("b", "altair", at(100))] + picks, _ = run_pick(cands, count=2, failures=lambda spec: {"altair": at(40)} if spec == "a" else {}) + assert picked(picks) == ["b/altair", "a/altair"] + assert picks[1].basis == "failed generation" + + def test_comments_count_only_real_failures(self): + comments = [ + fail_comment("a", "altair", 5), + fail_comment("a", "altair", 2, infra=True), # infrastructure: no attempt + fail_comment("a", "plotly", 1, infra=True), + fail_comment("other", "altair", 1), # another spec's marker + {"created_at": ts(1), "body": ""}, ] - assert picked(rp.pick(cands, records(), {}, NOW, MIN_AGE, 3)[0]) == ["a/altair", "a/plotly", "b/altair"] + assert rp.failed_generations(comments, "a") == {"altair": at(5)} + + def test_the_read_cap_stops_reading(self, capsys): + cands = [rp.Candidate(spec, "altair", at(100 + i)) for i, spec in enumerate("abc")] + asked: list[str] = [] + picks, _ = run_pick(cands, count=3, failures=lambda spec: asked.append(spec) or {}, max_reads=1) + assert len(picks) == 3 and asked == ["c"] + assert "::warning::Read the failed generations of 1 specs" in capsys.readouterr().out -class TestFold: - def test_pulls_keep_the_newest_attempt_and_mark_open_ones(self): +class TestFoldPulls: + def test_open_blocks_closed_and_merged_count(self): recs = records() rp.fold_pulls( [ - {"headRefName": "implementation/a/altair", "state": "MERGED", "updatedAt": ts(9)}, - {"headRefName": "implementation/a/altair", "state": "CLOSED", "updatedAt": ts(3)}, + {"headRefName": "implementation/a/altair", "state": "MERGED", "closedAt": ts(9), "updatedAt": ts(1)}, + {"headRefName": "implementation/a/altair", "state": "CLOSED", "closedAt": ts(40), "labels": []}, + { + "headRefName": "implementation/a/plotly", + "state": "CLOSED", + "closedAt": ts(3), + "labels": [{"name": "regen"}, {"name": "regen:kept"}], + }, {"headRefName": "implementation/b/plotly", "state": "OPEN", "updatedAt": ts(1)}, - {"headRefName": "review-writeback/a/altair/12", "state": "MERGED", "updatedAt": ts(0)}, - {"headRefName": "implementation/a/altair/extra", "state": "OPEN", "updatedAt": ts(0)}, + {"headRefName": "review-writeback/a/altair/12", "state": "MERGED", "closedAt": ts(0)}, + {"headRefName": "implementation/a/altair/extra", "state": "CLOSED", "closedAt": ts(0)}, ], - "implementation PR", recs, ) - assert recs.attempts == { - ("a", "altair"): (at(3), "implementation PR"), - ("b", "plotly"): (at(1), "implementation PR"), - } + assert recs.attempts == {("a", "altair"): (at(9), "merged PR"), ("a", "plotly"): (at(3), "kept PR")} assert recs.open_pairs == {("b", "plotly")} - def test_runs_parse_the_impl_generate_run_name(self): - recs = records() - rp.fold_runs( - [ - {"displayTitle": "Generate: plotly for line-basic", "createdAt": ts(4)}, - {"displayTitle": "Generate: generate:plotly for issue", "createdAt": ts(1)}, - {"displayTitle": "Review: PR #12", "createdAt": ts(1)}, - ], - recs, - ) - assert recs.attempts == {("line-basic", "plotly"): (at(4), "run")} - def test_issues(self): recs = records() rp.fold_issues([{"number": 3, "updatedAt": ts(2)}, {"number": "x", "updatedAt": ts(1)}], recs) assert recs.issue_activity == {3: at(2)} -# Answers each listing the picker makes from a JSON file named after it; a -# listing whose file is missing fails like an API error. Logs every call. +# Answers each listing the picker makes from a JSON file named after it, and an +# issue's comments (`api … --jq …`) from comments-.jsonl; a listing +# whose file is missing fails like an API error. Logs every call. FAKE_GH = """#!/bin/sh echo "$*" >> "$GH_LOG" case "$*" in - *"head:implementation/"*) f=prs ;; - *"regen:kept"*) f=kept ;; - "run list"*) f=runs ;; - "issue list"*) f=issues ;; + *"head:implementation/"*) f=prs.json ;; + *"regen:kept"*) f=kept.json ;; + "issue list"*) f=issues.json ;; + "api --paginate repos/o/r/issues/"*) n=${3#repos/o/r/issues/}; f=comments-${n%%/*}.jsonl ;; *) exit 1 ;; esac -[ -f "$GH_DATA/$f.json" ] || { echo "HTTP 502" >&2; exit 1; } -cat "$GH_DATA/$f.json" +[ -f "$GH_DATA/$f" ] || { echo "HTTP 502" >&2; exit 1; } +cat "$GH_DATA/$f" """ -def run_main(tmp_path: Path, data: dict[str, list], count: int = 3) -> tuple[subprocess.CompletedProcess, dict, str]: +def run_main(tmp_path: Path, data: dict[str, str], count: int = 3, *extra: str): bin_dir, data_dir = tmp_path / "bin", tmp_path / "data" bin_dir.mkdir() data_dir.mkdir() (bin_dir / "gh").write_text(FAKE_GH, encoding="utf-8") (bin_dir / "gh").chmod(0o755) - for name, rows in data.items(): - (data_dir / f"{name}.json").write_text(json.dumps(rows), encoding="utf-8") + for name, text in data.items(): + (data_dir / name).write_text(text, encoding="utf-8") out, summary = tmp_path / "out", tmp_path / "summary" env = { **os.environ, @@ -215,7 +252,7 @@ def run_main(tmp_path: Path, data: dict[str, list], count: int = 3) -> tuple[sub "GITHUB_OUTPUT": str(out), "GITHUB_STEP_SUMMARY": str(summary), } - args = ["--count", str(count), "--min-age-hours", "20", "--repo", "o/r", "--libraries", " ".join(LIBS)] + args = ["--count", str(count), "--min-age-hours", "20", "--repo", "o/r", "--libraries", " ".join(LIBS), *extra] result = subprocess.run( [sys.executable, str(SCRIPT), *args, "--plots", str(tmp_path / "plots")], env=env, @@ -228,30 +265,45 @@ def run_main(tmp_path: Path, data: dict[str, list], count: int = 3) -> tuple[sub class TestMain: def _plots(self, tmp_path: Path) -> None: + self.now = datetime.now(timezone.utc) plots = tmp_path / "plots" - now = datetime.now(timezone.utc) - - def old(days: float) -> str: - return (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ") - - write_spec(plots, "a-spec", {"altair": {"updated": old(90)}, "plotly": {"updated": old(60)}}, issue=11) - write_spec(plots, "b-spec", {"altair": {"updated": old(80)}, "matplotlib": {"updated": old(70)}}, issue=12) - self.recent = old(0.1) - self.week = old(7) + write_spec( + plots, "a-spec", {"altair": {"updated": ts(190, self.now)}, "plotly": {"updated": ts(160, self.now)}}, 11 + ) + write_spec( + plots, + "b-spec", + {"altair": {"updated": ts(180, self.now)}, "matplotlib": {"updated": ts(170, self.now)}}, + 12, + ) def test_writes_pairs_specs_and_summary(self, tmp_path): self._plots(tmp_path) + prs = [{"headRefName": "implementation/b-spec/matplotlib", "state": "OPEN", "updatedAt": ts(0.1, self.now)}] + kept = [ + { + "headRefName": "implementation/a-spec/altair", + "state": "CLOSED", + "closedAt": ts(40, self.now), + "labels": [{"name": "regen:kept"}], + } + ] + comments = [ + {"created_at": ts(5, self.now), "body": "\n"} + ] result, outputs, summary = run_main( tmp_path, { - "prs": [{"headRefName": "implementation/b-spec/matplotlib", "state": "OPEN", "updatedAt": self.recent}], - "kept": [{"headRefName": "implementation/a-spec/altair", "state": "CLOSED", "updatedAt": self.week}], - "runs": [], - "issues": [{"number": 11, "updatedAt": self.week}], + "prs.json": json.dumps(prs), + "kept.json": json.dumps(kept), + "issues.json": json.dumps([{"number": 11, "updatedAt": ts(7, self.now)}]), + "comments-11.jsonl": "", + "comments-12.jsonl": "\n".join(json.dumps(c) for c in comments), }, ) assert result.returncode == 0, result.stdout + result.stderr - # a-spec/altair is aged by its keep (7 d), b-spec/matplotlib has an open PR. + # b-spec/matplotlib has an open PR; a-spec/altair was kept 40 days ago, + # so it queues last; b-spec/altair's infrastructure failure is no attempt. assert json.loads(outputs["pairs_json"]) == [ {"spec": "b-spec", "library": "altair"}, {"spec": "a-spec", "library": "plotly"}, @@ -260,22 +312,35 @@ def test_writes_pairs_specs_and_summary(self, tmp_path): assert json.loads(outputs["specs_json"]) == ["b-spec", "a-spec"] assert outputs["specs"] == "b-spec a-spec" and outputs["count"] == "3" assert "| 3 | `a-spec` | `altair` |" in summary and "kept PR" in summary + log = (tmp_path / "gh.log").read_text() + assert "repos/o/r/issues/12/comments?per_page=100" in log and "run list" not in log + + def test_the_window_comes_from_the_argument(self, tmp_path): + self._plots(tmp_path) + kept = [{"headRefName": "implementation/a-spec/altair", "state": "CLOSED", "closedAt": ts(40, self.now)}] + data = {"prs.json": "[]", "kept.json": json.dumps(kept), "issues.json": "[]"} + data |= {"comments-11.jsonl": "", "comments-12.jsonl": ""} + _, outputs, _ = run_main(tmp_path, data, 5, "--attempt-window-days", "60") + # Kept 40 days ago is inside a 60-day window. + pairs = {(p["spec"], p["library"]) for p in json.loads(outputs["pairs_json"])} + assert pairs == {("a-spec", "plotly"), ("b-spec", "altair"), ("b-spec", "matplotlib")} def test_a_failed_listing_degrades_with_a_warning(self, tmp_path): self._plots(tmp_path) - result, outputs, _ = run_main(tmp_path, {"issues": []}, count=1) + result, outputs, _ = run_main(tmp_path, {"issues.json": "[]"}, 1) assert result.returncode == 0, result.stdout + result.stderr - assert result.stdout.count("::warning::Could not list") == 3 + assert result.stdout.count("::warning::Could not list") == 2 + assert "::warning::Could not read the comments of issue #11" in result.stdout assert json.loads(outputs["pairs_json"]) == [{"spec": "a-spec", "library": "altair"}] def test_nothing_eligible_writes_an_empty_pick(self, tmp_path): (tmp_path / "plots").mkdir() - result, outputs, _ = run_main(tmp_path, {"prs": [], "kept": [], "runs": [], "issues": []}) + result, outputs, _ = run_main(tmp_path, {"prs.json": "[]", "kept.json": "[]", "issues.json": "[]"}) assert result.returncode == 0, result.stdout + result.stderr assert outputs == {"pairs_json": "[]", "specs": "", "specs_json": "[]", "count": "0"} - @pytest.mark.parametrize("count", [0, -1]) - def test_refuses_a_count_below_one(self, tmp_path, count): + @pytest.mark.parametrize(("count", "window"), [(0, "30"), (-1, "30"), (1, "-1")]) + def test_refuses_bad_numbers(self, tmp_path, count, window): (tmp_path / "plots").mkdir() - result, _, _ = run_main(tmp_path, {}, count=count) + result, _, _ = run_main(tmp_path, {}, count, "--attempt-window-days", window) assert result.returncode == 1 and "::error::" in result.stdout diff --git a/tests/unit/skills/test_collect_week.py b/tests/unit/skills/test_collect_week.py index 4f2a01d2b94..d5c80b5df90 100644 --- a/tests/unit/skills/test_collect_week.py +++ b/tests/unit/skills/test_collect_week.py @@ -295,6 +295,11 @@ class TestSessionCost: def test_cost_lines_win_and_carry_the_credential(self, cw): assert cw.log_cost(self.LOG) == {"api": 0.2931} + def test_federation_is_api_credits_too(self, cw): + log = "claude-cost step=impl-review auth=oidc model=claude-opus-5-5 total_cost_usd=1.1\n" + assert cw.log_cost(log) == {"oidc": 1.1} + assert set(cw.API_MODES) == {"api", "oidc"} + def test_an_older_log_counts_as_subscription(self, cw): assert cw.log_cost(' "total_cost_usd": 1.1,\n "total_cost_usd": 0.4\n') == {"subscription": 1.5} diff --git a/tests/unit/workflows/test_claude_auth.py b/tests/unit/workflows/test_claude_auth.py index bad768a86ec..7b400c312de 100644 --- a/tests/unit/workflows/test_claude_auth.py +++ b/tests/unit/workflows/test_claude_auth.py @@ -1,19 +1,24 @@ -"""Claude credential switch (vars.CLAUDE_AUTH) and the per-session cost line. - -Every claude-code-action step passes exactly one non-empty credential: the -subscription's CLAUDE_CODE_OAUTH_TOKEN unless CLAUDE_AUTH is `api`, then -ANTHROPIC_API_KEY. The action's precedence with both set is undocumented, so -the other input is always the empty string. Every job that runs Claude starts -with the same check step, which stops on an unknown value or on `api` without -the key instead of falling back to the subscription, and every Claude step is -followed by the same cost step. There is no pre-merge smoke for these -workflows (claude-code-action refuses to run on a PR that changes them), so -the wiring is pinned here and the check step runs for real. - -Also here: daily-regen's mode switch (REGEN_PAIRS_PER_DAY / the `pairs` -input), its pair dispatch and the watchdog's rescue, which forwards the pair -count. The pick itself is tested in tests/unit/automation/scripts/ -test_regen_pick_pairs.py. +"""Claude credential switch (vars.CLAUDE_AUTH), the session summary and the spend limit. + +Every job that runs Claude starts with the same "Resolve Claude credentials" +step. It maps CLAUDE_AUTH (unset or `subscription`, `api`, `oidc`) to a mode, +stops the job on an unknown value or a missing credential instead of falling +back to the subscription, and hands the mode and, for `oidc`, the four +workload identity federation IDs to every Claude step as outputs. Each step +then passes exactly one credential; the action's precedence with several set +is undocumented (an API key or OAuth token silently shadows federation). + +After every Claude step, "Claude session summary" writes the cost line and +reads whether the session stopped at the Claude spend limit (api/oidc only); +the retry is skipped then, and the failure paths of impl-generate, +impl-review and impl-repair park the work as an infrastructure stop without +a retry or a counted attempt. + +There is no pre-merge smoke for these workflows (claude-code-action refuses +to run on a PR that changes them), so the wiring is pinned here and the +scripts run for real. Also here: daily-regen's mode switch, its pair +dispatch and the watchdog's rescue; the pick itself is tested in +tests/unit/automation/scripts/test_regen_pick_pairs.py. """ from __future__ import annotations @@ -30,22 +35,21 @@ WORKFLOWS_DIR = Path(__file__).parent.parent.parent.parent / ".github" / "workflows" ACTION = "anthropics/claude-code-action@" -API_KEY = "${{ vars.CLAUDE_AUTH == 'api' && secrets.ANTHROPIC_API_KEY || '' }}" -OAUTH = "${{ vars.CLAUDE_AUTH != 'api' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }}" -AUTH_MODE = "${{ vars.CLAUDE_AUTH == 'api' && 'api' || 'subscription' }}" -CHECK = "Check Claude credentials" -COST = "Claude cost to the job summary" -# Inputs that would select another credential or provider. -OTHER_CREDENTIALS = { - "anthropic_federation_rule_id", - "anthropic_organization_id", - "anthropic_service_account_id", - "anthropic_workspace_id", - "use_bedrock", - "use_vertex", - "use_foundry", - "claude_env", +RESOLVE = "Resolve Claude credentials" +SUMMARY = "Claude session summary" +STOP = "Stop at the Claude spend limit" +CREDENTIALS = { + "anthropic_api_key": "${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }}", + "claude_code_oauth_token": ( + "${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }}" + ), + "anthropic_federation_rule_id": "${{ steps.claude_auth.outputs.federation_rule_id }}", + "anthropic_organization_id": "${{ steps.claude_auth.outputs.organization_id }}", + "anthropic_service_account_id": "${{ steps.claude_auth.outputs.service_account_id }}", + "anthropic_workspace_id": "${{ steps.claude_auth.outputs.workspace_id }}", } +# Inputs that would select another provider or route credentials around the switch. +FOREIGN_INPUTS = {"use_bedrock", "use_vertex", "use_foundry", "claude_env", "anthropic_oidc_audience"} # The eight workflows (13 steps) that ran Claude on the OAuth token before the switch. CLAUDE_WORKFLOWS = { "daily-regen.yml": 2, @@ -57,19 +61,24 @@ "report-validate.yml": 1, "review-retest.yml": 1, } +WIF = { + "WIF_RULE_ID": "fdrl_01abc", + "WIF_ORG_ID": "00000000-0000-4000-8000-000000000000", + "WIF_SERVICE_ACCOUNT_ID": "svac_01abc", + "WIF_WORKSPACE_ID": "wrkspc_01abc", +} def _load(filename: str) -> dict[str, Any]: return yaml.safe_load((WORKFLOWS_DIR / filename).read_text(encoding="utf-8")) -def _claude_jobs() -> list[tuple[str, str, list[dict[str, Any]]]]: +def _claude_jobs() -> list[tuple[str, str, dict[str, Any]]]: jobs = [] for path in sorted(WORKFLOWS_DIR.glob("*.yml")): for name, job in (_load(path.name).get("jobs") or {}).items(): - steps = job.get("steps") or [] - if any(str(s.get("uses", "")).startswith(ACTION) for s in steps): - jobs.append((path.name, name, steps)) + if any(_is_claude(s) for s in job.get("steps") or []): + jobs.append((path.name, name, job)) return jobs @@ -77,192 +86,504 @@ def _is_claude(step: dict[str, Any]) -> bool: return str(step.get("uses", "")).startswith(ACTION) -def _check_steps() -> list[dict[str, Any]]: - return [s for _, _, steps in _claude_jobs() for s in steps if s.get("name") == CHECK] +def _named(name: str) -> list[dict[str, Any]]: + return [s for _, _, job in _claude_jobs() for s in job["steps"] if str(s.get("name", "")).startswith(name)] + + +def _step(filename: str, name: str) -> dict[str, Any]: + steps = [s for job in _load(filename)["jobs"].values() for s in job.get("steps", [])] + matches = [s for s in steps if s.get("name") == name] + assert len(matches) == 1, (filename, name) + return matches[0] + + +JOBS = _claude_jobs() +IDS = [f"{f}:{j}" for f, j, _ in JOBS] def test_every_claude_workflow_is_covered(): found: dict[str, int] = {} - for filename, _, steps in _claude_jobs(): - found[filename] = found.get(filename, 0) + sum(1 for s in steps if _is_claude(s)) + for filename, _, job in JOBS: + found[filename] = found.get(filename, 0) + sum(1 for s in job["steps"] if _is_claude(s)) assert found == CLAUDE_WORKFLOWS -@pytest.mark.parametrize(("filename", "job", "steps"), _claude_jobs(), ids=lambda v: v if isinstance(v, str) else "") -def test_each_claude_step_passes_exactly_the_switched_credentials(filename, job, steps): - for step in filter(_is_claude, steps): +# --- wiring ----------------------------------------------------------------- + + +@pytest.mark.parametrize(("filename", "name", "job"), JOBS, ids=IDS) +def test_each_claude_step_reads_the_resolved_credentials(filename, name, job): + for step in filter(_is_claude, job["steps"]): given = step.get("with") or {} - assert given.get("anthropic_api_key") == API_KEY, (filename, step.get("name")) - assert given.get("claude_code_oauth_token") == OAUTH, (filename, step.get("name")) - assert not OTHER_CREDENTIALS & set(given), (filename, step.get("name")) + assert {k: given.get(k) for k in CREDENTIALS} == CREDENTIALS, (filename, step.get("name")) + assert not FOREIGN_INPUTS & set(given), (filename, step.get("name")) env = step.get("env") or {} assert not {"ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"} & set(env), (filename, step.get("name")) -@pytest.mark.parametrize(("filename", "job", "steps"), _claude_jobs(), ids=lambda v: v if isinstance(v, str) else "") -def test_the_check_runs_before_any_claude_step(filename, job, steps): +@pytest.mark.parametrize(("filename", "name", "job"), JOBS, ids=IDS) +def test_the_credentials_are_resolved_before_any_claude_step(filename, name, job): + steps = job["steps"] names = [s.get("name") for s in steps] - assert names.count(CHECK) == 1, (filename, job) - check = steps[names.index(CHECK)] + assert names.count(RESOLVE) == 1, (filename, name) + resolve = steps[names.index(RESOLVE)] first_claude = next(i for i, s in enumerate(steps) if _is_claude(s)) - assert names.index(CHECK) < first_claude + assert names.index(RESOLVE) < first_claude + assert resolve["id"] == "claude_auth" # Unconditional, or gated exactly like the first Claude step (a label # event that never reaches Claude must not fail on a misconfigured switch). - assert check.get("if") in (None, steps[first_claude].get("if")), (filename, job) - assert "continue-on-error" not in check + assert resolve.get("if") in (None, steps[first_claude].get("if")), (filename, name) + assert "continue-on-error" not in resolve -def test_every_check_step_is_identical(): - checks = _check_steps() - assert len(checks) == len(_claude_jobs()) == 8 - reference = {k: v for k, v in checks[0].items() if k != "if"} - for check in checks[1:]: - assert {k: v for k, v in check.items() if k != "if"} == reference +def test_every_resolve_step_is_identical(): + steps = _named(RESOLVE) + assert len(steps) == len(JOBS) == 8 + reference = {k: v for k, v in steps[0].items() if k != "if"} + for step in steps[1:]: + assert {k: v for k, v in step.items() if k != "if"} == reference assert reference["env"] == { "CLAUDE_AUTH": "${{ vars.CLAUDE_AUTH }}", "API_KEY_SET": "${{ secrets.ANTHROPIC_API_KEY != '' }}", + "WIF_RULE_ID": "${{ vars.CLAUDE_WIF_RULE_ID }}", + "WIF_ORG_ID": "${{ vars.CLAUDE_WIF_ORG_ID }}", + "WIF_SERVICE_ACCOUNT_ID": "${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }}", + "WIF_WORKSPACE_ID": "${{ vars.CLAUDE_WIF_WORKSPACE_ID }}", } -@pytest.mark.parametrize(("filename", "job", "steps"), _claude_jobs(), ids=lambda v: v if isinstance(v, str) else "") -def test_each_claude_step_is_followed_by_its_cost_step(filename, job, steps): +@pytest.mark.parametrize(("filename", "name", "job"), JOBS, ids=IDS) +def test_id_token_is_where_it_was(filename, name, job): + """No job gains id-token in this change; review-retest stays without it (oidc stops there).""" + has_id_token = (job.get("permissions") or {}).get("id-token") == "write" + assert has_id_token is (filename != "review-retest.yml"), (filename, name) + + +@pytest.mark.parametrize(("filename", "name", "job"), JOBS, ids=IDS) +def test_each_claude_step_is_followed_by_its_session_summary(filename, name, job): """Read before the next Claude step: a retry writes the same execution file.""" + steps = job["steps"] for i, step in enumerate(steps): if not _is_claude(step): continue sid = step["id"] later = steps[i + 1 :] - cost = next(s for s in later if str(s.get("name", "")).startswith(COST)) + summary = next(s for s in later if str(s.get("name", "")).startswith(SUMMARY)) next_claude = next((j for j, s in enumerate(later) if _is_claude(s)), len(later)) - assert later.index(cost) < next_claude, (filename, step.get("name")) - assert cost["if"] == ( + assert later.index(summary) < next_claude, (filename, step.get("name")) + assert summary["id"] == f"{sid}_cost" + assert summary["if"] == ( f"${{{{ !cancelled() && (steps.{sid}.outcome == 'success' || steps.{sid}.outcome == 'failure') }}}}" ) - assert cost["continue-on-error"] is True - assert cost["env"]["EXECUTION_FILE"] == f"${{{{ steps.{sid}.outputs.execution_file }}}}" - assert cost["env"]["CLAUDE_AUTH_MODE"] == AUTH_MODE + assert summary["continue-on-error"] is True + assert summary["env"]["EXECUTION_FILE"] == f"${{{{ steps.{sid}.outputs.execution_file }}}}" + assert summary["env"]["CLAUDE_AUTH_MODE"] == "${{ steps.claude_auth.outputs.mode }}" -def test_cost_steps_share_one_script_and_unique_labels(): - costs = [s for _, _, steps in _claude_jobs() for s in steps if s.get("name", "").startswith(COST)] - labels = [s["env"]["CLAUDE_STEP"] for s in costs] +def test_summary_steps_share_one_script_and_unique_labels(): + summaries = _named(SUMMARY) + labels = [s["env"]["CLAUDE_STEP"] for s in summaries] assert len(labels) == len(set(labels)) == sum(CLAUDE_WORKFLOWS.values()) assert all(" " not in label for label in labels) - assert len({s["run"] for s in costs}) == 1 + shared = {(s["run"], s["env"]["SESSION_ERRORS"], s["env"]["SPEND_LIMIT_PATTERN"]) for s in summaries} + assert len(shared) == 1 # The line must not look like Claude Code's own `"total_cost_usd": n`, or # collect_week.py's fallback regex would count a session twice. - assert '"total_cost_usd":' not in costs[0]["run"] + assert '"total_cost_usd":' not in summaries[0]["run"] -def _run_cost(tmp_path: Path, execution: list[dict[str, Any]] | None, mode: str = "subscription") -> tuple[int, str]: - script = next(s for _, _, steps in _claude_jobs() for s in steps if s.get("name", "").startswith(COST))["run"] +@pytest.mark.parametrize(("filename", "name", "job"), JOBS, ids=IDS) +def test_a_spend_limit_skips_the_retry_and_stops_the_job(filename, name, job): + steps = job["steps"] + names = [s.get("name") for s in steps] + claude = [s for s in steps if _is_claude(s)] + if filename == "daily-regen.yml": + # Two independent pre-flight sessions, no retry: either one stops the dispatch. + stop = steps[names.index(STOP)] + assert stop["if"] == ( + "${{ steps.polish_cost.outputs.spend_limit == 'true' || steps.similarity_cost.outputs.spend_limit == 'true' }}" + ) + dispatches = [i for i, n in enumerate(names) if str(n).startswith("Dispatch ")] + assert len(dispatches) == 2 and min(dispatches) > names.index(STOP) > steps.index(claude[-1]) + elif len(claude) == 1: + # No retry to skip: the job's own failure path reads the summary. + assert STOP not in names + return + else: + first, retry = claude + assert retry["if"].endswith(f" && steps.{first['id']}_cost.outputs.spend_limit != 'true'") + stop = steps[names.index(STOP)] + assert names.index(STOP) > steps.index(retry) + assert stop["if"] == f"steps.{first['id']}_cost.outputs.spend_limit == 'true'" + assert "exit 1" in stop["run"] and "continue-on-error" not in stop + + +def test_the_api_key_reaches_nothing_but_the_action_input(): + """The key is never put into an env block, a script or another action.""" + allowed = { + f"anthropic_api_key: {CREDENTIALS['anthropic_api_key']}", + "API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }}", + } + for path in WORKFLOWS_DIR.glob("*.yml"): + for line in path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if "secrets.ANTHROPIC_API_KEY" in stripped and not stripped.startswith("#"): + assert stripped in allowed, (path.name, stripped) + + +def test_no_claude_workflow_runs_on_pull_request_events(): + """A fork PR never reaches a credential: no `pull_request` or `pull_request_target` trigger.""" + for filename in CLAUDE_WORKFLOWS: + workflow = _load(filename) + triggers = workflow.get(True) or workflow.get("on") or {} + assert "pull_request_target" not in triggers, filename + assert "pull_request" not in triggers, filename + + +# --- the resolve step, run for real ------------------------------------------ + + +def _resolve(tmp_path: Path, auth: str | None, key_set: bool = False, id_token: bool = True, **wif: str): + script = _named(RESOLVE)[0]["run"] + out = tmp_path / "out" + out.write_text("", encoding="utf-8") + env = {k: v for k, v in os.environ.items() if not k.startswith(("CLAUDE_", "WIF_", "ACTIONS_ID_TOKEN"))} + env |= {"API_KEY_SET": "true" if key_set else "false", "GITHUB_OUTPUT": str(out), **wif} + if auth is not None: + env["CLAUDE_AUTH"] = auth + if id_token: + env["ACTIONS_ID_TOKEN_REQUEST_URL"] = "https://token.actions.example/" + result = subprocess.run(["bash", "-e", "-c", script], cwd=tmp_path, env=env, capture_output=True, text=True) + outputs = dict(line.split("=", 1) for line in out.read_text().splitlines() if "=" in line) + return result, outputs + + +def _inputs(outputs: dict[str, str], key: str, token: str) -> dict[str, str]: + """The six action inputs as GitHub evaluates CREDENTIALS: `a && b || ''` yields b or ''.""" + mode = outputs.get("mode", "") + return { + "anthropic_api_key": key if mode == "api" and key else "", + "claude_code_oauth_token": token if mode == "subscription" and token else "", + "anthropic_federation_rule_id": outputs.get("federation_rule_id", ""), + "anthropic_organization_id": outputs.get("organization_id", ""), + "anthropic_service_account_id": outputs.get("service_account_id", ""), + "anthropic_workspace_id": outputs.get("workspace_id", ""), + } + + +@pytest.mark.parametrize( + ("auth", "key_set", "mode", "given"), + [ + (None, False, "subscription", {"claude_code_oauth_token"}), + ("", False, "subscription", {"claude_code_oauth_token"}), + ("Subscription", True, "subscription", {"claude_code_oauth_token"}), + ("api", True, "api", {"anthropic_api_key"}), + ("API", True, "api", {"anthropic_api_key"}), + ("oidc", True, "oidc", {"anthropic_federation_rule_id", "anthropic_organization_id"}), + ("OIDC", False, "oidc", {"anthropic_federation_rule_id", "anthropic_organization_id"}), + ], +) +def test_resolve_passes_exactly_one_credential(tmp_path, auth, key_set, mode, given): + result, outputs = _resolve(tmp_path, auth, key_set, **WIF) + assert result.returncode == 0, result.stdout + result.stderr + assert outputs["mode"] == mode and f"Claude credentials: {mode}" in result.stdout + inputs = _inputs(outputs, "sk-ant-key" if key_set else "", "oauth-token") + if mode == "oidc": + # Federation is one credential: rule + organization (+ service account, workspace), no key, no token. + assert set(given) <= {k for k, v in inputs.items() if v} + assert not inputs["anthropic_api_key"] and not inputs["claude_code_oauth_token"] + assert outputs["workspace_id"] == WIF["WIF_WORKSPACE_ID"] + else: + assert {k for k, v in inputs.items() if v} == given + assert not any(k in outputs for k in ("federation_rule_id", "organization_id")) + + +@pytest.mark.parametrize( + ("auth", "key_set", "wif", "id_token", "why"), + [ + ("api", False, WIF, True, "ANTHROPIC_API_KEY secret is missing"), + ("oidc", True, {**WIF, "WIF_RULE_ID": ""}, True, "CLAUDE_WIF_RULE_ID is missing"), + ("oidc", True, {k: v for k, v in WIF.items() if k != "WIF_WORKSPACE_ID"}, True, "CLAUDE_WIF_WORKSPACE_ID"), + ("oidc", True, {**WIF, "WIF_ORG_ID": "x\n::warning::y"}, True, "CLAUDE_WIF_ORG_ID is missing or not an ID"), + ("oidc", True, WIF, False, "no 'id-token: write' permission"), + ("apikey", True, WIF, True, "must be unset, 'subscription', 'api' or 'oidc'"), + (" api", True, WIF, True, "must be unset"), + ("api\n::warning::injected", True, WIF, True, "must be unset"), + ], +) +def test_resolve_stops_instead_of_falling_back(tmp_path, auth, key_set, wif, id_token, why): + result, outputs = _resolve(tmp_path, auth, key_set, id_token, **wif) + assert result.returncode == 1 + assert result.stdout.startswith("::error::") and why in result.stdout + assert "not falling back to the subscription" in result.stdout + assert "\n::warning::" not in result.stdout + assert "mode" not in outputs + + +# --- the session summary, run for real ---------------------------------------- + + +def _summary(tmp_path: Path, execution: list[dict[str, Any]] | None, mode: str = "subscription"): + step = _named(SUMMARY)[0] exec_file = tmp_path / "claude-execution-output.json" if execution is not None: exec_file.write_text(json.dumps(execution), encoding="utf-8") - summary = tmp_path / "summary" + summary, out = tmp_path / "summary", tmp_path / "out" env = { **os.environ, "EXECUTION_FILE": str(exec_file) if execution is not None else "", "CLAUDE_AUTH_MODE": mode, - "CLAUDE_STEP": "impl-generate", + "CLAUDE_STEP": "impl-review", + "SESSION_ERRORS": step["env"]["SESSION_ERRORS"], + "SPEND_LIMIT_PATTERN": step["env"]["SPEND_LIMIT_PATTERN"], "GITHUB_STEP_SUMMARY": str(summary), + "GITHUB_OUTPUT": str(out), } - result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True, text=True) - return result.returncode, summary.read_text() if summary.exists() else "" + result = subprocess.run(["bash", "-e", "-c", step["run"]], env=env, capture_output=True, text=True) + outputs = dict(line.split("=", 1) for line in out.read_text().splitlines()) if out.exists() else {} + return result, summary.read_text() if summary.exists() else "", outputs -def test_cost_step_reads_the_result_message(tmp_path): - execution = [ - {"type": "system", "subtype": "init", "model": "claude-sonnet-5-5"}, - {"type": "assistant"}, - {"type": "result", "subtype": "success", "total_cost_usd": 0.2931}, - ] - rc, summary = _run_cost(tmp_path, execution, mode="api") - assert rc == 0 - assert summary == "- `claude-cost step=impl-generate auth=api model=claude-sonnet-5-5 total_cost_usd=0.2931`\n" +INIT = {"type": "system", "subtype": "init", "model": "claude-opus-5-5"} + + +def _failed(text: str, error: str | None = "invalid_request", status: int = 400) -> list[dict[str, Any]]: + """A session the API stopped: Claude Code's synthetic error message and an error result.""" + assistant: dict[str, Any] = {"type": "assistant", "message": {"content": [{"type": "text", "text": text}]}} + if error: + assistant["error"] = error + result = {"type": "result", "subtype": "success", "is_error": True, "api_error_status": status} + return [INIT, assistant, result | {"result": text, "total_cost_usd": 0.41}] + + +# The API's own messages (platform.claude.com/docs/en/api/rate-limits, /api/errors). +WORKSPACE_LIMIT = ( + 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"You have reached your ' + 'specified workspace API usage limits. You will regain access on 2026-12-01 at 00:00 UTC."}}' +) +TIER_CAP = ( + 'API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"You have reached your API usage ' + 'limits: your organization has crossed its monthly API usage threshold.","details":{"error_code":' + '"enforced_spend_limit_reached"}}}' +) +CREDIT = "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing." + + +def test_summary_reads_cost_and_model(tmp_path): + execution = [INIT, {"type": "assistant"}, {"type": "result", "subtype": "success", "total_cost_usd": 0.2931}] + result, summary, outputs = _summary(tmp_path, execution, mode="oidc") + assert result.returncode == 0 + assert summary == "- `claude-cost step=impl-review auth=oidc model=claude-opus-5-5 total_cost_usd=0.2931`\n" + assert outputs == {"spend_limit": "false"} @pytest.mark.parametrize("execution", [None, [], [{"type": "assistant"}], [{"type": "result"}]]) -def test_cost_step_never_fails_without_a_cost(tmp_path, execution): - rc, summary = _run_cost(tmp_path, execution) - assert rc == 0 +def test_summary_never_fails_without_a_cost(tmp_path, execution): + result, summary, outputs = _summary(tmp_path, execution) + assert result.returncode == 0 assert "auth=subscription model=n/a total_cost_usd=n/a" in summary + assert outputs == {"spend_limit": "false"} -def test_the_api_key_reaches_nothing_but_the_action_input(): - """The key is never put into an env block, a script or another action.""" - for path in WORKFLOWS_DIR.glob("*.yml"): - for line in path.read_text(encoding="utf-8").splitlines(): - if "ANTHROPIC_API_KEY" not in line or line.lstrip().startswith("#"): - continue - stripped = line.strip() - assert ( - stripped in {f"anthropic_api_key: {API_KEY}", "API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }}"} - or "secrets.ANTHROPIC_API_KEY" not in stripped - ), (path.name, stripped) +@pytest.mark.parametrize( + "execution", + [ + _failed(WORKSPACE_LIMIT), + _failed(TIER_CAP, error="rate_limit", status=429), + _failed(CREDIT, error="billing_error"), + _failed("Credit balance is too low", error=None), + [INIT, {"type": "system", "subtype": "api_retry", "error": "billing_error", "error_status": 402}], + ], +) +@pytest.mark.parametrize("mode", ["api", "oidc"]) +def test_summary_detects_the_spend_limit(tmp_path, execution, mode): + result, summary, outputs = _summary(tmp_path, execution, mode) + assert result.returncode == 0 + assert outputs == {"spend_limit": "true"} + assert f"::error::Claude spend limit reached (mode {mode}) in impl-review" in result.stdout + assert f"- **Claude spend limit reached (mode {mode})** in `impl-review`" in summary -def test_no_pull_request_target_workflow_runs_claude(): - """A fork PR never reaches either credential (no secrets on `pull_request`).""" - for filename in CLAUDE_WORKFLOWS: - triggers = _load(filename).get(True) or _load(filename).get("on") - assert "pull_request_target" not in (triggers or {}), filename - assert "pull_request" not in (triggers or {}), filename +def test_the_subscription_never_reads_a_spend_limit(tmp_path): + """Subscription runs keep today's behaviour exactly: no new branch can fire.""" + assert _summary(tmp_path, _failed(WORKSPACE_LIMIT), "subscription")[2] == {"spend_limit": "false"} -# --- the check step, run for real ------------------------------------------ +@pytest.mark.parametrize( + "execution", + [ + # A transient rate limit is retried as before. + _failed( + 'API Error: 429 {"error":{"type":"rate_limit_error","message":"Number of request tokens has exceeded your per-minute rate limit"}}', + error="rate_limit", + status=429, + ), + _failed("API Error: 500 Internal server error", error="server_error", status=500), + # The session's own text may quote the messages (a doc it read, a review comment): not an error field. + [INIT, {"type": "assistant", "message": {"content": [{"type": "text", "text": WORKSPACE_LIMIT}]}}], + [INIT, {"type": "user", "message": {"content": [{"type": "tool_result", "content": CREDIT}]}}], + [INIT, {"type": "result", "subtype": "success", "is_error": False, "result": f"Docs say: {CREDIT}"}], + ], +) +def test_summary_ignores_everything_else(tmp_path, execution): + assert _summary(tmp_path, execution, "api")[2] == {"spend_limit": "false"} -def _run_check(tmp_path: Path, auth: str | None, key_set: bool) -> subprocess.CompletedProcess: - script = _check_steps()[0]["run"] - env = {k: v for k, v in os.environ.items() if k not in {"CLAUDE_AUTH", "API_KEY_SET"}} - env["API_KEY_SET"] = "true" if key_set else "false" - if auth is not None: - env["CLAUDE_AUTH"] = auth - return subprocess.run(["bash", "-e", "-c", script], cwd=tmp_path, env=env, capture_output=True, text=True) +# --- the failure paths at the spend limit --------------------------------------- + + +# Logs every call as one JSON line; `pr view`/`issue view` answer nothing. +FAKE_GH = """#!/usr/bin/env python3 +import json, os, sys +with open(os.environ["GH_LOG"], "a", encoding="utf-8") as f: + f.write(json.dumps(sys.argv[1:]) + "\\n") +""" -def _expression(auth: str, key: str, token: str) -> tuple[str, str]: - """The two inputs as GitHub evaluates them: `==` ignores case, `a && b || ''` yields b or ''.""" - is_api = auth.casefold() == "api" - return (key if is_api and key else "", token if not is_api and token else "") +def _run_handler(tmp_path: Path, filename: str, name: str, **env: str) -> tuple[subprocess.CompletedProcess, list]: + step = _step(filename, name) + script = step["run"] + for expr, value in { + "${{ github.repository }}": "owner/repo", + "${{ github.run_id }}": "1", + "${{ matrix.spec_id }}": "spec-a", + }.items(): + script = script.replace(expr, value) + assert "${{" not in script + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + (bin_dir / "gh").write_text(FAKE_GH, encoding="utf-8") + (bin_dir / "gh").chmod(0o755) + log = tmp_path / "gh.log" + base = {k: v for k, v in os.environ.items() if not k.startswith(("GIT_", "GH_"))} + result = subprocess.run( + ["bash", "-eo", "pipefail", "-c", script], + cwd=tmp_path, + env={**base, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", "GH_LOG": str(log), **env}, + capture_output=True, + text=True, + ) + calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] + return result, calls + + +def _bodies(calls: list[list[str]]) -> list[str]: + return [c[c.index("--body") + 1] for c in calls if "--body" in c] + + +@pytest.mark.parametrize(("regen_label", "live"), [("regen", True), ("", False)]) +def test_generation_at_the_spend_limit_is_parked(tmp_path, regen_label, live): + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + result, calls = _run_handler( + tmp_path, + "impl-generate.yml", + "Handle generation failure", + SPEC_ID="spec-a", + LIBRARY="plotly", + ISSUE="42", + MODEL="sonnet", + CHANGE_REQUEST="", + CAMPAIGN_WINDOW_H="12", + INFRA_WINDOW_CAP="5", + CLAUDE_RETRY_OUTCOME="skipped", + CLAUDE_EXEC_FILE="", + SPEND_LIMIT="true", + CLAUDE_MODE="oidc", + REGEN_LABEL=regen_label, + LANGUAGE="python", + EXT=".py", + REGEN_GATE="true", + SOURCE_REF="", + SOURCE_MODEL="", + ADOPT_CAUSE="", + TRIGGER_REF="main", + RUN_URL="https://github.com/owner/repo/actions/runs/1", + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "::error::Claude spend limit reached (mode oidc)" in result.stdout + assert not [c for c in calls if c[:2] == ["workflow", "run"]] # no auto-retry + edits = [c[3:5] for c in calls if c[:2] == ["issue", "edit"]] + assert ["--remove-label", "impl:plotly:pending"] in edits and ["--remove-label", "generate:plotly"] in edits + assert (["--add-label", "impl:plotly:done"] in edits) is live + assert not any("impl:plotly:failed" in c for c in edits) # never a capability verdict + (body,) = _bodies(calls) + # Its own marker: neither the 3-attempt cap nor daily-regen's attempt window counts it. + assert "" in body + assert "" in body + assert "" in body + assert " - ## :pause_button: ${LIBRARY} Paused (Claude spend limit) + gh issue comment "$ISSUE" --body " + ## :pause_button: ${LIBRARY} Paused (Claude spend/usage limit) - The Claude API refused the **${LIBRARY}** generation for \`${SPEC_ID}\` at the spend limit (credentials: \`${CLAUDE_MODE}\`). This is **not** a capability verdict: nothing was retried, no attempt was counted, and the pair is not marked failed. + Claude refused the **${LIBRARY}** generation for \`${SPEC_ID}\` at a spend or usage limit (auth=\`${CLAUDE_MODE}\`). This is **not** a capability verdict: nothing was retried, no attempt was counted, and the pair is not marked failed. - Once the limit is raised, run: + Once the limit resets or is raised, run: \`\`\` gh workflow run impl-generate.yml -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f issue_number=${ISSUE} -f model=${MODEL} \`\`\` diff --git a/.github/workflows/impl-repair.yml b/.github/workflows/impl-repair.yml index b42e0e5ab3c..a97ad6e6454 100644 --- a/.github/workflows/impl-repair.yml +++ b/.github/workflows/impl-repair.yml @@ -291,12 +291,10 @@ jobs: - BRANCH: ${{ env.branch }} # The session's cost (Claude Code's own estimate at API list price), - # model and credential, for the weekly regen review, and whether it - # stopped at the Claude spend limit (api/oidc only: the API's messages - # for a spend limit you set, the tier's spend cap and an empty credit - # balance, read only from the session's error fields, because its own - # text may quote them). Read right after the step: the retry writes the - # same execution file. Never fatal. + # model and credential mode, for the weekly regen review, and whether it + # stopped at a Claude spend or usage limit (output limit_reached). Read + # right after the step: the retry writes the same execution file. Never + # fatal. - name: Claude session summary id: claude_cost if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} @@ -305,10 +303,20 @@ jobs: EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: impl-repair + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -316,17 +324,17 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - # Not on a spend limit: the retry would be refused the same way. + # Not at a spend or usage limit: the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.spend_limit != 'true' + if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.limit_reached != 'true' id: claude_retry timeout-minutes: 45 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 @@ -358,10 +366,20 @@ jobs: EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: impl-repair/retry + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -369,21 +387,21 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - # The first session stopped at the spend limit and its retry was skipped: - # fail here, so "Handle repair failure" restores the PR without an - # auto-retry instead of the steps below running on an unrepaired file. - - name: Stop at the Claude spend limit - if: steps.claude_cost.outputs.spend_limit == 'true' + # The first session stopped at a spend or usage limit and its retry was + # skipped: fail here, so "Handle repair failure" restores the PR without + # an auto-retry instead of the steps below running on an unrepaired file. + - name: Stop at the Claude spend or usage limit + if: steps.claude_cost.outputs.limit_reached == 'true' run: | - echo "::error::Claude spend limit reached; the retry is skipped" + echo "::error::Claude spend/usage limit reached; the retry is skipped" exit 1 - name: Process repaired image @@ -541,27 +559,27 @@ jobs: MODEL: ${{ steps.model.outputs.model || inputs.model || 'auto' }} REPOSITORY: ${{ github.repository }} RUN_ID: ${{ github.run_id }} - # Either session stopped at the Claude spend limit (api/oidc only). - SPEND_LIMIT: ${{ steps.claude_cost.outputs.spend_limit == 'true' || steps.claude_retry_cost.outputs.spend_limit == 'true' }} + # Either session stopped at a Claude spend or usage limit. + LIMIT_REACHED: ${{ steps.claude_cost.outputs.limit_reached == 'true' || steps.claude_retry_cost.outputs.limit_reached == 'true' }} CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} run: | # Restore ai-rejected label that was removed at start of repair so the # PR state stays consistent (otherwise it looks "stuck approved"). gh pr edit "$PR_NUM" --add-label "ai-rejected" 2>/dev/null || true - # The Claude spend limit (api/oidc): an auto-retry now would be - # refused the same way. The comment carries its own marker, not the + # A Claude spend or usage limit: an auto-retry now would be refused + # the same way. The comment carries its own marker, not the # repair-retry one, so the attempt keeps its one crash retry for after - # the limit is raised; the watchdog's repair rescue (once per - # attempt) or a manual restart picks the PR up. - if [ "${SPEND_LIMIT}" = "true" ]; then - echo "::error::Claude spend limit reached (mode ${CLAUDE_MODE}) — repair attempt ${ATTEMPT} stopped, no auto-retry" - gh pr comment "$PR_NUM" --body " - ## :pause_button: Repair Paused (Claude spend limit, Attempt ${ATTEMPT}/4) + # the limit resets or is raised; the watchdog's repair rescue (once + # per attempt) or a manual restart picks the PR up. + if [ "${LIMIT_REACHED}" = "true" ]; then + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_MODE}) — repair attempt ${ATTEMPT} stopped, no auto-retry" + gh pr comment "$PR_NUM" --body " + ## :pause_button: Repair Paused (Claude spend/usage limit, Attempt ${ATTEMPT}/4) - The Claude API refused the repair at the spend limit (credentials: \`${CLAUDE_MODE}\`). Nothing was retried, and the attempt is not used up. + Claude refused the repair at a spend or usage limit (auth=\`${CLAUDE_MODE}\`). Nothing was retried, and the attempt is not used up. - **Restart once the limit is raised:** + **Restart once the limit resets or is raised:** \`\`\` gh workflow run impl-repair.yml -f pr_number=${PR_NUM} -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f attempt=${ATTEMPT} -f model=${MODEL} \`\`\` diff --git a/.github/workflows/impl-review.yml b/.github/workflows/impl-review.yml index a9850371335..87943bbe572 100644 --- a/.github/workflows/impl-review.yml +++ b/.github/workflows/impl-review.yml @@ -973,11 +973,9 @@ jobs: python3 "$RUNNER_TEMP/regen-tools/review_provenance.py" model --execution-file "${EXECUTION_FILE:-}" # The session's cost (Claude Code's own estimate at API list price), - # model and credential, for the weekly regen review, and whether it - # stopped at the Claude spend limit (api/oidc only: the API's messages - # for a spend limit you set, the tier's spend cap and an empty credit - # balance, read only from the session's error fields, because its own - # text may quote them). "Validate review output" reads it. Never fatal. + # model and credential mode, for the weekly regen review, and whether it + # stopped at a Claude spend or usage limit (output limit_reached, which + # "Validate review output" reads). Never fatal. - name: Claude session summary id: review_cost if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} @@ -986,10 +984,20 @@ jobs: EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: impl-review + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -997,13 +1005,13 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" # Undo "Hide the previous review", whatever the review step did: the # metadata step below reads the metadata file from disk and rewrites @@ -1108,37 +1116,37 @@ jobs: RUN_ID: ${{ github.run_id }} MODEL: ${{ steps.pr.outputs.model }} REVIEW_MODEL_ALIAS: ${{ steps.pr.outputs.review_model_alias }} - # The session stopped at the Claude spend limit (api/oidc only). - SPEND_LIMIT: ${{ steps.review_cost.outputs.spend_limit }} + # The session stopped at a Claude spend or usage limit. + LIMIT_REACHED: ${{ steps.review_cost.outputs.limit_reached }} CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} run: | echo "::error::AI Review did not produce valid output files (score=0)" - # The Claude spend limit (api/oidc): a re-review now would be refused - # the same way, and it says nothing about the implementation. Hand - # the PR to the watchdog with `ai-review-failed`, as an exhausted - # retry does, but without the review-retry marker, so the one - # auto-retry stays for after the limit is raised. No verdict, no + # A Claude spend or usage limit: a re-review now would be refused the + # same way, and it says nothing about the implementation. Hand the PR + # to the watchdog with `ai-review-failed`, as an exhausted retry + # does, but without the review-retry marker, so the one auto-retry + # stays for after the limit resets or is raised. No verdict, no # score, nothing merged or closed. - if [ "${SPEND_LIMIT}" = "true" ]; then - echo "::error::Claude spend limit reached (mode ${CLAUDE_MODE}) — no auto-retry for PR #${PR_NUM}" + if [ "${LIMIT_REACHED}" = "true" ]; then + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_MODE}) — no auto-retry for PR #${PR_NUM}" for attempt in 1 2 3; do gh pr edit "$PR_NUM" --add-label "ai-review-failed" 2>/dev/null && break [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) done || echo "::error::could not add ai-review-failed to PR #${PR_NUM} — left to the watchdog's scan of stalled PRs" - gh pr comment "$PR_NUM" --body " - ## :pause_button: AI Review Paused (Claude spend limit) + gh pr comment "$PR_NUM" --body " + ## :pause_button: AI Review Paused (Claude spend/usage limit) - The Claude API refused the review at the spend limit (credentials: \`${CLAUDE_MODE}\`). No score and no verdict were recorded, and the one automatic re-review is still unused. + Claude refused the review at a spend or usage limit (auth=\`${CLAUDE_MODE}\`). No score and no verdict were recorded, and the one automatic re-review is still unused. - **Rerun once the limit is raised:** + **Rerun once the limit resets or is raised:** \`\`\` gh workflow run impl-review.yml -f pr_number=$PR_NUM \`\`\` --- :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" \ - || echo "::error::spend-limit comment not posted on PR #${PR_NUM}" + || echo "::error::limit comment not posted on PR #${PR_NUM}" exit 1 fi diff --git a/.github/workflows/report-validate.yml b/.github/workflows/report-validate.yml index bedd269fa1e..a4396cee782 100644 --- a/.github/workflows/report-validate.yml +++ b/.github/workflows/report-validate.yml @@ -206,11 +206,9 @@ jobs: - Keep the analysis comment concise but informative # The session's cost (Claude Code's own estimate at API list price), - # model and credential, and whether it stopped at the Claude spend limit - # (api/oidc only: the API's messages for a spend limit you set, the - # tier's spend cap and an empty credit balance, read only from the - # session's error fields, because its own text may quote them). Never - # fatal; a spend-limit stop takes "Handle failure" like any other. + # model and credential mode, and whether it stopped at a Claude spend or + # usage limit (output limit_reached). Never fatal; a limit stop takes + # "Handle failure" like any other. - name: Claude session summary id: validate_cost if: ${{ !cancelled() && (steps.validate.outcome == 'success' || steps.validate.outcome == 'failure') }} @@ -219,10 +217,20 @@ jobs: EXECUTION_FILE: ${{ steps.validate.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: report-validate + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -230,13 +238,13 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - name: Add reaction on success if: steps.check.outputs.should_run == 'true' && steps.validate.outcome == 'success' diff --git a/.github/workflows/review-retest.yml b/.github/workflows/review-retest.yml index febc9a32488..793d3c1ce2d 100644 --- a/.github/workflows/review-retest.yml +++ b/.github/workflows/review-retest.yml @@ -383,11 +383,9 @@ jobs: instead. Do not run `gh` or `git push`; every other instruction applies unchanged. # The session's cost (Claude Code's own estimate at API list price), - # model and credential, and whether it stopped at the Claude spend limit - # (api/oidc only: the API's messages for a spend limit you set, the - # tier's spend cap and an empty credit balance, read only from the - # session's error fields, because its own text may quote them). Never - # fatal; a stopped cell is collected as a failed one. + # model and credential mode, and whether it stopped at a Claude spend or + # usage limit (output limit_reached). Never fatal; a stopped cell is + # collected as a failed one. - name: Claude session summary id: review_cost if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} @@ -396,10 +394,20 @@ jobs: EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: review-retest + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -407,13 +415,13 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - name: Collect the cell if: always() diff --git a/.github/workflows/spec-create.yml b/.github/workflows/spec-create.yml index bb6e53edc5c..0685b269914 100644 --- a/.github/workflows/spec-create.yml +++ b/.github/workflows/spec-create.yml @@ -297,12 +297,9 @@ jobs: - STOP after step 11 # The session's cost (Claude Code's own estimate at API list price), - # model and credential, and whether it stopped at the Claude spend limit - # (api/oidc only: the API's messages for a spend limit you set, the - # tier's spend cap and an empty credit balance, read only from the - # session's error fields, because its own text may quote them). Read - # right after the step: the retry writes the same execution file. Never - # fatal. + # model and credential mode, and whether it stopped at a Claude spend or + # usage limit (output limit_reached). Read right after the step: the + # retry writes the same execution file. Never fatal. - name: Claude session summary id: process_cost if: ${{ !cancelled() && (steps.process.outcome == 'success' || steps.process.outcome == 'failure') }} @@ -311,10 +308,20 @@ jobs: EXECUTION_FILE: ${{ steps.process.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: spec-create + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -322,17 +329,17 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - # Not on a spend limit: the retry would be refused the same way. + # Not at a spend or usage limit: the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' && steps.process_cost.outputs.spend_limit != 'true' + if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' && steps.process_cost.outputs.limit_reached != 'true' id: process_retry timeout-minutes: 30 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 @@ -513,10 +520,20 @@ jobs: EXECUTION_FILE: ${{ steps.process_retry.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: spec-create/retry + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -524,20 +541,20 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - # The first session stopped at the spend limit and its retry was skipped: - # fail here instead of reading outputs that were never written. - - name: Stop at the Claude spend limit - if: steps.process_cost.outputs.spend_limit == 'true' + # The first session stopped at a spend or usage limit and its retry was + # skipped: fail here instead of reading outputs that were never written. + - name: Stop at the Claude spend or usage limit + if: steps.process_cost.outputs.limit_reached == 'true' run: | - echo "::error::Claude spend limit reached; the retry is skipped" + echo "::error::Claude spend/usage limit reached; the retry is skipped" exit 1 - name: Extract outputs diff --git a/.github/workflows/util-claude.yml b/.github/workflows/util-claude.yml index d482202ad61..6418346836f 100644 --- a/.github/workflows/util-claude.yml +++ b/.github/workflows/util-claude.yml @@ -98,12 +98,9 @@ jobs: actions: read # The session's cost (Claude Code's own estimate at API list price), - # model and credential, and whether it stopped at the Claude spend limit - # (api/oidc only: the API's messages for a spend limit you set, the - # tier's spend cap and an empty credit balance, read only from the - # session's error fields, because its own text may quote them). Read - # right after the step: the retry writes the same execution file. Never - # fatal. + # model and credential mode, and whether it stopped at a Claude spend or + # usage limit (output limit_reached). Read right after the step: the + # retry writes the same execution file. Never fatal. - name: Claude session summary id: claude_cost if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} @@ -112,10 +109,20 @@ jobs: EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: util-claude + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -123,17 +130,17 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - # Not on a spend limit: the retry would be refused the same way. + # Not at a spend or usage limit: the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.spend_limit != 'true' + if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.limit_reached != 'true' id: claude_retry uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: @@ -156,10 +163,20 @@ jobs: EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} CLAUDE_STEP: util-claude/retry + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries. SESSION_ERRORS: >- - .[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end) - SPEND_LIMIT_PATTERN: 'reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance is too low|billing_error' + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + end + # The API's spend-limit, spend-cap and credit messages, Claude Code's + # billing_error and credit message, and its subscription usage-limit + # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -167,19 +184,19 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - spend_limit=false - if [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$SPEND_LIMIT_PATTERN" <<< "$errors"; then - spend_limit=true - echo "::error::Claude spend limit reached (mode ${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit is raised" - echo "- **Claude spend limit reached (mode ${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + limit_reached=false + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + limit_reached=true + echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" + echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" fi - echo "spend_limit=${spend_limit}" >> "$GITHUB_OUTPUT" + echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - # The first session stopped at the spend limit and its retry was skipped: - # the run fails instead of reporting a success it did not have. - - name: Stop at the Claude spend limit - if: steps.claude_cost.outputs.spend_limit == 'true' + # The first session stopped at a spend or usage limit and its retry was + # skipped: the run fails instead of reporting a success it did not have. + - name: Stop at the Claude spend or usage limit + if: steps.claude_cost.outputs.limit_reached == 'true' run: | - echo "::error::Claude spend limit reached; the retry is skipped" + echo "::error::Claude spend/usage limit reached; the retry is skipped" exit 1 diff --git a/agentic/docs/project-guide.md b/agentic/docs/project-guide.md index 70e1b4b9c41..d75358a1de6 100644 --- a/agentic/docs/project-guide.md +++ b/agentic/docs/project-guide.md @@ -799,7 +799,7 @@ The new architecture separates specification and implementation processes: - **Adopt mode.** The `source_ref` input (default empty) on `impl-generate.yml` and `bulk-generate.yml` takes each implementation file from that branch, tag or SHA instead of running Claude, for code written outside the pipeline (a `gemini/*` branch). The run renders the file with the generator's commands, checks the canvas with `core.canvas`, and is always a forced regeneration (`regen:forced`); the routed model only runs repair, and the PR body adds a `**Generation:** adopted from …` line above `**Model:**`. The optional `source_model` input (a model name such as `gemini-4-argon`, only with `source_ref`) names the model that wrote the code and goes on that line. The metadata records `generated_by: {source_model}`, or `generated_by: adopted:{source_ref}` without one, followed by `adopted_from: {source_ref} ({sha})` for every adopted file. Both workflows reject a malformed `source_ref` or `source_model` without echoing it. Retries keep both inputs; the watchdog forwards neither. Python and JavaScript only. A missing file or a failed render is never retried and never counts toward the 3-attempt cap. See `docs/workflows/overview.md#adopt-implementations-from-a-branch`. - **Review model.** Every quality review runs on Opus, whatever model generated the implementation: first implementations, reviews after a repair, and the regen gate's session (the predecessor's re-score and the new render's review). Only `impl-review.yml` chooses it; its `review_model` input (default `opus`) pins another model for one manual review and that run's own auto-retry, and no other workflow forwards one. The retest harness's `models=production` runs every cell on Opus to match (`review_retest.PRODUCTION_MODELS`, kept equal by `tests/unit/workflows/test_model_routing.py`). - Fixed models: `spec-create.yml` runs on Opus; the spec polish and cross-library similarity audit in `daily-regen.yml` run on Sonnet. See `docs/workflows/overview.md#pipeline-models`. -- **Credentials.** The repository variable `CLAUDE_AUTH` chooses what every Claude step runs on: unset or `subscription` = the `CLAUDE_CODE_OAUTH_TOKEN` secret (the owner's subscription), `oidc` = workload identity federation with the `CLAUDE_WIF_*` variables (API credits of the `ci-regen` workspace; accepts only runs on `main`), `api` = the `ANTHROPIC_API_KEY` secret (API credits, the fallback). The step "Resolve Claude credentials" (id `claude_auth`) at the start of every Claude job stops on any other value or a missing credential, never falling back to the subscription, and every Claude step reads its outputs, so each passes exactly one credential. After each Claude step, "Claude session summary" (id `_cost`) records `total_cost_usd`, the model and the mode, and flags a stop at the Claude spend limit (api/oidc): the retry is skipped and impl-generate, impl-review and impl-repair park the work as an infrastructure stop, without a retry or a counted attempt. A new Claude step copies all three patterns, which `tests/unit/workflows/test_claude_auth.py` enforces. See `docs/workflows/overview.md#claude-credentials`. +- **Credentials.** The repository variable `CLAUDE_AUTH` chooses what every Claude step runs on: unset or `subscription` = the `CLAUDE_CODE_OAUTH_TOKEN` secret (the owner's subscription), `oidc` = workload identity federation with the `CLAUDE_WIF_*` variables (API credits of the `ci-regen` workspace; accepts only runs on `main`), `api` = the `ANTHROPIC_API_KEY` secret (API credits, the fallback). The step "Resolve Claude credentials" (id `claude_auth`) at the start of every Claude job stops on any other value or a missing credential, never falling back to the subscription, and every Claude step reads its outputs, so each passes exactly one credential. After each Claude step, "Claude session summary" (id `_cost`) records `total_cost_usd`, the model and the mode, and flags a stop at a Claude spend or usage limit in any mode (output `limit_reached`: the API's spend limit, spend cap or credit balance, the subscription's usage limit): the retry is skipped and impl-generate, impl-review and impl-repair park the work as an infrastructure stop, without a retry, a counted attempt or `impl::failed`. A new Claude step copies all three patterns, which `tests/unit/workflows/test_claude_auth.py` enforces. See `docs/workflows/overview.md#claude-credentials`. ## GitHub Issue Labels diff --git a/automation/scripts/README.md b/automation/scripts/README.md index bae00b48d43..d27543aea20 100644 --- a/automation/scripts/README.md +++ b/automation/scripts/README.md @@ -261,7 +261,7 @@ The pick of `daily-regen.yml`'s pair mode (`REGEN_PAIRS_PER_DAY`, or the `pairs` input): the N oldest (spec, library) pairs across the catalogue, aged by the newer of their metadata `updated` and their last real attempt (a merged or closed implementation PR, `regen:kept` included, or a failed generation's -`impl-fail` comment; infrastructure failures and spend-limit stops don't +`impl-fail` comment; infrastructure failures and spend or usage limit stops don't count). It skips pairs touched within `--attempt-window-days` (default 30), pairs with an open implementation PR, and every pair of a spec whose issue saw activity within `--min-age-hours`. PyYAML plus `gh`; a failed read degrades diff --git a/automation/scripts/regen_pick_pairs.py b/automation/scripts/regen_pick_pairs.py index 12446140923..45d4adf62a3 100644 --- a/automation/scripts/regen_pick_pairs.py +++ b/automation/scripts/regen_pick_pairs.py @@ -17,8 +17,9 @@ on the spec issue. A merge moves ``updated`` anyway; the other two leave it alone, and without them the same pair would come back the next night. An infrastructure failure (its comment carries ``impl-fail-cause:infra``) and - a stop at the Claude spend limit (its own pause comment) are not attempts: - the pair stays in the queue and comes back on a later night. + a stop at a Claude spend or usage limit (its own pause comment, + ``impl-paused-claude-limit``) are not attempts: the pair stays in the queue + and comes back on a later night. - A pair with an open implementation PR is skipped: its attempt is still running or stuck, and the watchdog owns it. - Every pair of a spec whose issue saw activity within ``--min-age-hours`` is diff --git a/changelog.d/ci-claude-auth-switch.md b/changelog.d/ci-claude-auth-switch.md index bb25eb4358c..7af58a705dc 100644 --- a/changelog.d/ci-claude-auth-switch.md +++ b/changelog.d/ci-claude-auth-switch.md @@ -13,17 +13,20 @@ exactly one credential and a new mode changes one step. `docs/workflows/overview.md` has the Console setup, the switch commands and the runs federation refuses. -- **Every Claude session's cost in the job summary, and a clean stop at the - spend limit.** A step after each Claude step writes `total_cost_usd` - (Claude Code's estimate at API list price), the model and the credential - mode to the job summary and the log, and never fails the job. With `oidc` - or `api` it also reads the API's spend-limit and credit messages from the - session's error fields: the retry is skipped, `impl-generate.yml` parks the +- **Every Claude session's cost in the job summary, and a clean stop at a + spend or usage limit.** A step after each Claude step writes + `total_cost_usd` (Claude Code's estimate at API list price), the model and + the credential mode to the job summary and the log, and never fails the + job. It also reads, from the error fields of a failed session, the API's + spend-limit, spend-cap and credit messages and the subscription's + usage-limit messages: the retry is skipped, `impl-generate.yml` parks the pair, `impl-review.yml` and `impl-repair.yml` hand the PR on without - spending their automatic retry, and nothing counts as an attempt or a - failure verdict. The weekly regen review's `collect_week.py --costs` reads - the line, now counts the `daily-regen.yml` pre-flight too, and names the - part that ran on API credits. + spending their automatic retry, and nothing counts as an attempt or marks + `impl::failed`, a first implementation included. Before, a + subscription usage limit used up a pair's three attempts. The weekly regen + review's `collect_week.py --costs` reads the line, now counts the + `daily-regen.yml` pre-flight too, and names the part that ran on API + credits. - **Daily regeneration by implementation instead of by spec: the repository variable `REGEN_PAIRS_PER_DAY`.** Set to N, the scheduled `daily-regen.yml` run picks the N oldest (spec, library) pairs across the catalogue @@ -34,7 +37,8 @@ attempted (a closed PR, a kept regeneration, a failed generation) within the attempt window, 30 days unless `attempt_window_days` or `REGEN_ATTEMPT_WINDOW_DAYS` says otherwise, is not picked again, while an - infrastructure failure or a spend-limit stop leaves it in the queue. Unset, + infrastructure failure or a stop at a spend or usage limit leaves it in the + queue. Unset, the schedule regenerates whole specs as before; a manual run opts in with the new `pairs` input, and the watchdog's cron rescue forwards the variable. diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index e945c3d0784..4ffe34044ec 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -524,9 +524,10 @@ night: `REGEN_ATTEMPT_WINDOW_DAYS`, else 30 days. After the window, the pair queues behind every pair touched before it. - An infrastructure failure, whose comment carries - ``, and a stop at the Claude spend limit are - not attempts: the pair stays at its place in the queue and comes back on a - later night. + ``, and a stop at a Claude spend or usage + limit, whose comment carries ``, are not + attempts: the pair stays at its place in the queue and comes back on a later + night. - A pair with an open implementation PR is skipped: its attempt is still running or stuck. - Every pair of a spec whose issue saw activity within `min_age_hours` @@ -559,12 +560,15 @@ on for the schedule, run `gh variable set REGEN_PAIRS_PER_DAY --body 5`; to turn it off, run `gh variable delete REGEN_PAIRS_PER_DAY`. When a pre-flight session (the spec polish or the similarity audit) stops at -the [Claude spend limit](#at-the-spend-limit), the night's dispatch for that -spec is skipped and the run fails. `daily-regen.yml` dispatches every pair -within a few minutes, before the first generation is reviewed, so it can't -see a limit that a later review of the same night reaches. The pairs -dispatched after the limit was reached fail at their first refused request, -which costs nothing, and are parked without a retry or a counted attempt. +a [Claude spend or usage limit](#at-a-spend-or-usage-limit), the night's +dispatch for that spec is skipped and the run fails. In pair mode, the +similarity audit of each spec runs before that spec's pairs are dispatched, +one spec after another, so it works as the night's probe: once a pair's +generation, review, or repair has reached the limit, every later spec of the +night stops at its audit and dispatches nothing. Pairs that were already +dispatched fail at their first refused request, which costs nothing, and are +parked without a retry or a counted attempt. Their spec keeps its pairs in +the queue for a later night. --- @@ -816,30 +820,42 @@ put a generation on Sonnet at about $0.29 per implementation, a review on Opus at about $1.10, and a repair with its re-review at about $2.70 per implementation in all. -### At the spend limit +### At a spend or usage limit -With `oidc` or `api`, the workspace's spend limit stops the pipeline cleanly. -The API refuses every request once the limit is reached, and Claude Code's own -retries can't get past it: +A Claude spend or usage limit stops the pipeline cleanly in every mode. Once +the limit is reached, every request is refused until it resets or you raise +it, and Claude Code's own retries can't get past it. The pipeline recognizes +these texts: -- A spend limit you set on the workspace or the organization answers HTTP 400 - `invalid_request_error` with a message that starts with - `You have reached your specified workspace API usage limits` (or - `… specified API usage limits` for the organization). -- The usage tier's monthly spend cap answers HTTP 429 `rate_limit_error` with +- A spend limit you set on the workspace or the organization (`oidc`, `api`): + HTTP 400 `invalid_request_error`, with a message that starts with + `You have reached your specified workspace API usage limits` or + `You have reached your specified API usage limits`. +- The usage tier's monthly spend cap: HTTP 429 `rate_limit_error` with the `error_code` `enforced_spend_limit_reached` and no `retry-after` header. -- An empty credit balance answers `Your credit balance is too low`. - -"Claude session summary" reads these messages, and Claude Code's -`billing_error`, from the session's error fields only: the error result, Claude -Code's error messages, and its API retries. It never reads the conversation, -which may quote the same text. When it finds one, it writes an `::error::` -and the line **Claude spend limit reached (mode oidc)** (or `api`) to the job -summary, and every path treats the stop as an infrastructure failure: +- An empty credit balance: the API's `Your credit balance is too low`, Claude + Code's `Credit balance is too low` or `Credit balance too low · Add funds`, + and its error type `billing_error`. +- The subscription's usage limit (`subscription`): Claude Code's messages that + start with one of the prefixes its SDK exports as + `USAGE_LIMIT_ERROR_PREFIXES`, such as + `You've hit your session limit · resets 12:30pm`, and the older + `Claude AI usage limit reached`. + +A transient rate limit, such as a per-minute token limit, is none of these: +it keeps its retry and the infrastructure path it took before. + +"Claude session summary" reads these texts only from the error fields of a +session that failed: the error result, Claude Code's error messages, and its +API retries. It never reads the conversation, which may quote the same text, +and never flags a session that ended well. When it finds one, it writes an +`::error::` and the line **Claude spend/usage limit reached (auth=oidc)**, with +the mode, to the job summary, and every path treats the stop as an +infrastructure failure: | Workflow | What happens | |----------|--------------| -| `impl-generate.yml` | The retry is skipped, and "Stop at the Claude spend limit" fails the job. "Handle generation failure" parks the pair: it removes `generate:{library}` and `impl:{library}:pending`, puts `impl:{library}:done` back for a live implementation, and comments with its own marker. Nothing is retried, no attempt counts toward the cap of three, and the pair is never marked `impl:{library}:failed`. No PR exists; the next generation recreates the branch. | +| `impl-generate.yml` | The retry is skipped, and "Stop at the Claude spend or usage limit" fails the job. "Handle generation failure" parks the pair before it counts anything: it removes `generate:{library}` and `impl:{library}:pending`, puts `impl:{library}:done` back for a live implementation, and comments with its own marker. Nothing is retried, no attempt counts toward the cap of three, and the pair is never marked `impl:{library}:failed`, a first implementation included. No PR exists; the next generation recreates the branch. | | `impl-review.yml` | "Validate review output" adds `ai-review-failed` and comments, without the auto re-review and without spending it. No score, no verdict: the PR stays open, nothing is merged or closed. The watchdog re-dispatches the review once at its next scan. | | `impl-repair.yml` | The retry is skipped. "Handle repair failure" restores `ai-rejected` and comments, without the auto-retry and without spending it. The watchdog's repair rescue, once per attempt, or a manual restart picks the PR up. | | `daily-regen.yml` | A spec polish or similarity audit at the limit skips that spec's dispatch, and the entry fails. | @@ -852,8 +868,11 @@ review stopped keeps its open PR, which pair mode skips until the review finishes. A session that reaches the limit after it wrote its review files is scored on what it wrote, like any session that ends early. -With the subscription nothing changes: the summary never reads a spend limit -there, and the subscription's own usage limit takes the paths it took before. +Before this change, a subscription usage limit counted as the pair's own +failure: it used up the three attempts and marked a first implementation +`impl:{library}:failed`. A bulk run that reaches the subscription limit now +parks the rest of its pairs instead, and you re-dispatch them once the limit +resets. ### Test the spend limit @@ -869,10 +888,12 @@ Test the stop after the merge, with `CLAUDE_AUTH=oidc`: The generation on Sonnet costs about 0.29 USD and passes; the review on Opus costs about 1.10 USD and reaches the limit while it runs. -3. Check the `impl-review.yml` run: its summary shows **Claude spend limit - reached (mode oidc)**, the PR has `ai-review-failed` and the "AI Review - Paused (Claude spend limit)" comment, no second review run started, and no - verdict or `quality:` label was added. +3. Check the `impl-review.yml` run: its summary shows **Claude spend/usage + limit reached (auth=oidc)**, the PR has `ai-review-failed` and the "AI + Review Paused (Claude spend/usage limit)" comment, no second review run + started, and no verdict or `quality:` label was added. If the summary line + is missing, the run's error text didn't match: read it in the log and + extend `LIMIT_PATTERN`. 4. Check the spec issue: no `impl:{library}:failed` label, and no `impl-fail` comment for the pair. 5. Restore the spend limit, then rerun the review, or wait for the watchdog: diff --git a/tests/unit/automation/scripts/test_regen_pick_pairs.py b/tests/unit/automation/scripts/test_regen_pick_pairs.py index 4ee40ad9f3b..7971dad1c6a 100644 --- a/tests/unit/automation/scripts/test_regen_pick_pairs.py +++ b/tests/unit/automation/scripts/test_regen_pick_pairs.py @@ -6,7 +6,7 @@ of the queue, and postpones a spec whose issue just saw activity. The no-loop rule is pinned here: a pair the gate kept, or whose generation failed, does not come back the next night, and an infrastructure failure or -a stop at the spend limit is no attempt. ``main`` runs once against a fake +a stop at a spend or usage limit is no attempt. ``main`` runs once against a fake ``gh`` the way the workflow calls it. """ @@ -178,7 +178,7 @@ def test_comments_count_only_real_failures(self): fail_comment("a", "altair", 2, infra=True), # infrastructure: no attempt fail_comment("a", "plotly", 1, infra=True), fail_comment("other", "altair", 1), # another spec's marker - {"created_at": ts(1), "body": ""}, + {"created_at": ts(1), "body": ""}, # a spend or usage limit ] assert rp.failed_generations(comments, "a") == {"altair": at(5)} diff --git a/tests/unit/workflows/test_claude_auth.py b/tests/unit/workflows/test_claude_auth.py index 7b400c312de..8f8b11350f2 100644 --- a/tests/unit/workflows/test_claude_auth.py +++ b/tests/unit/workflows/test_claude_auth.py @@ -1,4 +1,4 @@ -"""Claude credential switch (vars.CLAUDE_AUTH), the session summary and the spend limit. +"""Claude credential switch (vars.CLAUDE_AUTH), the session summary and the spend/usage limit. Every job that runs Claude starts with the same "Resolve Claude credentials" step. It maps CLAUDE_AUTH (unset or `subscription`, `api`, `oidc`) to a mode, @@ -9,10 +9,11 @@ is undocumented (an API key or OAuth token silently shadows federation). After every Claude step, "Claude session summary" writes the cost line and -reads whether the session stopped at the Claude spend limit (api/oidc only); -the retry is skipped then, and the failure paths of impl-generate, -impl-review and impl-repair park the work as an infrastructure stop without -a retry or a counted attempt. +reads whether the session stopped at a Claude spend or usage limit (the API's +spend limit, spend cap or credit balance, the subscription's usage limit), in +every mode; the retry is skipped then, and the failure paths of +impl-generate, impl-review and impl-repair park the work as an infrastructure +stop without a retry, a counted attempt or a failure verdict. There is no pre-merge smoke for these workflows (claude-code-action refuses to run on a PR that changes them), so the wiring is pinned here and the @@ -37,7 +38,7 @@ ACTION = "anthropics/claude-code-action@" RESOLVE = "Resolve Claude credentials" SUMMARY = "Claude session summary" -STOP = "Stop at the Claude spend limit" +STOP = "Stop at the Claude spend or usage limit" CREDENTIALS = { "anthropic_api_key": "${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }}", "claude_code_oauth_token": ( @@ -185,7 +186,7 @@ def test_summary_steps_share_one_script_and_unique_labels(): labels = [s["env"]["CLAUDE_STEP"] for s in summaries] assert len(labels) == len(set(labels)) == sum(CLAUDE_WORKFLOWS.values()) assert all(" " not in label for label in labels) - shared = {(s["run"], s["env"]["SESSION_ERRORS"], s["env"]["SPEND_LIMIT_PATTERN"]) for s in summaries} + shared = {(s["run"], s["env"]["SESSION_ERRORS"], s["env"]["LIMIT_PATTERN"]) for s in summaries} assert len(shared) == 1 # The line must not look like Claude Code's own `"total_cost_usd": n`, or # collect_week.py's fallback regex would count a session twice. @@ -193,7 +194,7 @@ def test_summary_steps_share_one_script_and_unique_labels(): @pytest.mark.parametrize(("filename", "name", "job"), JOBS, ids=IDS) -def test_a_spend_limit_skips_the_retry_and_stops_the_job(filename, name, job): +def test_a_limit_skips_the_retry_and_stops_the_job(filename, name, job): steps = job["steps"] names = [s.get("name") for s in steps] claude = [s for s in steps if _is_claude(s)] @@ -201,7 +202,7 @@ def test_a_spend_limit_skips_the_retry_and_stops_the_job(filename, name, job): # Two independent pre-flight sessions, no retry: either one stops the dispatch. stop = steps[names.index(STOP)] assert stop["if"] == ( - "${{ steps.polish_cost.outputs.spend_limit == 'true' || steps.similarity_cost.outputs.spend_limit == 'true' }}" + "${{ steps.polish_cost.outputs.limit_reached == 'true' || steps.similarity_cost.outputs.limit_reached == 'true' }}" ) dispatches = [i for i, n in enumerate(names) if str(n).startswith("Dispatch ")] assert len(dispatches) == 2 and min(dispatches) > names.index(STOP) > steps.index(claude[-1]) @@ -211,10 +212,10 @@ def test_a_spend_limit_skips_the_retry_and_stops_the_job(filename, name, job): return else: first, retry = claude - assert retry["if"].endswith(f" && steps.{first['id']}_cost.outputs.spend_limit != 'true'") + assert retry["if"].endswith(f" && steps.{first['id']}_cost.outputs.limit_reached != 'true'") stop = steps[names.index(STOP)] assert names.index(STOP) > steps.index(retry) - assert stop["if"] == f"steps.{first['id']}_cost.outputs.spend_limit == 'true'" + assert stop["if"] == f"steps.{first['id']}_cost.outputs.limit_reached == 'true'" assert "exit 1" in stop["run"] and "continue-on-error" not in stop @@ -335,7 +336,7 @@ def _summary(tmp_path: Path, execution: list[dict[str, Any]] | None, mode: str = "CLAUDE_AUTH_MODE": mode, "CLAUDE_STEP": "impl-review", "SESSION_ERRORS": step["env"]["SESSION_ERRORS"], - "SPEND_LIMIT_PATTERN": step["env"]["SPEND_LIMIT_PATTERN"], + "LIMIT_PATTERN": step["env"]["LIMIT_PATTERN"], "GITHUB_STEP_SUMMARY": str(summary), "GITHUB_OUTPUT": str(out), } @@ -356,17 +357,86 @@ def _failed(text: str, error: str | None = "invalid_request", status: int = 400) return [INIT, assistant, result | {"result": text, "total_cost_usd": 0.41}] -# The API's own messages (platform.claude.com/docs/en/api/rate-limits, /api/errors). -WORKSPACE_LIMIT = ( - 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"You have reached your ' - 'specified workspace API usage limits. You will regain access on 2026-12-01 at 00:00 UTC."}}' -) -TIER_CAP = ( - 'API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"You have reached your API usage ' - 'limits: your organization has crossed its monthly API usage threshold.","details":{"error_code":' - '"enforced_spend_limit_reached"}}}' -) -CREDIT = "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing." +# Fixtures with the texts as the API and Claude Code produce them, by source. +LIMIT_FIXTURES = { + # A workspace spend limit you set: HTTP 400 invalid_request_error + # (platform.claude.com/docs/en/api/rate-limits; anthropics/claude-code#40664). + "workspace spend limit": _failed( + 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"You have reached your ' + 'specified workspace API usage limits. You will regain access on 2026-12-01 at 00:00 UTC."}}' + ), + # An organization spend limit you set (anthropics/claude-code#52077, the raw API error). + "org spend limit": _failed( + 'Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"You have reached your ' + 'specified API usage limits. You will regain access on 2026-05-01 at 00:00 UTC."}}' + ), + # The usage tier's monthly cap: HTTP 429 rate_limit_error with an error_code (the same docs page). + "tier spend cap": _failed( + 'API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"You have reached your API usage ' + 'limits: your organization has crossed its monthly API usage threshold.","details":{"error_code":' + '"enforced_spend_limit_reached"}}}', + error="rate_limit", + status=429, + ), + # Credits: the API's message, and Claude Code's non-interactive and interactive renderings + # (anthropics/claude-code#99371, #103854). + "credit, API": _failed( + "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or " + "purchase credits.", + error="billing_error", + ), + "credit, Claude Code result": _failed("Credit balance is too low", error=None), + "credit, Claude Code interactive": _failed( + "Credit balance too low · Add funds: https://platform.claude.com/settings/billing", error=None + ), + # The SDK's own billing error type on an API retry (SDKAssistantMessageError). + "billing_error retry": [ + INIT, + {"type": "system", "subtype": "api_retry", "error": "billing_error", "error_status": 402}, + ], + # Subscription: Claude Code's usage-limit messages, which start with one of the SDK's + # USAGE_LIMIT_ERROR_PREFIXES ("the error-path outputs of getLimitReachedText"); the + # session-limit text as reported in anthropics/claude-code#101965. + "session limit": _failed("You've hit your session limit · resets 12:30pm", error="rate_limit", status=429), + "session limit, curly apostrophe": _failed("You’ve hit your session limit · resets 4am (Europe/Berlin)", None), + "weekly limit": _failed("You've reached your weekly limit · resets Nov 3, 9am", error="rate_limit", status=429), + "out of extra usage": _failed("You're out of extra usage · resets 12:30pm", error="rate_limit", status=429), + "org out of usage": _failed("Your org is out of usage · contact your admin", error="rate_limit", status=429), + "older CLI wording": _failed("Claude AI usage limit reached|1761955200", error=None), +} + +NOT_A_LIMIT = { + # A transient rate limit keeps its retry and the infrastructure path it had. + "per-minute rate limit": _failed( + 'API Error: 429 {"error":{"type":"rate_limit_error","message":"Number of request tokens has exceeded your ' + 'per-minute rate limit"}}', + error="rate_limit", + status=429, + ), + "rate limit, CLI wording": _failed("API Error: Rate limit reached", error="rate_limit", status=429), + "server error": _failed("API Error: 500 Internal server error", error="server_error", status=500), + # A usage warning, not a stop (the SDK's USAGE_WARNING_PREFIXES). + "usage warning": _failed("You've used 95% of your weekly limit", error="unknown", status=500), + # The session's own text may quote the messages (a doc it read, a review comment): not an error field. + "quoted by the model": [ + INIT, + {"type": "assistant", "message": {"content": [{"type": "text", "text": "Docs: credit balance is too low"}]}}, + ], + "quoted by a tool": [ + INIT, + {"type": "user", "message": {"content": [{"type": "tool_result", "content": "You've hit your session limit"}]}}, + ], + "in a successful result": [ + INIT, + {"type": "result", "subtype": "success", "is_error": False, "result": "Docs say: credit balance is too low"}, + ], + # A retried billing error the session got past: it ended well, so it never stopped. + "recovered": [ + INIT, + {"type": "system", "subtype": "api_retry", "error": "billing_error", "error_status": 402}, + {"type": "result", "subtype": "success", "is_error": False, "result": "done"}, + ], +} def test_summary_reads_cost_and_model(tmp_path): @@ -374,7 +444,7 @@ def test_summary_reads_cost_and_model(tmp_path): result, summary, outputs = _summary(tmp_path, execution, mode="oidc") assert result.returncode == 0 assert summary == "- `claude-cost step=impl-review auth=oidc model=claude-opus-5-5 total_cost_usd=0.2931`\n" - assert outputs == {"spend_limit": "false"} + assert outputs == {"limit_reached": "false"} @pytest.mark.parametrize("execution", [None, [], [{"type": "assistant"}], [{"type": "result"}]]) @@ -382,54 +452,27 @@ def test_summary_never_fails_without_a_cost(tmp_path, execution): result, summary, outputs = _summary(tmp_path, execution) assert result.returncode == 0 assert "auth=subscription model=n/a total_cost_usd=n/a" in summary - assert outputs == {"spend_limit": "false"} + assert outputs == {"limit_reached": "false"} -@pytest.mark.parametrize( - "execution", - [ - _failed(WORKSPACE_LIMIT), - _failed(TIER_CAP, error="rate_limit", status=429), - _failed(CREDIT, error="billing_error"), - _failed("Credit balance is too low", error=None), - [INIT, {"type": "system", "subtype": "api_retry", "error": "billing_error", "error_status": 402}], - ], -) -@pytest.mark.parametrize("mode", ["api", "oidc"]) -def test_summary_detects_the_spend_limit(tmp_path, execution, mode): - result, summary, outputs = _summary(tmp_path, execution, mode) +@pytest.mark.parametrize("case", sorted(LIMIT_FIXTURES)) +@pytest.mark.parametrize("mode", ["subscription", "oidc", "api"]) +def test_summary_detects_a_spend_or_usage_limit(tmp_path, case, mode): + result, summary, outputs = _summary(tmp_path, LIMIT_FIXTURES[case], mode) assert result.returncode == 0 - assert outputs == {"spend_limit": "true"} - assert f"::error::Claude spend limit reached (mode {mode}) in impl-review" in result.stdout - assert f"- **Claude spend limit reached (mode {mode})** in `impl-review`" in summary - - -def test_the_subscription_never_reads_a_spend_limit(tmp_path): - """Subscription runs keep today's behaviour exactly: no new branch can fire.""" - assert _summary(tmp_path, _failed(WORKSPACE_LIMIT), "subscription")[2] == {"spend_limit": "false"} + assert outputs == {"limit_reached": "true"}, case + assert f"::error::Claude spend/usage limit reached (auth={mode}) in impl-review" in result.stdout + assert f"- **Claude spend/usage limit reached (auth={mode})** in `impl-review`" in summary -@pytest.mark.parametrize( - "execution", - [ - # A transient rate limit is retried as before. - _failed( - 'API Error: 429 {"error":{"type":"rate_limit_error","message":"Number of request tokens has exceeded your per-minute rate limit"}}', - error="rate_limit", - status=429, - ), - _failed("API Error: 500 Internal server error", error="server_error", status=500), - # The session's own text may quote the messages (a doc it read, a review comment): not an error field. - [INIT, {"type": "assistant", "message": {"content": [{"type": "text", "text": WORKSPACE_LIMIT}]}}], - [INIT, {"type": "user", "message": {"content": [{"type": "tool_result", "content": CREDIT}]}}], - [INIT, {"type": "result", "subtype": "success", "is_error": False, "result": f"Docs say: {CREDIT}"}], - ], -) -def test_summary_ignores_everything_else(tmp_path, execution): - assert _summary(tmp_path, execution, "api")[2] == {"spend_limit": "false"} +@pytest.mark.parametrize("case", sorted(NOT_A_LIMIT)) +def test_summary_ignores_everything_else(tmp_path, case): + result, summary, outputs = _summary(tmp_path, NOT_A_LIMIT[case], "api") + assert outputs == {"limit_reached": "false"}, case + assert "spend/usage limit" not in summary -# --- the failure paths at the spend limit --------------------------------------- +# --- the failure paths at a spend or usage limit ---------------------------------- # Logs every call as one JSON line; `pr view`/`issue view` answer nothing. @@ -471,8 +514,10 @@ def _bodies(calls: list[list[str]]) -> list[str]: return [c[c.index("--body") + 1] for c in calls if "--body" in c] +@pytest.mark.parametrize("mode", ["subscription", "oidc"]) @pytest.mark.parametrize(("regen_label", "live"), [("regen", True), ("", False)]) -def test_generation_at_the_spend_limit_is_parked(tmp_path, regen_label, live): +def test_generation_at_a_limit_is_parked(tmp_path, regen_label, live, mode): + """A first implementation too: no retry, no counted attempt, never `impl::failed`.""" subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) result, calls = _run_handler( tmp_path, @@ -487,8 +532,8 @@ def test_generation_at_the_spend_limit_is_parked(tmp_path, regen_label, live): INFRA_WINDOW_CAP="5", CLAUDE_RETRY_OUTCOME="skipped", CLAUDE_EXEC_FILE="", - SPEND_LIMIT="true", - CLAUDE_MODE="oidc", + LIMIT_REACHED="true", + CLAUDE_MODE=mode, REGEN_LABEL=regen_label, LANGUAGE="python", EXT=".py", @@ -500,19 +545,20 @@ def test_generation_at_the_spend_limit_is_parked(tmp_path, regen_label, live): RUN_URL="https://github.com/owner/repo/actions/runs/1", ) assert result.returncode == 0, result.stdout + result.stderr - assert "::error::Claude spend limit reached (mode oidc)" in result.stdout + assert f"::error::Claude spend/usage limit reached (auth={mode})" in result.stdout assert not [c for c in calls if c[:2] == ["workflow", "run"]] # no auto-retry edits = [c[3:5] for c in calls if c[:2] == ["issue", "edit"]] assert ["--remove-label", "impl:plotly:pending"] in edits and ["--remove-label", "generate:plotly"] in edits assert (["--add-label", "impl:plotly:done"] in edits) is live assert not any("impl:plotly:failed" in c for c in edits) # never a capability verdict + assert not [c for c in calls if c[:2] == ["api", "--paginate"]] # the attempt budget is not even read (body,) = _bodies(calls) # Its own marker: neither the 3-attempt cap nor daily-regen's attempt window counts it. - assert "" in body - assert "" in body + assert "" in body + assert "" in body assert "" in body + assert "" in body assert " - ## :pause_button: ${LIBRARY} Paused (Claude spend/usage limit) + gh issue comment "$ISSUE" --body " + ## :pause_button: ${LIBRARY} Paused (${WHY}) - Claude refused the **${LIBRARY}** generation for \`${SPEC_ID}\` at a spend or usage limit (auth=\`${CLAUDE_MODE}\`). This is **not** a capability verdict: nothing was retried, no attempt was counted, and the pair is not marked failed. + Claude stopped the **${LIBRARY}** generation for \`${SPEC_ID}\`: ${WHY} (auth=\`${CLAUDE_MODE}\`). This is **not** a capability verdict: nothing was retried, no attempt was counted, and the pair is not marked failed. - Once the limit resets or is raised, run: + ${WHEN}, run: \`\`\` gh workflow run impl-generate.yml -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f issue_number=${ISSUE} -f model=${MODEL} \`\`\` diff --git a/.github/workflows/impl-repair.yml b/.github/workflows/impl-repair.yml index a97ad6e6454..300424d65c3 100644 --- a/.github/workflows/impl-repair.yml +++ b/.github/workflows/impl-repair.yml @@ -68,10 +68,13 @@ jobs: env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} run: | fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } mode="${CLAUDE_AUTH,,}" @@ -79,16 +82,30 @@ jobs: ''|subscription) mode=subscription ;; api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; oidc) - for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do - [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" - done - [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" - { - echo "federation_rule_id=$WIF_RULE_ID" - echo "organization_id=$WIF_ORG_ID" - echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" - echo "workspace_id=$WIF_WORKSPACE_ID" - } >> "$GITHUB_OUTPUT" ;; + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac echo "mode=$mode" >> "$GITHUB_OUTPUT" @@ -259,11 +276,15 @@ jobs: echo "$FEEDBACK" > /tmp/ai_feedback.md + # `ai-repair-paused` goes too: this run resumes a repair that Claude + # stopped (a limit or refused credentials); the stop path puts it back + # if Claude stops this one as well. - name: Remove ai-rejected label env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh pr edit ${{ inputs.pr_number }} --remove-label "ai-rejected" 2>/dev/null || true + gh pr edit ${{ inputs.pr_number }} --remove-label "ai-repair-paused" 2>/dev/null || true - name: Run Claude Code to repair implementation id: claude @@ -291,10 +312,10 @@ jobs: - BRANCH: ${{ env.branch }} # The session's cost (Claude Code's own estimate at API list price), - # model and credential mode, for the weekly regen review, and whether it - # stopped at a Claude spend or usage limit (output limit_reached). Read - # right after the step: the retry writes the same execution file. Never - # fatal. + # model and credential mode, for the weekly regen review, and whether + # Claude stopped it: a spend or usage limit, or refused credentials + # (outputs stopped, stop_reason). Read right after the step: the retry + # writes the same execution file. Never fatal. - name: Claude session summary id: claude_cost if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} @@ -305,18 +326,25 @@ jobs: CLAUDE_STEP: impl-repair # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -324,17 +352,27 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" - # Not at a spend or usage limit: the retry would be refused the same way. + # Not after Claude stopped the session (a limit or refused credentials): + # the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.limit_reached != 'true' + if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.stopped != 'true' id: claude_retry timeout-minutes: 45 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 @@ -368,18 +406,25 @@ jobs: CLAUDE_STEP: impl-repair/retry # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -387,21 +432,31 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" - - # The first session stopped at a spend or usage limit and its retry was - # skipped: fail here, so "Handle repair failure" restores the PR without - # an auto-retry instead of the steps below running on an unrepaired file. - - name: Stop at the Claude spend or usage limit - if: steps.claude_cost.outputs.limit_reached == 'true' + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Claude stopped the first session (a limit or refused credentials) and + # its retry was skipped: fail here, so "Handle repair failure" pauses the + # PR without an auto-retry instead of the steps below running on an + # unrepaired file. + - name: Stop when Claude stopped the session + if: steps.claude_cost.outputs.stopped == 'true' run: | - echo "::error::Claude spend/usage limit reached; the retry is skipped" + echo "::error::Claude stopped the session (${{ steps.claude_cost.outputs.stop_reason }}); the retry is skipped" exit 1 - name: Process repaired image @@ -449,6 +504,9 @@ jobs: SPEC_ID: ${{ inputs.specification_id }} LIBRARY: ${{ inputs.library }} run: | + # This step runs model-written code: it gets no way to mint an OIDC + # token (the job has id-token for Claude and Google Cloud). + unset ACTIONS_ID_TOKEN_REQUEST_URL ACTIONS_ID_TOKEN_REQUEST_TOKEN OUT="$RUNNER_TEMP/adapt"; TOOLS="$RUNNER_TEMP/adapt-tools" rm -rf "$OUT" "$TOOLS"; mkdir -p "$OUT" "$TOOLS" # The runner comes from this workflow's ref, never from the PR branch. @@ -559,27 +617,40 @@ jobs: MODEL: ${{ steps.model.outputs.model || inputs.model || 'auto' }} REPOSITORY: ${{ github.repository }} RUN_ID: ${{ github.run_id }} - # Either session stopped at a Claude spend or usage limit. - LIMIT_REACHED: ${{ steps.claude_cost.outputs.limit_reached == 'true' || steps.claude_retry_cost.outputs.limit_reached == 'true' }} + # Claude stopped either session (a limit or refused credentials). + CLAUDE_STOPPED: ${{ steps.claude_cost.outputs.stopped == 'true' || steps.claude_retry_cost.outputs.stopped == 'true' }} + STOP_REASON: ${{ steps.claude_cost.outputs.stop_reason || steps.claude_retry_cost.outputs.stop_reason }} CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} run: | # Restore ai-rejected label that was removed at start of repair so the # PR state stays consistent (otherwise it looks "stuck approved"). gh pr edit "$PR_NUM" --add-label "ai-rejected" 2>/dev/null || true - # A Claude spend or usage limit: an auto-retry now would be refused - # the same way. The comment carries its own marker, not the - # repair-retry one, so the attempt keeps its one crash retry for after - # the limit resets or is raised; the watchdog's repair rescue (once - # per attempt) or a manual restart picks the PR up. - if [ "${LIMIT_REACHED}" = "true" ]; then - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_MODE}) — repair attempt ${ATTEMPT} stopped, no auto-retry" - gh pr comment "$PR_NUM" --body " - ## :pause_button: Repair Paused (Claude spend/usage limit, Attempt ${ATTEMPT}/4) + # Claude stopped the session (a spend or usage limit, or refused + # credentials): an auto-retry now would be refused the same way. + # `ai-repair-paused` keeps the watchdog's one-shot rescue off the PR; + # daily-regen's pair mode, or a manual restart, resumes it. The + # comment carries its own marker, not the repair-retry one, so the + # attempt keeps its one crash retry for after the stop is resolved. + if [ "${CLAUDE_STOPPED}" = "true" ]; then + if [ "${STOP_REASON}" = "auth" ]; then + WHY="Claude credentials refused" + WHEN="once the credentials are fixed" + else + WHY="Claude spend/usage limit reached" + WHEN="once the limit resets or is raised" + fi + echo "::error::${WHY} (auth=${CLAUDE_MODE}) — repair attempt ${ATTEMPT} paused, no auto-retry" + gh label create "ai-repair-paused" --color "fbca04" \ + --description "Repair paused: Claude stopped it (spend/usage limit or refused credentials)" 2>/dev/null || true + gh pr edit "$PR_NUM" --add-label "ai-repair-paused" 2>/dev/null \ + || echo "::warning::could not add ai-repair-paused to PR #${PR_NUM}" + gh pr comment "$PR_NUM" --body " + ## :pause_button: Repair Paused (${WHY}, Attempt ${ATTEMPT}/4) - Claude refused the repair at a spend or usage limit (auth=\`${CLAUDE_MODE}\`). Nothing was retried, and the attempt is not used up. + Claude stopped the repair: ${WHY} (auth=\`${CLAUDE_MODE}\`). Nothing was retried, and the attempt is not used up. The PR carries \`ai-repair-paused\`, which the watchdog leaves alone; daily-regen's pair mode resumes it. - **Restart once the limit resets or is raised:** + **Restart ${WHEN}:** \`\`\` gh workflow run impl-repair.yml -f pr_number=${PR_NUM} -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f attempt=${ATTEMPT} -f model=${MODEL} \`\`\` diff --git a/.github/workflows/impl-review.yml b/.github/workflows/impl-review.yml index 87943bbe572..0702c4891d6 100644 --- a/.github/workflows/impl-review.yml +++ b/.github/workflows/impl-review.yml @@ -64,10 +64,13 @@ jobs: env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} run: | fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } mode="${CLAUDE_AUTH,,}" @@ -75,16 +78,30 @@ jobs: ''|subscription) mode=subscription ;; api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; oidc) - for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do - [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" - done - [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" - { - echo "federation_rule_id=$WIF_RULE_ID" - echo "organization_id=$WIF_ORG_ID" - echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" - echo "workspace_id=$WIF_WORKSPACE_ID" - } >> "$GITHUB_OUTPUT" ;; + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac echo "mode=$mode" >> "$GITHUB_OUTPUT" @@ -259,6 +276,23 @@ jobs: echo "::warning::ai-review-failed could not be removed from PR #${PR_NUM} — the watchdog skips it while this review runs" exit 0 + # This run resumes a review that Claude stopped (a spend or usage limit, + # or refused credentials): `ai-review-paused` goes, and the stop path in + # "Validate review output" puts it back if Claude stops this one too. + # Best effort. + - name: Clear the ai-review-paused label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUM: ${{ steps.pr.outputs.pr_number }} + run: | + if gh pr view "$PR_NUM" --json labels -q '.labels[].name' 2>/dev/null | grep -qxF "ai-review-paused"; then + if gh pr edit "$PR_NUM" --remove-label "ai-review-paused" 2>/dev/null; then + echo "::notice::Removed ai-review-paused from PR #${PR_NUM}: this run resumes the review" + else + echo "::warning::could not remove ai-review-paused from PR #${PR_NUM}" + fi + fi + - name: Derive language + extension from library id: lang env: @@ -973,9 +1007,10 @@ jobs: python3 "$RUNNER_TEMP/regen-tools/review_provenance.py" model --execution-file "${EXECUTION_FILE:-}" # The session's cost (Claude Code's own estimate at API list price), - # model and credential mode, for the weekly regen review, and whether it - # stopped at a Claude spend or usage limit (output limit_reached, which - # "Validate review output" reads). Never fatal. + # model and credential mode, for the weekly regen review, and whether + # Claude stopped it: a spend or usage limit, or refused credentials + # (outputs stopped, stop_reason). "Extract quality score" and + # "Validate review output" read it. Never fatal. - name: Claude session summary id: review_cost if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} @@ -986,18 +1021,25 @@ jobs: CLAUDE_STEP: impl-review # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -1005,13 +1047,22 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" # Undo "Hide the previous review", whatever the review step did: the # metadata step below reads the metadata file from disk and rewrites @@ -1040,9 +1091,14 @@ jobs: git checkout -- "$META_FILE" 2>/dev/null || true fi + # Not from a session Claude stopped (a limit or refused credentials), + # even when it wrote quality_score.txt before the stop: a cut-off + # review gives no score, no verdict and no gate decision. The step is + # skipped, `has_output` stays empty, and "Validate review output" takes + # the paused path. - name: Extract quality score id: score - if: steps.review.conclusion == 'success' + if: steps.review.conclusion == 'success' && steps.review_cost.outputs.stopped != 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUM: ${{ steps.pr.outputs.pr_number }} @@ -1116,37 +1172,48 @@ jobs: RUN_ID: ${{ github.run_id }} MODEL: ${{ steps.pr.outputs.model }} REVIEW_MODEL_ALIAS: ${{ steps.pr.outputs.review_model_alias }} - # The session stopped at a Claude spend or usage limit. - LIMIT_REACHED: ${{ steps.review_cost.outputs.limit_reached }} + # Claude stopped the session (a limit or refused credentials). + CLAUDE_STOPPED: ${{ steps.review_cost.outputs.stopped }} + STOP_REASON: ${{ steps.review_cost.outputs.stop_reason }} CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} run: | echo "::error::AI Review did not produce valid output files (score=0)" - # A Claude spend or usage limit: a re-review now would be refused the - # same way, and it says nothing about the implementation. Hand the PR - # to the watchdog with `ai-review-failed`, as an exhausted retry - # does, but without the review-retry marker, so the one auto-retry - # stays for after the limit resets or is raised. No verdict, no - # score, nothing merged or closed. - if [ "${LIMIT_REACHED}" = "true" ]; then - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_MODE}) — no auto-retry for PR #${PR_NUM}" + # Claude stopped the session (a spend or usage limit, or refused + # credentials): a re-review now would be refused the same way, and + # it says nothing about the implementation. `ai-review-paused` (not + # `ai-review-failed`) keeps the watchdog's one-shot rescue off the + # PR; daily-regen's pair mode, or a manual rerun, resumes it. No + # review-retry marker, so the one auto-retry stays for after the stop + # is resolved. No verdict, no score, nothing merged or closed. + if [ "${CLAUDE_STOPPED}" = "true" ]; then + if [ "${STOP_REASON}" = "auth" ]; then + WHY="Claude credentials refused" + WHEN="once the credentials are fixed" + else + WHY="Claude spend/usage limit reached" + WHEN="once the limit resets or is raised" + fi + echo "::error::${WHY} (auth=${CLAUDE_MODE}) — review of PR #${PR_NUM} paused, no auto-retry" + gh label create "ai-review-paused" --color "fbca04" \ + --description "Review paused: Claude stopped it (spend/usage limit or refused credentials)" 2>/dev/null || true for attempt in 1 2 3; do - gh pr edit "$PR_NUM" --add-label "ai-review-failed" 2>/dev/null && break + gh pr edit "$PR_NUM" --add-label "ai-review-paused" 2>/dev/null && break [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) - done || echo "::error::could not add ai-review-failed to PR #${PR_NUM} — left to the watchdog's scan of stalled PRs" - gh pr comment "$PR_NUM" --body " - ## :pause_button: AI Review Paused (Claude spend/usage limit) + done || echo "::error::could not add ai-review-paused to PR #${PR_NUM} — resume it with a manual rerun" + gh pr comment "$PR_NUM" --body " + ## :pause_button: AI Review Paused (${WHY}) - Claude refused the review at a spend or usage limit (auth=\`${CLAUDE_MODE}\`). No score and no verdict were recorded, and the one automatic re-review is still unused. + Claude stopped the review: ${WHY} (auth=\`${CLAUDE_MODE}\`). No score and no verdict were recorded, and the one automatic re-review is still unused. The PR carries \`ai-review-paused\`, which the watchdog leaves alone; daily-regen's pair mode resumes it. - **Rerun once the limit resets or is raised:** + **Rerun ${WHEN}:** \`\`\` gh workflow run impl-review.yml -f pr_number=$PR_NUM \`\`\` --- :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" \ - || echo "::error::limit comment not posted on PR #${PR_NUM}" + || echo "::error::pause comment not posted on PR #${PR_NUM}" exit 1 fi diff --git a/.github/workflows/report-validate.yml b/.github/workflows/report-validate.yml index a4396cee782..634911b9a89 100644 --- a/.github/workflows/report-validate.yml +++ b/.github/workflows/report-validate.yml @@ -58,10 +58,13 @@ jobs: env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} run: | fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } mode="${CLAUDE_AUTH,,}" @@ -69,16 +72,30 @@ jobs: ''|subscription) mode=subscription ;; api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; oidc) - for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do - [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" - done - [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" - { - echo "federation_rule_id=$WIF_RULE_ID" - echo "organization_id=$WIF_ORG_ID" - echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" - echo "workspace_id=$WIF_WORKSPACE_ID" - } >> "$GITHUB_OUTPUT" ;; + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac echo "mode=$mode" >> "$GITHUB_OUTPUT" @@ -206,9 +223,9 @@ jobs: - Keep the analysis comment concise but informative # The session's cost (Claude Code's own estimate at API list price), - # model and credential mode, and whether it stopped at a Claude spend or - # usage limit (output limit_reached). Never fatal; a limit stop takes - # "Handle failure" like any other. + # model and credential mode, and whether Claude stopped it: a spend or + # usage limit, or refused credentials (outputs stopped, stop_reason). + # Never fatal; a stop takes "Handle failure" like any other failure. - name: Claude session summary id: validate_cost if: ${{ !cancelled() && (steps.validate.outcome == 'success' || steps.validate.outcome == 'failure') }} @@ -219,18 +236,25 @@ jobs: CLAUDE_STEP: report-validate # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -238,13 +262,22 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" - name: Add reaction on success if: steps.check.outputs.should_run == 'true' && steps.validate.outcome == 'success' diff --git a/.github/workflows/review-retest.yml b/.github/workflows/review-retest.yml index 793d3c1ce2d..fc6dbd86a81 100644 --- a/.github/workflows/review-retest.yml +++ b/.github/workflows/review-retest.yml @@ -235,17 +235,21 @@ jobs: # subscription. Identical in every job that runs Claude # (tests/unit/workflows/test_claude_auth.py); # docs/workflows/overview.md#claude-credentials. - # This job has no id-token on purpose (no Claude App token, so no - # session can write): under `oidc` it stops here. + # This job has no id-token on purpose (no session gets a token for the + # federation or Google Cloud): under `oidc` it runs on the + # subscription, logged in the job summary. - name: Resolve Claude credentials id: claude_auth env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} run: | fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } mode="${CLAUDE_AUTH,,}" @@ -253,16 +257,30 @@ jobs: ''|subscription) mode=subscription ;; api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; oidc) - for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do - [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" - done - [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" - { - echo "federation_rule_id=$WIF_RULE_ID" - echo "organization_id=$WIF_ORG_ID" - echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" - echo "workspace_id=$WIF_WORKSPACE_ID" - } >> "$GITHUB_OUTPUT" ;; + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac echo "mode=$mode" >> "$GITHUB_OUTPUT" @@ -383,9 +401,9 @@ jobs: instead. Do not run `gh` or `git push`; every other instruction applies unchanged. # The session's cost (Claude Code's own estimate at API list price), - # model and credential mode, and whether it stopped at a Claude spend or - # usage limit (output limit_reached). Never fatal; a stopped cell is - # collected as a failed one. + # model and credential mode, and whether Claude stopped it: a spend or + # usage limit, or refused credentials (outputs stopped, stop_reason). + # Never fatal; a stopped cell is collected as a failed one. - name: Claude session summary id: review_cost if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} @@ -396,18 +414,25 @@ jobs: CLAUDE_STEP: review-retest # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -415,13 +440,22 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" - name: Collect the cell if: always() diff --git a/.github/workflows/spec-create.yml b/.github/workflows/spec-create.yml index 0685b269914..35468587a48 100644 --- a/.github/workflows/spec-create.yml +++ b/.github/workflows/spec-create.yml @@ -86,10 +86,13 @@ jobs: env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} run: | fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } mode="${CLAUDE_AUTH,,}" @@ -97,16 +100,30 @@ jobs: ''|subscription) mode=subscription ;; api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; oidc) - for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do - [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" - done - [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" - { - echo "federation_rule_id=$WIF_RULE_ID" - echo "organization_id=$WIF_ORG_ID" - echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" - echo "workspace_id=$WIF_WORKSPACE_ID" - } >> "$GITHUB_OUTPUT" ;; + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac echo "mode=$mode" >> "$GITHUB_OUTPUT" @@ -297,9 +314,10 @@ jobs: - STOP after step 11 # The session's cost (Claude Code's own estimate at API list price), - # model and credential mode, and whether it stopped at a Claude spend or - # usage limit (output limit_reached). Read right after the step: the - # retry writes the same execution file. Never fatal. + # model and credential mode, and whether Claude stopped it: a spend or + # usage limit, or refused credentials (outputs stopped, stop_reason). + # Read right after the step: the retry writes the same execution file. + # Never fatal. - name: Claude session summary id: process_cost if: ${{ !cancelled() && (steps.process.outcome == 'success' || steps.process.outcome == 'failure') }} @@ -310,18 +328,25 @@ jobs: CLAUDE_STEP: spec-create # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -329,17 +354,27 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" - # Not at a spend or usage limit: the retry would be refused the same way. + # Not after Claude stopped the session (a limit or refused credentials): + # the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' && steps.process_cost.outputs.limit_reached != 'true' + if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' && steps.process_cost.outputs.stopped != 'true' id: process_retry timeout-minutes: 30 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 @@ -522,18 +557,25 @@ jobs: CLAUDE_STEP: spec-create/retry # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -541,20 +583,30 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" - # The first session stopped at a spend or usage limit and its retry was - # skipped: fail here instead of reading outputs that were never written. - - name: Stop at the Claude spend or usage limit - if: steps.process_cost.outputs.limit_reached == 'true' + # Claude stopped the first session (a limit or refused credentials) and + # its retry was skipped: fail here instead of reading outputs that were + # never written. + - name: Stop when Claude stopped the session + if: steps.process_cost.outputs.stopped == 'true' run: | - echo "::error::Claude spend/usage limit reached; the retry is skipped" + echo "::error::Claude stopped the session (${{ steps.process_cost.outputs.stop_reason }}); the retry is skipped" exit 1 - name: Extract outputs diff --git a/.github/workflows/util-claude.yml b/.github/workflows/util-claude.yml index 6418346836f..a479bc1a8e8 100644 --- a/.github/workflows/util-claude.yml +++ b/.github/workflows/util-claude.yml @@ -49,10 +49,13 @@ jobs: env: CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} run: | fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } mode="${CLAUDE_AUTH,,}" @@ -60,16 +63,30 @@ jobs: ''|subscription) mode=subscription ;; api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; oidc) - for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do - [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" - done - [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || fail "CLAUDE_AUTH is 'oidc' but this job has no 'id-token: write' permission" - { - echo "federation_rule_id=$WIF_RULE_ID" - echo "organization_id=$WIF_ORG_ID" - echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" - echo "workspace_id=$WIF_WORKSPACE_ID" - } >> "$GITHUB_OUTPUT" ;; + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; esac echo "mode=$mode" >> "$GITHUB_OUTPUT" @@ -98,9 +115,10 @@ jobs: actions: read # The session's cost (Claude Code's own estimate at API list price), - # model and credential mode, and whether it stopped at a Claude spend or - # usage limit (output limit_reached). Read right after the step: the - # retry writes the same execution file. Never fatal. + # model and credential mode, and whether Claude stopped it: a spend or + # usage limit, or refused credentials (outputs stopped, stop_reason). + # Read right after the step: the retry writes the same execution file. + # Never fatal. - name: Claude session summary id: claude_cost if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} @@ -111,18 +129,25 @@ jobs: CLAUDE_STEP: util-claude # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -130,17 +155,27 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" - # Not at a spend or usage limit: the retry would be refused the same way. + # Not after Claude stopped the session (a limit or refused credentials): + # the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.limit_reached != 'true' + if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.stopped != 'true' id: claude_retry uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: @@ -165,18 +200,25 @@ jobs: CLAUDE_STEP: util-claude/retry # Only the error fields of a session that failed, because its own # text may quote the same messages: the error result, Claude Code's - # error messages and its API retries. + # error messages and its API retries, with their HTTP status. SESSION_ERRORS: >- ([.[] | select(.type == "result")] | last) as $result | if $result != null and $result.is_error != true then empty else (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) - | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end)) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) end - # The API's spend-limit, spend-cap and credit messages, Claude Code's - # billing_error and credit message, and its subscription usage-limit - # messages (the SDK's USAGE_LIMIT_ERROR_PREFIXES). + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). LIMIT_PATTERN: >- reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] run: | cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) @@ -184,19 +226,29 @@ jobs: echo "$line" echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) - limit_reached=false + reason="" if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then - limit_reached=true - echo "::error::Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until the limit resets or is raised" - echo "- **Claude spend/usage limit reached (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" fi - echo "limit_reached=${limit_reached}" >> "$GITHUB_OUTPUT" + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" - # The first session stopped at a spend or usage limit and its retry was - # skipped: the run fails instead of reporting a success it did not have. - - name: Stop at the Claude spend or usage limit - if: steps.claude_cost.outputs.limit_reached == 'true' + # Claude stopped the first session (a limit or refused credentials) and + # its retry was skipped: the run fails instead of reporting a success it + # did not have. + - name: Stop when Claude stopped the session + if: steps.claude_cost.outputs.stopped == 'true' run: | - echo "::error::Claude spend/usage limit reached; the retry is skipped" + echo "::error::Claude stopped the session (${{ steps.claude_cost.outputs.stop_reason }}); the retry is skipped" exit 1 diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index 923a3718f93..9e60083fbbd 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -20,6 +20,10 @@ run-name: "Watchdog: scan ${{ github.event_name == 'workflow_dispatch' && '(manu # 4. daily-regen's cron silently starved by GitHub's scheduler (no run # for >26 h outside the quiet window) — re-dispatched manually. # +# PRs labelled `ai-review-paused` or `ai-repair-paused` (Claude stopped the +# review or repair at a spend or usage limit, or refused its credentials) are +# skipped: daily-regen's pair mode resumes them. +# # Retries are bounded per-cause via marker labels: # - `ai-review-rescued` (review failures) # - `watchdog:repair-rescued-` (repair failures, one per attempt#) @@ -200,6 +204,16 @@ jobs: spec_id=$(echo "$branch" | cut -d'/' -f2) library=$(echo "$branch" | cut -d'/' -f3) + # Paused: Claude stopped the PR's review or repair (a spend or + # usage limit, or refused credentials). A rescue now would be + # refused the same way and spend the PR's one-shot rescue on a + # stop that is still active. daily-regen's pair mode, or a manual + # rerun, resumes the PR, and the resumed run removes the label. + if echo " $labels " | grep -qE " (ai-review-paused|ai-repair-paused) "; then + echo "::notice::PR #$num: paused (Claude stopped its review or repair) — skipped" + continue + fi + # Case 0: regeneration the regen gate KEPT (live implementation # stays) whose PR is still open — impl-review's close was lost. # Re-close it; nothing else to do (main and GCS are untouched). diff --git a/agentic/docs/project-guide.md b/agentic/docs/project-guide.md index d75358a1de6..299bca9d84d 100644 --- a/agentic/docs/project-guide.md +++ b/agentic/docs/project-guide.md @@ -799,7 +799,7 @@ The new architecture separates specification and implementation processes: - **Adopt mode.** The `source_ref` input (default empty) on `impl-generate.yml` and `bulk-generate.yml` takes each implementation file from that branch, tag or SHA instead of running Claude, for code written outside the pipeline (a `gemini/*` branch). The run renders the file with the generator's commands, checks the canvas with `core.canvas`, and is always a forced regeneration (`regen:forced`); the routed model only runs repair, and the PR body adds a `**Generation:** adopted from …` line above `**Model:**`. The optional `source_model` input (a model name such as `gemini-4-argon`, only with `source_ref`) names the model that wrote the code and goes on that line. The metadata records `generated_by: {source_model}`, or `generated_by: adopted:{source_ref}` without one, followed by `adopted_from: {source_ref} ({sha})` for every adopted file. Both workflows reject a malformed `source_ref` or `source_model` without echoing it. Retries keep both inputs; the watchdog forwards neither. Python and JavaScript only. A missing file or a failed render is never retried and never counts toward the 3-attempt cap. See `docs/workflows/overview.md#adopt-implementations-from-a-branch`. - **Review model.** Every quality review runs on Opus, whatever model generated the implementation: first implementations, reviews after a repair, and the regen gate's session (the predecessor's re-score and the new render's review). Only `impl-review.yml` chooses it; its `review_model` input (default `opus`) pins another model for one manual review and that run's own auto-retry, and no other workflow forwards one. The retest harness's `models=production` runs every cell on Opus to match (`review_retest.PRODUCTION_MODELS`, kept equal by `tests/unit/workflows/test_model_routing.py`). - Fixed models: `spec-create.yml` runs on Opus; the spec polish and cross-library similarity audit in `daily-regen.yml` run on Sonnet. See `docs/workflows/overview.md#pipeline-models`. -- **Credentials.** The repository variable `CLAUDE_AUTH` chooses what every Claude step runs on: unset or `subscription` = the `CLAUDE_CODE_OAUTH_TOKEN` secret (the owner's subscription), `oidc` = workload identity federation with the `CLAUDE_WIF_*` variables (API credits of the `ci-regen` workspace; accepts only runs on `main`), `api` = the `ANTHROPIC_API_KEY` secret (API credits, the fallback). The step "Resolve Claude credentials" (id `claude_auth`) at the start of every Claude job stops on any other value or a missing credential, never falling back to the subscription, and every Claude step reads its outputs, so each passes exactly one credential. After each Claude step, "Claude session summary" (id `_cost`) records `total_cost_usd`, the model and the mode, and flags a stop at a Claude spend or usage limit in any mode (output `limit_reached`: the API's spend limit, spend cap or credit balance, the subscription's usage limit): the retry is skipped and impl-generate, impl-review and impl-repair park the work as an infrastructure stop, without a retry, a counted attempt or `impl::failed`. A new Claude step copies all three patterns, which `tests/unit/workflows/test_claude_auth.py` enforces. See `docs/workflows/overview.md#claude-credentials`. +- **Credentials.** The repository variable `CLAUDE_AUTH` chooses what every Claude step runs on: unset or `subscription` = the `CLAUDE_CODE_OAUTH_TOKEN` secret (the owner's subscription), `oidc` = workload identity federation with the `CLAUDE_WIF_*` variables (API credits of the `ci-regen` workspace; accepts only runs on `main`, so pull-request events, other refs and jobs without `id-token`, such as review-retest, run on the subscription with a logged notice), `api` = the `ANTHROPIC_API_KEY` secret (API credits, the fallback). The step "Resolve Claude credentials" (id `claude_auth`) at the start of every Claude job stops on any other value or a missing credential, never falling back to the subscription silently, and every Claude step reads its outputs, so each passes exactly one credential. After each Claude step, "Claude session summary" (id `_cost`) records `total_cost_usd`, the model and the mode, and flags when Claude stopped the session (outputs `stopped`, `stop_reason`): a spend or usage limit in any mode, or refused credentials with `api`/`oidc`. The retry is skipped; impl-generate parks the pair, impl-review and impl-repair pause the PR (`ai-review-paused`, `ai-repair-paused`, skipped by the watchdog, resumed first by daily-regen's pair mode), all without a retry, a counted attempt or `impl::failed`. A new Claude step copies all three patterns, which `tests/unit/workflows/test_claude_auth.py` enforces. See `docs/workflows/overview.md#claude-credentials`. ## GitHub Issue Labels diff --git a/automation/scripts/README.md b/automation/scripts/README.md index d27543aea20..2bf88a2b191 100644 --- a/automation/scripts/README.md +++ b/automation/scripts/README.md @@ -258,19 +258,23 @@ Closes a spec issue once every supported library is `impl::done` or ### `regen_pick_pairs.py` The pick of `daily-regen.yml`'s pair mode (`REGEN_PAIRS_PER_DAY`, or the -`pairs` input): the N oldest (spec, library) pairs across the catalogue, aged by -the newer of their metadata `updated` and their last real attempt (a merged or -closed implementation PR, `regen:kept` included, or a failed generation's -`impl-fail` comment; infrastructure failures and spend or usage limit stops don't -count). It skips pairs touched within `--attempt-window-days` (default 30), -pairs with an open implementation PR, and every pair of a spec whose issue saw -activity within `--min-age-hours`. PyYAML plus `gh`; a failed read degrades -with a warning. The library list comes from the caller. +`pairs` input). First the resumes: open implementation PRs whose review or +repair Claude stopped (`ai-review-paused`, `ai-repair-paused`), oldest first, +counted against N. Then the oldest (spec, library) pairs across the catalogue, +aged by the newer of their metadata `updated` and their last real attempt (a +merged or closed implementation PR, `regen:kept` included, or a failed +generation's `impl-fail` comment; infrastructure failures and generations +Claude stopped don't count). It skips pairs touched within +`--attempt-window-days` (default 30), pairs with an open implementation PR, +and every pair of a spec whose issue saw activity within `--min-age-hours`. +PyYAML plus `gh`. It fails closed: a failed implementation PR listing exits 1, +a spec whose comments can't be read sits out the night, and the kept and issue +listings degrade with a warning. The library list comes from the caller. ```bash python3 automation/scripts/regen_pick_pairs.py --count 5 --min-age-hours 20 \ --attempt-window-days 30 --repo OWNER/NAME --libraries "altair bokeh ..." -# writes pairs_json, specs, specs_json and count to $GITHUB_OUTPUT +# writes resume_json, resume_count, pairs_json, specs, specs_json and count to $GITHUB_OUTPUT ``` ### `spec_pr_guard.py` diff --git a/automation/scripts/regen_pick_pairs.py b/automation/scripts/regen_pick_pairs.py index 45d4adf62a3..f371000b2a3 100644 --- a/automation/scripts/regen_pick_pairs.py +++ b/automation/scripts/regen_pick_pairs.py @@ -1,9 +1,17 @@ #!/usr/bin/env python3 -"""Pick the oldest implementations for daily-regen's pair mode. +"""Pick the night's work for daily-regen's pair mode. Spec mode, the default, regenerates the N oldest specs with all their -libraries. Pair mode picks the N oldest implementations, one (spec, library) -pair each, across the catalogue: +libraries. Pair mode spends the night's N on two kinds of work: + +1. Resumes first. Open implementation PRs whose review or repair Claude + stopped (a spend or usage limit, or refused credentials) carry + ``ai-review-paused`` or ``ai-repair-paused``; the oldest of them are + re-dispatched before anything new. A refused request costs nothing, so a + stop that is still active only pauses them again, and they resume on + their own once it lifts. +2. Then the oldest implementations, one (spec, library) pair each, across the + catalogue: - A pair's age counts from its newest touch: its metadata ``updated`` (``created`` when there is none), or its last real regeneration attempt, @@ -17,11 +25,11 @@ on the spec issue. A merge moves ``updated`` anyway; the other two leave it alone, and without them the same pair would come back the next night. An infrastructure failure (its comment carries ``impl-fail-cause:infra``) and - a stop at a Claude spend or usage limit (its own pause comment, - ``impl-paused-claude-limit``) are not attempts: the pair stays in the queue - and comes back on a later night. + a generation Claude stopped (its own pause comment, ``impl-paused-claude``) + are not attempts: the pair stays in the queue and comes back on a later + night. - A pair with an open implementation PR is skipped: its attempt is still - running or stuck, and the watchdog owns it. + running, stuck or paused, and the watchdog or the resume owns it. - Every pair of a spec whose issue saw activity within ``--min-age-hours`` is skipped, like the spec mode's postponement: a comment, a label or a report holds the spec back, and a night does not stack a spec whose last batch is @@ -31,12 +39,13 @@ pair mode never writes a first implementation; gap backfill stays with ``bulk-generate.yml``. -The PR and issue listings are one ``gh`` call each, at most 1000 entries, most -recent first (kept regenerations get a listing of their own, which reaches -much further back). The failed-generation comments are read per spec issue, -only for the specs at the front of the queue, at most ``--max-issue-reads`` -issues per run. A read that fails degrades to the other records with a -warning, like the spec mode's issue listing. +The implementation-PR listing is the loop guard: when it fails, the run stops +with an error instead of picking without it. The kept-regeneration and issue +listings are best effort (one ``gh`` call each, at most 1000 entries, most +recent first; kept regenerations reach much further back than the PR +listing). The failed-generation comments are read per spec issue, only for +the specs at the front of the queue, at most ``--max-issue-reads`` issues per +run; a spec whose comments cannot be read is left out of the night. Usage, from the repository root:: @@ -44,9 +53,9 @@ --attempt-window-days 30 --repo OWNER/NAME --libraries "altair bokeh ..." The library list comes from the caller (``core.constants.SUPPORTED_LIBRARIES``), -so this file carries no copy that could drift. Writes ``pairs_json``, -``specs``, ``specs_json`` and ``count`` to ``$GITHUB_OUTPUT`` and a table of -the picks to ``$GITHUB_STEP_SUMMARY``. +so this file carries no copy that could drift. Writes ``resume_json``, +``resume_count``, ``pairs_json``, ``specs``, ``specs_json`` and ``count`` (new +pairs) to ``$GITHUB_OUTPUT`` and a table of both to ``$GITHUB_STEP_SUMMARY``. """ from __future__ import annotations @@ -59,7 +68,7 @@ import subprocess import sys from collections.abc import Callable, Iterable -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -70,14 +79,21 @@ ANCIENT = datetime.min.replace(tzinfo=timezone.utc) LIMIT = "1000" BRANCH_RE = re.compile(r"^implementation/(?P[a-z0-9][a-z0-9-]*)/(?P[a-z0-9]+)$") +ATTEMPT_RE = re.compile(r"^ai-attempt-(?P[0-9]+)$") # impl-generate.yml "Handle generation failure": every failure comment carries # the marker; an infrastructure failure's comment also carries the infra tag. FAIL_MARKER_RE = re.compile(r"") INFRA_TAG = "" +# impl-review.yml and impl-repair.yml set these when Claude stopped the session. +PAUSED_LABELS = {"ai-repair-paused": "repair", "ai-review-paused": "review"} Pair = tuple[str, str] +class ListingError(Exception): + """A listing the pick cannot do without failed.""" + + @dataclass(frozen=True) class Candidate: spec: str @@ -93,6 +109,16 @@ class Pick: basis: str +@dataclass(frozen=True) +class Resume: + pr: int + kind: str # "review" or "repair" + spec: str + library: str + attempt: int + updated: datetime + + @dataclass class Records: """What GitHub says about past attempts, per pair and per spec issue.""" @@ -100,6 +126,7 @@ class Records: attempts: dict[Pair, tuple[datetime, str]] open_pairs: set[Pair] issue_activity: dict[int, datetime] + paused: list[Resume] = field(default_factory=list) def parse_ts(raw: str) -> datetime | None: @@ -150,19 +177,25 @@ def _newer(records: dict[Pair, tuple[datetime, str]], pair: Pair, when: datetime def fold_pulls(pulls: Iterable[dict[str, Any]], records: Records) -> None: - """An open PR blocks its pair; a merged or closed one is the pair's attempt.""" + """An open PR blocks its pair (and is a resume when paused); a merged or closed one is an attempt.""" for pull in pulls: match = BRANCH_RE.match(str(pull.get("headRefName") or "")) if not match: continue pair = (match["spec"], match["lib"]) state = pull.get("state") + labels = {str(label.get("name")) for label in pull.get("labels") or [] if isinstance(label, dict)} if state == "OPEN": records.open_pairs.add(pair) + kind = next((PAUSED_LABELS[name] for name in PAUSED_LABELS if name in labels), None) + number = pull.get("number") + if kind and isinstance(number, int) and all(r.pr != number for r in records.paused): + attempts = [int(m["n"]) for name in labels if (m := ATTEMPT_RE.match(name))] + updated = parse_ts(str(pull.get("updatedAt") or "")) or ANCIENT + records.paused.append(Resume(number, kind, pair[0], pair[1], max(attempts, default=1), updated)) continue if state not in ("MERGED", "CLOSED"): continue - labels = {str(label.get("name")) for label in pull.get("labels") or [] if isinstance(label, dict)} basis = "merged PR" if state == "MERGED" else "kept PR" if "regen:kept" in labels else "closed PR" when = parse_ts(str(pull.get("closedAt") or pull.get("updatedAt") or "")) _newer(records.attempts, pair, when, basis) @@ -197,6 +230,11 @@ def _touched(pair: Pair, metadata: datetime | None, records: Records) -> tuple[d return touched, basis +def resumes(records: Records, count: int) -> list[Resume]: + """The oldest paused reviews and repairs, at most ``count``.""" + return sorted(records.paused, key=lambda r: (r.updated, r.pr))[:count] + + def pick( candidates: Iterable[Candidate], records: Records, @@ -205,15 +243,17 @@ def pick( min_age: timedelta, window: timedelta, count: int, - failures: Callable[[str], dict[str, datetime]] | None = None, + failures: Callable[[str], dict[str, datetime] | None] | None = None, max_reads: int = 100, ) -> tuple[list[Pick], int]: """The ``count`` oldest eligible pairs, oldest first, and how many were eligible. - ``failures(spec)`` returns the spec's failed generations per library. It is - asked lazily, once per spec, when one of the spec's pairs reaches the front - of the queue; a pair whose failure is newer than its age goes back into - the queue, or out of it while the failure is within the window. + ``failures(spec)`` returns the spec's failed generations per library, or + None when they cannot be read. It is asked lazily, once per spec, when one + of the spec's pairs reaches the front of the queue; a pair whose failure is + newer than its age goes back into the queue, or out of it while the + failure is within the window. A spec whose failures cannot be read is + left out of the night (fail closed: its pairs may have failed last night). """ metadata: dict[Pair, datetime | None] = {} queue: list[tuple[datetime, str, str, str]] = [] @@ -233,6 +273,7 @@ def pick( eligible = len(queue) read: set[str] = set() + unreadable: set[str] = set() picks: list[Pick] = [] while queue and len(picks) < count: key, spec, lib, basis = heapq.heappop(queue) @@ -242,8 +283,15 @@ def pick( failures = None else: read.add(spec) - for flib, when in failures(spec).items(): + found = failures(spec) + if found is None: + unreadable.add(spec) + print(f"::warning::Leaving {spec} out of tonight: its failed generations could not be read") + for flib, when in (found or {}).items(): _newer(records.attempts, (spec, flib), when, "failed generation") + if spec in unreadable: + eligible -= 1 + continue touched, new_basis = _touched((spec, lib), metadata[(spec, lib)], records) if (touched or ANCIENT) != key: if touched is not None and now - touched < window: @@ -260,18 +308,23 @@ def _gh_json(args: list[str], what: str) -> list[dict[str, Any]] | None: raw = subprocess.run(["gh", *args], check=True, capture_output=True, text=True).stdout data = json.loads(raw) except (subprocess.CalledProcessError, ValueError, OSError) as exc: - print(f"::warning::Could not list {what} ({' '.join(str(exc).split())[:200]}) — picking without them") + print(f"::warning::Could not list {what} ({' '.join(str(exc).split())[:200]})") return None return data if isinstance(data, list) else None def read_records(repo: str) -> Records: records = Records(attempts={}, open_pairs=set(), issue_activity={}) - pr_list = ["pr", "list", "--repo", repo, "--limit", LIMIT, "--json", "headRefName,state,updatedAt,closedAt,labels"] + fields = "number,headRefName,state,updatedAt,closedAt,labels" + pr_list = ["pr", "list", "--repo", repo, "--limit", LIMIT, "--json", fields] pulls = _gh_json( [*pr_list, "--state", "all", "--search", "head:implementation/ sort:updated-desc"], "implementation PRs" ) - fold_pulls(pulls or [], records) + if pulls is None: + # The loop guard: without it a kept, closed or paused pair would be + # picked again tonight. + raise ListingError("the implementation PRs could not be listed") + fold_pulls(pulls, records) # Kept regenerations again on their own: they are the attempts that leave # `updated` alone, and this listing reaches much further back. kept = _gh_json( @@ -285,16 +338,16 @@ def read_records(repo: str) -> Records: ) fold_issues(issues or [], records) print( - f"::notice::Attempt records for {len(records.attempts)} pair(s), {len(records.open_pairs)} open PR(s), " - f"activity for {len(records.issue_activity)} issue(s)" + f"::notice::Attempt records for {len(records.attempts)} pair(s), {len(records.open_pairs)} open PR(s) " + f"({len(records.paused)} paused), activity for {len(records.issue_activity)} issue(s)" ) return records -def issue_failures(repo: str, spec_issues: dict[str, int]) -> Callable[[str], dict[str, datetime]]: +def issue_failures(repo: str, spec_issues: dict[str, int]) -> Callable[[str], dict[str, datetime] | None]: """``failures(spec)`` backed by the spec issue's comments (one paginated read per spec).""" - def failures(spec: str) -> dict[str, datetime]: + def failures(spec: str) -> dict[str, datetime] | None: issue = spec_issues.get(spec) if issue is None: return {} @@ -309,19 +362,19 @@ def failures(spec: str) -> dict[str, datetime]: comments = [json.loads(line) for line in raw.splitlines() if line.strip()] except (subprocess.CalledProcessError, ValueError, OSError) as exc: print(f"::warning::Could not read the comments of issue #{issue} ({' '.join(str(exc).split())[:200]})") - return {} + return None return failed_generations(comments, spec) return failures -def summary_table(picks: list[Pick], eligible: int, count: int) -> str: - lines = [ - f"### Pair mode: {len(picks)} of {count} requested ({eligible} eligible)", - "", - "| # | Spec | Library | Last touched | From |", - "|---|------|---------|--------------|------|", - ] +def summary_table(resumed: list[Resume], picks: list[Pick], eligible: int, count: int) -> str: + lines = [f"### Pair mode: {len(resumed)} resumed, {len(picks)} new pair(s) of {count} ({eligible} eligible)", ""] + if resumed: + lines += ["| Resume | PR | Spec | Library |", "|--------|----|------|---------|"] + lines += [f"| {r.kind} | #{r.pr} | `{r.spec}` | `{r.library}` |" for r in resumed] + lines.append("") + lines += ["| # | Spec | Library | Last touched | From |", "|---|------|---------|--------------|------|"] for i, p in enumerate(picks, 1): touched = "never" if p.touched == ANCIENT else p.touched.strftime("%Y-%m-%dT%H:%M:%SZ") lines.append(f"| {i} | `{p.spec}` | `{p.library}` | {touched} | {p.basis} |") @@ -345,7 +398,12 @@ def main(argv: list[str] | None = None) -> int: return 1 candidates, spec_issues = load_candidates(Path(args.plots), libraries) - records = read_records(args.repo) + try: + records = read_records(args.repo) + except ListingError as exc: + print(f"::error::Pair mode stops: {exc}, so the pick would lose its record of last nights' attempts") + return 1 + resumed = resumes(records, args.count) picks, eligible = pick( candidates, records, @@ -353,11 +411,12 @@ def main(argv: list[str] | None = None) -> int: datetime.now(timezone.utc), timedelta(hours=args.min_age_hours), timedelta(days=args.attempt_window_days), - args.count, + args.count - len(resumed), failures=issue_failures(args.repo, spec_issues), max_reads=args.max_issue_reads, ) + print(f"::notice::Resumed: {[f'#{r.pr} {r.kind}' for r in resumed]}") print( f"::notice::Eligible pairs: {eligible} (attempt window {args.attempt_window_days} d) " f"picked: {[f'{p.spec}/{p.library}' for p in picks]}" @@ -367,7 +426,12 @@ def main(argv: list[str] | None = None) -> int: specs = list(dict.fromkeys(p.spec for p in picks)) pairs = [{"spec": p.spec, "library": p.library} for p in picks] + resume_rows = [ + {"pr": r.pr, "kind": r.kind, "spec": r.spec, "library": r.library, "attempt": r.attempt} for r in resumed + ] with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"resume_json={json.dumps(resume_rows, separators=(',', ':'))}\n") + f.write(f"resume_count={len(resumed)}\n") f.write(f"pairs_json={json.dumps(pairs, separators=(',', ':'))}\n") f.write(f"specs={' '.join(specs)}\n") f.write(f"specs_json={json.dumps(specs)}\n") @@ -375,7 +439,7 @@ def main(argv: list[str] | None = None) -> int: summary = os.environ.get("GITHUB_STEP_SUMMARY") if summary: with open(summary, "a", encoding="utf-8") as f: - f.write(summary_table(picks, eligible, args.count)) + f.write(summary_table(resumed, picks, eligible, args.count)) return 0 diff --git a/changelog.d/ci-claude-auth-switch.md b/changelog.d/ci-claude-auth-switch.md index 7af58a705dc..6f1c0b60960 100644 --- a/changelog.d/ci-claude-auth-switch.md +++ b/changelog.d/ci-claude-auth-switch.md @@ -10,19 +10,23 @@ "Resolve Claude credentials", stops the job on any other value or a missing credential instead of falling back to the subscription and the owner's interactive quota, and every Claude step reads its outputs, so each passes - exactly one credential and a new mode changes one step. - `docs/workflows/overview.md` has the Console setup, the switch commands and - the runs federation refuses. -- **Every Claude session's cost in the job summary, and a clean stop at a - spend or usage limit.** A step after each Claude step writes + exactly one credential and a new mode changes one step. Under `oidc`, the + runs federation refuses (pull-request review events, other refs, + `review-retest.yml`) use the subscription and say so in the job summary. + `docs/workflows/overview.md` has the Console setup, the switch commands, + and how to close the federation path when `oidc` is off. +- **Every Claude session's cost in the job summary, and a clean stop when + Claude refuses a session.** A step after each Claude step writes `total_cost_usd` (Claude Code's estimate at API list price), the model and the credential mode to the job summary and the log, and never fails the job. It also reads, from the error fields of a failed session, the API's - spend-limit, spend-cap and credit messages and the subscription's - usage-limit messages: the retry is skipped, `impl-generate.yml` parks the - pair, `impl-review.yml` and `impl-repair.yml` hand the PR on without - spending their automatic retry, and nothing counts as an attempt or marks - `impl::failed`, a first implementation included. Before, a + spend-limit, spend-cap and credit messages, the subscription's usage-limit + messages, and, with `api` or `oidc`, refused credentials: the retry is + skipped, `impl-generate.yml` parks the pair, `impl-review.yml` and + `impl-repair.yml` pause the PR with the new labels `ai-review-paused` and + `ai-repair-paused` without spending their automatic retry or recording a + score, the watchdog skips paused PRs, and nothing counts as an attempt or + marks `impl::failed`, a first implementation included. Before, a subscription usage limit used up a pair's three attempts. The weekly regen review's `collect_week.py --costs` reads the line, now counts the `daily-regen.yml` pre-flight too, and names the part that ran on API @@ -37,8 +41,8 @@ attempted (a closed PR, a kept regeneration, a failed generation) within the attempt window, 30 days unless `attempt_window_days` or `REGEN_ATTEMPT_WINDOW_DAYS` says otherwise, is not picked again, while an - infrastructure failure or a stop at a spend or usage limit leaves it in the - queue. Unset, - the schedule regenerates whole specs as before; a manual run opts in with - the new `pairs` input, and the watchdog's cron rescue forwards the - variable. + infrastructure failure or a generation Claude stopped leaves it in the + queue. Paused reviews and repairs are resumed first and count against N, + and the pick stops when it can't list the implementation PRs. Unset, the + schedule regenerates whole specs as before; a manual run opts in with the + new `pairs` input, and the watchdog's cron rescue forwards the variable. diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index 4ffe34044ec..28400d56815 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -228,7 +228,7 @@ PR. The same test checks that every pin resolves. | `generate:{library}` | Trigger generation for library | User | | `impl:{library}:pending` | Generation in progress | Workflow | | `impl:{library}:done` | Implementation merged to main | Workflow | -| `impl:{library}:failed` | Set by `impl-generate.yml` after three failed generation attempts for the pair within a 12-hour window, or by `impl-review.yml` when the PR still scores below 50 after four repair attempts. Infrastructure failures (provider incident, cloud auth, GitHub API) are retried on a separate cap of 5 and never set this label; a pair paused that way carries no label. An adopt-mode source failure never sets it either (see [Adopt implementations from a branch](#adopt-implementations-from-a-branch)). | Workflow | +| `impl:{library}:failed` | Set by `impl-generate.yml` after three failed generation attempts for the pair within a 12-hour window, or by `impl-review.yml` when the PR still scores below 50 after four repair attempts. Infrastructure failures (provider incident, cloud auth, GitHub API) are retried on a separate cap of 5 and never set this label; a pair paused that way carries no label. A generation Claude stopped never sets it (see [When Claude stops a session](#when-claude-stops-a-session)), and an adopt-mode source failure doesn't either (see [Adopt implementations from a branch](#adopt-implementations-from-a-branch)). | Workflow | ### PR labels (on Pull Requests) @@ -237,6 +237,8 @@ PR. The same test checks that every pin resolves. | `ai-approved` | Quality check passed (based on cascading thresholds) | Workflow | | `ai-rejected` | Quality check failed, triggers repair | Workflow | | `ai-attempt-1/2/3/4` | Retry counter | Workflow | +| `ai-review-paused` | Claude stopped the PR's review (a spend or usage limit, or refused credentials): no score, no verdict. The watchdog skips the PR; pair mode or a manual rerun resumes it, and the next review removes the label (see [When Claude stops a session](#when-claude-stops-a-session)) | Workflow | +| `ai-repair-paused` | Claude stopped the PR's repair, next to `ai-rejected`; the attempt isn't used up. The watchdog skips the PR; pair mode or a manual restart resumes it, and the next repair removes the label | Workflow | | `quality:XX` | Quality score (e.g., quality:92) | Workflow | | `quality-poor` | Score < 50, needs fundamental fixes | Workflow | | `regen` | Regeneration of an implementation that is live on main (marker for humans and the watchdog; `impl-review.yml` decides "regeneration" from `origin/main`) | Workflow | @@ -305,6 +307,8 @@ Both paths write the marker comment before they dispatch the retry, and they ret `ai-review-failed` is removed by the review it asks for. Every review run removes the label when it starts ("Clear a stale ai-review-failed label", best effort). The watchdog leaves a PR with that label alone while a review of it is queued or running, and drops the label without a re-review when the PR already has a verdict label. This matters when GitHub accepts a dispatch but reports an error: the step then adds `ai-review-failed` although the re-review is already queued, and without these two guards the watchdog would start a second review of the same PR. +A review that Claude stopped, at a spend or usage limit or on refused credentials, takes neither path: it gets `ai-review-paused`, spends no retry, and the watchdog skips it (see [When Claude stops a session](#when-claude-stops-a-session)). + `impl-repair.yml` has the same one-retry rule for a repair run that crashes, recorded by a `` marker. It sums the marker count over all comment pages and treats a count it can't read as a spent retry. The watchdog's crashed-repair case takes over from there. The generation retry of `impl-generate.yml` and the post-score review retry run on the ref of the run that failed: a run dispatched from a feature branch retries on that branch, and a production run retries on `main`. The no-score review retry still goes through `repository_dispatch`, which always runs on `main`. @@ -399,7 +403,7 @@ Located in `.github/workflows/`: | `bulk-generate.yml` | Batch implementation generation | | `review-retest.yml` | Dispatch-only measurement: re-runs the AI quality review on the frozen retest set (several fresh sessions per item, regen pairs in both orders) for one rules version, and reports score spread, verdict flips, weakness agreement, and gate order bias against a baseline arm. Read-only tokens; writes nothing outside its own artifacts. See [Review retest](review-retest.md) | | `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (once a day at 02:17 UTC, off the top of the hour to dodge GitHub's scheduler overload). A spec's age counts from the newer of its last merged update and the last activity on its spec issue (every regen touches the issue), so a kept regeneration is not re-picked the next night. Any activity on the issue — a comment, a label, a report — postpones that spec's regen the same way. With the repository variable `REGEN_PAIRS_PER_DAY`, the scheduled run regenerates single (spec, library) pairs instead of whole specs; see [Daily regeneration modes](#daily-regeneration-modes) | -| `watchdog-stuck-jobs.yml` | 6-hourly safety net: re-dispatches stuck reviews, repairs (including a repair that crashed after a rejection), merges and generations (straight to `impl-generate.yml`, marked only once the run exists), re-closes open `regen:kept` PRs, never rescues a regeneration into repair or re-dispatches its failed review (it flags it for manual attention instead; "regeneration" = the `regen` label or the implementation file on main, unless `regen:forced`), and rescues daily-regen when its cron is silently starved by GitHub (>26 h without a run) | +| `watchdog-stuck-jobs.yml` | 6-hourly safety net: re-dispatches stuck reviews, repairs (including a repair that crashed after a rejection), merges and generations (straight to `impl-generate.yml`, marked only once the run exists), re-closes open `regen:kept` PRs, never rescues a regeneration into repair or re-dispatches its failed review (it flags it for manual attention instead; "regeneration" = the `regen` label or the implementation file on main, unless `regen:forced`), skips a PR whose review or repair Claude stopped (`ai-review-paused`, `ai-repair-paused`), and rescues daily-regen when its cron is silently starved by GitHub (>26 h without a run) | | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main, and runs `alembic upgrade head` against Cloud SQL first; a push that only adds a revision under `alembic/versions/` triggers it too, and so does one that changes a pin the library versions are read from (`uv.lock`, `package-lock.json`, `Manifest.toml`, `.python-version`, a setup action) | | `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges | @@ -524,22 +528,34 @@ night: `REGEN_ATTEMPT_WINDOW_DAYS`, else 30 days. After the window, the pair queues behind every pair touched before it. - An infrastructure failure, whose comment carries - ``, and a stop at a Claude spend or usage - limit, whose comment carries ``, are not - attempts: the pair stays at its place in the queue and comes back on a later - night. + ``, and a generation Claude stopped, whose + comment carries ``, are not attempts: the pair + stays at its place in the queue and comes back on a later night. - A pair with an open implementation PR is skipped: its attempt is still - running or stuck. + running, stuck, or paused. - Every pair of a spec whose issue saw activity within `min_age_hours` (default 20) is skipped for that night, as in spec mode. - Only pairs with a metadata file are candidates, so pair mode never writes a first implementation. Gap backfill stays with `bulk-generate.yml`. +Before the new pairs, pair mode resumes the reviews and repairs that Claude +stopped: the open implementation PRs with `ai-review-paused` or +`ai-repair-paused`, oldest first. Each resume counts against N, so a night with +three paused PRs and `REGEN_PAIRS_PER_DAY=5` dispatches three resumes and two +new pairs. The job "Re-dispatch paused reviews and repairs" starts +`impl-review.yml` or `impl-repair.yml` with the arguments the watchdog's +rescues use, at the pace of the pair dispatches. While the stop lasts, a resume +is refused at its first request, which costs nothing, and pauses again. + +The picker fails closed. When the implementation PR listing fails, the run +stops with an error, because that listing records the kept, closed, open, and +paused PRs. The kept-regeneration and issue listings degrade with a warning. The picker reads the failed-generation comments only for the specs at the -front of the queue, at most 100 issues per run. Spec mode keeps its own rule: -a spec's age is the newer of its metadata and any activity on its issue, so a -kept or failed regeneration moves the spec to the back of the queue, and so -does an infrastructure failure or any comment. +front of the queue, at most 100 issues per run, and a spec whose comments +can't be read sits out the night. Spec mode keeps its own rule: a spec's age is +the newer of its metadata and any activity on its issue, so a kept or failed +regeneration moves the spec to the back of the queue, and so does an +infrastructure failure or any comment. Pair mode skips the spec polish, because a spec rewritten for one library would leave its other implementations written against the old text. It @@ -555,20 +571,24 @@ The picks go to the log and the run summary, so a dry run shows them: gh workflow run daily-regen.yml -f pairs=5 -f dry_run=true ``` -The dry run still runs the similarity audit, as in spec mode. To turn pair mode -on for the schedule, run `gh variable set REGEN_PAIRS_PER_DAY --body 5`; to -turn it off, run `gh variable delete REGEN_PAIRS_PER_DAY`. - -When a pre-flight session (the spec polish or the similarity audit) stops at -a [Claude spend or usage limit](#at-a-spend-or-usage-limit), the night's -dispatch for that spec is skipped and the run fails. In pair mode, the -similarity audit of each spec runs before that spec's pairs are dispatched, -one spec after another, so it works as the night's probe: once a pair's -generation, review, or repair has reached the limit, every later spec of the -night stops at its audit and dispatches nothing. Pairs that were already -dispatched fail at their first refused request, which costs nothing, and are -parked without a retry or a counted attempt. Their spec keeps its pairs in -the queue for a later night. +The dry run still runs the similarity audit, as in spec mode. With +`CLAUDE_AUTH=oidc`, run [Test the spend limit](#test-the-spend-limit) first, +before you set the variable. To turn pair mode on for the schedule, run +`gh variable set REGEN_PAIRS_PER_DAY --body 5`; to turn it off, run +`gh variable delete REGEN_PAIRS_PER_DAY`. + +When Claude stops a pre-flight session (the spec polish or the similarity +audit), the night's dispatch for that spec is skipped and the run fails (see +[When Claude stops a session](#when-claude-stops-a-session)). In pair mode, +the similarity audit of each spec runs before that spec's pairs are +dispatched, one spec after another, so it works as the night's probe: once a +pair's generation, review, or repair has reached the limit, every later spec +of the night stops at its audit and dispatches nothing. Pairs that were +already dispatched fail at their first refused request, which costs nothing, +and are parked without a retry or a counted attempt. Their spec keeps its +pairs in the queue for a later night. The resumes don't wait for the audit: +they start in a job of their own, and while the stop lasts each one pauses +again at no cost. --- @@ -701,7 +721,10 @@ interactive quota: - `CLAUDE_AUTH` has an unknown value. - `CLAUDE_AUTH` is `api` and the `ANTHROPIC_API_KEY` secret is missing. - `CLAUDE_AUTH` is `oidc` and one of the four `CLAUDE_WIF_*` variables is - missing or isn't an ID, or the job has no `id-token: write` permission. + missing or isn't an ID. +- `CLAUDE_AUTH` is `oidc`, the run is one that federation refuses, and the + `CLAUDE_CODE_OAUTH_TOKEN` secret is missing (see + [Which runs federation accepts](#which-runs-federation-accepts)). In `spec-create.yml` and `report-validate.yml`, the step runs only for an issue that reaches Claude. In `impl-generate.yml`, it runs before the issue @@ -728,19 +751,24 @@ the Claude steps of these runs work: | `spec-create.yml`, `report-validate.yml` | `issues` (a label) | `…:ref:refs/heads/main` | | `util-claude.yml` | `issue_comment`, also on a pull request | `…:ref:refs/heads/main` | -Federation refuses these runs with `401 Authentication failed`, so they need -`subscription` or `api`: +Federation would refuse the other runs with `401 Authentication failed`, so +under `oidc` they use the subscription on purpose: - `util-claude.yml` started by a `pull_request_review_comment` or a `pull_request_review`: the subject is `repo:MarkusNeusinger/anyplot:pull_request`. - Any workflow dispatched on another branch, such as a branch smoke with `--ref feature/x` and the retries it dispatches on that branch: the subject - is `…:ref:refs/heads/feature/x`. `impl-generate.yml` counts such a refusal as - a failed attempt, like any other authentication error. + is `…:ref:refs/heads/feature/x`. - `review-retest.yml`: its review job has no `id-token` permission on purpose, because without it the action can't mint the Claude GitHub App - token, so no session can write to the repository. Under `oidc`, "Resolve - Claude credentials" stops the job. + token, so no session can write to the repository. A retest arm on `oidc` + runs on the subscription. + +"Resolve Claude credentials" recognizes these runs by the event name, the +ref, and the missing token request URL. It logs +`oidc not available for /; using subscription` as a notice and in +the job summary, then runs the job on the `CLAUDE_CODE_OAUTH_TOKEN` secret. If +the secret is missing, it stops the job instead. Every other job that runs Claude already has `id-token: write`, for the Claude GitHub App token and, in some of them, Google Cloud. The federation @@ -820,12 +848,19 @@ put a generation on Sonnet at about $0.29 per implementation, a review on Opus at about $1.10, and a repair with its re-review at about $2.70 per implementation in all. -### At a spend or usage limit +### When Claude stops a session + +Two stops end the pipeline's work cleanly instead of counting as a failure of +the implementation: -A Claude spend or usage limit stops the pipeline cleanly in every mode. Once -the limit is reached, every request is refused until it resets or you raise -it, and Claude Code's own retries can't get past it. The pipeline recognizes -these texts: +- A Claude spend or usage limit, in every mode. Once the limit is reached, + every request is refused until it resets or you raise it, and Claude Code's + own retries can't get past it. +- Refused credentials, with `api` and `oidc` only: an invalid or revoked API + key, or a federation exchange that the rule refuses. With the subscription, + an authentication error keeps the handling it had before the switch. + +For a limit, the pipeline recognizes these texts: - A spend limit you set on the workspace or the organization (`oidc`, `api`): HTTP 400 `invalid_request_error`, with a message that starts with @@ -845,28 +880,49 @@ these texts: A transient rate limit, such as a per-minute token limit, is none of these: it keeps its retry and the infrastructure path it took before. +For refused credentials, it recognizes the API's error types +`authentication_error` and `permission_error`, HTTP status 401 or 403 on the +failed request, Claude Code's error type `authentication_failed`, and the +texts `Authentication failed` and `invalid x-api-key`. A limit wins over its +status code: a 403 with a usage-limit message counts as a limit. + "Claude session summary" reads these texts only from the error fields of a session that failed: the error result, Claude Code's error messages, and its API retries. It never reads the conversation, which may quote the same text, and never flags a session that ended well. When it finds one, it writes an -`::error::` and the line **Claude spend/usage limit reached (auth=oidc)**, with -the mode, to the job summary, and every path treats the stop as an -infrastructure failure: +`::error::` and the line **Claude spend/usage limit reached (auth=oidc)** or +**Claude credentials refused (auth=oidc)**, with the mode, to the job summary, +and every path treats the stop as an infrastructure failure: | Workflow | What happens | |----------|--------------| -| `impl-generate.yml` | The retry is skipped, and "Stop at the Claude spend or usage limit" fails the job. "Handle generation failure" parks the pair before it counts anything: it removes `generate:{library}` and `impl:{library}:pending`, puts `impl:{library}:done` back for a live implementation, and comments with its own marker. Nothing is retried, no attempt counts toward the cap of three, and the pair is never marked `impl:{library}:failed`, a first implementation included. No PR exists; the next generation recreates the branch. | -| `impl-review.yml` | "Validate review output" adds `ai-review-failed` and comments, without the auto re-review and without spending it. No score, no verdict: the PR stays open, nothing is merged or closed. The watchdog re-dispatches the review once at its next scan. | -| `impl-repair.yml` | The retry is skipped. "Handle repair failure" restores `ai-rejected` and comments, without the auto-retry and without spending it. The watchdog's repair rescue, once per attempt, or a manual restart picks the PR up. | -| `daily-regen.yml` | A spec polish or similarity audit at the limit skips that spec's dispatch, and the entry fails. | +| `impl-generate.yml` | The retry is skipped, and "Stop when Claude stopped the session" fails the job. "Handle generation failure" parks the pair before it counts anything: it removes `generate:{library}` and `impl:{library}:pending`, puts `impl:{library}:done` back for a live implementation, and comments with the marker ``. Nothing is retried, no attempt counts toward the cap of three, and the pair is never marked `impl:{library}:failed`, a first implementation included. No PR exists; the next generation recreates the branch. | +| `impl-review.yml` | No score, no verdict, and no regen gate decision, even when the session wrote `quality_score.txt` before it stopped: "Extract quality score" is skipped. "Validate review output" adds `ai-review-paused` and the "AI Review Paused" comment, without the auto re-review and without spending it. The PR stays open, and nothing is merged or closed. | +| `impl-repair.yml` | The retry is skipped. "Handle repair failure" restores `ai-rejected`, adds `ai-repair-paused`, and comments, without the auto-retry and without spending it. The attempt isn't used up. | +| `daily-regen.yml` | A spec polish or similarity audit that Claude stopped skips that spec's dispatch, and the entry fails. | | `spec-create.yml`, `util-claude.yml` | The retry is skipped, and the job fails. | | `report-validate.yml`, `review-retest.yml` | The step fails as before: the issue gets the "Validation Failed" comment, and the cell is collected as failed. | -In pair mode, a parked pair keeps its place in the queue: neither a parked -generation nor a review stopped on an open PR is a real attempt. A pair whose -review stopped keeps its open PR, which pair mode skips until the review -finishes. A session that reaches the limit after it wrote its review files is -scored on what it wrote, like any session that ends early. +The watchdog skips every PR with `ai-review-paused` or `ai-repair-paused`, so +it never spends its one-shot rescue on a stop. In pair mode, daily-regen +resumes the paused PRs first, oldest first (see +[Daily regeneration modes](#daily-regeneration-modes)), and the resumed review +or repair removes its label when it starts. A parked pair keeps its place in +the queue: neither a parked generation nor a paused review or repair is a real +attempt. + +In spec mode, or to resume sooner, resume the paused PRs yourself once the +limit resets or the credentials are fixed: + +1. Rerun every paused review: + + ```bash + gh pr list --label ai-review-paused --json number --jq '.[].number' \ + | xargs -I{} gh workflow run impl-review.yml -f pr_number={} + ``` + +2. Restart every paused repair with the command in its "Repair Paused" + comment, which names the PR, the pair, and the attempt. Before this change, a subscription usage limit counted as the pair's own failure: it used up the three attempts and marked a first implementation @@ -876,7 +932,9 @@ resets. ### Test the spend limit -Test the stop after the merge, with `CLAUDE_AUTH=oidc`: +Test the stop on the first day of `CLAUDE_AUTH=oidc`, before you set +`REGEN_PAIRS_PER_DAY`, so that no nightly run spends the workspace while you +test: 1. In the Claude Console, set the `ci-regen` workspace's spend limit to the month's spend so far plus about 0.50 USD. @@ -889,20 +947,24 @@ Test the stop after the merge, with `CLAUDE_AUTH=oidc`: The generation on Sonnet costs about 0.29 USD and passes; the review on Opus costs about 1.10 USD and reaches the limit while it runs. 3. Check the `impl-review.yml` run: its summary shows **Claude spend/usage - limit reached (auth=oidc)**, the PR has `ai-review-failed` and the "AI - Review Paused (Claude spend/usage limit)" comment, no second review run - started, and no verdict or `quality:` label was added. If the summary line - is missing, the run's error text didn't match: read it in the log and + limit reached (auth=oidc)**, the PR has `ai-review-paused` and the "AI + Review Paused (Claude spend/usage limit reached)" comment, no second review + run started, and no verdict or `quality:` label was added. If the summary + line is missing, the run's error text didn't match: read it in the log and extend `LIMIT_PATTERN`. 4. Check the spec issue: no `impl:{library}:failed` label, and no `impl-fail` comment for the pair. -5. Restore the spend limit, then rerun the review, or wait for the watchdog: +5. Check the next watchdog scan: its log reads + `PR #PR_NUMBER: paused (Claude stopped its review or repair) — skipped`, + and it dispatched nothing for the PR. +6. Restore the spend limit, then rerun the review: ```bash gh workflow run impl-review.yml -f pr_number=PR_NUMBER ``` -6. Check that the review finishes and the regen gate decides as usual. +7. Check that the review removes `ai-review-paused`, finishes, and the regen + gate decides as usual. To test the generation path as well, set the limit to the spend so far plus about 0.05 USD: the generation stops, the pair is parked as described above, @@ -918,10 +980,33 @@ or `pull_request_target`, so a pull request from a fork never gets a credential. `util-claude.yml` runs only for comments by owners, members, and collaborators. `spec-create.yml` and `report-validate.yml` run on a label, which only maintainers can add, but they read issue text that anyone can -write. Federation adds no stored secret: any step of a job with -`id-token: write` can mint an OIDC token, but the federation rule accepts only -`main`, so a token from a pull-request event or another branch is refused. A -leaked federated token expires after 600 seconds, and an API key is billed +write. + +Federation adds no stored secret, but the trust lives in the Claude Console, +not in the repository: + +- Any step of a job with `id-token: write` can mint an OIDC token. The + federation rule accepts only `main`, so a token from a pull-request event or + another branch is refused, but a step on `main` can exchange its token for a + Claude API token whatever `CLAUDE_AUTH` says. +- Switching `CLAUDE_AUTH` back to `subscription` or `api` doesn't close that + path. While you don't use `oidc`, disable the federation rule on the + [Workload identity](https://platform.claude.com/settings/workload-identity-federation) + page. +- The rule's claims name the repository and the ref, not the workflow. GitHub + also signs the `job_workflow_ref` claim, the workflow file the job runs + from, so adding it to the rule could narrow federation to the workflows + that run Claude. This is unverified: check that the Console accepts the + claim before you rely on it. +- The steps that run model-written code in a job with `id-token: write` + unset `ACTIONS_ID_TOKEN_REQUEST_URL` and `ACTIONS_ID_TOKEN_REQUEST_TOKEN` + first, so that code can't mint a token: "Adopt implementation from source + ref" and "Adaptability smoke run (record-only)" in `impl-generate.yml`, and + "Adaptability smoke run (record-only)" in `impl-repair.yml`. The + adaptability check in `impl-review.yml` reads the code without running it. + The switch doesn't change what a Claude session's own tools can reach. + +A leaked federated token expires after 600 seconds, and an API key is billed until you revoke it; the workspace's spend limit bounds either. --- diff --git a/tests/unit/automation/scripts/test_regen_pick_pairs.py b/tests/unit/automation/scripts/test_regen_pick_pairs.py index 7971dad1c6a..da52cc3cbac 100644 --- a/tests/unit/automation/scripts/test_regen_pick_pairs.py +++ b/tests/unit/automation/scripts/test_regen_pick_pairs.py @@ -1,13 +1,14 @@ """Tests for automation.scripts.regen_pick_pairs — daily-regen's pair mode. -The picker ages each (spec, library) pair by its newest touch, the metadata -``updated`` or the last real attempt (a merged or closed implementation PR, a -failed generation), keeps every pair touched within the attempt window out -of the queue, and postpones a spec whose issue just saw activity. The -no-loop rule is pinned here: a pair the gate kept, or whose generation -failed, does not come back the next night, and an infrastructure failure or -a stop at a spend or usage limit is no attempt. ``main`` runs once against a fake -``gh`` the way the workflow calls it. +The picker resumes the paused reviews and repairs first, then ages each +(spec, library) pair by its newest touch, the metadata ``updated`` or the +last real attempt (a merged or closed implementation PR, a failed +generation), keeps every pair touched within the attempt window out of the +queue, and postpones a spec whose issue just saw activity. The no-loop rule +is pinned here: a pair the gate kept, or whose generation failed, does not +come back the next night, an infrastructure failure or a generation Claude +stopped is no attempt, and a pick that cannot see its records fails closed. +``main`` runs against a fake ``gh`` the way the workflow calls it. """ from __future__ import annotations @@ -178,10 +179,18 @@ def test_comments_count_only_real_failures(self): fail_comment("a", "altair", 2, infra=True), # infrastructure: no attempt fail_comment("a", "plotly", 1, infra=True), fail_comment("other", "altair", 1), # another spec's marker - {"created_at": ts(1), "body": ""}, # a spend or usage limit + {"created_at": ts(1), "body": ""}, # Claude stopped the session ] assert rp.failed_generations(comments, "a") == {"altair": at(5)} + def test_unreadable_failures_leave_the_spec_out(self, capsys): + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("a", "plotly", at(190))] + cands.append(rp.Candidate("b", "altair", at(100))) + picks, eligible = run_pick(cands, count=3, failures=lambda spec: None if spec == "a" else {}) + # Fail closed: a's pairs may have failed last night. + assert picked(picks) == ["b/altair"] and eligible == 1 + assert "::warning::Leaving a out of tonight" in capsys.readouterr().out + def test_the_read_cap_stops_reading(self, capsys): cands = [rp.Candidate(spec, "altair", at(100 + i)) for i, spec in enumerate("abc")] asked: list[str] = [] @@ -211,6 +220,34 @@ def test_open_blocks_closed_and_merged_count(self): ) assert recs.attempts == {("a", "altair"): (at(9), "merged PR"), ("a", "plotly"): (at(3), "kept PR")} assert recs.open_pairs == {("b", "plotly")} + assert recs.paused == [] + + def test_paused_prs_become_resumes(self): + recs = records() + review = { + "number": 21, + "headRefName": "implementation/a/plotly", + "state": "OPEN", + "updatedAt": ts(2), + "labels": [{"name": "ai-review-paused"}], + } + repair = { + "number": 22, + "headRefName": "implementation/b/altair", + "state": "OPEN", + "updatedAt": ts(3), + "labels": [{"name": n} for n in ("ai-rejected", "ai-attempt-1", "ai-attempt-2", "ai-repair-paused")], + } + # A paused PR that is closed meanwhile is an attempt, not a resume. + closed = {**review, "number": 23, "headRefName": "implementation/c/plotly", "state": "CLOSED"} + rp.fold_pulls([review, repair, closed, review], recs) # the same PR twice: listed once + assert recs.open_pairs == {("a", "plotly"), ("b", "altair")} + assert recs.paused == [ + rp.Resume(21, "review", "a", "plotly", 1, at(2)), + rp.Resume(22, "repair", "b", "altair", 2, at(3)), + ] + assert [r.pr for r in rp.resumes(recs, 5)] == [22, 21] # oldest first + assert [r.pr for r in rp.resumes(recs, 1)] == [22] def test_issues(self): recs = records() @@ -325,19 +362,76 @@ def test_the_window_comes_from_the_argument(self, tmp_path): pairs = {(p["spec"], p["library"]) for p in json.loads(outputs["pairs_json"])} assert pairs == {("a-spec", "plotly"), ("b-spec", "altair"), ("b-spec", "matplotlib")} - def test_a_failed_listing_degrades_with_a_warning(self, tmp_path): + def _paused(self) -> dict[str, str]: + prs = [ + { + "number": 21, + "headRefName": "implementation/a-spec/plotly", + "state": "OPEN", + "updatedAt": ts(2, self.now), + "labels": [{"name": "ai-review-paused"}], + }, + { + "number": 22, + "headRefName": "implementation/b-spec/altair", + "state": "OPEN", + "updatedAt": ts(3, self.now), + "labels": [{"name": "ai-rejected"}, {"name": "ai-attempt-2"}, {"name": "ai-repair-paused"}], + }, + ] + data = {"prs.json": json.dumps(prs), "kept.json": "[]", "issues.json": "[]"} + return data | {"comments-11.jsonl": "", "comments-12.jsonl": ""} + + def test_paused_prs_are_resumed_before_new_pairs(self, tmp_path): self._plots(tmp_path) - result, outputs, _ = run_main(tmp_path, {"issues.json": "[]"}, 1) + result, outputs, summary = run_main(tmp_path, self._paused(), 3) + assert result.returncode == 0, result.stdout + result.stderr + # Both resumes count against the night's three; their pairs are open, so + # the one new pair is the oldest of the rest. + assert json.loads(outputs["resume_json"]) == [ + {"pr": 22, "kind": "repair", "spec": "b-spec", "library": "altair", "attempt": 2}, + {"pr": 21, "kind": "review", "spec": "a-spec", "library": "plotly", "attempt": 1}, + ] + assert outputs["resume_count"] == "2" + assert json.loads(outputs["pairs_json"]) == [{"spec": "a-spec", "library": "altair"}] + assert outputs["count"] == "1" + assert "Pair mode: 2 resumed, 1 new pair(s) of 3" in summary and "| repair | #22 |" in summary + + def test_resumes_can_take_the_whole_night(self, tmp_path): + self._plots(tmp_path) + _, outputs, _ = run_main(tmp_path, self._paused(), 1) + assert json.loads(outputs["resume_json"])[0]["pr"] == 22 + assert outputs["resume_count"] == "1" and outputs["count"] == "0" and outputs["pairs_json"] == "[]" + + def test_a_failed_implementation_pr_listing_stops_the_pick(self, tmp_path): + """The loop guard: without it a kept, closed or paused pair would be picked again.""" + self._plots(tmp_path) + result, outputs, _ = run_main(tmp_path, {"kept.json": "[]", "issues.json": "[]"}, 1) + assert result.returncode == 1 + assert "::error::Pair mode stops: the implementation PRs could not be listed" in result.stdout + assert outputs == {} + + def test_the_other_listings_degrade_with_a_warning(self, tmp_path): + self._plots(tmp_path) + result, outputs, _ = run_main(tmp_path, {"prs.json": "[]", "comments-12.jsonl": ""}, 1) assert result.returncode == 0, result.stdout + result.stderr assert result.stdout.count("::warning::Could not list") == 2 + # a-spec's comments cannot be read: it is left out, b-spec/altair is next. assert "::warning::Could not read the comments of issue #11" in result.stdout - assert json.loads(outputs["pairs_json"]) == [{"spec": "a-spec", "library": "altair"}] + assert json.loads(outputs["pairs_json"]) == [{"spec": "b-spec", "library": "altair"}] def test_nothing_eligible_writes_an_empty_pick(self, tmp_path): (tmp_path / "plots").mkdir() result, outputs, _ = run_main(tmp_path, {"prs.json": "[]", "kept.json": "[]", "issues.json": "[]"}) assert result.returncode == 0, result.stdout + result.stderr - assert outputs == {"pairs_json": "[]", "specs": "", "specs_json": "[]", "count": "0"} + assert outputs == { + "resume_json": "[]", + "resume_count": "0", + "pairs_json": "[]", + "specs": "", + "specs_json": "[]", + "count": "0", + } @pytest.mark.parametrize(("count", "window"), [(0, "30"), (-1, "30"), (1, "-1")]) def test_refuses_bad_numbers(self, tmp_path, count, window): diff --git a/tests/unit/workflows/test_claude_auth.py b/tests/unit/workflows/test_claude_auth.py index 8f8b11350f2..70cc7719d29 100644 --- a/tests/unit/workflows/test_claude_auth.py +++ b/tests/unit/workflows/test_claude_auth.py @@ -1,4 +1,4 @@ -"""Claude credential switch (vars.CLAUDE_AUTH), the session summary and the spend/usage limit. +"""Claude credential switch (vars.CLAUDE_AUTH), the session summary, and the stops Claude makes. Every job that runs Claude starts with the same "Resolve Claude credentials" step. It maps CLAUDE_AUTH (unset or `subscription`, `api`, `oidc`) to a mode, @@ -7,18 +7,21 @@ workload identity federation IDs to every Claude step as outputs. Each step then passes exactly one credential; the action's precedence with several set is undocumented (an API key or OAuth token silently shadows federation). +Under `oidc`, the runs the federation rule refuses (a pull-request event, +another ref, a job without id-token) use the subscription by design. After every Claude step, "Claude session summary" writes the cost line and -reads whether the session stopped at a Claude spend or usage limit (the API's -spend limit, spend cap or credit balance, the subscription's usage limit), in -every mode; the retry is skipped then, and the failure paths of -impl-generate, impl-review and impl-repair park the work as an infrastructure -stop without a retry, a counted attempt or a failure verdict. +reads whether Claude stopped the session: a spend or usage limit (the API's +spend limit, spend cap or credit balance, the subscription's usage limit) in +every mode, or refused credentials with api/oidc. The retry is skipped then; +impl-generate parks the pair, impl-review and impl-repair pause the PR +(`ai-review-paused`, `ai-repair-paused`, which the watchdog skips), and +nothing counts as an attempt or a failure verdict. Pair mode resumes the +paused PRs first. There is no pre-merge smoke for these workflows (claude-code-action refuses to run on a PR that changes them), so the wiring is pinned here and the -scripts run for real. Also here: daily-regen's mode switch, its pair -dispatch and the watchdog's rescue; the pick itself is tested in +scripts run for real. The pick itself is tested in tests/unit/automation/scripts/test_regen_pick_pairs.py. """ @@ -38,7 +41,7 @@ ACTION = "anthropics/claude-code-action@" RESOLVE = "Resolve Claude credentials" SUMMARY = "Claude session summary" -STOP = "Stop at the Claude spend or usage limit" +STOP = "Stop when Claude stopped the session" CREDENTIALS = { "anthropic_api_key": "${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }}", "claude_code_oauth_token": ( @@ -98,6 +101,10 @@ def _step(filename: str, name: str) -> dict[str, Any]: return matches[0] +def _names(filename: str, job: str) -> list[str]: + return [s.get("name", "") for s in _load(filename)["jobs"][job]["steps"]] + + JOBS = _claude_jobs() IDS = [f"{f}:{j}" for f, j, _ in JOBS] @@ -146,16 +153,19 @@ def test_every_resolve_step_is_identical(): assert reference["env"] == { "CLAUDE_AUTH": "${{ vars.CLAUDE_AUTH }}", "API_KEY_SET": "${{ secrets.ANTHROPIC_API_KEY != '' }}", + "OAUTH_TOKEN_SET": "${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }}", "WIF_RULE_ID": "${{ vars.CLAUDE_WIF_RULE_ID }}", "WIF_ORG_ID": "${{ vars.CLAUDE_WIF_ORG_ID }}", "WIF_SERVICE_ACCOUNT_ID": "${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }}", "WIF_WORKSPACE_ID": "${{ vars.CLAUDE_WIF_WORKSPACE_ID }}", + "EVENT": "${{ github.event_name }}", + "REF": "${{ github.ref }}", } @pytest.mark.parametrize(("filename", "name", "job"), JOBS, ids=IDS) def test_id_token_is_where_it_was(filename, name, job): - """No job gains id-token in this change; review-retest stays without it (oidc stops there).""" + """No job gains id-token in this change; review-retest stays without it (oidc routes it to the subscription).""" has_id_token = (job.get("permissions") or {}).get("id-token") == "write" assert has_id_token is (filename != "review-retest.yml"), (filename, name) @@ -186,15 +196,15 @@ def test_summary_steps_share_one_script_and_unique_labels(): labels = [s["env"]["CLAUDE_STEP"] for s in summaries] assert len(labels) == len(set(labels)) == sum(CLAUDE_WORKFLOWS.values()) assert all(" " not in label for label in labels) - shared = {(s["run"], s["env"]["SESSION_ERRORS"], s["env"]["LIMIT_PATTERN"]) for s in summaries} - assert len(shared) == 1 + keys = ("SESSION_ERRORS", "LIMIT_PATTERN", "AUTH_PATTERN") + assert len({(s["run"], *(s["env"][k] for k in keys)) for s in summaries}) == 1 # The line must not look like Claude Code's own `"total_cost_usd": n`, or # collect_week.py's fallback regex would count a session twice. assert '"total_cost_usd":' not in summaries[0]["run"] @pytest.mark.parametrize(("filename", "name", "job"), JOBS, ids=IDS) -def test_a_limit_skips_the_retry_and_stops_the_job(filename, name, job): +def test_a_stop_skips_the_retry_and_fails_the_job(filename, name, job): steps = job["steps"] names = [s.get("name") for s in steps] claude = [s for s in steps if _is_claude(s)] @@ -202,7 +212,7 @@ def test_a_limit_skips_the_retry_and_stops_the_job(filename, name, job): # Two independent pre-flight sessions, no retry: either one stops the dispatch. stop = steps[names.index(STOP)] assert stop["if"] == ( - "${{ steps.polish_cost.outputs.limit_reached == 'true' || steps.similarity_cost.outputs.limit_reached == 'true' }}" + "${{ steps.polish_cost.outputs.stopped == 'true' || steps.similarity_cost.outputs.stopped == 'true' }}" ) dispatches = [i for i, n in enumerate(names) if str(n).startswith("Dispatch ")] assert len(dispatches) == 2 and min(dispatches) > names.index(STOP) > steps.index(claude[-1]) @@ -212,10 +222,10 @@ def test_a_limit_skips_the_retry_and_stops_the_job(filename, name, job): return else: first, retry = claude - assert retry["if"].endswith(f" && steps.{first['id']}_cost.outputs.limit_reached != 'true'") + assert retry["if"].endswith(f" && steps.{first['id']}_cost.outputs.stopped != 'true'") stop = steps[names.index(STOP)] assert names.index(STOP) > steps.index(retry) - assert stop["if"] == f"steps.{first['id']}_cost.outputs.limit_reached == 'true'" + assert stop["if"] == f"steps.{first['id']}_cost.outputs.stopped == 'true'" assert "exit 1" in stop["run"] and "continue-on-error" not in stop @@ -241,22 +251,53 @@ def test_no_claude_workflow_runs_on_pull_request_events(): assert "pull_request" not in triggers, filename +@pytest.mark.parametrize( + ("filename", "name"), + [ + ("impl-generate.yml", "Adopt implementation from source ref"), + ("impl-generate.yml", "Adaptability smoke run (record-only)"), + ("impl-repair.yml", "Adaptability smoke run (record-only)"), + ], +) +def test_steps_running_model_written_code_get_no_oidc_request(filename, name): + """The jobs have id-token; the code they run must not be able to mint a token.""" + first = next(line for line in _step(filename, name)["run"].splitlines() if not line.lstrip().startswith("#")) + assert first == "unset ACTIONS_ID_TOKEN_REQUEST_URL ACTIONS_ID_TOKEN_REQUEST_TOKEN" + + # --- the resolve step, run for real ------------------------------------------ -def _resolve(tmp_path: Path, auth: str | None, key_set: bool = False, id_token: bool = True, **wif: str): +def _resolve( + tmp_path: Path, + auth: str | None, + key_set: bool = False, + id_token: bool = True, + oauth_set: bool = True, + event: str = "schedule", + ref: str = "refs/heads/main", + **wif: str, +): script = _named(RESOLVE)[0]["run"] - out = tmp_path / "out" + out, summary = tmp_path / "out", tmp_path / "summary" out.write_text("", encoding="utf-8") env = {k: v for k, v in os.environ.items() if not k.startswith(("CLAUDE_", "WIF_", "ACTIONS_ID_TOKEN"))} - env |= {"API_KEY_SET": "true" if key_set else "false", "GITHUB_OUTPUT": str(out), **wif} + env |= { + "API_KEY_SET": "true" if key_set else "false", + "OAUTH_TOKEN_SET": "true" if oauth_set else "false", + "EVENT": event, + "REF": ref, + "GITHUB_OUTPUT": str(out), + "GITHUB_STEP_SUMMARY": str(summary), + **wif, + } if auth is not None: env["CLAUDE_AUTH"] = auth if id_token: env["ACTIONS_ID_TOKEN_REQUEST_URL"] = "https://token.actions.example/" result = subprocess.run(["bash", "-e", "-c", script], cwd=tmp_path, env=env, capture_output=True, text=True) outputs = dict(line.split("=", 1) for line in out.read_text().splitlines() if "=" in line) - return result, outputs + return result, outputs, summary.read_text() if summary.exists() else "" def _inputs(outputs: dict[str, str], key: str, token: str) -> dict[str, str]: @@ -285,7 +326,7 @@ def _inputs(outputs: dict[str, str], key: str, token: str) -> dict[str, str]: ], ) def test_resolve_passes_exactly_one_credential(tmp_path, auth, key_set, mode, given): - result, outputs = _resolve(tmp_path, auth, key_set, **WIF) + result, outputs, _ = _resolve(tmp_path, auth, key_set, **WIF) assert result.returncode == 0, result.stdout + result.stderr assert outputs["mode"] == mode and f"Claude credentials: {mode}" in result.stdout inputs = _inputs(outputs, "sk-ant-key" if key_set else "", "oauth-token") @@ -300,20 +341,49 @@ def test_resolve_passes_exactly_one_credential(tmp_path, auth, key_set, mode, gi @pytest.mark.parametrize( - ("auth", "key_set", "wif", "id_token", "why"), + ("event", "ref", "id_token", "where"), [ - ("api", False, WIF, True, "ANTHROPIC_API_KEY secret is missing"), - ("oidc", True, {**WIF, "WIF_RULE_ID": ""}, True, "CLAUDE_WIF_RULE_ID is missing"), - ("oidc", True, {k: v for k, v in WIF.items() if k != "WIF_WORKSPACE_ID"}, True, "CLAUDE_WIF_WORKSPACE_ID"), - ("oidc", True, {**WIF, "WIF_ORG_ID": "x\n::warning::y"}, True, "CLAUDE_WIF_ORG_ID is missing or not an ID"), - ("oidc", True, WIF, False, "no 'id-token: write' permission"), - ("apikey", True, WIF, True, "must be unset, 'subscription', 'api' or 'oidc'"), - (" api", True, WIF, True, "must be unset"), - ("api\n::warning::injected", True, WIF, True, "must be unset"), + # util-claude's review events present the `…:pull_request` subject. + ("pull_request_review", "refs/pull/7/merge", True, "pull_request_review/refs/pull/7/merge"), + ("pull_request_review_comment", "refs/pull/7/merge", True, "pull_request_review_comment/refs/pull/7/merge"), + # A branch smoke: `…:ref:refs/heads/feature/x`. + ("workflow_dispatch", "refs/heads/feature/x", True, "workflow_dispatch/refs/heads/feature/x"), + # review-retest: no id-token on purpose. + ("workflow_dispatch", "refs/heads/main", False, "workflow_dispatch/refs/heads/main (no id-token)"), ], ) -def test_resolve_stops_instead_of_falling_back(tmp_path, auth, key_set, wif, id_token, why): - result, outputs = _resolve(tmp_path, auth, key_set, id_token, **wif) +def test_oidc_routes_the_runs_federation_refuses_to_the_subscription(tmp_path, event, ref, id_token, where): + result, outputs, summary = _resolve(tmp_path, "oidc", id_token=id_token, event=event, ref=ref, **WIF) + assert result.returncode == 0, result.stdout + result.stderr + assert outputs["mode"] == "subscription" and "federation_rule_id" not in outputs + assert f"::notice::oidc not available for {where}; using subscription" in result.stdout + assert f"- oidc not available for {where}; using subscription" in summary + assert {k for k, v in _inputs(outputs, "", "oauth-token").items() if v} == {"claude_code_oauth_token"} + + +def test_oidc_routing_fails_fast_without_the_oauth_token(tmp_path): + result, outputs, _ = _resolve( + tmp_path, "oidc", oauth_set=False, event="pull_request_review", ref="refs/pull/7/merge" + ) + assert result.returncode == 1 and "mode" not in outputs + assert result.stdout.startswith("::error::oidc not available for pull_request_review/refs/pull/7/merge") + assert "CLAUDE_CODE_OAUTH_TOKEN secret is missing" in result.stdout + + +@pytest.mark.parametrize( + ("auth", "key_set", "wif", "why"), + [ + ("api", False, WIF, "ANTHROPIC_API_KEY secret is missing"), + ("oidc", True, {**WIF, "WIF_RULE_ID": ""}, "CLAUDE_WIF_RULE_ID is missing"), + ("oidc", True, {k: v for k, v in WIF.items() if k != "WIF_WORKSPACE_ID"}, "CLAUDE_WIF_WORKSPACE_ID"), + ("oidc", True, {**WIF, "WIF_ORG_ID": "x\n::warning::y"}, "CLAUDE_WIF_ORG_ID is missing or not an ID"), + ("apikey", True, WIF, "must be unset, 'subscription', 'api' or 'oidc'"), + (" api", True, WIF, "must be unset"), + ("api\n::warning::injected", True, WIF, "must be unset"), + ], +) +def test_resolve_stops_instead_of_falling_back(tmp_path, auth, key_set, wif, why): + result, outputs, _ = _resolve(tmp_path, auth, key_set, **wif) assert result.returncode == 1 assert result.stdout.startswith("::error::") and why in result.stdout assert "not falling back to the subscription" in result.stdout @@ -335,8 +405,7 @@ def _summary(tmp_path: Path, execution: list[dict[str, Any]] | None, mode: str = "EXECUTION_FILE": str(exec_file) if execution is not None else "", "CLAUDE_AUTH_MODE": mode, "CLAUDE_STEP": "impl-review", - "SESSION_ERRORS": step["env"]["SESSION_ERRORS"], - "LIMIT_PATTERN": step["env"]["LIMIT_PATTERN"], + **{k: step["env"][k] for k in ("SESSION_ERRORS", "LIMIT_PATTERN", "AUTH_PATTERN")}, "GITHUB_STEP_SUMMARY": str(summary), "GITHUB_OUTPUT": str(out), } @@ -346,6 +415,7 @@ def _summary(tmp_path: Path, execution: list[dict[str, Any]] | None, mode: str = INIT = {"type": "system", "subtype": "init", "model": "claude-opus-5-5"} +NOT_STOPPED = {"stopped": "false", "stop_reason": ""} def _failed(text: str, error: str | None = "invalid_request", status: int = 400) -> list[dict[str, Any]]: @@ -405,7 +475,29 @@ def _failed(text: str, error: str | None = "invalid_request", status: int = 400) "older CLI wording": _failed("Claude AI usage limit reached|1761955200", error=None), } -NOT_A_LIMIT = { +# Refused credentials (the API's error types, docs/en/api/errors; the federation's +# fixed `401 Authentication failed`, docs/en/manage-claude/wif-providers/github-actions). +AUTH_FIXTURES = { + "bad API key": _failed( + 'API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}', + error="authentication_failed", + status=401, + ), + "federation refused": _failed( + 'API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Authentication failed"}}', + error="authentication_failed", + status=401, + ), + "permission": _failed( + 'API Error: 403 {"type":"error","error":{"type":"permission_error","message":"Your API key does not have ' + 'permission to use the specified resource."}}', + error="unknown", + status=403, + ), + "status only": _failed("Request failed", error=None, status=401), +} + +NOT_A_STOP = { # A transient rate limit keeps its retry and the infrastructure path it had. "per-minute rate limit": _failed( 'API Error: 429 {"error":{"type":"rate_limit_error","message":"Number of request tokens has exceeded your ' @@ -428,7 +520,7 @@ def _failed(text: str, error: str | None = "invalid_request", status: int = 400) ], "in a successful result": [ INIT, - {"type": "result", "subtype": "success", "is_error": False, "result": "Docs say: credit balance is too low"}, + {"type": "result", "subtype": "success", "is_error": False, "result": "Docs say: Authentication failed"}, ], # A retried billing error the session got past: it ended well, so it never stopped. "recovered": [ @@ -444,7 +536,7 @@ def test_summary_reads_cost_and_model(tmp_path): result, summary, outputs = _summary(tmp_path, execution, mode="oidc") assert result.returncode == 0 assert summary == "- `claude-cost step=impl-review auth=oidc model=claude-opus-5-5 total_cost_usd=0.2931`\n" - assert outputs == {"limit_reached": "false"} + assert outputs == NOT_STOPPED @pytest.mark.parametrize("execution", [None, [], [{"type": "assistant"}], [{"type": "result"}]]) @@ -452,7 +544,7 @@ def test_summary_never_fails_without_a_cost(tmp_path, execution): result, summary, outputs = _summary(tmp_path, execution) assert result.returncode == 0 assert "auth=subscription model=n/a total_cost_usd=n/a" in summary - assert outputs == {"limit_reached": "false"} + assert outputs == NOT_STOPPED @pytest.mark.parametrize("case", sorted(LIMIT_FIXTURES)) @@ -460,32 +552,53 @@ def test_summary_never_fails_without_a_cost(tmp_path, execution): def test_summary_detects_a_spend_or_usage_limit(tmp_path, case, mode): result, summary, outputs = _summary(tmp_path, LIMIT_FIXTURES[case], mode) assert result.returncode == 0 - assert outputs == {"limit_reached": "true"}, case + assert outputs == {"stopped": "true", "stop_reason": "limit"}, case assert f"::error::Claude spend/usage limit reached (auth={mode}) in impl-review" in result.stdout assert f"- **Claude spend/usage limit reached (auth={mode})** in `impl-review`" in summary -@pytest.mark.parametrize("case", sorted(NOT_A_LIMIT)) -def test_summary_ignores_everything_else(tmp_path, case): - result, summary, outputs = _summary(tmp_path, NOT_A_LIMIT[case], "api") - assert outputs == {"limit_reached": "false"}, case - assert "spend/usage limit" not in summary +@pytest.mark.parametrize("case", sorted(AUTH_FIXTURES)) +@pytest.mark.parametrize("mode", ["oidc", "api"]) +def test_summary_detects_refused_credentials_with_api_and_oidc(tmp_path, case, mode): + result, summary, outputs = _summary(tmp_path, AUTH_FIXTURES[case], mode) + assert outputs == {"stopped": "true", "stop_reason": "auth"}, case + assert f"::error::Claude credentials refused (auth={mode}) in impl-review" in result.stdout + assert f"- **Claude credentials refused (auth={mode})** in `impl-review`" in summary + + +@pytest.mark.parametrize("case", sorted(AUTH_FIXTURES)) +def test_the_subscription_keeps_its_handling_of_refused_credentials(tmp_path, case): + assert _summary(tmp_path, AUTH_FIXTURES[case], "subscription")[2] == NOT_STOPPED + + +def test_a_limit_wins_over_its_status_code(tmp_path): + """A 403 with a usage-limit message is a limit, not refused credentials.""" + execution = _failed("You've hit your session limit · resets 12:30pm", error="rate_limit", status=403) + assert _summary(tmp_path, execution, "oidc")[2] == {"stopped": "true", "stop_reason": "limit"} + + +@pytest.mark.parametrize("case", sorted(NOT_A_STOP)) +@pytest.mark.parametrize("mode", ["subscription", "api"]) +def test_summary_ignores_everything_else(tmp_path, case, mode): + result, summary, outputs = _summary(tmp_path, NOT_A_STOP[case], mode) + assert outputs == NOT_STOPPED, case + assert "limit reached" not in summary and "refused" not in summary -# --- the failure paths at a spend or usage limit ---------------------------------- +# --- the failure paths after a stop ------------------------------------------- -# Logs every call as one JSON line; `pr view`/`issue view` answer nothing. +# Logs every call as one JSON line; `pr view` answers with $GH_LABELS. FAKE_GH = """#!/usr/bin/env python3 import json, os, sys with open(os.environ["GH_LOG"], "a", encoding="utf-8") as f: f.write(json.dumps(sys.argv[1:]) + "\\n") +if sys.argv[1:3] == ["pr", "view"]: + print(os.environ.get("GH_LABELS", "")) """ -def _run_handler(tmp_path: Path, filename: str, name: str, **env: str) -> tuple[subprocess.CompletedProcess, list]: - step = _step(filename, name) - script = step["run"] +def _run(tmp_path: Path, script: str, **env: str) -> tuple[subprocess.CompletedProcess, list]: for expr, value in { "${{ github.repository }}": "owner/repo", "${{ github.run_id }}": "1", @@ -495,8 +608,9 @@ def _run_handler(tmp_path: Path, filename: str, name: str, **env: str) -> tuple[ assert "${{" not in script bin_dir = tmp_path / "bin" bin_dir.mkdir(exist_ok=True) - (bin_dir / "gh").write_text(FAKE_GH, encoding="utf-8") - (bin_dir / "gh").chmod(0o755) + for name, body in {"gh": FAKE_GH, "sleep": "#!/bin/sh\nexit 0\n"}.items(): + (bin_dir / name).write_text(body, encoding="utf-8") + (bin_dir / name).chmod(0o755) log = tmp_path / "gh.log" base = {k: v for k, v in os.environ.items() if not k.startswith(("GIT_", "GH_"))} result = subprocess.run( @@ -510,42 +624,52 @@ def _run_handler(tmp_path: Path, filename: str, name: str, **env: str) -> tuple[ return result, calls +def _run_step(tmp_path: Path, filename: str, name: str, **env: str): + return _run(tmp_path, _step(filename, name)["run"], **env) + + def _bodies(calls: list[list[str]]) -> list[str]: return [c[c.index("--body") + 1] for c in calls if "--body" in c] -@pytest.mark.parametrize("mode", ["subscription", "oidc"]) +GENERATION_ENV = { + "SPEC_ID": "spec-a", + "LIBRARY": "plotly", + "ISSUE": "42", + "MODEL": "sonnet", + "CHANGE_REQUEST": "", + "CAMPAIGN_WINDOW_H": "12", + "INFRA_WINDOW_CAP": "5", + "CLAUDE_RETRY_OUTCOME": "skipped", + "CLAUDE_EXEC_FILE": "", + "CLAUDE_STOPPED": "true", + "LANGUAGE": "python", + "EXT": ".py", + "REGEN_GATE": "true", + "SOURCE_REF": "", + "SOURCE_MODEL": "", + "ADOPT_CAUSE": "", + "TRIGGER_REF": "main", + "RUN_URL": "https://github.com/owner/repo/actions/runs/1", +} + + +@pytest.mark.parametrize( + ("mode", "reason", "why"), + [ + ("subscription", "limit", "Claude spend/usage limit reached"), + ("oidc", "limit", "Claude spend/usage limit reached"), + ("oidc", "auth", "Claude credentials refused"), + ], +) @pytest.mark.parametrize(("regen_label", "live"), [("regen", True), ("", False)]) -def test_generation_at_a_limit_is_parked(tmp_path, regen_label, live, mode): +def test_a_stopped_generation_is_parked(tmp_path, regen_label, live, mode, reason, why): """A first implementation too: no retry, no counted attempt, never `impl::failed`.""" subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) - result, calls = _run_handler( - tmp_path, - "impl-generate.yml", - "Handle generation failure", - SPEC_ID="spec-a", - LIBRARY="plotly", - ISSUE="42", - MODEL="sonnet", - CHANGE_REQUEST="", - CAMPAIGN_WINDOW_H="12", - INFRA_WINDOW_CAP="5", - CLAUDE_RETRY_OUTCOME="skipped", - CLAUDE_EXEC_FILE="", - LIMIT_REACHED="true", - CLAUDE_MODE=mode, - REGEN_LABEL=regen_label, - LANGUAGE="python", - EXT=".py", - REGEN_GATE="true", - SOURCE_REF="", - SOURCE_MODEL="", - ADOPT_CAUSE="", - TRIGGER_REF="main", - RUN_URL="https://github.com/owner/repo/actions/runs/1", - ) + env = GENERATION_ENV | {"CLAUDE_MODE": mode, "STOP_REASON": reason, "REGEN_LABEL": regen_label} + result, calls = _run_step(tmp_path, "impl-generate.yml", "Handle generation failure", **env) assert result.returncode == 0, result.stdout + result.stderr - assert f"::error::Claude spend/usage limit reached (auth={mode})" in result.stdout + assert f"::error::{why} (auth={mode})" in result.stdout assert not [c for c in calls if c[:2] == ["workflow", "run"]] # no auto-retry edits = [c[3:5] for c in calls if c[:2] == ["issue", "edit"]] assert ["--remove-label", "impl:plotly:pending"] in edits and ["--remove-label", "generate:plotly"] in edits @@ -554,12 +678,13 @@ def test_generation_at_a_limit_is_parked(tmp_path, regen_label, live, mode): assert not [c for c in calls if c[:2] == ["api", "--paginate"]] # the attempt budget is not even read (body,) = _bodies(calls) # Its own marker: neither the 3-attempt cap nor daily-regen's attempt window counts it. - assert "" in body + assert "" in body and why in body assert "" in body + assert "" in body assert "" in body + assert "" in body assert " + # A resume while the stop lasts pauses again: one comment per + # pause, so the label alone stands when the PR's latest bot + # comment is already this attempt's pause (its first line is the + # marker). A read that fails posts it. + PAUSE_MARKER="" + LAST_BOT=$(gh api --paginate "repos/${REPOSITORY}/issues/${PR_NUM}/comments?per_page=100" \ + --jq '.[] | select(.user.type == "Bot") | (.body // "") | split("\n") | .[0] // "" | rtrimstr("\r")' 2>/dev/null \ + | tail -n 1) || LAST_BOT="" + if [ "$LAST_BOT" = "$PAUSE_MARKER" ]; then + echo "::notice::PR #${PR_NUM} already shows this pause; no new comment" + exit 0 + fi + gh pr comment "$PR_NUM" --body "${PAUSE_MARKER} ## :pause_button: Repair Paused (${WHY}, Attempt ${ATTEMPT}/4) Claude stopped the repair: ${WHY} (auth=\`${CLAUDE_MODE}\`). Nothing was retried, and the attempt is not used up. The PR carries \`ai-repair-paused\`, which the watchdog leaves alone; daily-regen's pair mode resumes it. diff --git a/.github/workflows/impl-review.yml b/.github/workflows/impl-review.yml index 0702c4891d6..ee34b3aef83 100644 --- a/.github/workflows/impl-review.yml +++ b/.github/workflows/impl-review.yml @@ -1201,7 +1201,19 @@ jobs: gh pr edit "$PR_NUM" --add-label "ai-review-paused" 2>/dev/null && break [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) done || echo "::error::could not add ai-review-paused to PR #${PR_NUM} — resume it with a manual rerun" - gh pr comment "$PR_NUM" --body " + # A resume while the stop lasts pauses again: one comment per + # pause, so the label alone stands when the PR's latest bot + # comment is already this pause (its first line is the marker). + # A read that fails posts it. + PAUSE_MARKER="" + LAST_BOT=$(gh api --paginate "repos/${REPOSITORY}/issues/${PR_NUM}/comments?per_page=100" \ + --jq '.[] | select(.user.type == "Bot") | (.body // "") | split("\n") | .[0] // "" | rtrimstr("\r")' 2>/dev/null \ + | tail -n 1) || LAST_BOT="" + if [ "$LAST_BOT" = "$PAUSE_MARKER" ]; then + echo "::notice::PR #${PR_NUM} already shows this pause; no new comment" + exit 1 + fi + gh pr comment "$PR_NUM" --body "${PAUSE_MARKER} ## :pause_button: AI Review Paused (${WHY}) Claude stopped the review: ${WHY} (auth=\`${CLAUDE_MODE}\`). No score and no verdict were recorded, and the one automatic re-review is still unused. The PR carries \`ai-review-paused\`, which the watchdog leaves alone; daily-regen's pair mode resumes it. diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index 28400d56815..24ded3972a7 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -907,7 +907,9 @@ The watchdog skips every PR with `ai-review-paused` or `ai-repair-paused`, so it never spends its one-shot rescue on a stop. In pair mode, daily-regen resumes the paused PRs first, oldest first (see [Daily regeneration modes](#daily-regeneration-modes)), and the resumed review -or repair removes its label when it starts. A parked pair keeps its place in +or repair removes its label when it starts. A resume that pauses again while +the stop lasts puts the label back but posts no second comment when the PR's +latest bot comment is already the same pause. A parked pair keeps its place in the queue: neither a parked generation nor a paused review or repair is a real attempt. diff --git a/tests/unit/workflows/test_claude_auth.py b/tests/unit/workflows/test_claude_auth.py index 70cc7719d29..de94b958896 100644 --- a/tests/unit/workflows/test_claude_auth.py +++ b/tests/unit/workflows/test_claude_auth.py @@ -29,6 +29,8 @@ import json import os +import re +import shutil import subprocess from pathlib import Path from typing import Any @@ -588,14 +590,22 @@ def test_summary_ignores_everything_else(tmp_path, case, mode): # --- the failure paths after a stop ------------------------------------------- -# Logs every call as one JSON line; `pr view` answers with $GH_LABELS. +# Logs every call as one JSON line; `pr view` answers with $GH_LABELS, and a +# read of an issue's comments with $GH_COMMENTS (what its --jq would print), +# failing when that is FAIL. FAKE_GH = """#!/usr/bin/env python3 import json, os, sys with open(os.environ["GH_LOG"], "a", encoding="utf-8") as f: f.write(json.dumps(sys.argv[1:]) + "\\n") if sys.argv[1:3] == ["pr", "view"]: print(os.environ.get("GH_LABELS", "")) +elif sys.argv[1] == "api" and "/comments" in " ".join(sys.argv): + if os.environ.get("GH_COMMENTS") == "FAIL": + sys.exit(1) + print(os.environ.get("GH_COMMENTS", "")) """ +# The pause paths' only read: the PR's bot comments, newest last. +LAST_BOT_READ = "repos/owner/repo/issues/7/comments?per_page=100" def _run(tmp_path: Path, script: str, **env: str) -> tuple[subprocess.CompletedProcess, list]: @@ -705,7 +715,8 @@ def test_a_stopped_repair_is_paused_and_keeps_its_retry(tmp_path, reason, why): assert ["pr", "edit", "7", "--add-label", "ai-repair-paused"] in calls assert any(c[:3] == ["label", "create", "ai-repair-paused"] for c in calls) assert not [c for c in calls if c[:2] == ["workflow", "run"]] - assert not [c for c in calls if c[:1] == ["api"]] # the retry budget is not even read + # The retry budget is not even read: the one read is the latest bot comment. + assert [c[2] for c in calls if c[:1] == ["api"]] == [LAST_BOT_READ] (body,) = _bodies(calls) assert "" in body assert "" in body assert "", + 1, + ), + "repair": ( + "impl-repair.yml", + "Handle repair failure", + {"MODEL": "sonnet", "ATTEMPT": "2"}, + "ai-repair-paused", + "", + 0, + ), +} + + +@pytest.mark.parametrize("kind", sorted(PAUSE_PATHS)) +@pytest.mark.parametrize( + ("latest", "posts"), + [ + # A resume that paused again: the label alone, no second comment. + ("same pause", False), + # The PR moved on since (another bot comment), or nothing to compare. + ("other comment after the pause", True), + ("no comments", True), + ("read fails", True), + ], +) +def test_a_repeated_pause_posts_no_second_comment(tmp_path, kind, latest, posts): + filename, name, extra, label, marker, rc = PAUSE_PATHS[kind] + # The first line of each bot comment, oldest first; a pause starts with its marker. + comments = { + "same pause": f"## AI Review - Attempt 1/3\n{marker}", + "other comment after the pause": f"{marker}\n## AI Review - Attempt 1/3", + "no comments": "", + "read fails": "FAIL", + }[latest] + env = {"PR_NUM": "7", "SPEC_ID": "spec-a", "LIBRARY": "plotly", "REPOSITORY": "owner/repo", "RUN_ID": "1"} + env |= extra | {"CLAUDE_STOPPED": "true", "STOP_REASON": "limit", "CLAUDE_MODE": "oidc", "GH_COMMENTS": comments} + result, calls = _run_step(tmp_path, filename, name, **env) + assert result.returncode == rc, result.stdout + result.stderr + assert ["pr", "edit", "7", "--add-label", label] in calls # the label is kept either way + assert (len(_bodies(calls)) == 1) is posts + assert ("already shows this pause" in result.stdout) is not posts + (read,) = [c for c in calls if c[:1] == ["api"]] + assert read[read.index("--jq") + 1].startswith('.[] | select(.user.type == "Bot") | (.body // "")') + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq") +@pytest.mark.parametrize("kind", sorted(PAUSE_PATHS)) +def test_the_latest_bot_comment_read_yields_first_lines(kind): + """The real filter on REST-shaped comments: bots only, first line, CRLF and empty bodies safe.""" + filename, name, *_ = PAUSE_PATHS[kind] + match = re.search(r"--jq '([^']+)'", _step(filename, name)["run"]) + assert match + comments = [ + {"user": {"type": "Bot"}, "body": "## AI Review - Attempt 1/3\nscore"}, + {"user": {"type": "User"}, "body": ""}, + {"user": {"type": "Bot"}, "body": None}, + {"user": {"type": "Bot"}, "body": "\r\n## Paused & "}, + ] + out = subprocess.run( + ["jq", "-r", match[1]], input=json.dumps(comments), capture_output=True, text=True, check=True + ).stdout + assert out.splitlines() == ["## AI Review - Attempt 1/3", "", ""] + + +def test_a_new_repair_attempt_gets_its_own_pause_comment(tmp_path): + earlier = "" + env = {"PR_NUM": "7", "SPEC_ID": "spec-a", "LIBRARY": "plotly", "REPOSITORY": "owner/repo", "RUN_ID": "1"} + env |= {"MODEL": "sonnet", "ATTEMPT": "2", "CLAUDE_STOPPED": "true", "STOP_REASON": "limit", "CLAUDE_MODE": "oidc"} + _, calls = _run_step(tmp_path, "impl-repair.yml", "Handle repair failure", GH_COMMENTS=earlier, **env) + (body,) = _bodies(calls) + assert "" in body + + @pytest.mark.parametrize( ("filename", "name"), [ From 3140e3a01883953764778aae32e210e079f5e24c Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:33:04 +0100 Subject: [PATCH 6/7] docs(changelog): add the PR number to the CI auth fragment (#12487) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H7HSkGyea1k2v293XiAMo8 --- changelog.d/ci-claude-auth-switch.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/ci-claude-auth-switch.md b/changelog.d/ci-claude-auth-switch.md index 6f1c0b60960..03cdc60a834 100644 --- a/changelog.d/ci-claude-auth-switch.md +++ b/changelog.d/ci-claude-auth-switch.md @@ -14,7 +14,7 @@ runs federation refuses (pull-request review events, other refs, `review-retest.yml`) use the subscription and say so in the job summary. `docs/workflows/overview.md` has the Console setup, the switch commands, - and how to close the federation path when `oidc` is off. + and how to close the federation path when `oidc` is off. (#12487) - **Every Claude session's cost in the job summary, and a clean stop when Claude refuses a session.** A step after each Claude step writes `total_cost_usd` (Claude Code's estimate at API list price), the model and @@ -30,7 +30,7 @@ subscription usage limit used up a pair's three attempts. The weekly regen review's `collect_week.py --costs` reads the line, now counts the `daily-regen.yml` pre-flight too, and names the part that ran on API - credits. + credits. (#12487) - **Daily regeneration by implementation instead of by spec: the repository variable `REGEN_PAIRS_PER_DAY`.** Set to N, the scheduled `daily-regen.yml` run picks the N oldest (spec, library) pairs across the catalogue @@ -45,4 +45,4 @@ queue. Paused reviews and repairs are resumed first and count against N, and the pick stops when it can't list the implementation PRs. Unset, the schedule regenerates whole specs as before; a manual run opts in with the - new `pairs` input, and the watchdog's cron rescue forwards the variable. + new `pairs` input, and the watchdog's cron rescue forwards the variable. (#12487) From 3a6d00a042b7f35ef1d7a5ea6eb461cf6f773398 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:59:31 +0100 Subject: [PATCH 7/7] test(ci): cover the pair-mode picker in process The main() tests ran the picker as a subprocess, which coverage doesn't see. In-process tests with a fake runner now cover the gh listing wrappers, read_records and its fail-closed implementation-PR listing, issue_failures, the paused-label parsing edges, summary_table, argument checks and the GITHUB_OUTPUT and step-summary writing. No behaviour change; regen_pick_pairs.py is at 100 % line and branch coverage. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H7HSkGyea1k2v293XiAMo8 --- .../scripts/test_regen_pick_pairs.py | 298 ++++++++++++++++++ 1 file changed, 298 insertions(+) diff --git a/tests/unit/automation/scripts/test_regen_pick_pairs.py b/tests/unit/automation/scripts/test_regen_pick_pairs.py index da52cc3cbac..9b0c43b6bad 100644 --- a/tests/unit/automation/scripts/test_regen_pick_pairs.py +++ b/tests/unit/automation/scripts/test_regen_pick_pairs.py @@ -438,3 +438,301 @@ def test_refuses_bad_numbers(self, tmp_path, count, window): (tmp_path / "plots").mkdir() result, _, _ = run_main(tmp_path, {}, count, "--attempt-window-days", window) assert result.returncode == 1 and "::error::" in result.stdout + + +# --- the same paths in process ------------------------------------------------ + + +class FakeRunner: + """Stands in for ``subprocess.run``: answers each ``gh`` call the picker makes from ``data``. + + The keys are ``prs``, ``kept``, ``issues`` and ``comments-``; a value + that is an exception is raised, and a missing key fails like an API error. + """ + + def __init__(self, data: dict[str, str | BaseException]): + self.data = data + self.calls: list[list[str]] = [] + + @staticmethod + def key(args: list[str]) -> str: + joined = " ".join(args) + if "head:implementation/" in joined: + return "prs" + if "regen:kept" in joined: + return "kept" + if args[1:3] == ["issue", "list"]: + return "issues" + if args[1:3] == ["api", "--paginate"] and "/comments" in args[3]: + return "comments-" + args[3].split("/issues/")[1].split("/")[0] + return "unknown" + + def __call__(self, args: list[str], **kwargs) -> subprocess.CompletedProcess: + assert args[0] == "gh" and kwargs == {"check": True, "capture_output": True, "text": True} + self.calls.append(args) + value = self.data.get(self.key(args)) + if value is None: + raise subprocess.CalledProcessError(1, args, stderr="HTTP 502") + if isinstance(value, BaseException): + raise value + return subprocess.CompletedProcess(args, 0, stdout=value, stderr="") + + +@pytest.fixture +def runner(monkeypatch): + def install(data: dict[str, str | BaseException]) -> FakeRunner: + fake = FakeRunner(data) + monkeypatch.setattr(rp.subprocess, "run", fake) + return fake + + return install + + +def pr(branch: str, state: str, number: int = 1, labels: tuple[str, ...] = (), **times: str) -> dict: + return {"number": number, "headRefName": branch, "state": state, "labels": [{"name": n} for n in labels], **times} + + +class TestLoadCandidatesEdges: + @pytest.mark.parametrize( + "spec_yaml", + [ + None, # no specification.yaml at all + "issue: [unclosed\n", # not YAML + "- a list\n", # YAML, but no mapping + "issue: '7'\n", # not a number + ], + ) + def test_a_spec_without_a_readable_issue_number_is_still_a_candidate(self, tmp_path, spec_yaml): + write_spec(tmp_path, "a", {"altair": {"updated": ts(10)}}, issue=7) + if spec_yaml is None: + (tmp_path / "a" / "specification.yaml").unlink() + else: + (tmp_path / "a" / "specification.yaml").write_text(spec_yaml, encoding="utf-8") + cands, issues = rp.load_candidates(tmp_path, LIBS) + assert [c.library for c in cands] == ["altair"] and issues == {} + + def test_unreadable_metadata_and_stray_entries_are_skipped(self, tmp_path): + write_spec(tmp_path, "a", {"altair": "updated: [unclosed\n", "plotly": {"updated": ts(10)}}) + (tmp_path / ".hidden").mkdir() + (tmp_path / ".hidden" / "specification.md").write_text("# s\n", encoding="utf-8") + (tmp_path / "README.md").write_text("not a spec\n", encoding="utf-8") + assert [(c.spec, c.library) for c in rp.load_candidates(tmp_path, LIBS)[0]] == [("a", "plotly")] + + +class TestFoldPullsEdges: + def test_other_states_and_unnumbered_pauses(self): + recs = records() + rp.fold_pulls( + [ + pr("implementation/a/altair", "DRAFT", closedAt=ts(1)), # neither open nor done + {"headRefName": "implementation/a/plotly", "state": "OPEN", "labels": [{"name": "ai-review-paused"}]}, + pr("implementation/b/plotly", "OPEN", 9, ("ai-repair-paused", "ai-attempt-x")), + {"headRefName": None, "state": "MERGED"}, + pr("implementation/c/plotly", "MERGED", 3), # no timestamp: no attempt + ], + recs, + ) + assert recs.attempts == {} + assert recs.open_pairs == {("a", "plotly"), ("b", "plotly")} + # The unnumbered pause blocks its pair but can't be resumed; a malformed + # attempt label falls back to attempt 1, a missing time to ANCIENT. + assert recs.paused == [rp.Resume(9, "repair", "b", "plotly", 1, rp.ANCIENT)] + + def test_both_pause_labels_resume_the_repair(self): + """PAUSED_LABELS lists the repair first; a resumed repair reviews again anyway.""" + recs = records() + rp.fold_pulls([pr("implementation/a/plotly", "OPEN", 5, ("ai-review-paused", "ai-repair-paused"))], recs) + assert [r.kind for r in recs.paused] == ["repair"] + + +class TestGhJson: + def test_a_list_comes_back(self, runner): + fake = runner({"issues": '[{"number": 1}]'}) + assert rp._gh_json(["issue", "list"], "issues") == [{"number": 1}] + assert fake.calls == [["gh", "issue", "list"]] + + @pytest.mark.parametrize( + "answer", + [None, "not json", FileNotFoundError("gh"), subprocess.CalledProcessError(1, ["gh"], stderr="HTTP 502")], + ) + def test_a_failure_warns_and_yields_none(self, runner, capsys, answer): + runner({} if answer is None else {"issues": answer}) + assert rp._gh_json(["issue", "list"], "issues") is None + assert capsys.readouterr().out.startswith("::warning::Could not list issues (") + + def test_a_non_list_yields_none(self, runner, capsys): + runner({"issues": '{"message": "Not Found"}'}) + assert rp._gh_json(["issue", "list"], "issues") is None + assert capsys.readouterr().out == "" + + +class TestReadRecords: + def test_folds_every_listing(self, runner, capsys): + fake = runner( + { + "prs": json.dumps( + [ + pr("implementation/a/altair", "MERGED", closedAt=ts(9)), + pr("implementation/a/plotly", "OPEN", 7, ("ai-review-paused",), updatedAt=ts(1)), + ] + ), + "kept": json.dumps([pr("implementation/b/altair", "CLOSED", 8, ("regen:kept",), closedAt=ts(200))]), + "issues": json.dumps([{"number": 11, "updatedAt": ts(2)}]), + } + ) + recs = rp.read_records("o/r") + assert set(recs.attempts) == {("a", "altair"), ("b", "altair")} + assert recs.attempts[("b", "altair")][1] == "kept PR" + assert recs.open_pairs == {("a", "plotly")} and [r.pr for r in recs.paused] == [7] + assert set(recs.issue_activity) == {11} + assert ( + "Attempt records for 2 pair(s), 1 open PR(s) (1 paused), activity for 1 issue(s)" in capsys.readouterr().out + ) + prs_call, kept_call, issues_call = fake.calls + assert prs_call[prs_call.index("--json") + 1] == "number,headRefName,state,updatedAt,closedAt,labels" + assert prs_call[prs_call.index("--state") + 1] == "all" and "--repo" in prs_call and "o/r" in prs_call + assert kept_call[kept_call.index("--state") + 1] == "closed" + assert issues_call[issues_call.index("--json") + 1] == "number,updatedAt" + + @pytest.mark.parametrize("prs", [None, '{"message": "rate limited"}']) + def test_no_implementation_prs_is_an_error(self, runner, prs): + fake = runner({"kept": "[]", "issues": "[]"} | ({} if prs is None else {"prs": prs})) + with pytest.raises(rp.ListingError, match="implementation PRs could not be listed"): + rp.read_records("o/r") + assert len(fake.calls) == 1 # nothing else is read + + def test_the_other_listings_are_best_effort(self, runner, capsys): + runner({"prs": json.dumps([pr("implementation/a/altair", "CLOSED", closedAt=ts(5))])}) + recs = rp.read_records("o/r") + assert recs.attempts == {("a", "altair"): (at(5), "closed PR")} and recs.issue_activity == {} + out = capsys.readouterr().out + assert "Could not list kept regenerations" in out and "Could not list issues" in out + + +class TestIssueFailures: + def test_reads_the_spec_issue_once_per_call(self, runner): + comments = [ + fail_comment("a", "altair", 4), + fail_comment("a", "plotly", 2, infra=True), + fail_comment("b", "altair", 1), + ] + fake = runner({"comments-11": "\n".join(json.dumps(c) for c in comments) + "\n\n"}) + failures = rp.issue_failures("o/r", {"a": 11}) + assert failures("a") == {"altair": at(4)} + (call,) = fake.calls + assert call[3] == "repos/o/r/issues/11/comments?per_page=100" + assert '"