diff --git a/.claude/skills/review-regen-week/SKILL.md b/.claude/skills/review-regen-week/SKILL.md index 6ae55be0ccf..640e0c91fe2 100644 --- a/.claude/skills/review-regen-week/SKILL.md +++ b/.claude/skills/review-regen-week/SKILL.md @@ -5,7 +5,8 @@ description: Weekly review of what daily-regen produced — collect every regene # Weekly regen review -`daily-regen.yml` regenerates one spec per night (02:17 UTC): 15 pairs, +`daily-regen.yml` regenerates one spec per night (02:17 UTC): 15 pairs +(in pair mode, `REGEN_PAIRS_PER_DAY` single pairs across the catalogue), each reviewed once on Opus, and the regen gate either merges the new version (`regen:improved`) or keeps the live one (`regen:kept`, with a write-back of the re-score). Once a week you look at what that did to @@ -65,8 +66,10 @@ uv run python -m automation.scripts.review_retest first-reviews \ no gate record; it is listed so you can check why it was forced. The collector also counts the window's pipeline runs by conclusion, prints the failed ones under the metric table, and with `--costs` - sums `total_cost_usd` from every generate, review and repair job log - (about one minute per 50 runs; leave it off for a quick look). Its + sums `total_cost_usd` from every daily-regen pre-flight, generate, + review and repair job log, and names the part that ran on API credits + (`CLAUDE_AUTH=oidc` or `api`, read from each step's `claude-cost` line; + about one minute per 50 runs; leave it off for a quick look). Its stdout is the metric table and the PR table the report starts from; `◆` marks the render sample. - `gate-report` adds the aggregate view and its soft alarms, among diff --git a/.claude/skills/review-regen-week/collect_week.py b/.claude/skills/review-regen-week/collect_week.py index c5926d66776..2ed1f212a94 100644 --- a/.claude/skills/review-regen-week/collect_week.py +++ b/.claude/skills/review-regen-week/collect_week.py @@ -52,6 +52,15 @@ WRITEBACK_BRANCH_RE = re.compile(r"^review-writeback/(?P[^/]+)/(?P[^/]+)/(?P\d+)$") RUN_URL_RE = re.compile(r"/actions/runs/(\d+)") COST_RE = re.compile(r'"total_cost_usd":\s*([0-9.]+)') +# The line every Claude step's "Claude session summary" step prints, with the +# credential mode it ran on (vars.CLAUDE_AUTH). A job from before that step +# carries only Claude Code's own `"total_cost_usd": n` and ran on the +# subscription. +COST_LINE_RE = re.compile( + r"claude-cost step=\S+ auth=(?Papi|oidc|subscription) model=\S+ total_cost_usd=(?P[0-9.]+)" +) +# The modes billed to Claude API credits; the subscription is not billed per session. +API_MODES = ("api", "oidc") GCS_PUBLIC = "https://storage.googleapis.com/anyplot-images/plots" VERDICT_LABELS = ("regen:improved", "regen:kept", "ai-approved", "ai-rejected") PIPELINE_WORKFLOWS = ( @@ -62,7 +71,7 @@ "impl-repair.yml", "impl-merge.yml", ) -CLAUDE_WORKFLOWS = ("impl-generate.yml", "impl-review.yml", "impl-repair.yml") +CLAUDE_WORKFLOWS = ("daily-regen.yml", "impl-generate.yml", "impl-review.yml", "impl-repair.yml") # A keep whose re-score sits this far below the stored score is a render look. # Under the same judge (both reviews from one model family) a few points is @@ -309,7 +318,9 @@ def previously_reported(rows: Sequence[dict[str, Any]]) -> set[int]: return {int(r["pr"]) for r in rows if r.get("state") != "OPEN"} -def summarize(rows: Sequence[dict[str, Any]], runs: dict[str, Counter], cost: float | None) -> dict[str, Any]: +def summarize( + rows: Sequence[dict[str, Any]], runs: dict[str, Counter], cost: float | None, api_cost: float | None = None +) -> dict[str, Any]: decided = [r for r in rows if r["verdict"]] merges = [r for r in decided if r["verdict"] == "merge"] keeps = [r for r in decided if r["verdict"] == "keep"] @@ -337,6 +348,7 @@ def summarize(rows: Sequence[dict[str, Any]], runs: dict[str, Counter], cost: fl "identical_vectors": identical_vectors(rows), "runs": {wf: dict(counter) for wf, counter in runs.items()}, "cost_usd": round(cost, 2) if cost is not None else None, + "cost_usd_api": round(api_cost, 2) if api_cost is not None else None, } @@ -364,10 +376,18 @@ def summarize(rows: Sequence[dict[str, Any]], runs: dict[str, Counter], cost: fl ("Missing pair artifact", "0", lambda s: s["flags"].get("no_pair_artifact", 0)), ("daily-regen runs (success/all)", "7/7", lambda s: _success(s["runs"].get("daily-regen.yml", {}))), ("Failed pipeline runs", "few", lambda s: sum(c.get("failure", 0) for c in s["runs"].values())), - ("Session cost (USD)", "", lambda s: "not collected" if s["cost_usd"] is None else f"${s['cost_usd']:.0f}"), + ("Session cost (USD)", "", lambda s: _cost(s)), ) +def _cost(summary: dict[str, Any]) -> str: + """The week's session cost, and the part that ran on API credits when there is one.""" + if summary["cost_usd"] is None: + return "not collected" + api = summary.get("cost_usd_api") + return f"${summary['cost_usd']:.0f}" + (f" (API ${api:.0f})" if api else "") + + def _cell(value: Any) -> str: """A record key that older records lack (pre-P3.1 kinds, pre-P8 code) reads as a dash.""" return "–" if value is None else str(value) @@ -544,9 +564,20 @@ def pipeline_runs(since: str) -> dict[str, list[dict[str, Any]]]: return runs -def session_cost(runs: dict[str, list[dict[str, Any]]]) -> float: - """Sum of ``total_cost_usd`` over every job log of the Claude-running workflows.""" - total = 0.0 +def log_cost(log: str) -> dict[str, float]: + """One job log's session cost by credential: the cost lines, else Claude Code's own lines.""" + lines = COST_LINE_RE.findall(log) + if not lines: + return {"subscription": sum(float(m) for m in COST_RE.findall(log))} + by_auth: dict[str, float] = {} + for auth, usd in lines: + by_auth[auth] = by_auth.get(auth, 0.0) + float(usd) + return by_auth + + +def session_cost(runs: dict[str, list[dict[str, Any]]]) -> dict[str, float]: + """``total_cost_usd`` over every job log of the Claude-running workflows, by credential.""" + total: dict[str, float] = {} for workflow in CLAUDE_WORKFLOWS: for run in runs.get(workflow, []): jobs = _gh_json(["api", f"repos/{{owner}}/{{repo}}/actions/runs/{run['databaseId']}/jobs"]) or {} @@ -555,7 +586,8 @@ def session_cost(runs: dict[str, list[dict[str, Any]]]) -> float: log = _gh(["api", f"repos/{{owner}}/{{repo}}/actions/jobs/{job['id']}/logs"]) except subprocess.CalledProcessError: continue # an expired or missing log costs nothing to skip - total += sum(float(m) for m in COST_RE.findall(log)) + for auth, usd in log_cost(log).items(): + total[auth] = total.get(auth, 0.0) + usd return total @@ -602,7 +634,13 @@ def main(argv: Sequence[str] | None = None) -> int: ] for wf, listed in runs.items() } - summary = summarize(rows, run_counts, session_cost(runs) if args.costs else None) + costs = session_cost(runs) if args.costs else None + summary = summarize( + rows, + run_counts, + sum(costs.values()) if costs is not None else None, + sum(costs.get(mode, 0.0) for mode in API_MODES) if costs is not None else None, + ) sample, not_opened = render_sample(rows, seed=args.since) previous = previous_payload.get("summary") payload = { diff --git a/.github/workflows/daily-regen.yml b/.github/workflows/daily-regen.yml index 03b1a7bc10a..4a9587bd265 100644 --- a/.github/workflows/daily-regen.yml +++ b/.github/workflows/daily-regen.yml @@ -1,9 +1,41 @@ name: "Scheduled: Regen oldest specs" -run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.event.inputs.count || '1' }} / ${{ github.event.inputs.model || 'auto' }})" +run-name: "Scheduled regen (${{ github.event.inputs.specification_id || (github.event.inputs.pairs && format('{0} pairs', github.event.inputs.pairs)) || github.event.inputs.count || '1' }} / ${{ github.event.inputs.model || 'auto' }})" # Picks the N oldest specs (by most-recent implementation `updated` timestamp) # and re-dispatches `bulk-generate.yml` for each. Default N=1 per cron tick. # +# Pair mode (off unless configured): with the repository variable +# REGEN_PAIRS_PER_DAY set to N >= 1, the scheduled run instead picks the N +# oldest implementations, single (spec, library) pairs across the catalogue +# (automation/scripts/regen_pick_pairs.py), and dispatches each pair straight +# to impl-generate.yml, skipping the spec polish. A manual run opts in with the +# `pairs` input; the variable never changes a manual run, except the +# watchdog's cron rescue, which forwards it as `pairs` because it replaces the +# missed scheduled run. Unset, the schedule runs spec mode exactly as before. +# +# No re-pick loop: a pair's age is its newest touch, the metadata `updated` +# or its last real attempt (a merged or closed implementation PR, `regen:kept` +# included, or a failed generation's `impl-fail` comment on the spec issue). +# A pair touched within the attempt window (`attempt_window_days`, else the +# REGEN_ATTEMPT_WINDOW_DAYS variable, else 30 days) is not picked, and after +# it queues behind every pair touched before it. Infrastructure failures and +# generations Claude stopped are not attempts: the pair comes back on a later +# night. Spec mode has no window; a spec's age is the newer of its metadata +# and ANY activity on its issue, infrastructure failures included. +# +# When Claude stops a session (a spend or usage limit — the API's spend limit +# or credit balance, the subscription's usage limit — or, with api/oidc, +# refused credentials): a pre-flight session that is stopped skips the +# night's dispatch for its spec. In pair mode each spec's audit runs before +# its pairs are dispatched, so once a pair's chain is stopped, the later specs +# of the night stop at their audit and dispatch nothing. Pairs already +# dispatched fail fast at their first refused request: impl-generate parks +# the pair without a retry or a counted attempt, and impl-review and +# impl-repair pause the PR (`ai-review-paused`, `ai-repair-paused`, which the +# watchdog skips). Pair mode resumes paused PRs first, oldest first, within +# the night's N (job resume-paused), so they continue once the stop lifts +# (docs/workflows/overview.md#when-claude-stops-a-session). +# # Schedule: once per day at 02:17 UTC (04:17 Berlin under CEST, 03:17 under # CET — Actions cron is always UTC). Until 2026-09 it ticked every 2 hours # (10× per day); the owner cut it to one tick when re-enabling the workflow @@ -29,8 +61,9 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.e # # Triggers: # - schedule: once daily at 02:17 UTC → auto (Sonnet regenerations, Opus reviews) -# - workflow_dispatch: manual, with inputs for spec id, model, count, dry-run -# (also used by the watchdog's cron-liveness rescue, with all defaults) +# - workflow_dispatch: manual, with inputs for spec id, model, count, dry-run, +# pairs (also used by the watchdog's cron-liveness rescue, with all +# defaults — plus `pairs` when REGEN_PAIRS_PER_DAY is set) on: schedule: @@ -63,6 +96,14 @@ on: description: "Just print picks, do not trigger bulk-generate" type: boolean default: false + pairs: + description: "Pair mode: regen this many of the oldest implementations (spec × library) instead of whole specs, without spec polish (empty = spec mode; ignored when specification_id is set)" + required: false + default: '' + attempt_window_days: + description: "Pair mode only: skip pairs merged or attempted within this many days (empty = the REGEN_ATTEMPT_WINDOW_DAYS variable, else 30)" + required: false + default: '' permissions: contents: read @@ -77,11 +118,16 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - issues: read # spec issue updatedAt = last regen attempt per spec + issues: read # spec issue updatedAt = last regen attempt per spec + pull-requests: read # pair mode: implementation PRs = last attempt per pair outputs: - specs: ${{ steps.pick.outputs.specs }} - specs_json: ${{ steps.pick.outputs.specs_json }} - count: ${{ steps.pick.outputs.count }} + mode: ${{ steps.mode.outputs.mode }} + specs: ${{ steps.pick.outputs.specs || steps.pick_pairs.outputs.specs }} + specs_json: ${{ steps.pick.outputs.specs_json || steps.pick_pairs.outputs.specs_json }} + count: ${{ steps.pick.outputs.count || steps.pick_pairs.outputs.count }} + pairs_json: ${{ steps.pick_pairs.outputs.pairs_json }} + resume_json: ${{ steps.pick_pairs.outputs.resume_json }} + resume_count: ${{ steps.pick_pairs.outputs.resume_count }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -93,8 +139,32 @@ jobs: - name: Install PyYAML run: pip install pyyaml + # Spec mode unless a pair count is given: the `pairs` input on a manual + # run (the watchdog's cron rescue forwards REGEN_PAIRS_PER_DAY there), + # the REGEN_PAIRS_PER_DAY variable on the schedule. A spec override + # always wins. Anything but a positive integer stops the run instead of + # falling back to a whole spec. + - name: Resolve regen mode + id: mode + env: + PAIRS: ${{ github.event_name == 'schedule' && vars.REGEN_PAIRS_PER_DAY || inputs.pairs }} + SPEC_OVERRIDE: ${{ inputs.specification_id }} + run: | + if [ -n "$SPEC_OVERRIDE" ] || [ -z "$PAIRS" ]; then + echo "mode=spec" >> "$GITHUB_OUTPUT" + echo "Regen mode: spec" + elif [[ "$PAIRS" =~ ^[1-9][0-9]*$ ]]; then + echo "mode=pairs" >> "$GITHUB_OUTPUT" + echo "pairs=$PAIRS" >> "$GITHUB_OUTPUT" + echo "::notice::Regen mode: pairs (${PAIRS} per run)" + else + echo "::error::The pair count (REGEN_PAIRS_PER_DAY or the pairs input) must be a positive integer, got '${PAIRS//[^A-Za-z0-9 ._-]/?}'" + exit 1 + fi + - name: Pick oldest spec(s) (or use override) id: pick + if: steps.mode.outputs.mode == 'spec' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} @@ -235,6 +305,81 @@ jobs: f.write(f"count={len(picks)}\n") PY + # Pair mode: the paused reviews and repairs to resume first, then the + # oldest (spec, library) pairs for the rest of the night's N (see the + # script's docstring and the header above). The picks go to the log and + # the run summary, also on a dry run. The script fails when the + # implementation PRs cannot be listed: they are the loop guard. + - name: Pick oldest pairs (pair mode) + id: pick_pairs + if: steps.mode.outputs.mode == 'pairs' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COUNT: ${{ steps.mode.outputs.pairs }} + MIN_AGE_HOURS: ${{ inputs.min_age_hours || '20' }} + ATTEMPT_WINDOW_DAYS: ${{ inputs.attempt_window_days || vars.REGEN_ATTEMPT_WINDOW_DAYS || '30' }} + run: | + LIBRARIES=$(python3 -c "from core.constants import SUPPORTED_LIBRARIES; print(' '.join(sorted(SUPPORTED_LIBRARIES)))") + python3 automation/scripts/regen_pick_pairs.py --count "$COUNT" --min-age-hours "$MIN_AGE_HOURS" \ + --attempt-window-days "$ATTEMPT_WINDOW_DAYS" --repo "$GH_REPO" --libraries "$LIBRARIES" + + # ============================================================================ + # Pair mode: resume the reviews and repairs that Claude stopped (a spend or + # usage limit, or refused credentials; labels `ai-review-paused` and + # `ai-repair-paused`, which the watchdog leaves alone), oldest first, before + # the new pairs; they count against the night's N. A refused request costs + # nothing, so a stop that is still active only pauses them again, and they + # resume on their own once the limit resets, is raised, or the credentials + # are fixed. The resumed run removes the pause label. + # ============================================================================ + resume-paused: + needs: pick + if: ${{ needs.pick.outputs.mode == 'pairs' && needs.pick.outputs.resume_count != '0' && needs.pick.outputs.resume_count != '' && !inputs.dry_run }} + runs-on: ubuntu-latest + permissions: + contents: read + actions: write # dispatch impl-review / impl-repair + steps: + - name: Re-dispatch paused reviews and repairs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RESUME_JSON: ${{ needs.pick.outputs.resume_json }} + PACE_SECONDS: ${{ (inputs.model == 'sonnet' || inputs.model == 'haiku') && '120' || '180' }} + run: | + set -u + failed=0 + i=0 + while IFS=$'\t' read -r kind pr spec library attempt; do + i=$((i + 1)) + if [ "$i" -gt 1 ]; then + echo "Sleeping ${PACE_SECONDS}s before the next dispatch…" + sleep "$PACE_SECONDS" + fi + if [ "$kind" = "review" ]; then + args=(impl-review.yml -f pr_number="$pr") + else + args=(impl-repair.yml -f pr_number="$pr" -f specification_id="$spec" -f library="$library" -f attempt="$attempt") + fi + echo "::notice::resuming the paused ${kind} of PR #${pr} (${spec}/${library})" + dispatched=0 + for try in 1 2 3; do + if gh workflow run "${args[@]}" --repo "${{ github.repository }}"; then + dispatched=1 + break + fi + echo "::warning::Dispatch attempt $try failed for PR #${pr}, retrying in 10s" + sleep 10 + done + if [ "$dispatched" = "0" ]; then + echo "::error::Failed to resume PR #${pr} after 3 attempts" + failed=$((failed + 1)) + fi + done < <(jq -r '.[] | [.kind, (.pr | tostring), .spec, .library, (.attempt | tostring)] | @tsv' <<< "$RESUME_JSON") + if [ "$failed" -gt 0 ]; then + exit 1 + fi + # ============================================================================ # Pre-flight: per spec, run autonomous spec polish + cross-library similarity # audit, then dispatch bulk-generate with the resulting change_requests. @@ -248,6 +393,15 @@ jobs: # model ("auto" unless a manual run pins one) is passed through to # bulk-generate (and from there to impl-generate and repair) unchanged; # reviews always run on Opus. + # + # Pair mode: one matrix entry per spec among the picked pairs. The spec + # polish is skipped (a spec rewritten for one library would leave its other + # implementations written against the old text), the similarity audit runs + # as in spec mode, and each picked pair of the spec is dispatched straight + # to impl-generate.yml with bulk-generate's arguments and pacing — not via + # bulk-generate.yml, whose single concurrency group keeps at most one + # pending run and cancels the older one (see the watchdog's + # dispatch_generate). # ============================================================================ preflight-dispatch: needs: pick @@ -268,6 +422,65 @@ jobs: # Side-effect steps (polish, dispatch) are individually gated on # `!inputs.dry_run` below. steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; + esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -294,14 +507,20 @@ jobs: fi - name: Spec polish (autonomous, opens PR with auto-merge) - if: ${{ steps.gate.outputs.skip_polish == '0' && !inputs.dry_run }} + id: polish + if: ${{ steps.gate.outputs.skip_polish == '0' && !inputs.dry_run && needs.pick.outputs.mode != 'pairs' }} # Optional quality pass: a transient action failure here must not # block the main regeneration pipeline. Skip cleanly and continue. continue-on-error: true timeout-minutes: 15 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: '--model sonnet --settings ${{ github.workspace }}/.claude/settings.json' allowed_bots: '*' prompt: | @@ -310,14 +529,77 @@ jobs: Variables for this run: - SPEC_ID: ${{ matrix.spec_id }} + # The session's cost (Claude Code's own estimate at API list price), + # model and credential mode, for the weekly regen review, and whether + # Claude stopped it: a spend or usage limit, or refused credentials + # (outputs stopped, stop_reason). Never fatal. + - name: Claude session summary (spec polish) + id: polish_cost + if: ${{ !cancelled() && (steps.polish.outcome == 'success' || steps.polish.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.polish.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: daily-regen/polish + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + - name: Cross-library similarity audit + id: similarity # Read-only audit; if it fails, fall back to empty change_requests # rather than aborting the dispatch. continue-on-error: true timeout-minutes: 15 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: '--model sonnet --settings ${{ github.workspace }}/.claude/settings.json' allowed_bots: '*' prompt: | @@ -326,6 +608,59 @@ jobs: Variables for this run: - SPEC_ID: ${{ matrix.spec_id }} + - name: Claude session summary (similarity audit) + id: similarity_cost + if: ${{ !cancelled() && (steps.similarity.outcome == 'success' || steps.similarity.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.similarity.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: daily-regen/similarity + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + - name: Collect change_requests id: collect run: | @@ -347,8 +682,21 @@ jobs: flagged=$(echo "$CR_COMPACT" | jq 'length') echo "::notice::change_requests for ${{ matrix.spec_id }}: ${flagged} lib(s) flagged — ${CR_COMPACT}" + # Claude stopped a pre-flight session (a spend or usage limit, or + # refused credentials): every generation dispatched now would be + # refused the same way, so this spec's dispatch is skipped and the entry + # fails. In pair mode the audit of each spec runs before that spec's + # pairs are dispatched, so once a pair's chain has hit the stop, every + # later spec of the night stops here (a refused request costs nothing), + # and its pairs stay in the queue for a later night. + - name: Stop when Claude stopped the session + if: ${{ steps.polish_cost.outputs.stopped == 'true' || steps.similarity_cost.outputs.stopped == 'true' }} + run: | + echo "::error::Claude stopped the pre-flight (${{ steps.polish_cost.outputs.stop_reason || steps.similarity_cost.outputs.stop_reason }}); nothing dispatched for ${{ matrix.spec_id }}" + exit 1 + - name: Dispatch bulk-generate with change_requests - if: ${{ !inputs.dry_run }} + if: ${{ !inputs.dry_run && needs.pick.outputs.mode != 'pairs' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SPEC_ID: ${{ matrix.spec_id }} @@ -366,3 +714,57 @@ jobs: # the next matrix entry's dispatch (matrix is serialized via # max-parallel: 1, so this is between specs). sleep 5 + + # Pair mode: bulk-generate's per-pair dispatch for each picked library of + # this spec — the issue from specification.yaml, the library's + # divergence hint, the regen gate on, three dispatch attempts — with + # bulk-generate's default pace (180 s, 120 s for a pinned sonnet or + # haiku) before every dispatch but the run's first. impl-generate sets + # `impl::pending` itself, so this job needs no issue permission. + - name: Dispatch impl-generate for each picked pair (pair mode) + if: ${{ !inputs.dry_run && needs.pick.outputs.mode == 'pairs' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SPEC_ID: ${{ matrix.spec_id }} + MODEL: ${{ inputs.model || 'auto' }} + CHANGE_REQUESTS: ${{ steps.collect.outputs.change_requests }} + PAIRS_JSON: ${{ needs.pick.outputs.pairs_json }} + PACE_SECONDS: ${{ (inputs.model == 'sonnet' || inputs.model == 'haiku') && '120' || '180' }} + run: | + set -u + ISSUE="" + if [ -f "plots/${SPEC_ID}/specification.yaml" ]; then + ISSUE=$(yq '.issue' "plots/${SPEC_ID}/specification.yaml" 2>/dev/null || echo "") + [ "$ISSUE" = "null" ] && ISSUE="" + fi + FIRST=$(echo "$PAIRS_JSON" | jq -r '.[0] | "\(.spec)/\(.library)"') + failed=0 + for LIBRARY in $(echo "$PAIRS_JSON" | jq -r --arg spec "$SPEC_ID" '.[] | select(.spec == $spec) | .library'); do + if [ "${SPEC_ID}/${LIBRARY}" != "$FIRST" ]; then + echo "Sleeping ${PACE_SECONDS}s before the next dispatch…" + sleep "$PACE_SECONDS" + fi + HINT=$(echo "$CHANGE_REQUESTS" | jq -r --arg lib "$LIBRARY" '.[$lib] // ""') + args=(-f specification_id="${SPEC_ID}" -f library="${LIBRARY}" -f model="${MODEL}" + -f change_request="${HINT}" -f regen_gate=true -f source_ref= -f source_model=) + if [ -n "$ISSUE" ]; then + args+=(-f issue_number="${ISSUE}") + fi + echo "::notice::$(date -u +%H:%M:%SZ) dispatching impl-generate for ${SPEC_ID}/${LIBRARY} (issue: ${ISSUE:-none}, model=${MODEL}, change_request: ${HINT:-none})" + dispatched=0 + for attempt in 1 2 3; do + if gh workflow run impl-generate.yml --repo "${{ github.repository }}" "${args[@]}"; then + dispatched=1 + break + fi + echo "::warning::Dispatch attempt $attempt failed for ${SPEC_ID}/${LIBRARY}, retrying in 10s" + sleep 10 + done + if [ "$dispatched" = "0" ]; then + echo "::error::Failed to dispatch ${SPEC_ID}/${LIBRARY} after 3 attempts — continuing" + failed=$((failed + 1)) + fi + done + if [ "$failed" -gt 0 ]; then + exit 1 + fi diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml index b17dce9091c..2e00b069c93 100644 --- a/.github/workflows/impl-generate.yml +++ b/.github/workflows/impl-generate.yml @@ -98,6 +98,67 @@ jobs: pr_number: ${{ steps.pr.outputs.pr_number }} steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + # First step: a stop here precedes "Set final issue number", so "Handle + # generation failure" never counts it against the pair's attempts. + - name: Resolve Claude credentials + id: claude_auth + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; + esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" + # ======================================================================== # Setup: Extract inputs and validate # ======================================================================== @@ -552,6 +613,9 @@ jobs: EXT: ${{ steps.inputs.outputs.ext }} BRANCH: ${{ steps.branch.outputs.branch }} run: | + # This step runs the adopted, model-written code: it gets no way to + # mint an OIDC token (the job has id-token for Claude and Google Cloud). + unset ACTIONS_ID_TOKEN_REQUEST_URL ACTIONS_ID_TOKEN_REQUEST_TOKEN IMPL_DIR="plots/${SPEC_ID}/implementations/${LANGUAGE}" IMPL_FILE="${IMPL_DIR}/${LIBRARY}${EXT}" AUTHOR="${SOURCE_MODEL:+, written by ${SOURCE_MODEL}}" @@ -651,7 +715,12 @@ jobs: timeout-minutes: 60 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.inputs.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -665,13 +734,78 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} + # The session's cost (Claude Code's own estimate at API list price), + # model and credential mode, for the weekly regen review, and whether + # Claude stopped it: a spend or usage limit, or refused credentials + # (outputs stopped, stop_reason). Read right after the step: the retry + # writes the same execution file. Never fatal. + - name: Claude session summary + id: claude_cost + if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: impl-generate + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Not after Claude stopped the session (a limit or refused credentials): + # the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' && steps.inputs.outputs.source_ref == '' + if: steps.claude.outcome == 'failure' && steps.inputs.outputs.source_ref == '' && steps.claude_cost.outputs.stopped != 'true' id: claude_retry timeout-minutes: 60 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.inputs.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -685,6 +819,68 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} + - name: Claude session summary (retry) + id: claude_retry_cost + if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: impl-generate/retry + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Claude stopped the first session (a limit or refused credentials) and + # its retry was skipped: fail here, so "Handle generation failure" parks + # the pair instead of the steps below running without an implementation. + - name: Stop when Claude stopped the session + if: steps.claude_cost.outputs.stopped == 'true' + run: | + echo "::error::Claude stopped the session (${{ steps.claude_cost.outputs.stop_reason }}); the retry is skipped" + exit 1 + # ======================================================================== # Create metadata file (before PR) # ======================================================================== @@ -1088,6 +1284,9 @@ jobs: SPEC_ID: ${{ steps.inputs.outputs.specification_id }} LIBRARY: ${{ steps.inputs.outputs.library }} run: | + # This step runs model-written code: it gets no way to mint an OIDC + # token (the job has id-token for Claude and Google Cloud). + unset ACTIONS_ID_TOKEN_REQUEST_URL ACTIONS_ID_TOKEN_REQUEST_TOKEN OUT="$RUNNER_TEMP/adapt"; TOOLS="$RUNNER_TEMP/adapt-tools" rm -rf "$OUT" "$TOOLS"; mkdir -p "$OUT/prev-src" "$TOOLS" # The runner comes from this workflow's ref, never from the branch the @@ -1485,6 +1684,12 @@ jobs: # job by itself). CLAUDE_RETRY_OUTCOME: ${{ steps.claude_retry.outcome }} CLAUDE_EXEC_FILE: ${{ steps.claude_retry.outputs.execution_file }} + # Claude stopped either session ("Claude session summary" reads it: + # a spend or usage limit, or refused credentials with api/oidc): + # parked below, never retried, never counted. + CLAUDE_STOPPED: ${{ steps.claude_cost.outputs.stopped == 'true' || steps.claude_retry_cost.outputs.stopped == 'true' }} + STOP_REASON: ${{ steps.claude_cost.outputs.stop_reason || steps.claude_retry_cost.outputs.stop_reason }} + CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} GCP_AUTH_OUTCOME: ${{ steps.gcp_auth.outcome }} GCLOUD_OUTCOME: ${{ steps.gcloud.outcome }} GCS_OUTCOME: ${{ steps.gcs.outcome }} @@ -1596,6 +1801,48 @@ jobs: exit 0 fi + # Claude stopped the session: a spend or usage limit (the API's + # spend limit or credit balance, the subscription's usage limit) or, + # with api/oidc, refused credentials. An infrastructure stop that a + # retry cannot get past until it is resolved, so the pair is parked + # at once — for a first implementation too. No auto-retry, no + # attempt counted (the comment carries its own marker, never the + # impl-fail one, so neither the 3-attempt cap nor daily-regen's + # attempt window sees it), never `impl::failed`. The pair stays + # in daily-regen's queue. + if [ "${CLAUDE_STOPPED}" = "true" ]; then + if [ "${STOP_REASON}" = "auth" ]; then + WHY="Claude credentials refused" + WHEN="Once the credentials are fixed" + else + WHY="Claude spend/usage limit reached" + WHEN="Once the limit resets or is raised" + fi + echo "::error::${WHY} (auth=${CLAUDE_MODE}) — ${LIBRARY}/${SPEC_ID} parked: no auto-retry, no attempt counted, not marked failed" + for stale in "generate:${LIBRARY}" "impl:${LIBRARY}:pending"; do + gh issue edit "$ISSUE" --remove-label "$stale" 2>/dev/null || true + done + if [ "$IS_LIVE" = "true" ]; then + gh issue edit "$ISSUE" --add-label "impl:${LIBRARY}:done" 2>/dev/null || true + fi + gh issue comment "$ISSUE" --body " + ## :pause_button: ${LIBRARY} Paused (${WHY}) + + Claude stopped the **${LIBRARY}** generation for \`${SPEC_ID}\`: ${WHY} (auth=\`${CLAUDE_MODE}\`). This is **not** a capability verdict: nothing was retried, no attempt was counted, and the pair is not marked failed. + + ${WHEN}, run: + \`\`\` + gh workflow run impl-generate.yml -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f issue_number=${ISSUE} -f model=${MODEL} + \`\`\` + + --- + :robot: *[impl-generate](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*" + if [ "$IS_LIVE" = "true" ]; then + close_issue_if_complete + fi + exit 0 + fi + # Classify the failure. The retry cap exists to stop re-running a pair # the model cannot solve; a provider incident, a cloud token refresh # or a GitHub 5xx says nothing about the pair. During the Claude diff --git a/.github/workflows/impl-repair.yml b/.github/workflows/impl-repair.yml index c91dc9810a0..b8a3afb0b1b 100644 --- a/.github/workflows/impl-repair.yml +++ b/.github/workflows/impl-repair.yml @@ -52,6 +52,65 @@ jobs: id-token: write steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; + esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -217,11 +276,15 @@ jobs: echo "$FEEDBACK" > /tmp/ai_feedback.md + # `ai-repair-paused` goes too: this run resumes a repair that Claude + # stopped (a limit or refused credentials); the stop path puts it back + # if Claude stops this one as well. - name: Remove ai-rejected label env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh pr edit ${{ inputs.pr_number }} --remove-label "ai-rejected" 2>/dev/null || true + gh pr edit ${{ inputs.pr_number }} --remove-label "ai-repair-paused" 2>/dev/null || true - name: Run Claude Code to repair implementation id: claude @@ -229,7 +292,12 @@ jobs: timeout-minutes: 45 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.model.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -243,13 +311,78 @@ jobs: - ATTEMPT: ${{ inputs.attempt }} - BRANCH: ${{ env.branch }} + # The session's cost (Claude Code's own estimate at API list price), + # model and credential mode, for the weekly regen review, and whether + # Claude stopped it: a spend or usage limit, or refused credentials + # (outputs stopped, stop_reason). Read right after the step: the retry + # writes the same execution file. Never fatal. + - name: Claude session summary + id: claude_cost + if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: impl-repair + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Not after Claude stopped the session (a limit or refused credentials): + # the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' + if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.stopped != 'true' id: claude_retry timeout-minutes: 45 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.model.outputs.model }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -263,6 +396,69 @@ jobs: - ATTEMPT: ${{ inputs.attempt }} - BRANCH: ${{ env.branch }} + - name: Claude session summary (retry) + id: claude_retry_cost + if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: impl-repair/retry + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Claude stopped the first session (a limit or refused credentials) and + # its retry was skipped: fail here, so "Handle repair failure" pauses the + # PR without an auto-retry instead of the steps below running on an + # unrepaired file. + - name: Stop when Claude stopped the session + if: steps.claude_cost.outputs.stopped == 'true' + run: | + echo "::error::Claude stopped the session (${{ steps.claude_cost.outputs.stop_reason }}); the retry is skipped" + exit 1 + - name: Process repaired image if: success() env: @@ -308,6 +504,9 @@ jobs: SPEC_ID: ${{ inputs.specification_id }} LIBRARY: ${{ inputs.library }} run: | + # This step runs model-written code: it gets no way to mint an OIDC + # token (the job has id-token for Claude and Google Cloud). + unset ACTIONS_ID_TOKEN_REQUEST_URL ACTIONS_ID_TOKEN_REQUEST_TOKEN OUT="$RUNNER_TEMP/adapt"; TOOLS="$RUNNER_TEMP/adapt-tools" rm -rf "$OUT" "$TOOLS"; mkdir -p "$OUT" "$TOOLS" # The runner comes from this workflow's ref, never from the PR branch. @@ -418,11 +617,61 @@ jobs: MODEL: ${{ steps.model.outputs.model || inputs.model || 'auto' }} REPOSITORY: ${{ github.repository }} RUN_ID: ${{ github.run_id }} + # Claude stopped either session (a limit or refused credentials). + CLAUDE_STOPPED: ${{ steps.claude_cost.outputs.stopped == 'true' || steps.claude_retry_cost.outputs.stopped == 'true' }} + STOP_REASON: ${{ steps.claude_cost.outputs.stop_reason || steps.claude_retry_cost.outputs.stop_reason }} + CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} run: | # Restore ai-rejected label that was removed at start of repair so the # PR state stays consistent (otherwise it looks "stuck approved"). gh pr edit "$PR_NUM" --add-label "ai-rejected" 2>/dev/null || true + # Claude stopped the session (a spend or usage limit, or refused + # credentials): an auto-retry now would be refused the same way. + # `ai-repair-paused` keeps the watchdog's one-shot rescue off the PR; + # daily-regen's pair mode, or a manual restart, resumes it. The + # comment carries its own marker, not the repair-retry one, so the + # attempt keeps its one crash retry for after the stop is resolved. + if [ "${CLAUDE_STOPPED}" = "true" ]; then + if [ "${STOP_REASON}" = "auth" ]; then + WHY="Claude credentials refused" + WHEN="once the credentials are fixed" + else + WHY="Claude spend/usage limit reached" + WHEN="once the limit resets or is raised" + fi + echo "::error::${WHY} (auth=${CLAUDE_MODE}) — repair attempt ${ATTEMPT} paused, no auto-retry" + gh label create "ai-repair-paused" --color "fbca04" \ + --description "Repair paused: Claude stopped it (spend/usage limit or refused credentials)" 2>/dev/null || true + gh pr edit "$PR_NUM" --add-label "ai-repair-paused" 2>/dev/null \ + || echo "::warning::could not add ai-repair-paused to PR #${PR_NUM}" + # A resume while the stop lasts pauses again: one comment per + # pause, so the label alone stands when the PR's latest bot + # comment is already this attempt's pause (its first line is the + # marker). A read that fails posts it. + PAUSE_MARKER="" + LAST_BOT=$(gh api --paginate "repos/${REPOSITORY}/issues/${PR_NUM}/comments?per_page=100" \ + --jq '.[] | select(.user.type == "Bot") | (.body // "") | split("\n") | .[0] // "" | rtrimstr("\r")' 2>/dev/null \ + | tail -n 1) || LAST_BOT="" + if [ "$LAST_BOT" = "$PAUSE_MARKER" ]; then + echo "::notice::PR #${PR_NUM} already shows this pause; no new comment" + exit 0 + fi + gh pr comment "$PR_NUM" --body "${PAUSE_MARKER} + ## :pause_button: Repair Paused (${WHY}, Attempt ${ATTEMPT}/4) + + Claude stopped the repair: ${WHY} (auth=\`${CLAUDE_MODE}\`). Nothing was retried, and the attempt is not used up. The PR carries \`ai-repair-paused\`, which the watchdog leaves alone; daily-regen's pair mode resumes it. + + **Restart ${WHEN}:** + \`\`\` + gh workflow run impl-repair.yml -f pr_number=${PR_NUM} -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f attempt=${ATTEMPT} -f model=${MODEL} + \`\`\` + + --- + :robot: *[impl-repair](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" || true + exit 0 + fi + MARKER="" # Paginate so the marker is found even on PRs with >30 comments. # `--paginate --jq` prints one count per page, so the pages are diff --git a/.github/workflows/impl-review.yml b/.github/workflows/impl-review.yml index 2f402e32f66..ee34b3aef83 100644 --- a/.github/workflows/impl-review.yml +++ b/.github/workflows/impl-review.yml @@ -48,6 +48,65 @@ jobs: actions: write steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; + esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -217,6 +276,23 @@ jobs: echo "::warning::ai-review-failed could not be removed from PR #${PR_NUM} — the watchdog skips it while this review runs" exit 0 + # This run resumes a review that Claude stopped (a spend or usage limit, + # or refused credentials): `ai-review-paused` goes, and the stop path in + # "Validate review output" puts it back if Claude stops this one too. + # Best effort. + - name: Clear the ai-review-paused label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUM: ${{ steps.pr.outputs.pr_number }} + run: | + if gh pr view "$PR_NUM" --json labels -q '.labels[].name' 2>/dev/null | grep -qxF "ai-review-paused"; then + if gh pr edit "$PR_NUM" --remove-label "ai-review-paused" 2>/dev/null; then + echo "::notice::Removed ai-review-paused from PR #${PR_NUM}: this run resumes the review" + else + echo "::warning::could not remove ai-review-paused from PR #${PR_NUM}" + fi + fi + - name: Derive language + extension from library id: lang env: @@ -890,7 +966,12 @@ jobs: PR_NUMBER: ${{ steps.pr.outputs.pr_number }} ATTEMPT: ${{ steps.attempts.outputs.display }} with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model ${{ steps.pr.outputs.review_model_alias }} --settings ${{ github.workspace }}/.claude/settings.json" allowed_bots: '*' prompt: | @@ -925,6 +1006,64 @@ jobs: run: | python3 "$RUNNER_TEMP/regen-tools/review_provenance.py" model --execution-file "${EXECUTION_FILE:-}" + # The session's cost (Claude Code's own estimate at API list price), + # model and credential mode, for the weekly regen review, and whether + # Claude stopped it: a spend or usage limit, or refused credentials + # (outputs stopped, stop_reason). "Extract quality score" and + # "Validate review output" read it. Never fatal. + - name: Claude session summary + id: review_cost + if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: impl-review + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + # Undo "Hide the previous review", whatever the review step did: the # metadata step below reads the metadata file from disk and rewrites # the header. The implementation file goes back to the branch's copy @@ -952,9 +1091,14 @@ jobs: git checkout -- "$META_FILE" 2>/dev/null || true fi + # Not from a session Claude stopped (a limit or refused credentials), + # even when it wrote quality_score.txt before the stop: a cut-off + # review gives no score, no verdict and no gate decision. The step is + # skipped, `has_output` stays empty, and "Validate review output" takes + # the paused path. - name: Extract quality score id: score - if: steps.review.conclusion == 'success' + if: steps.review.conclusion == 'success' && steps.review_cost.outputs.stopped != 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUM: ${{ steps.pr.outputs.pr_number }} @@ -1028,9 +1172,63 @@ jobs: RUN_ID: ${{ github.run_id }} MODEL: ${{ steps.pr.outputs.model }} REVIEW_MODEL_ALIAS: ${{ steps.pr.outputs.review_model_alias }} + # Claude stopped the session (a limit or refused credentials). + CLAUDE_STOPPED: ${{ steps.review_cost.outputs.stopped }} + STOP_REASON: ${{ steps.review_cost.outputs.stop_reason }} + CLAUDE_MODE: ${{ steps.claude_auth.outputs.mode }} run: | echo "::error::AI Review did not produce valid output files (score=0)" + # Claude stopped the session (a spend or usage limit, or refused + # credentials): a re-review now would be refused the same way, and + # it says nothing about the implementation. `ai-review-paused` (not + # `ai-review-failed`) keeps the watchdog's one-shot rescue off the + # PR; daily-regen's pair mode, or a manual rerun, resumes it. No + # review-retry marker, so the one auto-retry stays for after the stop + # is resolved. No verdict, no score, nothing merged or closed. + if [ "${CLAUDE_STOPPED}" = "true" ]; then + if [ "${STOP_REASON}" = "auth" ]; then + WHY="Claude credentials refused" + WHEN="once the credentials are fixed" + else + WHY="Claude spend/usage limit reached" + WHEN="once the limit resets or is raised" + fi + echo "::error::${WHY} (auth=${CLAUDE_MODE}) — review of PR #${PR_NUM} paused, no auto-retry" + gh label create "ai-review-paused" --color "fbca04" \ + --description "Review paused: Claude stopped it (spend/usage limit or refused credentials)" 2>/dev/null || true + for attempt in 1 2 3; do + gh pr edit "$PR_NUM" --add-label "ai-review-paused" 2>/dev/null && break + [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) + done || echo "::error::could not add ai-review-paused to PR #${PR_NUM} — resume it with a manual rerun" + # A resume while the stop lasts pauses again: one comment per + # pause, so the label alone stands when the PR's latest bot + # comment is already this pause (its first line is the marker). + # A read that fails posts it. + PAUSE_MARKER="" + LAST_BOT=$(gh api --paginate "repos/${REPOSITORY}/issues/${PR_NUM}/comments?per_page=100" \ + --jq '.[] | select(.user.type == "Bot") | (.body // "") | split("\n") | .[0] // "" | rtrimstr("\r")' 2>/dev/null \ + | tail -n 1) || LAST_BOT="" + if [ "$LAST_BOT" = "$PAUSE_MARKER" ]; then + echo "::notice::PR #${PR_NUM} already shows this pause; no new comment" + exit 1 + fi + gh pr comment "$PR_NUM" --body "${PAUSE_MARKER} + ## :pause_button: AI Review Paused (${WHY}) + + Claude stopped the review: ${WHY} (auth=\`${CLAUDE_MODE}\`). No score and no verdict were recorded, and the one automatic re-review is still unused. The PR carries \`ai-review-paused\`, which the watchdog leaves alone; daily-regen's pair mode resumes it. + + **Rerun ${WHEN}:** + \`\`\` + gh workflow run impl-review.yml -f pr_number=$PR_NUM + \`\`\` + + --- + :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" \ + || echo "::error::pause comment not posted on PR #${PR_NUM}" + exit 1 + fi + MARKER="" # Paginate so the marker is found even on PRs with >30 comments. # `--paginate --jq` prints one count per page, so the pages are diff --git a/.github/workflows/report-validate.yml b/.github/workflows/report-validate.yml index 9e0d9512ca9..634911b9a89 100644 --- a/.github/workflows/report-validate.yml +++ b/.github/workflows/report-validate.yml @@ -41,6 +41,66 @@ jobs: fi echo "should_run=true" >> $GITHUB_OUTPUT + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth + if: steps.check.outputs.should_run == 'true' + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; + esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" + - name: Checkout repository if: steps.check.outputs.should_run == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -65,7 +125,12 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Validate Issue Report @@ -157,6 +222,63 @@ jobs: - Do NOT trigger any fixes - Keep the analysis comment concise but informative + # The session's cost (Claude Code's own estimate at API list price), + # model and credential mode, and whether Claude stopped it: a spend or + # usage limit, or refused credentials (outputs stopped, stop_reason). + # Never fatal; a stop takes "Handle failure" like any other failure. + - name: Claude session summary + id: validate_cost + if: ${{ !cancelled() && (steps.validate.outcome == 'success' || steps.validate.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.validate.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: report-validate + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + - name: Add reaction on success if: steps.check.outputs.should_run == 'true' && steps.validate.outcome == 'success' env: diff --git a/.github/workflows/review-retest.yml b/.github/workflows/review-retest.yml index 8ce9efac753..fc6dbd86a81 100644 --- a/.github/workflows/review-retest.yml +++ b/.github/workflows/review-retest.yml @@ -224,6 +224,68 @@ jobs: CELL_JSON: ${{ toJSON(matrix.cell) }} RULES_SHA: ${{ needs.prep.outputs.rules_sha }} steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + # This job has no id-token on purpose (no session gets a token for the + # federation or Google Cloud): under `oidc` it runs on the + # subscription, logged in the job summary. + - name: Resolve Claude credentials + id: claude_auth + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; + esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" + - name: Checkout harness uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -309,7 +371,12 @@ jobs: PR_NUMBER: '0' ATTEMPT: '1' with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} github_token: ${{ github.token }} claude_args: "--model ${{ matrix.cell.model }} --settings ${{ github.workspace }}/automation/retest/claude-settings.json" prompt: | @@ -333,6 +400,63 @@ jobs: to the PR, write the exact comment body to `review_comment.md` in the repository root instead. Do not run `gh` or `git push`; every other instruction applies unchanged. + # The session's cost (Claude Code's own estimate at API list price), + # model and credential mode, and whether Claude stopped it: a spend or + # usage limit, or refused credentials (outputs stopped, stop_reason). + # Never fatal; a stopped cell is collected as a failed one. + - name: Claude session summary + id: review_cost + if: ${{ !cancelled() && (steps.review.outcome == 'success' || steps.review.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: review-retest + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + - name: Collect the cell if: always() env: diff --git a/.github/workflows/spec-create.yml b/.github/workflows/spec-create.yml index c99c68987b4..35468587a48 100644 --- a/.github/workflows/spec-create.yml +++ b/.github/workflows/spec-create.yml @@ -69,6 +69,66 @@ jobs: fi echo "should_run=true" >> "$GITHUB_OUTPUT" + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth + if: steps.check.outputs.should_run == 'true' + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; + esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" + - name: Checkout repository if: steps.check.outputs.should_run == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -99,7 +159,12 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Create New Specification @@ -248,8 +313,68 @@ jobs: - Do NOT close the issue - STOP after step 11 + # The session's cost (Claude Code's own estimate at API list price), + # model and credential mode, and whether Claude stopped it: a spend or + # usage limit, or refused credentials (outputs stopped, stop_reason). + # Read right after the step: the retry writes the same execution file. + # Never fatal. + - name: Claude session summary + id: process_cost + if: ${{ !cancelled() && (steps.process.outcome == 'success' || steps.process.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.process.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: spec-create + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Not after Claude stopped the session (a limit or refused credentials): + # the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' + if: steps.check.outputs.should_run == 'true' && steps.process.outcome == 'failure' && steps.process_cost.outputs.stopped != 'true' id: process_retry timeout-minutes: 30 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 @@ -259,7 +384,12 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} ISSUE_USER: ${{ github.event.issue.user.login }} with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" prompt: | ## Task: Create New Specification @@ -417,6 +547,68 @@ jobs: - Do NOT close the issue - STOP after step 11 + - name: Claude session summary (retry) + id: process_retry_cost + if: ${{ !cancelled() && (steps.process_retry.outcome == 'success' || steps.process_retry.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.process_retry.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: spec-create/retry + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Claude stopped the first session (a limit or refused credentials) and + # its retry was skipped: fail here instead of reading outputs that were + # never written. + - name: Stop when Claude stopped the session + if: steps.process_cost.outputs.stopped == 'true' + run: | + echo "::error::Claude stopped the session (${{ steps.process_cost.outputs.stop_reason }}); the retry is skipped" + exit 1 + - name: Extract outputs if: steps.check.outputs.should_run == 'true' id: extract diff --git a/.github/workflows/util-claude.yml b/.github/workflows/util-claude.yml index a4dd6fb9a7f..a479bc1a8e8 100644 --- a/.github/workflows/util-claude.yml +++ b/.github/workflows/util-claude.yml @@ -33,6 +33,65 @@ jobs: id-token: write actions: read # Required for Claude to read CI results on PRs steps: + # Claude credentials: the repository variable CLAUDE_AUTH picks one mode + # per run, and every Claude step below reads this step's outputs, so a + # new mode changes only this step. Unset or `subscription`: + # CLAUDE_CODE_OAUTH_TOKEN, as before; `api`: ANTHROPIC_API_KEY; `oidc`: + # workload identity federation with the four CLAUDE_WIF_* variables, + # which needs `id-token: write` on the job. Each Claude step gets + # exactly one credential. An unknown value, or a mode whose credential + # is missing, stops the job here, never a silent fallback to the + # subscription. Identical in every job that runs Claude + # (tests/unit/workflows/test_claude_auth.py); + # docs/workflows/overview.md#claude-credentials. + - name: Resolve Claude credentials + id: claude_auth + env: + CLAUDE_AUTH: ${{ vars.CLAUDE_AUTH }} + API_KEY_SET: ${{ secrets.ANTHROPIC_API_KEY != '' }} + OAUTH_TOKEN_SET: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN != '' }} + WIF_RULE_ID: ${{ vars.CLAUDE_WIF_RULE_ID }} + WIF_ORG_ID: ${{ vars.CLAUDE_WIF_ORG_ID }} + WIF_SERVICE_ACCOUNT_ID: ${{ vars.CLAUDE_WIF_SERVICE_ACCOUNT_ID }} + WIF_WORKSPACE_ID: ${{ vars.CLAUDE_WIF_WORKSPACE_ID }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + fail() { echo "::error::$1; not falling back to the subscription"; exit 1; } + mode="${CLAUDE_AUTH,,}" + case "$mode" in + ''|subscription) mode=subscription ;; + api) [ "$API_KEY_SET" = "true" ] || fail "CLAUDE_AUTH is 'api' but the ANTHROPIC_API_KEY secret is missing" ;; + oidc) + # The federation rule accepts only `…:ref:refs/heads/main`. A + # pull-request event (`…:pull_request`), another ref, and a job + # without id-token run on the subscription instead, by design + # and logged, never on an error. + if [[ "$EVENT" == pull_request* || "$REF" != "refs/heads/main" || -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]]; then + where="${EVENT}/${REF//[^A-Za-z0-9\/._-]/?}$([ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || echo ' (no id-token)')" + if [ "$OAUTH_TOKEN_SET" != "true" ]; then + echo "::error::oidc not available for ${where}, and the CLAUDE_CODE_OAUTH_TOKEN secret is missing" + exit 1 + fi + echo "::notice::oidc not available for ${where}; using subscription" + echo "- oidc not available for ${where}; using subscription" >> "$GITHUB_STEP_SUMMARY" + mode=subscription + else + for name in WIF_RULE_ID WIF_ORG_ID WIF_SERVICE_ACCOUNT_ID WIF_WORKSPACE_ID; do + [[ "${!name}" =~ ^[A-Za-z0-9_-]+$ ]] || fail "CLAUDE_AUTH is 'oidc' but the variable CLAUDE_${name} is missing or not an ID" + done + { + echo "federation_rule_id=$WIF_RULE_ID" + echo "organization_id=$WIF_ORG_ID" + echo "service_account_id=$WIF_SERVICE_ACCOUNT_ID" + echo "workspace_id=$WIF_WORKSPACE_ID" + } >> "$GITHUB_OUTPUT" + fi ;; + *) fail "CLAUDE_AUTH must be unset, 'subscription', 'api' or 'oidc', got '${CLAUDE_AUTH//[^A-Za-z0-9 ._-]/?}'" ;; + esac + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Claude credentials: $mode" + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -43,21 +102,153 @@ jobs: continue-on-error: true uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" # This is an optional setting that allows Claude to read CI results on PRs additional_permissions: | actions: read + # The session's cost (Claude Code's own estimate at API list price), + # model and credential mode, and whether Claude stopped it: a spend or + # usage limit, or refused credentials (outputs stopped, stop_reason). + # Read right after the step: the retry writes the same execution file. + # Never fatal. + - name: Claude session summary + id: claude_cost + if: ${{ !cancelled() && (steps.claude.outcome == 'success' || steps.claude.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: util-claude + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Not after Claude stopped the session (a limit or refused credentials): + # the retry would be refused the same way. - name: Retry Claude (on failure) - if: steps.claude.outcome == 'failure' + if: steps.claude.outcome == 'failure' && steps.claude_cost.outputs.stopped != 'true' id: claude_retry uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model opus --settings ${{ github.workspace }}/.claude/settings.json" additional_permissions: | actions: read + - name: Claude session summary (retry) + id: claude_retry_cost + if: ${{ !cancelled() && (steps.claude_retry.outcome == 'success' || steps.claude_retry.outcome == 'failure') }} + continue-on-error: true + env: + EXECUTION_FILE: ${{ steps.claude_retry.outputs.execution_file }} + CLAUDE_AUTH_MODE: ${{ steps.claude_auth.outputs.mode }} + CLAUDE_STEP: util-claude/retry + # Only the error fields of a session that failed, because its own + # text may quote the same messages: the error result, Claude Code's + # error messages and its API retries, with their HTTP status. + SESSION_ERRORS: >- + ([.[] | select(.type == "result")] | last) as $result + | if $result != null and $result.is_error != true then empty else + (.[] | select((.type == "result" and .is_error == true) or (.type == "assistant" and .error != null) or (.type == "system" and .subtype == "api_retry")) + | (.result // ""), ((.errors // [])[] | tostring), (.error // "" | tostring), (if .type == "assistant" then ((.message.content // [])[] | .text? // "") else "" end), (if (.api_error_status // .error_status) then "api_error_status=\(.api_error_status // .error_status)" else "" end)) + end + # A spend or usage limit, in every mode: the API's spend-limit, + # spend-cap and credit messages, Claude Code's billing_error and + # credit message, and its subscription usage-limit messages (the + # SDK's USAGE_LIMIT_ERROR_PREFIXES). + LIMIT_PATTERN: >- + reached your (specified )?(workspace )?API usage limits|enforced_spend_limit_reached|credit balance (is )?too low|billing_error|usage limit reached|You.{1,3}ve (hit|reached) your|You.{1,3}re out of (usage credits|extra usage)|Your org is out of usage|Your seat type doesn.{1,3}t include|Your usage allocation has been disabled|Your group.{1,3}s usage limit is set to \$0|requires usage credits + # Refused credentials, with api and oidc only (the subscription keeps + # its handling): HTTP 401 or 403, the API's authentication_error and + # permission_error, Claude Code's authentication_failed, a refused + # federation exchange ("Authentication failed"), a bad API key. + AUTH_PATTERN: >- + authentication_error|authentication_failed|permission_error|Authentication failed|invalid x-api-key|account_on_hold|api_error_status=40[13] + run: | + cost=$(jq -r '[.[] | select(.type == "result")] | last | .total_cost_usd // empty' "$EXECUTION_FILE" 2>/dev/null || true) + model=$(jq -r '[.[] | select(.type == "system" and .subtype == "init")] | first | .model // empty' "$EXECUTION_FILE" 2>/dev/null || true) + line="claude-cost step=${CLAUDE_STEP} auth=${CLAUDE_AUTH_MODE} model=${model:-n/a} total_cost_usd=${cost:-n/a}" + echo "$line" + echo "- \`${line}\`" >> "$GITHUB_STEP_SUMMARY" + errors=$(jq -r "$SESSION_ERRORS" "$EXECUTION_FILE" 2>/dev/null || true) + reason="" + if grep -qiE "$LIMIT_PATTERN" <<< "$errors"; then + reason=limit + what="Claude spend/usage limit reached" + elif [[ "$CLAUDE_AUTH_MODE" =~ ^(api|oidc)$ ]] && grep -qiE "$AUTH_PATTERN" <<< "$errors"; then + reason=auth + what="Claude credentials refused" + fi + if [ -n "$reason" ]; then + echo "::error::${what} (auth=${CLAUDE_AUTH_MODE}) in ${CLAUDE_STEP}: no retry until it is resolved" + echo "- **${what} (auth=${CLAUDE_AUTH_MODE})** in \`${CLAUDE_STEP}\`" >> "$GITHUB_STEP_SUMMARY" + echo "stopped=true" >> "$GITHUB_OUTPUT" + else + echo "stopped=false" >> "$GITHUB_OUTPUT" + fi + echo "stop_reason=${reason}" >> "$GITHUB_OUTPUT" + + # Claude stopped the first session (a limit or refused credentials) and + # its retry was skipped: the run fails instead of reporting a success it + # did not have. + - name: Stop when Claude stopped the session + if: steps.claude_cost.outputs.stopped == 'true' + run: | + echo "::error::Claude stopped the session (${{ steps.claude_cost.outputs.stop_reason }}); the retry is skipped" + exit 1 + diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index 541f8c74108..9e60083fbbd 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -20,6 +20,10 @@ run-name: "Watchdog: scan ${{ github.event_name == 'workflow_dispatch' && '(manu # 4. daily-regen's cron silently starved by GitHub's scheduler (no run # for >26 h outside the quiet window) — re-dispatched manually. # +# PRs labelled `ai-review-paused` or `ai-repair-paused` (Claude stopped the +# review or repair at a spend or usage limit, or refused its credentials) are +# skipped: daily-regen's pair mode resumes them. +# # Retries are bounded per-cause via marker labels: # - `ai-review-rescued` (review failures) # - `watchdog:repair-rescued-` (repair failures, one per attempt#) @@ -69,6 +73,10 @@ jobs: GH_REPO: ${{ github.repository }} STALE_HOURS: ${{ inputs.stale_hours || '4' }} DRY_RUN: ${{ inputs.dry_run || 'false' }} + # daily-regen's pair mode: the cron rescue replaces a missed + # scheduled run, so it forwards the pair count the schedule would + # have used. Empty = spec mode, the dispatch as before. + REGEN_PAIRS: ${{ vars.REGEN_PAIRS_PER_DAY }} run: | set -euo pipefail STALE_SEC=$(( STALE_HOURS * 3600 )) @@ -196,6 +204,16 @@ jobs: spec_id=$(echo "$branch" | cut -d'/' -f2) library=$(echo "$branch" | cut -d'/' -f3) + # Paused: Claude stopped the PR's review or repair (a spend or + # usage limit, or refused credentials). A rescue now would be + # refused the same way and spend the PR's one-shot rescue on a + # stop that is still active. daily-regen's pair mode, or a manual + # rerun, resumes the PR, and the resumed run removes the label. + if echo " $labels " | grep -qE " (ai-review-paused|ai-repair-paused) "; then + echo "::notice::PR #$num: paused (Claude stopped its review or repair) — skipped" + continue + fi + # Case 0: regeneration the regen gate KEPT (live implementation # stays) whose PR is still open — impl-review's close was lost. # Re-close it; nothing else to do (main and GCS are untouched). @@ -568,7 +586,7 @@ jobs: gap_hm=$(printf '%dh%02dm' $(( gap / 3600 )) $(( (gap % 3600) / 60 ))) if (( gap > LIVENESS_HOURS * 3600 )); then echo "::warning::daily-regen: no new run on main for ${gap_hm} (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" - dispatch "daily-regen: cron starved (${gap_hm} silent)" daily-regen.yml || true + dispatch "daily-regen: cron starved (${gap_hm} silent)" daily-regen.yml ${REGEN_PAIRS:+-f "pairs=$REGEN_PAIRS"} || true else echo "daily-regen liveness: ${ref_label} ${gap_hm} ago — healthy" fi diff --git a/agentic/docs/project-guide.md b/agentic/docs/project-guide.md index bed00e977f3..299bca9d84d 100644 --- a/agentic/docs/project-guide.md +++ b/agentic/docs/project-guide.md @@ -768,7 +768,7 @@ Issue ready for maintainer review | **sync-labels.yml** | Auto-syncs labels after manual PR merges (spec-ready, impl:*:done) | | **notify-deployment.yml** | Deployment notifications | | **util-claude.yml** | Claude utility workflow | -| **daily-regen.yml** | Scheduled: picks the oldest specs, runs spec polish + cross-library similarity audit, then dispatches bulk-generate | +| **daily-regen.yml** | Scheduled: picks the oldest specs, runs spec polish + cross-library similarity audit, then dispatches bulk-generate. With the repository variable `REGEN_PAIRS_PER_DAY` (or the manual `pairs` input) it picks the N oldest (spec, library) pairs instead (`automation/scripts/regen_pick_pairs.py`), skipping pairs merged or really attempted within the attempt window (30 days by default), skips the polish and dispatches each pair straight to impl-generate; see `docs/workflows/overview.md#daily-regeneration-modes` | | **watchdog-stuck-jobs.yml** | Periodic safety net: detects and unsticks stalled impl-pipeline PRs | | **bot-serving-check.yml** | Daily synthetic monitor for the nginx bot -> seo-proxy path; routes and titles derived from `api/routers/seo.py` and the spec files, and a failure opens/comments the fixed-title issue "Bot serving check is red" (closed again by the next green run) | @@ -799,6 +799,7 @@ The new architecture separates specification and implementation processes: - **Adopt mode.** The `source_ref` input (default empty) on `impl-generate.yml` and `bulk-generate.yml` takes each implementation file from that branch, tag or SHA instead of running Claude, for code written outside the pipeline (a `gemini/*` branch). The run renders the file with the generator's commands, checks the canvas with `core.canvas`, and is always a forced regeneration (`regen:forced`); the routed model only runs repair, and the PR body adds a `**Generation:** adopted from …` line above `**Model:**`. The optional `source_model` input (a model name such as `gemini-4-argon`, only with `source_ref`) names the model that wrote the code and goes on that line. The metadata records `generated_by: {source_model}`, or `generated_by: adopted:{source_ref}` without one, followed by `adopted_from: {source_ref} ({sha})` for every adopted file. Both workflows reject a malformed `source_ref` or `source_model` without echoing it. Retries keep both inputs; the watchdog forwards neither. Python and JavaScript only. A missing file or a failed render is never retried and never counts toward the 3-attempt cap. See `docs/workflows/overview.md#adopt-implementations-from-a-branch`. - **Review model.** Every quality review runs on Opus, whatever model generated the implementation: first implementations, reviews after a repair, and the regen gate's session (the predecessor's re-score and the new render's review). Only `impl-review.yml` chooses it; its `review_model` input (default `opus`) pins another model for one manual review and that run's own auto-retry, and no other workflow forwards one. The retest harness's `models=production` runs every cell on Opus to match (`review_retest.PRODUCTION_MODELS`, kept equal by `tests/unit/workflows/test_model_routing.py`). - Fixed models: `spec-create.yml` runs on Opus; the spec polish and cross-library similarity audit in `daily-regen.yml` run on Sonnet. See `docs/workflows/overview.md#pipeline-models`. +- **Credentials.** The repository variable `CLAUDE_AUTH` chooses what every Claude step runs on: unset or `subscription` = the `CLAUDE_CODE_OAUTH_TOKEN` secret (the owner's subscription), `oidc` = workload identity federation with the `CLAUDE_WIF_*` variables (API credits of the `ci-regen` workspace; accepts only runs on `main`, so pull-request events, other refs and jobs without `id-token`, such as review-retest, run on the subscription with a logged notice), `api` = the `ANTHROPIC_API_KEY` secret (API credits, the fallback). The step "Resolve Claude credentials" (id `claude_auth`) at the start of every Claude job stops on any other value or a missing credential, never falling back to the subscription silently, and every Claude step reads its outputs, so each passes exactly one credential. After each Claude step, "Claude session summary" (id `_cost`) records `total_cost_usd`, the model and the mode, and flags when Claude stopped the session (outputs `stopped`, `stop_reason`): a spend or usage limit in any mode, or refused credentials with `api`/`oidc`. The retry is skipped; impl-generate parks the pair, impl-review and impl-repair pause the PR (`ai-review-paused`, `ai-repair-paused`, skipped by the watchdog, resumed first by daily-regen's pair mode), all without a retry, a counted attempt or `impl::failed`. A new Claude step copies all three patterns, which `tests/unit/workflows/test_claude_auth.py` enforces. See `docs/workflows/overview.md#claude-credentials`. ## GitHub Issue Labels diff --git a/automation/scripts/README.md b/automation/scripts/README.md index 71d18cd31a4..2bf88a2b191 100644 --- a/automation/scripts/README.md +++ b/automation/scripts/README.md @@ -256,6 +256,27 @@ Closes a spec issue once every supported library is `impl::done` or `:failed`. Shared by `impl-merge.yml` and the kept-regen branch of `impl-review.yml`. +### `regen_pick_pairs.py` +The pick of `daily-regen.yml`'s pair mode (`REGEN_PAIRS_PER_DAY`, or the +`pairs` input). First the resumes: open implementation PRs whose review or +repair Claude stopped (`ai-review-paused`, `ai-repair-paused`), oldest first, +counted against N. Then the oldest (spec, library) pairs across the catalogue, +aged by the newer of their metadata `updated` and their last real attempt (a +merged or closed implementation PR, `regen:kept` included, or a failed +generation's `impl-fail` comment; infrastructure failures and generations +Claude stopped don't count). It skips pairs touched within +`--attempt-window-days` (default 30), pairs with an open implementation PR, +and every pair of a spec whose issue saw activity within `--min-age-hours`. +PyYAML plus `gh`. It fails closed: a failed implementation PR listing exits 1, +a spec whose comments can't be read sits out the night, and the kept and issue +listings degrade with a warning. The library list comes from the caller. + +```bash +python3 automation/scripts/regen_pick_pairs.py --count 5 --min-age-hours 20 \ + --attempt-window-days 30 --repo OWNER/NAME --libraries "altair bokeh ..." +# writes resume_json, resume_count, pairs_json, specs, specs_json and count to $GITHUB_OUTPUT +``` + ### `spec_pr_guard.py` Fail-closed file allowlist for the spec PR that the merge job in `spec-create.yml` merges: the diff of the PR head against its merge base with `main` may only add or diff --git a/automation/scripts/regen_pick_pairs.py b/automation/scripts/regen_pick_pairs.py new file mode 100644 index 00000000000..f371000b2a3 --- /dev/null +++ b/automation/scripts/regen_pick_pairs.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Pick the night's work for daily-regen's pair mode. + +Spec mode, the default, regenerates the N oldest specs with all their +libraries. Pair mode spends the night's N on two kinds of work: + +1. Resumes first. Open implementation PRs whose review or repair Claude + stopped (a spend or usage limit, or refused credentials) carry + ``ai-review-paused`` or ``ai-repair-paused``; the oldest of them are + re-dispatched before anything new. A refused request costs nothing, so a + stop that is still active only pauses them again, and they resume on + their own once it lifts. +2. Then the oldest implementations, one (spec, library) pair each, across the + catalogue: + +- A pair's age counts from its newest touch: its metadata ``updated`` + (``created`` when there is none), or its last real regeneration attempt, + whichever is newer. A pair touched within ``--attempt-window-days`` + (default 30) is not picked, so an attempted pair waits at least that long + and then queues behind every pair touched before it. +- A real attempt is one that reached a verdict or burned a generation attempt: + the pair's implementation PR (head ``implementation//``) + once it is merged or closed (a ``regen:kept`` PR included), and a failed + generation, which leaves an ```` comment + on the spec issue. A merge moves ``updated`` anyway; the other two leave it + alone, and without them the same pair would come back the next night. An + infrastructure failure (its comment carries ``impl-fail-cause:infra``) and + a generation Claude stopped (its own pause comment, ``impl-paused-claude``) + are not attempts: the pair stays in the queue and comes back on a later + night. +- A pair with an open implementation PR is skipped: its attempt is still + running, stuck or paused, and the watchdog or the resume owns it. +- Every pair of a spec whose issue saw activity within ``--min-age-hours`` is + skipped, like the spec mode's postponement: a comment, a label or a report + holds the spec back, and a night does not stack a spec whose last batch is + still in flight. +- An unparseable metadata timestamp skips the pair, as in spec mode. Only + pairs with a metadata file and a ``specification.md`` are candidates, so + pair mode never writes a first implementation; gap backfill stays with + ``bulk-generate.yml``. + +The implementation-PR listing is the loop guard: when it fails, the run stops +with an error instead of picking without it. The kept-regeneration and issue +listings are best effort (one ``gh`` call each, at most 1000 entries, most +recent first; kept regenerations reach much further back than the PR +listing). The failed-generation comments are read per spec issue, only for +the specs at the front of the queue, at most ``--max-issue-reads`` issues per +run; a spec whose comments cannot be read is left out of the night. + +Usage, from the repository root:: + + python3 automation/scripts/regen_pick_pairs.py --count 5 --min-age-hours 20 \\ + --attempt-window-days 30 --repo OWNER/NAME --libraries "altair bokeh ..." + +The library list comes from the caller (``core.constants.SUPPORTED_LIBRARIES``), +so this file carries no copy that could drift. Writes ``resume_json``, +``resume_count``, ``pairs_json``, ``specs``, ``specs_json`` and ``count`` (new +pairs) to ``$GITHUB_OUTPUT`` and a table of both to ``$GITHUB_STEP_SUMMARY``. +""" + +from __future__ import annotations + +import argparse +import heapq +import json +import os +import re +import subprocess +import sys +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import yaml + + +ANCIENT = datetime.min.replace(tzinfo=timezone.utc) +LIMIT = "1000" +BRANCH_RE = re.compile(r"^implementation/(?P[a-z0-9][a-z0-9-]*)/(?P[a-z0-9]+)$") +ATTEMPT_RE = re.compile(r"^ai-attempt-(?P[0-9]+)$") +# impl-generate.yml "Handle generation failure": every failure comment carries +# the marker; an infrastructure failure's comment also carries the infra tag. +FAIL_MARKER_RE = re.compile(r"") +INFRA_TAG = "" +# impl-review.yml and impl-repair.yml set these when Claude stopped the session. +PAUSED_LABELS = {"ai-repair-paused": "repair", "ai-review-paused": "review"} + +Pair = tuple[str, str] + + +class ListingError(Exception): + """A listing the pick cannot do without failed.""" + + +@dataclass(frozen=True) +class Candidate: + spec: str + library: str + metadata: datetime | None # None: the metadata file names no timestamp + + +@dataclass(frozen=True) +class Pick: + spec: str + library: str + touched: datetime + basis: str + + +@dataclass(frozen=True) +class Resume: + pr: int + kind: str # "review" or "repair" + spec: str + library: str + attempt: int + updated: datetime + + +@dataclass +class Records: + """What GitHub says about past attempts, per pair and per spec issue.""" + + attempts: dict[Pair, tuple[datetime, str]] + open_pairs: set[Pair] + issue_activity: dict[int, datetime] + paused: list[Resume] = field(default_factory=list) + + +def parse_ts(raw: str) -> datetime | None: + try: + dt = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + return None + return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc) + + +def load_candidates(plots: Path, libraries: Iterable[str]) -> tuple[list[Candidate], dict[str, int]]: + """Every pair with a metadata file, plus each spec's issue number.""" + supported = set(libraries) + candidates: list[Candidate] = [] + issues: dict[str, int] = {} + for spec_dir in sorted(plots.iterdir()): + if not spec_dir.is_dir() or spec_dir.name.startswith(".") or not (spec_dir / "specification.md").is_file(): + continue + try: + issue = (yaml.safe_load((spec_dir / "specification.yaml").read_text(encoding="utf-8")) or {}).get("issue") + except OSError, yaml.YAMLError, AttributeError: + issue = None + if isinstance(issue, int): + issues[spec_dir.name] = issue + for meta in sorted(spec_dir.glob("metadata/*/*.yaml")): + if meta.stem not in supported: + continue + try: + data = yaml.safe_load(meta.read_text(encoding="utf-8")) or {} + except OSError, yaml.YAMLError: + continue + if not isinstance(data, dict): + continue + raw = data.get("updated") or data.get("created") + if not raw: + candidates.append(Candidate(spec_dir.name, meta.stem, None)) + continue + dt = parse_ts(str(raw)) + if dt is None: + continue # unparseable timestamp: skipped, as in spec mode + candidates.append(Candidate(spec_dir.name, meta.stem, dt)) + return candidates, issues + + +def _newer(records: dict[Pair, tuple[datetime, str]], pair: Pair, when: datetime | None, basis: str) -> None: + if when is not None and (pair not in records or when > records[pair][0]): + records[pair] = (when, basis) + + +def fold_pulls(pulls: Iterable[dict[str, Any]], records: Records) -> None: + """An open PR blocks its pair (and is a resume when paused); a merged or closed one is an attempt.""" + for pull in pulls: + match = BRANCH_RE.match(str(pull.get("headRefName") or "")) + if not match: + continue + pair = (match["spec"], match["lib"]) + state = pull.get("state") + labels = {str(label.get("name")) for label in pull.get("labels") or [] if isinstance(label, dict)} + if state == "OPEN": + records.open_pairs.add(pair) + kind = next((PAUSED_LABELS[name] for name in PAUSED_LABELS if name in labels), None) + number = pull.get("number") + if kind and isinstance(number, int) and all(r.pr != number for r in records.paused): + attempts = [int(m["n"]) for name in labels if (m := ATTEMPT_RE.match(name))] + updated = parse_ts(str(pull.get("updatedAt") or "")) or ANCIENT + records.paused.append(Resume(number, kind, pair[0], pair[1], max(attempts, default=1), updated)) + continue + if state not in ("MERGED", "CLOSED"): + continue + basis = "merged PR" if state == "MERGED" else "kept PR" if "regen:kept" in labels else "closed PR" + when = parse_ts(str(pull.get("closedAt") or pull.get("updatedAt") or "")) + _newer(records.attempts, pair, when, basis) + + +def fold_issues(issues: Iterable[dict[str, Any]], records: Records) -> None: + for issue in issues: + ts = parse_ts(str(issue.get("updatedAt") or "")) + if ts and isinstance(issue.get("number"), int): + records.issue_activity[issue["number"]] = ts + + +def failed_generations(comments: Iterable[dict[str, Any]], spec: str) -> dict[str, datetime]: + """Newest failed generation per library of ``spec``, from the spec issue's comments.""" + newest: dict[str, datetime] = {} + for comment in comments: + body = str(comment.get("body") or "") + if INFRA_TAG in body: + continue # an infrastructure failure, not an attempt + when = parse_ts(str(comment.get("created_at") or "")) + for match in FAIL_MARKER_RE.finditer(body): + if match["spec"] == spec and when and (match["lib"] not in newest or when > newest[match["lib"]]): + newest[match["lib"]] = when + return newest + + +def _touched(pair: Pair, metadata: datetime | None, records: Records) -> tuple[datetime | None, str]: + touched, basis = (metadata, "metadata") if metadata else (None, "none") + attempt = records.attempts.get(pair) + if attempt and (touched is None or attempt[0] > touched): + touched, basis = attempt + return touched, basis + + +def resumes(records: Records, count: int) -> list[Resume]: + """The oldest paused reviews and repairs, at most ``count``.""" + return sorted(records.paused, key=lambda r: (r.updated, r.pr))[:count] + + +def pick( + candidates: Iterable[Candidate], + records: Records, + spec_issues: dict[str, int], + now: datetime, + min_age: timedelta, + window: timedelta, + count: int, + failures: Callable[[str], dict[str, datetime] | None] | None = None, + max_reads: int = 100, +) -> tuple[list[Pick], int]: + """The ``count`` oldest eligible pairs, oldest first, and how many were eligible. + + ``failures(spec)`` returns the spec's failed generations per library, or + None when they cannot be read. It is asked lazily, once per spec, when one + of the spec's pairs reaches the front of the queue; a pair whose failure is + newer than its age goes back into the queue, or out of it while the + failure is within the window. A spec whose failures cannot be read is + left out of the night (fail closed: its pairs may have failed last night). + """ + metadata: dict[Pair, datetime | None] = {} + queue: list[tuple[datetime, str, str, str]] = [] + for cand in candidates: + pair = (cand.spec, cand.library) + if pair in records.open_pairs: + continue + issue_ts = records.issue_activity.get(spec_issues.get(cand.spec, -1)) + if issue_ts is not None and now - issue_ts < min_age: + continue + touched, basis = _touched(pair, cand.metadata, records) + if touched is not None and now - touched < window: + continue # touched within the window: not yet + metadata[pair] = cand.metadata + queue.append((touched or ANCIENT, cand.spec, cand.library, basis)) + heapq.heapify(queue) + eligible = len(queue) + + read: set[str] = set() + unreadable: set[str] = set() + picks: list[Pick] = [] + while queue and len(picks) < count: + key, spec, lib, basis = heapq.heappop(queue) + if failures is not None and spec not in read: + if len(read) == max_reads: + print(f"::warning::Read the failed generations of {max_reads} specs; picking the rest without them") + failures = None + else: + read.add(spec) + found = failures(spec) + if found is None: + unreadable.add(spec) + print(f"::warning::Leaving {spec} out of tonight: its failed generations could not be read") + for flib, when in (found or {}).items(): + _newer(records.attempts, (spec, flib), when, "failed generation") + if spec in unreadable: + eligible -= 1 + continue + touched, new_basis = _touched((spec, lib), metadata[(spec, lib)], records) + if (touched or ANCIENT) != key: + if touched is not None and now - touched < window: + eligible -= 1 + continue + heapq.heappush(queue, (touched or ANCIENT, spec, lib, new_basis)) + continue + picks.append(Pick(spec, lib, key, basis)) + return picks, eligible + + +def _gh_json(args: list[str], what: str) -> list[dict[str, Any]] | None: + try: + raw = subprocess.run(["gh", *args], check=True, capture_output=True, text=True).stdout + data = json.loads(raw) + except (subprocess.CalledProcessError, ValueError, OSError) as exc: + print(f"::warning::Could not list {what} ({' '.join(str(exc).split())[:200]})") + return None + return data if isinstance(data, list) else None + + +def read_records(repo: str) -> Records: + records = Records(attempts={}, open_pairs=set(), issue_activity={}) + fields = "number,headRefName,state,updatedAt,closedAt,labels" + pr_list = ["pr", "list", "--repo", repo, "--limit", LIMIT, "--json", fields] + pulls = _gh_json( + [*pr_list, "--state", "all", "--search", "head:implementation/ sort:updated-desc"], "implementation PRs" + ) + if pulls is None: + # The loop guard: without it a kept, closed or paused pair would be + # picked again tonight. + raise ListingError("the implementation PRs could not be listed") + fold_pulls(pulls, records) + # Kept regenerations again on their own: they are the attempts that leave + # `updated` alone, and this listing reaches much further back. + kept = _gh_json( + [*pr_list, "--state", "closed", "--label", "regen:kept", "--search", "sort:updated-desc"], "kept regenerations" + ) + fold_pulls(kept or [], records) + issues = _gh_json( + ["issue", "list", "--repo", repo, "--state", "all", "--search", "sort:updated-desc", "--limit", LIMIT] + + ["--json", "number,updatedAt"], + "issues", + ) + fold_issues(issues or [], records) + print( + f"::notice::Attempt records for {len(records.attempts)} pair(s), {len(records.open_pairs)} open PR(s) " + f"({len(records.paused)} paused), activity for {len(records.issue_activity)} issue(s)" + ) + return records + + +def issue_failures(repo: str, spec_issues: dict[str, int]) -> Callable[[str], dict[str, datetime] | None]: + """``failures(spec)`` backed by the spec issue's comments (one paginated read per spec).""" + + def failures(spec: str) -> dict[str, datetime] | None: + issue = spec_issues.get(spec) + if issue is None: + return {} + jq = '.[] | select(.body != null and (.body | contains("` marker. It sums the marker count over all comment pages and treats a count it can't read as a spent retry. The watchdog's crashed-repair case takes over from there. The generation retry of `impl-generate.yml` and the post-score review retry run on the ref of the run that failed: a run dispatched from a feature branch retries on that branch, and a production run retries on `main`. The no-score review retry still goes through `repository_dispatch`, which always runs on `main`. @@ -398,8 +402,8 @@ Located in `.github/workflows/`: | `impl-merge.yml` | Merges approved PRs, and the review write-back PRs a kept regeneration opens (job `writeback`) | | `bulk-generate.yml` | Batch implementation generation | | `review-retest.yml` | Dispatch-only measurement: re-runs the AI quality review on the frozen retest set (several fresh sessions per item, regen pairs in both orders) for one rules version, and reports score spread, verdict flips, weakness agreement, and gate order bias against a baseline arm. Read-only tokens; writes nothing outside its own artifacts. See [Review retest](review-retest.md) | -| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (once a day at 02:17 UTC, off the top of the hour to dodge GitHub's scheduler overload). A spec's age counts from the newer of its last merged update and the last activity on its spec issue (every regen touches the issue), so a kept regeneration is not re-picked the next night. Any activity on the issue — a comment, a label, a report — postpones that spec's regen the same way | -| `watchdog-stuck-jobs.yml` | 6-hourly safety net: re-dispatches stuck reviews, repairs (including a repair that crashed after a rejection), merges and generations (straight to `impl-generate.yml`, marked only once the run exists), re-closes open `regen:kept` PRs, never rescues a regeneration into repair or re-dispatches its failed review (it flags it for manual attention instead; "regeneration" = the `regen` label or the implementation file on main, unless `regen:forced`), and rescues daily-regen when its cron is silently starved by GitHub (>26 h without a run) | +| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (once a day at 02:17 UTC, off the top of the hour to dodge GitHub's scheduler overload). A spec's age counts from the newer of its last merged update and the last activity on its spec issue (every regen touches the issue), so a kept regeneration is not re-picked the next night. Any activity on the issue — a comment, a label, a report — postpones that spec's regen the same way. With the repository variable `REGEN_PAIRS_PER_DAY`, the scheduled run regenerates single (spec, library) pairs instead of whole specs; see [Daily regeneration modes](#daily-regeneration-modes) | +| `watchdog-stuck-jobs.yml` | 6-hourly safety net: re-dispatches stuck reviews, repairs (including a repair that crashed after a rejection), merges and generations (straight to `impl-generate.yml`, marked only once the run exists), re-closes open `regen:kept` PRs, never rescues a regeneration into repair or re-dispatches its failed review (it flags it for manual attention instead; "regeneration" = the `regen` label or the implementation file on main, unless `regen:forced`), skips a PR whose review or repair Claude stopped (`ai-review-paused`, `ai-repair-paused`), and rescues daily-regen when its cron is silently starved by GitHub (>26 h without a run) | | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main, and runs `alembic upgrade head` against Cloud SQL first; a push that only adds a revision under `alembic/versions/` triggers it too, and so does one that changes a pin the library versions are read from (`uv.lock`, `package-lock.json`, `Manifest.toml`, `.python-version`, a setup action) | | `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges | @@ -493,6 +497,99 @@ on the source ref gets an "Adoption Failed" comment. The watchdog's generation rescues don't carry `source_ref` or `source_model`: a pair the watchdog re-dispatches is generated by Claude. +### Daily regeneration modes + +`daily-regen.yml` has two modes. The repository variable +`REGEN_PAIRS_PER_DAY` chooses the mode of the scheduled run: + +| `REGEN_PAIRS_PER_DAY` | Scheduled run | +|-----------------------|---------------| +| Unset or empty | Spec mode, as before: the oldest spec (`count`, default 1), spec polish, the cross-library similarity audit, then `bulk-generate.yml` for all 15 libraries | +| A positive integer N | Pair mode: the N oldest implementations across the catalogue, the similarity audit for each spec among them, then one `impl-generate.yml` dispatch per pair | +| Anything else | The run stops with an error before it picks anything | + +The variable changes only the scheduled run. A manual run keeps its inputs and +their meaning, and switches to pair mode only with the `pairs` input. The +watchdog's cron rescue replaces a missed scheduled run, so it forwards the +variable as `pairs`. A `specification_id` always wins over a pair count. + +Pair mode picks the oldest pairs and never picks the same pair night after +night: + +- A pair's age is its newest touch: its metadata `updated` timestamp + (`created` when there's none), or its last real attempt, whichever is newer. + A real attempt is an `implementation/{spec-id}/{library}` PR that was merged + or closed, a `regen:kept` PR included, or a failed generation, which leaves + an `` comment on the spec issue. A + kept regeneration and a failed generation leave `updated` alone, so without + the attempt record the same pair would come back the next night. +- A pair touched within the attempt window is skipped. The window is the + `attempt_window_days` input, else the repository variable + `REGEN_ATTEMPT_WINDOW_DAYS`, else 30 days. After the window, the pair queues + behind every pair touched before it. +- An infrastructure failure, whose comment carries + ``, and a generation Claude stopped, whose + comment carries ``, are not attempts: the pair + stays at its place in the queue and comes back on a later night. +- A pair with an open implementation PR is skipped: its attempt is still + running, stuck, or paused. +- Every pair of a spec whose issue saw activity within `min_age_hours` + (default 20) is skipped for that night, as in spec mode. +- Only pairs with a metadata file are candidates, so pair mode never writes a + first implementation. Gap backfill stays with `bulk-generate.yml`. + +Before the new pairs, pair mode resumes the reviews and repairs that Claude +stopped: the open implementation PRs with `ai-review-paused` or +`ai-repair-paused`, oldest first. Each resume counts against N, so a night with +three paused PRs and `REGEN_PAIRS_PER_DAY=5` dispatches three resumes and two +new pairs. The job "Re-dispatch paused reviews and repairs" starts +`impl-review.yml` or `impl-repair.yml` with the arguments the watchdog's +rescues use, at the pace of the pair dispatches. While the stop lasts, a resume +is refused at its first request, which costs nothing, and pauses again. + +The picker fails closed. When the implementation PR listing fails, the run +stops with an error, because that listing records the kept, closed, open, and +paused PRs. The kept-regeneration and issue listings degrade with a warning. +The picker reads the failed-generation comments only for the specs at the +front of the queue, at most 100 issues per run, and a spec whose comments +can't be read sits out the night. Spec mode keeps its own rule: a spec's age is +the newer of its metadata and any activity on its issue, so a kept or failed +regeneration moves the spec to the back of the queue, and so does an +infrastructure failure or any comment. + +Pair mode skips the spec polish, because a spec rewritten for one library +would leave its other implementations written against the old text. It +dispatches each pair straight to `impl-generate.yml` with the arguments and +pace of `bulk-generate.yml` (180 seconds between dispatches, 120 seconds for a +pinned `sonnet` or `haiku`), and never through `bulk-generate.yml`: that +workflow's single concurrency group keeps one pending run and cancels the +older one. The regen gate, the review, and the merge are unchanged. + +The picks go to the log and the run summary, so a dry run shows them: + +```bash +gh workflow run daily-regen.yml -f pairs=5 -f dry_run=true +``` + +The dry run still runs the similarity audit, as in spec mode. With +`CLAUDE_AUTH=oidc`, run [Test the spend limit](#test-the-spend-limit) first, +before you set the variable. To turn pair mode on for the schedule, run +`gh variable set REGEN_PAIRS_PER_DAY --body 5`; to turn it off, run +`gh variable delete REGEN_PAIRS_PER_DAY`. + +When Claude stops a pre-flight session (the spec polish or the similarity +audit), the night's dispatch for that spec is skipped and the run fails (see +[When Claude stops a session](#when-claude-stops-a-session)). In pair mode, +the similarity audit of each spec runs before that spec's pairs are +dispatched, one spec after another, so it works as the night's probe: once a +pair's generation, review, or repair has reached the limit, every later spec +of the night stops at its audit and dispatches nothing. Pairs that were +already dispatched fail at their first refused request, which costs nothing, +and are parked without a retry or a counted attempt. Their spec keeps its +pairs in the queue for a later night. The resumes don't wait for the audit: +they start in a job of their own, and while the stop lasts each one pauses +again at no cost. + --- ## Pipeline models @@ -594,6 +691,328 @@ gate record and notice line show `model=n/a`. --- +## Claude credentials + +Every Claude step of the pipeline runs on one of three credentials. The +repository variable `CLAUDE_AUTH` chooses it for every run: + +| `CLAUDE_AUTH` | Credential | Billed to | +|---------------|------------|-----------| +| Unset, empty, or `subscription` | The `CLAUDE_CODE_OAUTH_TOKEN` secret | The owner's Claude subscription, as before the switch | +| `oidc` | Workload identity federation: the job's GitHub OIDC token, exchanged for a short-lived Claude API token. The repository variables `CLAUDE_WIF_RULE_ID`, `CLAUDE_WIF_ORG_ID`, `CLAUDE_WIF_SERVICE_ACCOUNT_ID`, and `CLAUDE_WIF_WORKSPACE_ID` name the federation. | The API credits of the federation's workspace | +| `api` | The `ANTHROPIC_API_KEY` secret, the fallback when federation isn't available | The API credits of the key's workspace | +| Anything else | None: the job stops | Nothing | + +The value ignores case. The switch covers all 13 Claude steps in the 8 +workflows that run Claude: `daily-regen.yml`, `impl-generate.yml`, +`impl-repair.yml`, `impl-review.yml`, `report-validate.yml`, +`review-retest.yml`, `spec-create.yml`, and `util-claude.yml`. The models don't +change: the aliases `sonnet` and `opus` resolve to the same models on the API. + +Every job that runs Claude starts with the step "Resolve Claude credentials". +It turns `CLAUDE_AUTH` into a mode, and every Claude step reads the step's +outputs, so a new mode changes only that step. Each Claude step gets exactly +one credential and leaves the others empty, because an API key or an OAuth +token silently shadows federation, and the action doesn't document which of +the two wins. The step stops the job with an error in these cases, and never +falls back to the subscription, so a broken switch can't spend the owner's +interactive quota: + +- `CLAUDE_AUTH` has an unknown value. +- `CLAUDE_AUTH` is `api` and the `ANTHROPIC_API_KEY` secret is missing. +- `CLAUDE_AUTH` is `oidc` and one of the four `CLAUDE_WIF_*` variables is + missing or isn't an ID. +- `CLAUDE_AUTH` is `oidc`, the run is one that federation refuses, and the + `CLAUDE_CODE_OAUTH_TOKEN` secret is missing (see + [Which runs federation accepts](#which-runs-federation-accepts)). + +In `spec-create.yml` and `report-validate.yml`, the step runs only for an +issue that reaches Claude. In `impl-generate.yml`, it runs before the issue +number is known, so a stop never counts against the pair's three attempts. + +With an API key, the action also classifies Claude's unconfirmed inline PR +comments with Haiku on that key and drops the ones it reads as test probes. +With the subscription or federation, it posts all of them. Only +`util-claude.yml` on a pull request writes inline comments. + +### Which runs federation accepts + +GitHub mints the OIDC token per job, and its `sub` claim names the trigger. +The federation rule accepts only the subject +`repo:MarkusNeusinger/anyplot:ref:refs/heads/main`, so with `CLAUDE_AUTH=oidc` +the Claude steps of these runs work: + +| Workflow | Triggers | Subject | +|----------|----------|---------| +| `daily-regen.yml` | `schedule`, `workflow_dispatch` on `main` | `…:ref:refs/heads/main` | +| `impl-generate.yml` | `issues` (a `generate:` label), `workflow_dispatch` on `main` from bulk-generate, daily-regen, the watchdog, and its own retries | `…:ref:refs/heads/main` | +| `impl-review.yml` | `repository_dispatch`, `workflow_dispatch` on `main` | `…:ref:refs/heads/main` | +| `impl-repair.yml` | `workflow_dispatch` on `main` | `…:ref:refs/heads/main` | +| `spec-create.yml`, `report-validate.yml` | `issues` (a label) | `…:ref:refs/heads/main` | +| `util-claude.yml` | `issue_comment`, also on a pull request | `…:ref:refs/heads/main` | + +Federation would refuse the other runs with `401 Authentication failed`, so +under `oidc` they use the subscription on purpose: + +- `util-claude.yml` started by a `pull_request_review_comment` or a + `pull_request_review`: the subject is `repo:MarkusNeusinger/anyplot:pull_request`. +- Any workflow dispatched on another branch, such as a branch smoke with + `--ref feature/x` and the retries it dispatches on that branch: the subject + is `…:ref:refs/heads/feature/x`. +- `review-retest.yml`: its review job has no `id-token` permission on + purpose, because without it the action can't mint the Claude GitHub App + token, so no session can write to the repository. A retest arm on `oidc` + runs on the subscription. + +"Resolve Claude credentials" recognizes these runs by the event name, the +ref, and the missing token request URL. It logs +`oidc not available for /; using subscription` as a notice and in +the job summary, then runs the job on the `CLAUDE_CODE_OAUTH_TOKEN` secret. If +the secret is missing, it stops the job instead. + +Every other job that runs Claude already has `id-token: write`, for the +Claude GitHub App token and, in some of them, Google Cloud. The federation +didn't add the permission to any job. + +### Set up workload identity federation + +1. In the Claude Console, open + [Workload identity](https://platform.claude.com/settings/workload-identity-federation), + click **Connect workload**, and select **GitHub Actions**. The wizard + registers GitHub's issuer, creates a service account, and creates the + federation rule. +2. Give the rule these values: + - Subject prefix: `repo:MarkusNeusinger/anyplot:ref:refs/heads/main`. + - Audience: `https://api.anthropic.com`, the action's default. + - Claims: `repository_owner=MarkusNeusinger`, `repository=MarkusNeusinger/anyplot`, + and `ref=refs/heads/main`. + - Workspace: `ci-regen` only, with the scope `workspace:developer` and a + token lifetime of 600 seconds. The action fetches a fresh GitHub token + every 4 minutes and Claude Code exchanges it again, so a session longer + than the lifetime keeps running. +3. Remove the `event_name=push` claim that the wizard fills in. With it, the + rule refuses every scheduled, dispatched, and issue-triggered run, which is + every run of the pipeline. +4. Set a spend limit on the `ci-regen` workspace. +5. Store the four IDs as repository variables. They're identifiers, not + secrets. The organization ID is on the + [Organization](https://platform.claude.com/settings/organization) page; the + rule ID (`fdrl_…`), the service account ID (`svac_…`), and the workspace ID + (`wrkspc_…`) belong to the resources the wizard created: + + ```bash + gh variable set CLAUDE_WIF_RULE_ID --body fdrl_... + gh variable set CLAUDE_WIF_ORG_ID --body 00000000-0000-0000-0000-000000000000 + gh variable set CLAUDE_WIF_SERVICE_ACCOUNT_ID --body svac_... + gh variable set CLAUDE_WIF_WORKSPACE_ID --body wrkspc_... + ``` + +6. Switch the pipeline to federation, as described in + [Switch modes](#switch-modes). + +A refused exchange always answers `401 Authentication failed`. The reason, such +as `match_subject_prefix`, is in the authentication history on the Workload +identity page. + +### Switch modes + +Switch with one command. Each takes effect with the next job that starts. + +- Federation: `gh variable set CLAUDE_AUTH --body oidc` +- API key: run `gh secret set ANTHROPIC_API_KEY` once, which prompts for the + value so the key stays out of your shell history, then + `gh variable set CLAUDE_AUTH --body api` +- Subscription: `gh variable set CLAUDE_AUTH --body subscription`, or + `gh variable delete CLAUDE_AUTH` + +To check a switch, open the next Claude run: the step "Resolve Claude +credentials" logs `Claude credentials: `, and the cost line in the job +summary reads `auth=`. For an API key, create it in a workspace of its +own with a spend limit. + +### Session cost + +After every Claude step, the step "Claude session summary" writes one line to +the job summary and the log: + +```text +claude-cost step=impl-generate auth=api model=claude-sonnet-5-5 total_cost_usd=0.2931 +``` + +`total_cost_usd` is Claude Code's own estimate at API list price. On the +subscription it's what the session would have cost on the API. It reads +`n/a` when the session wrote no result, and the step never fails the job. The +weekly regen review sums these lines (`collect_week.py --costs`) and shows the +part that ran on API credits. Job logs measured from 2026-10-28 to 2026-10-31 +put a generation on Sonnet at about $0.29 per implementation, a review on +Opus at about $1.10, and a repair with its re-review at about $2.70 per +implementation in all. + +### When Claude stops a session + +Two stops end the pipeline's work cleanly instead of counting as a failure of +the implementation: + +- A Claude spend or usage limit, in every mode. Once the limit is reached, + every request is refused until it resets or you raise it, and Claude Code's + own retries can't get past it. +- Refused credentials, with `api` and `oidc` only: an invalid or revoked API + key, or a federation exchange that the rule refuses. With the subscription, + an authentication error keeps the handling it had before the switch. + +For a limit, the pipeline recognizes these texts: + +- A spend limit you set on the workspace or the organization (`oidc`, `api`): + HTTP 400 `invalid_request_error`, with a message that starts with + `You have reached your specified workspace API usage limits` or + `You have reached your specified API usage limits`. +- The usage tier's monthly spend cap: HTTP 429 `rate_limit_error` with the + `error_code` `enforced_spend_limit_reached` and no `retry-after` header. +- An empty credit balance: the API's `Your credit balance is too low`, Claude + Code's `Credit balance is too low` or `Credit balance too low · Add funds`, + and its error type `billing_error`. +- The subscription's usage limit (`subscription`): Claude Code's messages that + start with one of the prefixes its SDK exports as + `USAGE_LIMIT_ERROR_PREFIXES`, such as + `You've hit your session limit · resets 12:30pm`, and the older + `Claude AI usage limit reached`. + +A transient rate limit, such as a per-minute token limit, is none of these: +it keeps its retry and the infrastructure path it took before. + +For refused credentials, it recognizes the API's error types +`authentication_error` and `permission_error`, HTTP status 401 or 403 on the +failed request, Claude Code's error type `authentication_failed`, and the +texts `Authentication failed` and `invalid x-api-key`. A limit wins over its +status code: a 403 with a usage-limit message counts as a limit. + +"Claude session summary" reads these texts only from the error fields of a +session that failed: the error result, Claude Code's error messages, and its +API retries. It never reads the conversation, which may quote the same text, +and never flags a session that ended well. When it finds one, it writes an +`::error::` and the line **Claude spend/usage limit reached (auth=oidc)** or +**Claude credentials refused (auth=oidc)**, with the mode, to the job summary, +and every path treats the stop as an infrastructure failure: + +| Workflow | What happens | +|----------|--------------| +| `impl-generate.yml` | The retry is skipped, and "Stop when Claude stopped the session" fails the job. "Handle generation failure" parks the pair before it counts anything: it removes `generate:{library}` and `impl:{library}:pending`, puts `impl:{library}:done` back for a live implementation, and comments with the marker ``. Nothing is retried, no attempt counts toward the cap of three, and the pair is never marked `impl:{library}:failed`, a first implementation included. No PR exists; the next generation recreates the branch. | +| `impl-review.yml` | No score, no verdict, and no regen gate decision, even when the session wrote `quality_score.txt` before it stopped: "Extract quality score" is skipped. "Validate review output" adds `ai-review-paused` and the "AI Review Paused" comment, without the auto re-review and without spending it. The PR stays open, and nothing is merged or closed. | +| `impl-repair.yml` | The retry is skipped. "Handle repair failure" restores `ai-rejected`, adds `ai-repair-paused`, and comments, without the auto-retry and without spending it. The attempt isn't used up. | +| `daily-regen.yml` | A spec polish or similarity audit that Claude stopped skips that spec's dispatch, and the entry fails. | +| `spec-create.yml`, `util-claude.yml` | The retry is skipped, and the job fails. | +| `report-validate.yml`, `review-retest.yml` | The step fails as before: the issue gets the "Validation Failed" comment, and the cell is collected as failed. | + +The watchdog skips every PR with `ai-review-paused` or `ai-repair-paused`, so +it never spends its one-shot rescue on a stop. In pair mode, daily-regen +resumes the paused PRs first, oldest first (see +[Daily regeneration modes](#daily-regeneration-modes)), and the resumed review +or repair removes its label when it starts. A resume that pauses again while +the stop lasts puts the label back but posts no second comment when the PR's +latest bot comment is already the same pause. A parked pair keeps its place in +the queue: neither a parked generation nor a paused review or repair is a real +attempt. + +In spec mode, or to resume sooner, resume the paused PRs yourself once the +limit resets or the credentials are fixed: + +1. Rerun every paused review: + + ```bash + gh pr list --label ai-review-paused --json number --jq '.[].number' \ + | xargs -I{} gh workflow run impl-review.yml -f pr_number={} + ``` + +2. Restart every paused repair with the command in its "Repair Paused" + comment, which names the PR, the pair, and the attempt. + +Before this change, a subscription usage limit counted as the pair's own +failure: it used up the three attempts and marked a first implementation +`impl:{library}:failed`. A bulk run that reaches the subscription limit now +parks the rest of its pairs instead, and you re-dispatch them once the limit +resets. + +### Test the spend limit + +Test the stop on the first day of `CLAUDE_AUTH=oidc`, before you set +`REGEN_PAIRS_PER_DAY`, so that no nightly run spends the workspace while you +test: + +1. In the Claude Console, set the `ci-regen` workspace's spend limit to the + month's spend so far plus about 0.50 USD. +2. Dispatch one regeneration of a pair whose implementation is live on main: + + ```bash + gh workflow run impl-generate.yml -f specification_id=SPEC_ID -f library=LIBRARY + ``` + + The generation on Sonnet costs about 0.29 USD and passes; the review on + Opus costs about 1.10 USD and reaches the limit while it runs. +3. Check the `impl-review.yml` run: its summary shows **Claude spend/usage + limit reached (auth=oidc)**, the PR has `ai-review-paused` and the "AI + Review Paused (Claude spend/usage limit reached)" comment, no second review + run started, and no verdict or `quality:` label was added. If the summary + line is missing, the run's error text didn't match: read it in the log and + extend `LIMIT_PATTERN`. +4. Check the spec issue: no `impl:{library}:failed` label, and no + `impl-fail` comment for the pair. +5. Check the next watchdog scan: its log reads + `PR #PR_NUMBER: paused (Claude stopped its review or repair) — skipped`, + and it dispatched nothing for the PR. +6. Restore the spend limit, then rerun the review: + + ```bash + gh workflow run impl-review.yml -f pr_number=PR_NUMBER + ``` + +7. Check that the review removes `ai-review-paused`, finishes, and the regen + gate decides as usual. + +To test the generation path as well, set the limit to the spend so far plus +about 0.05 USD: the generation stops, the pair is parked as described above, +and its retry is skipped. + +### Exposure + +Each credential reaches only the Claude steps, and nothing else: +`tests/unit/workflows/test_claude_auth.py` checks that the API key appears only +in the action input and in the resolve step's `!= ''` test, and that no job +gained `id-token: write`. None of the eight workflows runs on `pull_request` +or `pull_request_target`, so a pull request from a fork never gets a +credential. `util-claude.yml` runs only for comments by owners, members, and +collaborators. `spec-create.yml` and `report-validate.yml` run on a label, +which only maintainers can add, but they read issue text that anyone can +write. + +Federation adds no stored secret, but the trust lives in the Claude Console, +not in the repository: + +- Any step of a job with `id-token: write` can mint an OIDC token. The + federation rule accepts only `main`, so a token from a pull-request event or + another branch is refused, but a step on `main` can exchange its token for a + Claude API token whatever `CLAUDE_AUTH` says. +- Switching `CLAUDE_AUTH` back to `subscription` or `api` doesn't close that + path. While you don't use `oidc`, disable the federation rule on the + [Workload identity](https://platform.claude.com/settings/workload-identity-federation) + page. +- The rule's claims name the repository and the ref, not the workflow. GitHub + also signs the `job_workflow_ref` claim, the workflow file the job runs + from, so adding it to the rule could narrow federation to the workflows + that run Claude. This is unverified: check that the Console accepts the + claim before you rely on it. +- The steps that run model-written code in a job with `id-token: write` + unset `ACTIONS_ID_TOKEN_REQUEST_URL` and `ACTIONS_ID_TOKEN_REQUEST_TOKEN` + first, so that code can't mint a token: "Adopt implementation from source + ref" and "Adaptability smoke run (record-only)" in `impl-generate.yml`, and + "Adaptability smoke run (record-only)" in `impl-repair.yml`. The + adaptability check in `impl-review.yml` reads the code without running it. + The switch doesn't change what a Claude session's own tools can reach. + +A leaked federated token expires after 600 seconds, and an API key is billed +until you revoke it; the workspace's spend limit bounds either. + +--- + ## CLI model tiers The agentic workflows use abstract model tiers (`small`, `medium`, `large`) instead of CLI-specific model names. This allows the same command to work across different AI tools. diff --git a/prompts/README.md b/prompts/README.md index 52cbc7a6855..8c117708e40 100644 --- a/prompts/README.md +++ b/prompts/README.md @@ -47,7 +47,14 @@ See `workflow-prompts/README.md` for the wiring pattern + per-variable reference # Example: from .github/workflows/impl-review.yml - uses: anthropics/claude-code-action@ # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Exactly one credential, from the job's "Resolve Claude credentials" step + # (docs/workflows/overview.md#claude-credentials) + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model sonnet" prompt: | Read `prompts/workflow-prompts/ai-quality-review.md` and follow those instructions. diff --git a/prompts/workflow-prompts/README.md b/prompts/workflow-prompts/README.md index b5a3a4f73df..6011cbb5c29 100644 --- a/prompts/workflow-prompts/README.md +++ b/prompts/workflow-prompts/README.md @@ -20,7 +20,14 @@ Example (from `.github/workflows/impl-review.yml`): ```yaml - uses: anthropics/claude-code-action@ # v1 with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Exactly one credential, from the job's "Resolve Claude credentials" step + # (docs/workflows/overview.md#claude-credentials) + anthropic_api_key: ${{ steps.claude_auth.outputs.mode == 'api' && secrets.ANTHROPIC_API_KEY || '' }} + claude_code_oauth_token: ${{ steps.claude_auth.outputs.mode == 'subscription' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + anthropic_federation_rule_id: ${{ steps.claude_auth.outputs.federation_rule_id }} + anthropic_organization_id: ${{ steps.claude_auth.outputs.organization_id }} + anthropic_service_account_id: ${{ steps.claude_auth.outputs.service_account_id }} + anthropic_workspace_id: ${{ steps.claude_auth.outputs.workspace_id }} claude_args: "--model sonnet" prompt: | Read `prompts/workflow-prompts/ai-quality-review.md` and follow those instructions. diff --git a/tests/unit/automation/scripts/test_regen_pick_pairs.py b/tests/unit/automation/scripts/test_regen_pick_pairs.py new file mode 100644 index 00000000000..9b0c43b6bad --- /dev/null +++ b/tests/unit/automation/scripts/test_regen_pick_pairs.py @@ -0,0 +1,738 @@ +"""Tests for automation.scripts.regen_pick_pairs — daily-regen's pair mode. + +The picker resumes the paused reviews and repairs first, then ages each +(spec, library) pair by its newest touch, the metadata ``updated`` or the +last real attempt (a merged or closed implementation PR, a failed +generation), keeps every pair touched within the attempt window out of the +queue, and postpones a spec whose issue just saw activity. The no-loop rule +is pinned here: a pair the gate kept, or whose generation failed, does not +come back the next night, an infrastructure failure or a generation Claude +stopped is no attempt, and a pick that cannot see its records fails closed. +``main`` runs against a fake ``gh`` the way the workflow calls it. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +import yaml + +from automation.scripts import regen_pick_pairs as rp + + +NOW = datetime(2026, 10, 31, 2, 17, tzinfo=timezone.utc) +MIN_AGE = timedelta(hours=20) +WINDOW = timedelta(days=30) +LIBS = ["altair", "matplotlib", "plotly"] +SCRIPT = Path(rp.__file__) + + +def ts(days: float, now: datetime = NOW) -> str: + return (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def at(days: float) -> datetime: + return NOW - timedelta(days=days) + + +def write_spec(root: Path, spec: str, metadata: dict[str, dict | str | None], issue: int | None = None) -> None: + spec_dir = root / spec + spec_dir.mkdir(parents=True) + (spec_dir / "specification.md").write_text("# s\n", encoding="utf-8") + (spec_dir / "specification.yaml").write_text(yaml.safe_dump({"issue": issue} if issue else {}), encoding="utf-8") + for lib, data in metadata.items(): + lang = "javascript" if lib == "chartjs" else "python" + meta = spec_dir / "metadata" / lang + meta.mkdir(parents=True, exist_ok=True) + text = data if isinstance(data, str) else yaml.safe_dump(data or {}) + (meta / f"{lib}.yaml").write_text(text, encoding="utf-8") + + +def records(**kw) -> rp.Records: + return rp.Records( + attempts=kw.get("attempts", {}), open_pairs=kw.get("open_pairs", set()), issue_activity=kw.get("issues", {}) + ) + + +def run_pick(cands, recs=None, issues=None, count=5, window=WINDOW, failures=None, max_reads=100): + return rp.pick(cands, recs or records(), issues or {}, NOW, MIN_AGE, window, count, failures, max_reads) + + +def picked(picks: list[rp.Pick]) -> list[str]: + return [f"{p.spec}/{p.library}" for p in picks] + + +def fail_comment(spec: str, lib: str, days: float, infra: bool = False) -> dict[str, str]: + body = f"\n" + (f"{rp.INFRA_TAG}\n" if infra else "") + "## Failed" + return {"created_at": ts(days), "body": body} + + +class TestLoadCandidates: + def test_reads_every_supported_pair_with_its_timestamp(self, tmp_path): + write_spec(tmp_path, "a", {"altair": {"updated": ts(10)}, "plotly": {"created": ts(30)}}, issue=7) + cands, issues = rp.load_candidates(tmp_path, LIBS) + assert {(c.spec, c.library, c.metadata) for c in cands} == {("a", "altair", at(10)), ("a", "plotly", at(30))} + assert issues == {"a": 7} + + def test_skips_unsupported_libraries_and_specs_without_specification(self, tmp_path): + write_spec(tmp_path, "a", {"chartjs": {"updated": ts(10)}, "legacy": {"updated": ts(10)}}) + (tmp_path / "no-spec" / "metadata" / "python").mkdir(parents=True) + (tmp_path / "no-spec" / "metadata" / "python" / "altair.yaml").write_text("updated: x\n", encoding="utf-8") + assert rp.load_candidates(tmp_path, LIBS)[0] == [] + assert [c.library for c in rp.load_candidates(tmp_path, [*LIBS, "chartjs"])[0]] == ["chartjs"] + + def test_unparseable_timestamp_skips_the_pair_and_missing_one_keeps_it(self, tmp_path): + write_spec(tmp_path, "a", {"altair": {"updated": "not a date"}, "plotly": {}, "matplotlib": "- a list\n"}) + cands, _ = rp.load_candidates(tmp_path, LIBS) + assert [(c.library, c.metadata) for c in cands] == [("plotly", None)] + + +class TestPick: + def test_oldest_first_across_specs(self): + cands = [ + rp.Candidate("a", "altair", at(40)), + rp.Candidate("b", "plotly", at(90)), + rp.Candidate("a", "plotly", at(60)), + ] + picks, eligible = run_pick(cands, count=2) + assert picked(picks) == ["b/plotly", "a/plotly"] + assert eligible == 3 + + def test_a_kept_pair_waits_out_the_window(self): + # The gate kept a/altair 10 days ago: `updated` is still 200 days old. + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("b", "plotly", at(60))] + recs = records(attempts={("a", "altair"): (at(10), "kept PR")}) + assert picked(run_pick(cands, recs)[0]) == ["b/plotly"] + + def test_after_the_window_an_attempt_queues_behind_older_pairs(self): + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("b", "plotly", at(60))] + recs = records(attempts={("a", "altair"): (at(45), "kept PR")}) + picks, _ = run_pick(cands, recs) + assert picked(picks) == ["b/plotly", "a/altair"] + assert picks[1].basis == "kept PR" and picks[1].touched == at(45) + + def test_a_recent_merge_is_inside_the_window_too(self): + cands = [rp.Candidate("a", "altair", at(12)), rp.Candidate("a", "plotly", at(31))] + assert picked(run_pick(cands)[0]) == ["a/plotly"] + + def test_window_zero_orders_only(self): + cands = [rp.Candidate("a", "altair", at(2)), rp.Candidate("a", "plotly", at(1))] + assert picked(run_pick(cands, window=timedelta(0))[0]) == ["a/altair", "a/plotly"] + + def test_open_pr_skips_only_its_pair(self): + cands = [rp.Candidate("a", "altair", at(60)), rp.Candidate("a", "plotly", at(70))] + assert picked(run_pick(cands, records(open_pairs={("a", "plotly")}))[0]) == ["a/altair"] + + def test_recent_issue_activity_postpones_every_pair_of_the_spec(self): + cands = [rp.Candidate("a", "altair", at(60)), rp.Candidate("b", "altair", at(40))] + recs = records(issues={1: at(0.3), 2: at(3)}) + assert picked(run_pick(cands, recs, {"a": 1, "b": 2})[0]) == ["b/altair"] + + def test_issue_activity_is_a_gate_not_an_age(self): + cands = [rp.Candidate("a", "altair", at(60)), rp.Candidate("b", "altair", at(50))] + picks, _ = run_pick(cands, records(issues={1: at(2)}), {"a": 1}, count=1) + assert picked(picks) == ["a/altair"] and picks[0].touched == at(60) + + def test_no_timestamp_at_all_is_ancient(self): + cands = [rp.Candidate("a", "altair", at(300)), rp.Candidate("b", "altair", None)] + picks, _ = run_pick(cands, count=2) + assert picked(picks) == ["b/altair", "a/altair"] + assert picks[0].touched == rp.ANCIENT and picks[0].basis == "none" + + def test_ties_break_by_spec_then_library(self): + cands = [rp.Candidate("b", "altair", at(90)), rp.Candidate("a", "plotly", at(90))] + cands.append(rp.Candidate("a", "altair", at(90))) + assert picked(run_pick(cands, count=3)[0]) == ["a/altair", "a/plotly", "b/altair"] + + +class TestFailedGenerations: + def test_a_failed_generation_is_an_attempt(self): + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("a", "plotly", at(150))] + cands.append(rp.Candidate("b", "altair", at(100))) + asked: list[str] = [] + + def failures(spec): + asked.append(spec) + return {"altair": at(3)} if spec == "a" else {} + + picks, eligible = run_pick(cands, count=2, failures=failures) + # a/altair failed three days ago: out for the window; its sibling is not. + assert picked(picks) == ["a/plotly", "b/altair"] + assert eligible == 2 + assert asked == ["a", "b"] # once per spec, only for the front of the queue + + def test_an_old_failure_re_ages_the_pair(self): + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("b", "altair", at(100))] + picks, _ = run_pick(cands, count=2, failures=lambda spec: {"altair": at(40)} if spec == "a" else {}) + assert picked(picks) == ["b/altair", "a/altair"] + assert picks[1].basis == "failed generation" + + def test_comments_count_only_real_failures(self): + comments = [ + fail_comment("a", "altair", 5), + fail_comment("a", "altair", 2, infra=True), # infrastructure: no attempt + fail_comment("a", "plotly", 1, infra=True), + fail_comment("other", "altair", 1), # another spec's marker + {"created_at": ts(1), "body": ""}, # Claude stopped the session + ] + assert rp.failed_generations(comments, "a") == {"altair": at(5)} + + def test_unreadable_failures_leave_the_spec_out(self, capsys): + cands = [rp.Candidate("a", "altair", at(200)), rp.Candidate("a", "plotly", at(190))] + cands.append(rp.Candidate("b", "altair", at(100))) + picks, eligible = run_pick(cands, count=3, failures=lambda spec: None if spec == "a" else {}) + # Fail closed: a's pairs may have failed last night. + assert picked(picks) == ["b/altair"] and eligible == 1 + assert "::warning::Leaving a out of tonight" in capsys.readouterr().out + + def test_the_read_cap_stops_reading(self, capsys): + cands = [rp.Candidate(spec, "altair", at(100 + i)) for i, spec in enumerate("abc")] + asked: list[str] = [] + picks, _ = run_pick(cands, count=3, failures=lambda spec: asked.append(spec) or {}, max_reads=1) + assert len(picks) == 3 and asked == ["c"] + assert "::warning::Read the failed generations of 1 specs" in capsys.readouterr().out + + +class TestFoldPulls: + def test_open_blocks_closed_and_merged_count(self): + recs = records() + rp.fold_pulls( + [ + {"headRefName": "implementation/a/altair", "state": "MERGED", "closedAt": ts(9), "updatedAt": ts(1)}, + {"headRefName": "implementation/a/altair", "state": "CLOSED", "closedAt": ts(40), "labels": []}, + { + "headRefName": "implementation/a/plotly", + "state": "CLOSED", + "closedAt": ts(3), + "labels": [{"name": "regen"}, {"name": "regen:kept"}], + }, + {"headRefName": "implementation/b/plotly", "state": "OPEN", "updatedAt": ts(1)}, + {"headRefName": "review-writeback/a/altair/12", "state": "MERGED", "closedAt": ts(0)}, + {"headRefName": "implementation/a/altair/extra", "state": "CLOSED", "closedAt": ts(0)}, + ], + recs, + ) + assert recs.attempts == {("a", "altair"): (at(9), "merged PR"), ("a", "plotly"): (at(3), "kept PR")} + assert recs.open_pairs == {("b", "plotly")} + assert recs.paused == [] + + def test_paused_prs_become_resumes(self): + recs = records() + review = { + "number": 21, + "headRefName": "implementation/a/plotly", + "state": "OPEN", + "updatedAt": ts(2), + "labels": [{"name": "ai-review-paused"}], + } + repair = { + "number": 22, + "headRefName": "implementation/b/altair", + "state": "OPEN", + "updatedAt": ts(3), + "labels": [{"name": n} for n in ("ai-rejected", "ai-attempt-1", "ai-attempt-2", "ai-repair-paused")], + } + # A paused PR that is closed meanwhile is an attempt, not a resume. + closed = {**review, "number": 23, "headRefName": "implementation/c/plotly", "state": "CLOSED"} + rp.fold_pulls([review, repair, closed, review], recs) # the same PR twice: listed once + assert recs.open_pairs == {("a", "plotly"), ("b", "altair")} + assert recs.paused == [ + rp.Resume(21, "review", "a", "plotly", 1, at(2)), + rp.Resume(22, "repair", "b", "altair", 2, at(3)), + ] + assert [r.pr for r in rp.resumes(recs, 5)] == [22, 21] # oldest first + assert [r.pr for r in rp.resumes(recs, 1)] == [22] + + def test_issues(self): + recs = records() + rp.fold_issues([{"number": 3, "updatedAt": ts(2)}, {"number": "x", "updatedAt": ts(1)}], recs) + assert recs.issue_activity == {3: at(2)} + + +# Answers each listing the picker makes from a JSON file named after it, and an +# issue's comments (`api … --jq …`) from comments-.jsonl; a listing +# whose file is missing fails like an API error. Logs every call. +FAKE_GH = """#!/bin/sh +echo "$*" >> "$GH_LOG" +case "$*" in + *"head:implementation/"*) f=prs.json ;; + *"regen:kept"*) f=kept.json ;; + "issue list"*) f=issues.json ;; + "api --paginate repos/o/r/issues/"*) n=${3#repos/o/r/issues/}; f=comments-${n%%/*}.jsonl ;; + *) exit 1 ;; +esac +[ -f "$GH_DATA/$f" ] || { echo "HTTP 502" >&2; exit 1; } +cat "$GH_DATA/$f" +""" + + +def run_main(tmp_path: Path, data: dict[str, str], count: int = 3, *extra: str): + bin_dir, data_dir = tmp_path / "bin", tmp_path / "data" + bin_dir.mkdir() + data_dir.mkdir() + (bin_dir / "gh").write_text(FAKE_GH, encoding="utf-8") + (bin_dir / "gh").chmod(0o755) + for name, text in data.items(): + (data_dir / name).write_text(text, encoding="utf-8") + out, summary = tmp_path / "out", tmp_path / "summary" + env = { + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "GH_LOG": str(tmp_path / "gh.log"), + "GH_DATA": str(data_dir), + "GITHUB_OUTPUT": str(out), + "GITHUB_STEP_SUMMARY": str(summary), + } + args = ["--count", str(count), "--min-age-hours", "20", "--repo", "o/r", "--libraries", " ".join(LIBS), *extra] + result = subprocess.run( + [sys.executable, str(SCRIPT), *args, "--plots", str(tmp_path / "plots")], + env=env, + capture_output=True, + text=True, + ) + outputs = dict(line.split("=", 1) for line in out.read_text().splitlines()) if out.exists() else {} + return result, outputs, summary.read_text() if summary.exists() else "" + + +class TestMain: + def _plots(self, tmp_path: Path) -> None: + self.now = datetime.now(timezone.utc) + plots = tmp_path / "plots" + write_spec( + plots, "a-spec", {"altair": {"updated": ts(190, self.now)}, "plotly": {"updated": ts(160, self.now)}}, 11 + ) + write_spec( + plots, + "b-spec", + {"altair": {"updated": ts(180, self.now)}, "matplotlib": {"updated": ts(170, self.now)}}, + 12, + ) + + def test_writes_pairs_specs_and_summary(self, tmp_path): + self._plots(tmp_path) + prs = [{"headRefName": "implementation/b-spec/matplotlib", "state": "OPEN", "updatedAt": ts(0.1, self.now)}] + kept = [ + { + "headRefName": "implementation/a-spec/altair", + "state": "CLOSED", + "closedAt": ts(40, self.now), + "labels": [{"name": "regen:kept"}], + } + ] + comments = [ + {"created_at": ts(5, self.now), "body": "\n"} + ] + result, outputs, summary = run_main( + tmp_path, + { + "prs.json": json.dumps(prs), + "kept.json": json.dumps(kept), + "issues.json": json.dumps([{"number": 11, "updatedAt": ts(7, self.now)}]), + "comments-11.jsonl": "", + "comments-12.jsonl": "\n".join(json.dumps(c) for c in comments), + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + # b-spec/matplotlib has an open PR; a-spec/altair was kept 40 days ago, + # so it queues last; b-spec/altair's infrastructure failure is no attempt. + assert json.loads(outputs["pairs_json"]) == [ + {"spec": "b-spec", "library": "altair"}, + {"spec": "a-spec", "library": "plotly"}, + {"spec": "a-spec", "library": "altair"}, + ] + assert json.loads(outputs["specs_json"]) == ["b-spec", "a-spec"] + assert outputs["specs"] == "b-spec a-spec" and outputs["count"] == "3" + assert "| 3 | `a-spec` | `altair` |" in summary and "kept PR" in summary + log = (tmp_path / "gh.log").read_text() + assert "repos/o/r/issues/12/comments?per_page=100" in log and "run list" not in log + + def test_the_window_comes_from_the_argument(self, tmp_path): + self._plots(tmp_path) + kept = [{"headRefName": "implementation/a-spec/altair", "state": "CLOSED", "closedAt": ts(40, self.now)}] + data = {"prs.json": "[]", "kept.json": json.dumps(kept), "issues.json": "[]"} + data |= {"comments-11.jsonl": "", "comments-12.jsonl": ""} + _, outputs, _ = run_main(tmp_path, data, 5, "--attempt-window-days", "60") + # Kept 40 days ago is inside a 60-day window. + pairs = {(p["spec"], p["library"]) for p in json.loads(outputs["pairs_json"])} + assert pairs == {("a-spec", "plotly"), ("b-spec", "altair"), ("b-spec", "matplotlib")} + + def _paused(self) -> dict[str, str]: + prs = [ + { + "number": 21, + "headRefName": "implementation/a-spec/plotly", + "state": "OPEN", + "updatedAt": ts(2, self.now), + "labels": [{"name": "ai-review-paused"}], + }, + { + "number": 22, + "headRefName": "implementation/b-spec/altair", + "state": "OPEN", + "updatedAt": ts(3, self.now), + "labels": [{"name": "ai-rejected"}, {"name": "ai-attempt-2"}, {"name": "ai-repair-paused"}], + }, + ] + data = {"prs.json": json.dumps(prs), "kept.json": "[]", "issues.json": "[]"} + return data | {"comments-11.jsonl": "", "comments-12.jsonl": ""} + + def test_paused_prs_are_resumed_before_new_pairs(self, tmp_path): + self._plots(tmp_path) + result, outputs, summary = run_main(tmp_path, self._paused(), 3) + assert result.returncode == 0, result.stdout + result.stderr + # Both resumes count against the night's three; their pairs are open, so + # the one new pair is the oldest of the rest. + assert json.loads(outputs["resume_json"]) == [ + {"pr": 22, "kind": "repair", "spec": "b-spec", "library": "altair", "attempt": 2}, + {"pr": 21, "kind": "review", "spec": "a-spec", "library": "plotly", "attempt": 1}, + ] + assert outputs["resume_count"] == "2" + assert json.loads(outputs["pairs_json"]) == [{"spec": "a-spec", "library": "altair"}] + assert outputs["count"] == "1" + assert "Pair mode: 2 resumed, 1 new pair(s) of 3" in summary and "| repair | #22 |" in summary + + def test_resumes_can_take_the_whole_night(self, tmp_path): + self._plots(tmp_path) + _, outputs, _ = run_main(tmp_path, self._paused(), 1) + assert json.loads(outputs["resume_json"])[0]["pr"] == 22 + assert outputs["resume_count"] == "1" and outputs["count"] == "0" and outputs["pairs_json"] == "[]" + + def test_a_failed_implementation_pr_listing_stops_the_pick(self, tmp_path): + """The loop guard: without it a kept, closed or paused pair would be picked again.""" + self._plots(tmp_path) + result, outputs, _ = run_main(tmp_path, {"kept.json": "[]", "issues.json": "[]"}, 1) + assert result.returncode == 1 + assert "::error::Pair mode stops: the implementation PRs could not be listed" in result.stdout + assert outputs == {} + + def test_the_other_listings_degrade_with_a_warning(self, tmp_path): + self._plots(tmp_path) + result, outputs, _ = run_main(tmp_path, {"prs.json": "[]", "comments-12.jsonl": ""}, 1) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.count("::warning::Could not list") == 2 + # a-spec's comments cannot be read: it is left out, b-spec/altair is next. + assert "::warning::Could not read the comments of issue #11" in result.stdout + assert json.loads(outputs["pairs_json"]) == [{"spec": "b-spec", "library": "altair"}] + + def test_nothing_eligible_writes_an_empty_pick(self, tmp_path): + (tmp_path / "plots").mkdir() + result, outputs, _ = run_main(tmp_path, {"prs.json": "[]", "kept.json": "[]", "issues.json": "[]"}) + assert result.returncode == 0, result.stdout + result.stderr + assert outputs == { + "resume_json": "[]", + "resume_count": "0", + "pairs_json": "[]", + "specs": "", + "specs_json": "[]", + "count": "0", + } + + @pytest.mark.parametrize(("count", "window"), [(0, "30"), (-1, "30"), (1, "-1")]) + def test_refuses_bad_numbers(self, tmp_path, count, window): + (tmp_path / "plots").mkdir() + result, _, _ = run_main(tmp_path, {}, count, "--attempt-window-days", window) + assert result.returncode == 1 and "::error::" in result.stdout + + +# --- the same paths in process ------------------------------------------------ + + +class FakeRunner: + """Stands in for ``subprocess.run``: answers each ``gh`` call the picker makes from ``data``. + + The keys are ``prs``, ``kept``, ``issues`` and ``comments-``; a value + that is an exception is raised, and a missing key fails like an API error. + """ + + def __init__(self, data: dict[str, str | BaseException]): + self.data = data + self.calls: list[list[str]] = [] + + @staticmethod + def key(args: list[str]) -> str: + joined = " ".join(args) + if "head:implementation/" in joined: + return "prs" + if "regen:kept" in joined: + return "kept" + if args[1:3] == ["issue", "list"]: + return "issues" + if args[1:3] == ["api", "--paginate"] and "/comments" in args[3]: + return "comments-" + args[3].split("/issues/")[1].split("/")[0] + return "unknown" + + def __call__(self, args: list[str], **kwargs) -> subprocess.CompletedProcess: + assert args[0] == "gh" and kwargs == {"check": True, "capture_output": True, "text": True} + self.calls.append(args) + value = self.data.get(self.key(args)) + if value is None: + raise subprocess.CalledProcessError(1, args, stderr="HTTP 502") + if isinstance(value, BaseException): + raise value + return subprocess.CompletedProcess(args, 0, stdout=value, stderr="") + + +@pytest.fixture +def runner(monkeypatch): + def install(data: dict[str, str | BaseException]) -> FakeRunner: + fake = FakeRunner(data) + monkeypatch.setattr(rp.subprocess, "run", fake) + return fake + + return install + + +def pr(branch: str, state: str, number: int = 1, labels: tuple[str, ...] = (), **times: str) -> dict: + return {"number": number, "headRefName": branch, "state": state, "labels": [{"name": n} for n in labels], **times} + + +class TestLoadCandidatesEdges: + @pytest.mark.parametrize( + "spec_yaml", + [ + None, # no specification.yaml at all + "issue: [unclosed\n", # not YAML + "- a list\n", # YAML, but no mapping + "issue: '7'\n", # not a number + ], + ) + def test_a_spec_without_a_readable_issue_number_is_still_a_candidate(self, tmp_path, spec_yaml): + write_spec(tmp_path, "a", {"altair": {"updated": ts(10)}}, issue=7) + if spec_yaml is None: + (tmp_path / "a" / "specification.yaml").unlink() + else: + (tmp_path / "a" / "specification.yaml").write_text(spec_yaml, encoding="utf-8") + cands, issues = rp.load_candidates(tmp_path, LIBS) + assert [c.library for c in cands] == ["altair"] and issues == {} + + def test_unreadable_metadata_and_stray_entries_are_skipped(self, tmp_path): + write_spec(tmp_path, "a", {"altair": "updated: [unclosed\n", "plotly": {"updated": ts(10)}}) + (tmp_path / ".hidden").mkdir() + (tmp_path / ".hidden" / "specification.md").write_text("# s\n", encoding="utf-8") + (tmp_path / "README.md").write_text("not a spec\n", encoding="utf-8") + assert [(c.spec, c.library) for c in rp.load_candidates(tmp_path, LIBS)[0]] == [("a", "plotly")] + + +class TestFoldPullsEdges: + def test_other_states_and_unnumbered_pauses(self): + recs = records() + rp.fold_pulls( + [ + pr("implementation/a/altair", "DRAFT", closedAt=ts(1)), # neither open nor done + {"headRefName": "implementation/a/plotly", "state": "OPEN", "labels": [{"name": "ai-review-paused"}]}, + pr("implementation/b/plotly", "OPEN", 9, ("ai-repair-paused", "ai-attempt-x")), + {"headRefName": None, "state": "MERGED"}, + pr("implementation/c/plotly", "MERGED", 3), # no timestamp: no attempt + ], + recs, + ) + assert recs.attempts == {} + assert recs.open_pairs == {("a", "plotly"), ("b", "plotly")} + # The unnumbered pause blocks its pair but can't be resumed; a malformed + # attempt label falls back to attempt 1, a missing time to ANCIENT. + assert recs.paused == [rp.Resume(9, "repair", "b", "plotly", 1, rp.ANCIENT)] + + def test_both_pause_labels_resume_the_repair(self): + """PAUSED_LABELS lists the repair first; a resumed repair reviews again anyway.""" + recs = records() + rp.fold_pulls([pr("implementation/a/plotly", "OPEN", 5, ("ai-review-paused", "ai-repair-paused"))], recs) + assert [r.kind for r in recs.paused] == ["repair"] + + +class TestGhJson: + def test_a_list_comes_back(self, runner): + fake = runner({"issues": '[{"number": 1}]'}) + assert rp._gh_json(["issue", "list"], "issues") == [{"number": 1}] + assert fake.calls == [["gh", "issue", "list"]] + + @pytest.mark.parametrize( + "answer", + [None, "not json", FileNotFoundError("gh"), subprocess.CalledProcessError(1, ["gh"], stderr="HTTP 502")], + ) + def test_a_failure_warns_and_yields_none(self, runner, capsys, answer): + runner({} if answer is None else {"issues": answer}) + assert rp._gh_json(["issue", "list"], "issues") is None + assert capsys.readouterr().out.startswith("::warning::Could not list issues (") + + def test_a_non_list_yields_none(self, runner, capsys): + runner({"issues": '{"message": "Not Found"}'}) + assert rp._gh_json(["issue", "list"], "issues") is None + assert capsys.readouterr().out == "" + + +class TestReadRecords: + def test_folds_every_listing(self, runner, capsys): + fake = runner( + { + "prs": json.dumps( + [ + pr("implementation/a/altair", "MERGED", closedAt=ts(9)), + pr("implementation/a/plotly", "OPEN", 7, ("ai-review-paused",), updatedAt=ts(1)), + ] + ), + "kept": json.dumps([pr("implementation/b/altair", "CLOSED", 8, ("regen:kept",), closedAt=ts(200))]), + "issues": json.dumps([{"number": 11, "updatedAt": ts(2)}]), + } + ) + recs = rp.read_records("o/r") + assert set(recs.attempts) == {("a", "altair"), ("b", "altair")} + assert recs.attempts[("b", "altair")][1] == "kept PR" + assert recs.open_pairs == {("a", "plotly")} and [r.pr for r in recs.paused] == [7] + assert set(recs.issue_activity) == {11} + assert ( + "Attempt records for 2 pair(s), 1 open PR(s) (1 paused), activity for 1 issue(s)" in capsys.readouterr().out + ) + prs_call, kept_call, issues_call = fake.calls + assert prs_call[prs_call.index("--json") + 1] == "number,headRefName,state,updatedAt,closedAt,labels" + assert prs_call[prs_call.index("--state") + 1] == "all" and "--repo" in prs_call and "o/r" in prs_call + assert kept_call[kept_call.index("--state") + 1] == "closed" + assert issues_call[issues_call.index("--json") + 1] == "number,updatedAt" + + @pytest.mark.parametrize("prs", [None, '{"message": "rate limited"}']) + def test_no_implementation_prs_is_an_error(self, runner, prs): + fake = runner({"kept": "[]", "issues": "[]"} | ({} if prs is None else {"prs": prs})) + with pytest.raises(rp.ListingError, match="implementation PRs could not be listed"): + rp.read_records("o/r") + assert len(fake.calls) == 1 # nothing else is read + + def test_the_other_listings_are_best_effort(self, runner, capsys): + runner({"prs": json.dumps([pr("implementation/a/altair", "CLOSED", closedAt=ts(5))])}) + recs = rp.read_records("o/r") + assert recs.attempts == {("a", "altair"): (at(5), "closed PR")} and recs.issue_activity == {} + out = capsys.readouterr().out + assert "Could not list kept regenerations" in out and "Could not list issues" in out + + +class TestIssueFailures: + def test_reads_the_spec_issue_once_per_call(self, runner): + comments = [ + fail_comment("a", "altair", 4), + fail_comment("a", "plotly", 2, infra=True), + fail_comment("b", "altair", 1), + ] + fake = runner({"comments-11": "\n".join(json.dumps(c) for c in comments) + "\n\n"}) + failures = rp.issue_failures("o/r", {"a": 11}) + assert failures("a") == {"altair": at(4)} + (call,) = fake.calls + assert call[3] == "repos/o/r/issues/11/comments?per_page=100" + assert '"" in body and why in body + assert "" in body + assert "" in body + assert "", + 1, + ), + "repair": ( + "impl-repair.yml", + "Handle repair failure", + {"MODEL": "sonnet", "ATTEMPT": "2"}, + "ai-repair-paused", + "", + 0, + ), +} + + +@pytest.mark.parametrize("kind", sorted(PAUSE_PATHS)) +@pytest.mark.parametrize( + ("latest", "posts"), + [ + # A resume that paused again: the label alone, no second comment. + ("same pause", False), + # The PR moved on since (another bot comment), or nothing to compare. + ("other comment after the pause", True), + ("no comments", True), + ("read fails", True), + ], +) +def test_a_repeated_pause_posts_no_second_comment(tmp_path, kind, latest, posts): + filename, name, extra, label, marker, rc = PAUSE_PATHS[kind] + # The first line of each bot comment, oldest first; a pause starts with its marker. + comments = { + "same pause": f"## AI Review - Attempt 1/3\n{marker}", + "other comment after the pause": f"{marker}\n## AI Review - Attempt 1/3", + "no comments": "", + "read fails": "FAIL", + }[latest] + env = {"PR_NUM": "7", "SPEC_ID": "spec-a", "LIBRARY": "plotly", "REPOSITORY": "owner/repo", "RUN_ID": "1"} + env |= extra | {"CLAUDE_STOPPED": "true", "STOP_REASON": "limit", "CLAUDE_MODE": "oidc", "GH_COMMENTS": comments} + result, calls = _run_step(tmp_path, filename, name, **env) + assert result.returncode == rc, result.stdout + result.stderr + assert ["pr", "edit", "7", "--add-label", label] in calls # the label is kept either way + assert (len(_bodies(calls)) == 1) is posts + assert ("already shows this pause" in result.stdout) is not posts + (read,) = [c for c in calls if c[:1] == ["api"]] + assert read[read.index("--jq") + 1].startswith('.[] | select(.user.type == "Bot") | (.body // "")') + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="needs jq") +@pytest.mark.parametrize("kind", sorted(PAUSE_PATHS)) +def test_the_latest_bot_comment_read_yields_first_lines(kind): + """The real filter on REST-shaped comments: bots only, first line, CRLF and empty bodies safe.""" + filename, name, *_ = PAUSE_PATHS[kind] + match = re.search(r"--jq '([^']+)'", _step(filename, name)["run"]) + assert match + comments = [ + {"user": {"type": "Bot"}, "body": "## AI Review - Attempt 1/3\nscore"}, + {"user": {"type": "User"}, "body": ""}, + {"user": {"type": "Bot"}, "body": None}, + {"user": {"type": "Bot"}, "body": "\r\n## Paused & "}, + ] + out = subprocess.run( + ["jq", "-r", match[1]], input=json.dumps(comments), capture_output=True, text=True, check=True + ).stdout + assert out.splitlines() == ["## AI Review - Attempt 1/3", "", ""] + + +def test_a_new_repair_attempt_gets_its_own_pause_comment(tmp_path): + earlier = "" + env = {"PR_NUM": "7", "SPEC_ID": "spec-a", "LIBRARY": "plotly", "REPOSITORY": "owner/repo", "RUN_ID": "1"} + env |= {"MODEL": "sonnet", "ATTEMPT": "2", "CLAUDE_STOPPED": "true", "STOP_REASON": "limit", "CLAUDE_MODE": "oidc"} + _, calls = _run_step(tmp_path, "impl-repair.yml", "Handle repair failure", GH_COMMENTS=earlier, **env) + (body,) = _bodies(calls) + assert "" in body + + +@pytest.mark.parametrize( + ("filename", "name"), + [ + ("impl-generate.yml", "Handle generation failure"), + ("impl-repair.yml", "Handle repair failure"), + ("impl-review.yml", "Validate review output"), + ], +) +def test_failure_paths_read_the_stop(filename, name): + env = _step(filename, name)["env"] + assert env["CLAUDE_MODE"] == "${{ steps.claude_auth.outputs.mode }}" + if filename == "impl-review.yml": + assert env["CLAUDE_STOPPED"] == "${{ steps.review_cost.outputs.stopped }}" + assert env["STOP_REASON"] == "${{ steps.review_cost.outputs.stop_reason }}" + else: + assert env["CLAUDE_STOPPED"] == ( + "${{ steps.claude_cost.outputs.stopped == 'true' || steps.claude_retry_cost.outputs.stopped == 'true' }}" + ) + assert env["STOP_REASON"] == ( + "${{ steps.claude_cost.outputs.stop_reason || steps.claude_retry_cost.outputs.stop_reason }}" + ) + + +def test_generation_park_comes_before_any_attempt_counting(): + """The park exits before the infra/agent split reads or writes a single impl-fail marker.""" + script = _step("impl-generate.yml", "Handle generation failure")["run"] + park = script.index('if [ "${CLAUDE_STOPPED}" = "true" ]; then') + assert park < script.index('INFRA_CAUSE=""') < script.index('MARKER="