From 61116ecdce491e6325f8e865e700cf49d0491880 Mon Sep 17 00:00:00 2001 From: Joris Wouter Jonkers Date: Sun, 30 Aug 2026 15:50:00 +0200 Subject: [PATCH] fix(budget): refuse to report a percentage the watchdog cannot substantiate The previous change made the watchdog go green reporting "0 of 3000 (0%)". It could read neither billing nor workflow runs, warned about it, and then reported a comfortable zero as a pass. That is worse than the red it replaced, and worse in the specific way my own commit message had warned against two commits earlier: a budget watchdog that reports no usage because it cannot see usage looks like good news. A false all-clear on a budget alarm is the failure mode the whole thing exists to prevent. Two changes. A partial or empty read is now refused rather than averaged into a percentage. If any run-listing query is rejected, or if zero runs are found across every private repo in a month, the run fails and says what to grant. Zero runs org-wide is not a small number, it is evidence the measurement did not happen. The capable token is now minted best-effort with continue-on-error instead of behind an opt-in flag. create-github-app-token fails the job when asked for a permission the installation lacks, which is why the flag existed -- but allowing that one step to fail achieves the same protection without anyone having to set anything, and an App that does hold the grants is used automatically. It asks for organization-plan: read for the exact billing figure and actions: read for the fallback, and falls back to a minimal token that is only used to file the alert issue. Verified by reproducing the production condition -- a token that lists repos but is refused both billing and runs -- which now exits 1 naming the permissions, where before it exited 0 claiming 0%. Refs #126 --- .github/workflows/actions-budget-watchdog.yml | 71 +++++++++++-------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/.github/workflows/actions-budget-watchdog.yml b/.github/workflows/actions-budget-watchdog.yml index 556daf0..11fe8e0 100644 --- a/.github/workflows/actions-budget-watchdog.yml +++ b/.github/workflows/actions-budget-watchdog.yml @@ -36,14 +36,6 @@ name: 'Actions Budget Watchdog' 'required': false 'type': 'string' 'default': '' - 'app-has-billing-permission': - 'description': > - Set true only once the App has actually been granted "Organization plan" (read). - create-github-app-token fails outright when asked for a permission the installation - does not hold, so requesting it optimistically takes the whole watchdog down. - 'required': false - 'type': 'boolean' - 'default': false 'secrets': 'BUDGET_APP_ID': 'description': 'GitHub App id able to read org billing and repo metadata.' @@ -67,18 +59,18 @@ name: 'Actions Budget Watchdog' 'BUDGET_APP_ID': '${{ secrets.BUDGET_APP_ID }}' 'BUDGET_APP_PRIVATE_KEY': '${{ secrets.BUDGET_APP_PRIVATE_KEY }}' 'steps': - # Two minting steps, because create-github-app-token FAILS THE JOB when - # asked for a permission the installation does not hold -- it does not - # degrade. Requesting billing optimistically therefore takes the whole - # watchdog down at token minting, before it can even report why, which is - # strictly worse than reading no billing at all. + # create-github-app-token FAILS THE JOB when asked for a permission the + # installation does not hold -- it does not degrade. So the capable mint + # is allowed to fail instead: continue-on-error means an App without these + # grants costs a skipped step rather than the whole watchdog, and an App + # that has them is used automatically with nothing to configure. # - # Billing sits behind its own App permission: "Organization plan" - # (read-only). Until the App is granted it, leave - # app-has-billing-permission false and supply BUDGET_TOKEN instead. - - 'name': 'Mint organization token (billing permission granted)' - 'id': 'app-token-billing' - 'if': "${{ env.BUDGET_APP_ID != '' && env.BUDGET_APP_PRIVATE_KEY != '' && inputs.app-has-billing-permission }}" + # organization-plan: read -> the exact billing figure + # actions: read -> pricing runs when billing is unreachable + - 'name': 'Mint organization token (best effort, capable)' + 'id': 'app-token-capable' + 'if': "${{ env.BUDGET_APP_ID != '' && env.BUDGET_APP_PRIVATE_KEY != '' }}" + 'continue-on-error': true 'uses': 'actions/create-github-app-token@v3' 'with': 'app-id': '${{ env.BUDGET_APP_ID }}' @@ -87,10 +79,13 @@ name: 'Actions Budget Watchdog' 'permission-metadata': 'read' 'permission-issues': 'write' 'permission-organization-plan': 'read' + 'permission-actions': 'read' - - 'name': 'Mint organization token' + # Always mintable: these two permissions the App certainly holds. Used + # only to file the alert issue when the capable mint was refused. + - 'name': 'Mint organization token (minimal)' 'id': 'app-token' - 'if': "${{ env.BUDGET_APP_ID != '' && env.BUDGET_APP_PRIVATE_KEY != '' && !inputs.app-has-billing-permission }}" + 'if': "${{ env.BUDGET_APP_ID != '' && env.BUDGET_APP_PRIVATE_KEY != '' }}" 'uses': 'actions/create-github-app-token@v3' 'with': 'app-id': '${{ env.BUDGET_APP_ID }}' @@ -101,7 +96,7 @@ name: 'Actions Budget Watchdog' - 'name': 'Check the Actions minute budget' 'env': - 'GH_TOKEN': "${{ steps.app-token-billing.outputs.token || secrets.BUDGET_TOKEN || steps.app-token.outputs.token || github.token }}" + 'GH_TOKEN': "${{ steps.app-token-capable.outputs.token || secrets.BUDGET_TOKEN || steps.app-token.outputs.token || github.token }}" 'OWNER': '${{ github.repository_owner }}' 'ALLOWANCE': '${{ inputs.allowance-minutes }}' 'WARN_PCT': '${{ inputs.warn-percent }}' @@ -209,7 +204,7 @@ name: 'Actions Budget Watchdog' return out def run_ids(repo): - ids = [] + ids, failures = [], 0 for window in month_windows(): proc = subprocess.run( ['gh', 'api', '--paginate', @@ -217,11 +212,10 @@ name: 'Actions Budget Watchdog' '--jq', '.workflow_runs[].id'], capture_output=True, text=True, check=False) if proc.returncode != 0: - print(f'::warning::could not list runs for {repo} in {window}; ' - 'this month is undercounted') + failures += 1 continue ids.extend(x for x in proc.stdout.split() if x.strip()) - return ids + return ids, failures def price_run(job_spec): repo, run_id = job_spec @@ -264,10 +258,31 @@ name: 'Actions Budget Watchdog' # trips a little early rather than a little late. But do not quote # this figure as fact. Fix the billing permission and the exact # number comes back. - specs = [] + specs, list_failures = [], 0 for repo in sorted(private): - ids = run_ids(repo) + ids, failures = run_ids(repo) + list_failures += failures specs.extend((repo, rid) for rid in ids) + + # A partial read is not a budget figure. Reporting one as a + # percentage is how a watchdog produces a reassuring number it has + # no basis for, so refuse instead -- and refuse loudest when the + # answer would have been a comfortable zero. + if list_failures: + print(f'::error::could not list workflow runs ({list_failures} query/queries ' + 'refused), so the month cannot be priced.') + print('::error::This needs actions: read across the org. Grant the App ' + '"Actions" (read) and "Organization plan" (read), accept the updated ' + 'installation permissions, or supply BUDGET_TOKEN.') + sys.exit(1) + if not specs: + print(f'::error::found zero workflow runs across {len(private)} private ' + 'repo(s) this month, which is not a plausible measurement.') + print('::error::Refusing to report 0% -- a budget watchdog that reports no ' + 'usage because it cannot see usage is worse than one that is absent, ' + 'because it looks like good news.') + sys.exit(1) + print(f'::notice::pricing {len(specs)} run(s) from the Actions API') acc = {} with ThreadPoolExecutor(max_workers=8) as pool: