Skip to content

fix(budget): refuse to report a percentage the watchdog cannot substantiate #133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ExtraToast merged 1 commit into from
Aug 30, 2026
Merged

fix(budget): refuse to report a percentage the watchdog cannot substantiate #133

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 43 additions & 28 deletions .github/workflows/actions-budget-watchdog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,6 @@ name: 'Actions Budget Watchdog'
'required': false
'type': 'string'
'default': ''
'app-has-billing-permission':
'description': >
Set true only once the App has actually been granted "Organization plan" (read).
create-github-app-token fails outright when asked for a permission the installation
does not hold, so requesting it optimistically takes the whole watchdog down.
'required': false
'type': 'boolean'
'default': false
'secrets':
'BUDGET_APP_ID':
'description': 'GitHub App id able to read org billing and repo metadata.'
Expand All @@ -67,18 +59,18 @@ name: 'Actions Budget Watchdog'
'BUDGET_APP_ID': '${{ secrets.BUDGET_APP_ID }}'
'BUDGET_APP_PRIVATE_KEY': '${{ secrets.BUDGET_APP_PRIVATE_KEY }}'
'steps':
# Two minting steps, because create-github-app-token FAILS THE JOB when
# asked for a permission the installation does not hold -- it does not
# degrade. Requesting billing optimistically therefore takes the whole
# watchdog down at token minting, before it can even report why, which is
# strictly worse than reading no billing at all.
# create-github-app-token FAILS THE JOB when asked for a permission the
# installation does not hold -- it does not degrade. So the capable mint
# is allowed to fail instead: continue-on-error means an App without these
# grants costs a skipped step rather than the whole watchdog, and an App
# that has them is used automatically with nothing to configure.
#
# Billing sits behind its own App permission: "Organization plan"
# (read-only). Until the App is granted it, leave
# app-has-billing-permission false and supply BUDGET_TOKEN instead.
- 'name': 'Mint organization token (billing permission granted)'
'id': 'app-token-billing'
'if': "${{ env.BUDGET_APP_ID != '' && env.BUDGET_APP_PRIVATE_KEY != '' && inputs.app-has-billing-permission }}"
# organization-plan: read -> the exact billing figure
# actions: read -> pricing runs when billing is unreachable
- 'name': 'Mint organization token (best effort, capable)'
'id': 'app-token-capable'
'if': "${{ env.BUDGET_APP_ID != '' && env.BUDGET_APP_PRIVATE_KEY != '' }}"
'continue-on-error': true
'uses': 'actions/create-github-app-token@v3'
'with':
'app-id': '${{ env.BUDGET_APP_ID }}'
Expand All @@ -87,10 +79,13 @@ name: 'Actions Budget Watchdog'
'permission-metadata': 'read'
'permission-issues': 'write'
'permission-organization-plan': 'read'
'permission-actions': 'read'

- 'name': 'Mint organization token'
# Always mintable: these two permissions the App certainly holds. Used
# only to file the alert issue when the capable mint was refused.
- 'name': 'Mint organization token (minimal)'
'id': 'app-token'
'if': "${{ env.BUDGET_APP_ID != '' && env.BUDGET_APP_PRIVATE_KEY != '' && !inputs.app-has-billing-permission }}"
'if': "${{ env.BUDGET_APP_ID != '' && env.BUDGET_APP_PRIVATE_KEY != '' }}"
'uses': 'actions/create-github-app-token@v3'
'with':
'app-id': '${{ env.BUDGET_APP_ID }}'
Expand All @@ -101,7 +96,7 @@ name: 'Actions Budget Watchdog'

- 'name': 'Check the Actions minute budget'
'env':
'GH_TOKEN': "${{ steps.app-token-billing.outputs.token || secrets.BUDGET_TOKEN || steps.app-token.outputs.token || github.token }}"
'GH_TOKEN': "${{ steps.app-token-capable.outputs.token || secrets.BUDGET_TOKEN || steps.app-token.outputs.token || github.token }}"
'OWNER': '${{ github.repository_owner }}'
'ALLOWANCE': '${{ inputs.allowance-minutes }}'
'WARN_PCT': '${{ inputs.warn-percent }}'
Expand Down Expand Up @@ -209,19 +204,18 @@ name: 'Actions Budget Watchdog'
return out

def run_ids(repo):
ids = []
ids, failures = [], 0
for window in month_windows():
proc = subprocess.run(
['gh', 'api', '--paginate',
f'/repos/{owner}/{repo}/actions/runs?per_page=100&created={window}',
'--jq', '.workflow_runs[].id'],
capture_output=True, text=True, check=False)
if proc.returncode != 0:
print(f'::warning::could not list runs for {repo} in {window}; '
'this month is undercounted')
failures += 1
continue
ids.extend(x for x in proc.stdout.split() if x.strip())
return ids
return ids, failures

def price_run(job_spec):
repo, run_id = job_spec
Expand Down Expand Up @@ -264,10 +258,31 @@ name: 'Actions Budget Watchdog'
# trips a little early rather than a little late. But do not quote
# this figure as fact. Fix the billing permission and the exact
# number comes back.
specs = []
specs, list_failures = [], 0
for repo in sorted(private):
ids = run_ids(repo)
ids, failures = run_ids(repo)
list_failures += failures
specs.extend((repo, rid) for rid in ids)

# A partial read is not a budget figure. Reporting one as a
# percentage is how a watchdog produces a reassuring number it has
# no basis for, so refuse instead -- and refuse loudest when the
# answer would have been a comfortable zero.
if list_failures:
print(f'::error::could not list workflow runs ({list_failures} query/queries '
'refused), so the month cannot be priced.')
print('::error::This needs actions: read across the org. Grant the App '
'"Actions" (read) and "Organization plan" (read), accept the updated '
'installation permissions, or supply BUDGET_TOKEN.')
sys.exit(1)
if not specs:
print(f'::error::found zero workflow runs across {len(private)} private '
'repo(s) this month, which is not a plausible measurement.')
print('::error::Refusing to report 0% -- a budget watchdog that reports no '
'usage because it cannot see usage is worse than one that is absent, '
'because it looks like good news.')
sys.exit(1)

print(f'::notice::pricing {len(specs)} run(s) from the Actions API')
acc = {}
with ThreadPoolExecutor(max_workers=8) as pool:
Expand Down
Loading