From 26933b263745b249982880201e21be089e18e6f9 Mon Sep 17 00:00:00 2001 From: Pascal Garber Date: Thu, 10 Sep 2026 22:30:21 +0200 Subject: [PATCH 1/5] fix(release): kill a publish that outran its timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publish timeout rejected its promise and left `npm publish` running. Nothing else killed it. A live child holds its piped stdio open, those handles keep Node's event loop referenced, and the process then refuses to exit once `main()` is done โ€” the step goes silent with the sweep already finished. Measured against the exact structure this replaces, a 2 s timeout over a 600 s child: `main()` finished at 2.0 s, the process was still alive at 25 s. With the kill in place, driven end to end through the real script with a fake `npm` that sleeps 600 s and NPM_TIMEOUT_SEC=3: the run ends after 3 s of wall clock, names the package and the limit, and leaves zero surviving children. Be precise about what the leak costs TODAY, or the next reader deletes this as paranoia: it cannot currently hang a run, because a timeout is not in `isRetryablePublishError`, so every path that produces a live child also lands in `errors > 0` and exits through `process.exit(1)`, which does not wait for handles. This is a landmine, not a live defect โ€” and it is armed the moment someone makes timeouts retryable, which is the obvious next change, since killing the child is precisely what makes a timed-out publish safe to retry. stdout is drained for the same reason. npm writes its notices to stderr โ€” measured 167,468 bytes of stderr against 28 bytes of stdout for a 6000-file package โ€” so that pipe is near-empty today, but an unread pipe is a 64 KiB deadlock waiting for the release npm decides to say something on stdout. Claude-Session: https://claude.ai/code/session_01GtCcG3LLsz9voXAG9DVjhD --- .github/release-script/src/index.ts | 83 +++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/.github/release-script/src/index.ts b/.github/release-script/src/index.ts index 2ca102ea83..77fe92cc8b 100644 --- a/.github/release-script/src/index.ts +++ b/.github/release-script/src/index.ts @@ -76,6 +76,9 @@ const RETRY_MAX_MS = Math.max(RETRY_BASE_MS, getEnvInt("NPM_RETRY_MAX_MS", 60000 const API_TIMEOUT_MS = 10000; +/** How long a timed-out `npm publish` gets to honour SIGTERM before it is SIGKILLed. */ +const KILL_GRACE_MS = 10_000; + /** Run async tasks with a concurrency limit */ async function pMap(items: T[], fn: (item: T, index: number) => Promise, concurrency: number): Promise { const results: R[] = new Array(items.length); @@ -597,10 +600,6 @@ async function publishPackageOnce(pkg: Package, config: Config): Promise { console.log(`๐Ÿš€ Publishing ${pkg.name}@${pkg.version}...`); return new Promise((resolve, reject) => { - const timeoutId = setTimeout(() => { - reject(new Error(`Timeout after ${config.timeoutSec}s for ${pkg.name}`)); - }, config.timeoutSec * 1000); - // In OIDC mode the token must be ABSENT, not empty โ€” an unset secret still // exports `NODE_AUTH_TOKEN=""` into this process. const env = { ...process.env } as NodeJS.ProcessEnv; @@ -616,22 +615,84 @@ async function publishPackageOnce(pkg: Package, config: Config): Promise { }); let stderr = ""; + let settled = false; + let killTimer: NodeJS.Timeout | undefined; + + const clearTimers = (): void => { + clearTimeout(timeoutId); + if (killTimer) clearTimeout(killTimer); + }; + const settleResolve = (): void => { + if (settled) return; + settled = true; + resolve(); + }; + const settleReject = (err: Error): void => { + if (settled) return; + settled = true; + reject(err); + }; + + // The timeout TERMINATES the attempt; it used to only reject. + // + // Rejecting left `npm publish` running and nothing else ever killed it. A live + // child holds its piped stdio open, those handles keep Node's event loop + // referenced, and the process then does not exit when `main()` is done โ€” the + // step goes silent with the sweep already finished. Measured against the exact + // structure this replaces, a 2 s timeout over a 600 s child: `main()` finished + // at 2.0 s, the process was still alive at 25 s. + // + // BE PRECISE ABOUT WHAT THAT COSTS TODAY, or the next reader deletes this as + // paranoia: right now the leak cannot actually hang a run, because a timeout is + // not in `isRetryablePublishError`, so every path that produces a live child + // also lands in `errors > 0` and exits through `process.exit(1)`, which does not + // wait for handles. This is a landmine, not a live defect โ€” and it is armed the + // moment someone makes timeouts retryable, which is the obvious next change, + // since killing the child is exactly what makes a timed-out publish safe to + // retry (npm answers the duplicate with EPUBLISHCONFLICT, which this script + // already resolves as already-published). + // + // `shell: true` puts a shell between us and npm, so SIGTERM reaches the shell + // first; the SIGKILL escalation is what reclaims a child wedged in a write or an + // unanswered socket read. The escalation timer is `unref`'d so it can never + // itself be the handle that keeps the loop alive. + const timeoutId = setTimeout(() => { + proc.kill("SIGTERM"); + killTimer = setTimeout(() => proc.kill("SIGKILL"), KILL_GRACE_MS); + killTimer.unref(); + settleReject( + new Error( + `Timeout after ${config.timeoutSec}s for ${pkg.name}@${pkg.version} โ€” publish killed. ` + + "Raise NPM_TIMEOUT_SEC if the package is genuinely slow; otherwise the registry stalled.", + ), + ); + }, config.timeoutSec * 1000); proc.stderr.on("data", (data) => { stderr += data.toString(); }); + // npm writes its notices to STDERR โ€” measured 167,468 bytes of stderr against + // 28 bytes of stdout for a 6000-file package โ€” so this pipe is near-empty + // today. It is drained anyway: an unread pipe is a 64 KiB deadlock waiting for + // the release npm decides to say something on stdout, and that deadlock would + // present as exactly the silent stall this timeout now has to clean up. + proc.stdout.on("data", () => {}); + proc.on("error", (err) => { - clearTimeout(timeoutId); - reject(new Error(`Spawn error for ${pkg.name}: ${err.message}`)); + clearTimers(); + settleReject(new Error(`Spawn error for ${pkg.name}: ${err.message}`)); }); proc.on("exit", (code) => { - clearTimeout(timeoutId); + clearTimers(); + // The timeout already decided this attempt; what arrives now is the corpse + // of the process it killed, and its exit code says nothing about the publish. + if (settled) return; if (code === 0) { console.log(`โœ… Published ${pkg.name}@${pkg.version}`); - resolve(); + settleResolve(); return; } @@ -641,17 +702,17 @@ async function publishPackageOnce(pkg: Package, config: Config): Promise { stderr.includes("Cannot publish over existing version") ) { console.log(`โš ๏ธ ${pkg.name}@${pkg.version} already published`); - resolve(); + settleResolve(); return; } if (stderr.includes("404 Not Found") && stderr.includes("organization")) { const orgName = pkg.name.split("/")[0]; - reject(new Error(`Organization '${orgName}' not found. Create it at https://www.npmjs.com/org/create`)); + settleReject(new Error(`Organization '${orgName}' not found. Create it at https://www.npmjs.com/org/create`)); return; } - reject(new Error(`Failed to publish ${pkg.name}: ${stderr.trim() || `exit code ${code}`}`)); + settleReject(new Error(`Failed to publish ${pkg.name}: ${stderr.trim() || `exit code ${code}`}`)); }); }); } From 9a80d8db8b508e1f69c41b801628b708b60dd93d Mon Sep 17 00:00:00 2001 From: Pascal Garber Date: Thu, 10 Sep 2026 22:30:48 +0200 Subject: [PATCH 2/5] feat(release): bound the sweep and show it moving MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hung sweep and a working one were indistinguishable from outside, and that cost the 4.8.0 release three hours. Measured on run 34509558730, attempt 1: the job published 276 packages between 17:42:24Z and 18:31:49Z, 10.78 s each, largest gap between any two consecutive log lines 13.33 s. It was never stuck. What was stuck was the run-level `updatedAt` the GitHub API reports, which does not advance while a job streams logs โ€” so the run looked frozen at 17:41:39Z, was cancelled at 38 %, and the half-published set stayed unresolvable until the restart finished at 19:54Z. So each batch line now carries elapsed and ETA. A reader who wants to know whether a sweep is alive should not have to consult a field that cannot answer. And the sweep gets a wall-clock budget, NPM_DEADLINE_MIN, default 300 min. The job's `timeout-minutes: 360` was the only other bound, and a job killed by it reports "the runner has exceeded the maximum execution time" โ€” naming the runner, not the package the sweep was on. The default sits under it on purpose so that this message is the one a human reads. For scale: 716 packages at the measured cadence is 2.14 h. Claude-Session: https://claude.ai/code/session_01GtCcG3LLsz9voXAG9DVjhD --- .github/release-script/src/index.ts | 44 ++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/release-script/src/index.ts b/.github/release-script/src/index.ts index 77fe92cc8b..206968a091 100644 --- a/.github/release-script/src/index.ts +++ b/.github/release-script/src/index.ts @@ -79,6 +79,18 @@ const API_TIMEOUT_MS = 10000; /** How long a timed-out `npm publish` gets to honour SIGTERM before it is SIGKILLed. */ const KILL_GRACE_MS = 10_000; +/** + * Wall-clock budget for the whole sweep, after which the run FAILS BY NAME. + * + * The job's `timeout-minutes` is the only other bound, and a job killed by it + * says "The job running on runner โ€ฆ has exceeded the maximum execution time" โ€” + * which names the runner, not the package the sweep was on. The default sits + * under release.yml's 360 minutes on purpose, so this message is the one a human + * reads. Measured for scale: the 4.8.0 sweep published 716 packages at 10.78 s + * each, 2.14 h end to end. + */ +const DEADLINE_MIN = Math.max(0, getEnvInt("NPM_DEADLINE_MIN", 300)); + /** Run async tasks with a concurrency limit */ async function pMap(items: T[], fn: (item: T, index: number) => Promise, concurrency: number): Promise { const results: R[] = new Array(items.length); @@ -100,6 +112,14 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +function formatDuration(seconds: number): string { + const s = Math.max(0, Math.round(seconds)); + if (s < 60) return `${s}s`; + const m = Math.floor(s / 60); + if (m < 60) return `${m}m${String(s % 60).padStart(2, "0")}s`; + return `${Math.floor(m / 60)}h${String(m % 60).padStart(2, "0")}m`; +} + function calcBackoffMs(attempt: number, baseMs: number, maxMs: number): number { const exp = Math.min(maxMs, baseMs * 2 ** attempt); // Add jitter (+/-20%) to avoid thundering herd @@ -929,8 +949,22 @@ async function publishPendingPackages( let processed = 0; let errors = 0; + const startedAt = Date.now(); + const deadlineAt = DEADLINE_MIN > 0 ? startedAt + DEADLINE_MIN * 60_000 : undefined; for (let i = 0; i < needsPublish.length; i += BATCH_SIZE) { + // Checked between batches rather than only at the end: a sweep that will not + // finish should say so while it still has a runner to say it on. This is fatal + // even under `--continue-on-error`, which governs a failing PACKAGE, not a run + // that has stopped fitting in its job. + if (deadlineAt !== undefined && Date.now() > deadlineAt) { + throw new Error( + `sweep deadline of ${DEADLINE_MIN} min reached with ${needsPublish.length - i} of ` + + `${needsPublish.length} package(s) unpublished (published: ${processed}, errors: ${errors}). ` + + "Raise NPM_DEADLINE_MIN, or find why the registry got slow.", + ); + } + const batch = needsPublish.slice(i, i + BATCH_SIZE); const batchNum = Math.floor(i / BATCH_SIZE) + 1; const totalBatches = Math.ceil(needsPublish.length / BATCH_SIZE); @@ -976,9 +1010,17 @@ async function publishPendingPackages( } } + // Elapsed and ETA on every batch line, because the run-level `updatedAt` GitHub + // exposes does NOT advance while a job streams logs โ€” from the API a sweep that + // is working looks exactly like one that is wedged. That reading is what got the + // 4.8.0 release cancelled at 38 % while it was publishing normally. const progress = (((i + BATCH_SIZE) / needsPublish.length) * 100).toFixed(1); + const elapsedS = (Date.now() - startedAt) / 1000; + const done = i + batch.length; + const etaS = done > 0 ? (elapsedS / done) * (needsPublish.length - done) : 0; console.log( - `โœ… Batch ${batchNum}/${totalBatches} done (${progress}%) - Processed: ${processed}, Errors: ${errors}\n`, + `โœ… Batch ${batchNum}/${totalBatches} done (${progress}%) - Processed: ${processed}, Errors: ${errors}` + + ` - elapsed ${formatDuration(elapsedS)}, ETA ${formatDuration(etaS)}\n`, ); // Delay between batches (not after the last one) From 09b0f4f1d2c43d479fec5ab9c07d1845adaf561e Mon Sep 17 00:00:00 2001 From: Pascal Garber Date: Thu, 10 Sep 2026 22:31:25 +0200 Subject: [PATCH 3/5] feat(release): verify the published set resolves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A release could report success over a set that does not install, and on 2026-09-10 it did. ts-for-gir tagged v4.8.0, run 34509558730 swept the tree alphabetically at 10.78 s per package, and every package it published declared `^4.8.0` for its siblings. For the 2.14 h that takes, the registry served packages demanding versions that were not there yet: npm error code ETARGET npm error notarget No matching version found for @girs/gjs@^4.8.0. Measured from the run's own log, the window is not theoretical: `@girs/gdkpixbuf-2.0` went out at 18:23:33Z and `@girs/gjs` at 18:30:00Z, so for six and a half minutes every consumer of the first was broken by the absence of the second. The job was green throughout and would have ended green. Phase 3 now asks the question a consumer asks: does every dependency range of every package in the set resolve against the registry? WHY RANGE-BY-RANGE, NOT `npm install --dry-run`. The install is what a human reaches for and it is how this was found, but as a gate it checks almost nothing: it exercises only the dependency closure of whichever root it is pointed at. Measured on this tree โ€” 716 packages, 7637 edges โ€” a root like `@gjsify/cli` reaches a double-digit handful; the incident would have been caught only because `@girs/gjs` happens to sit in that closure. It would also need a root package from another repository, published after this one, which points a dependency the wrong way down the release train. Asking the registry per range covers every edge, names both ends of a failure, and is cheaper: the 7637 edges carry 280 distinct dependency names, so it costs 280 packument reads and runs in seconds. REGISTRY LAG IS MEASURED, NOT GUESSED. npm is read-after-write inconsistent and not monotonic about it. From the incident, comparing what attempt 1 published against what attempt 2's status check could see two minutes later: @girs/glib-2.0 published 18:31:07Z -> INVISIBLE at 18:33:09Z (>122 s) @girs/glibunix-2.0 published 18:31:17Z -> INVISIBLE at 18:33:09Z (>112 s) @girs/glibwin32-2.0 published 18:31:28Z -> visible @girs/gly-2 published 18:31:49Z -> visible Two packages published EARLIER were still hidden while two published later were already served, so "wait until the last publish is N seconds old" is not a rule the registry honours. The grace is therefore a bounded per-name poll, and it is granted only to a name THIS run published at a version that satisfies the range. Everything else fails at once โ€” otherwise the one failure this gate exists to catch, a sibling that is never coming, becomes a ten-minute wait followed by the same red, and the check reads as flaky. NOT VACUOUS. It asserts packages > 0 and edges > 0 before believing itself, and prints both counts on every run. That guard is not hypothetical: pointed at an empty root, the script previously printed "Total: 0" and "completed successfully" and exited 0. The range matcher is a deliberately small subset โ€” `*`, exact, `^`, `~` โ€” which is all this tree contains (measured: the 7637 entries use exactly `^4.8.0` and `typescript: "*"`), and it REFUSES any other spelling rather than guessing. Dependency-free on purpose: sdk-types.yml runs this publisher on its dry-run path without `npm ci`, so a runtime import would crash a path nobody exercises before a release. Both it and the lag/defect split run as always-on self-tests, same reasoning as the retry classifier above. Negative control, driving the real script against a fake registry holding exactly the incident state โ€” tree wants ^4.8.0, registry serves only 4.7.0: the published set does not resolve: 1 of 2 dependency range(s) across 1 package(s) cannot be satisfied. @girs/fixture-a@4.7.0 needs @girs/fixture-gjs@^4.8.0 (dependencies); registry has: 4.7.0 exit 1. The same rig with the range satisfied exits 0. Against the real registry through a read-only proxy, the full tree passes: 7637 ranges over 716 packages. Claude-Session: https://claude.ai/code/session_01GtCcG3LLsz9voXAG9DVjhD --- .github/release-script/src/index.ts | 480 +++++++++++++++++++++++++++- 1 file changed, 476 insertions(+), 4 deletions(-) diff --git a/.github/release-script/src/index.ts b/.github/release-script/src/index.ts index 206968a091..4c1aca4055 100644 --- a/.github/release-script/src/index.ts +++ b/.github/release-script/src/index.ts @@ -37,6 +37,8 @@ interface Package { name: string; version: string; rootFolder: string; + /** The dependency maps a CONSUMER resolves, kept so Phase 3 can check every range. */ + dependencies: Partial>>; } interface PackageStatus { @@ -508,10 +510,19 @@ async function parsePackageJson(packageFile: string): Promise { throw new Error(`Invalid package.json at ${packageFile}: missing name or version`); } + const dependencies: Package["dependencies"] = {}; + for (const field of CONSUMER_DEPENDENCY_FIELDS) { + const value = data[field]; + if (value && typeof value === "object") { + dependencies[field] = value as Record; + } + } + return { name: data.name, version: data.version, rootFolder: dirname(packageFile), + dependencies, }; } @@ -925,7 +936,7 @@ async function publishPendingPackages( packages: Package[], statuses: Map, config: Config, -): Promise<{ alreadyPublished: number; processed: number; errors: number }> { +): Promise<{ alreadyPublished: number; processed: number; errors: number; publishedNow: Map }> { // Split into already-published and needs-publish const needsPublish: { pkg: Package; isUpdate: boolean }[] = []; let alreadyPublished = 0; @@ -941,8 +952,12 @@ async function publishPendingPackages( console.log(`๐Ÿ“Š ${alreadyPublished} already published, ${needsPublish.length} to publish\n`); + // Every name this run put on the registry, at the version it put there. Phase 3 + // needs it to tell a propagation lag apart from a genuinely missing sibling. + const publishedNow = new Map(); + if (needsPublish.length === 0) { - return { alreadyPublished, processed: 0, errors: 0 }; + return { alreadyPublished, processed: 0, errors: 0, publishedNow }; } console.log(`๐Ÿš€ Phase 2: Publishing ${needsPublish.length} packages (batch size: ${BATCH_SIZE})...\n`); @@ -982,6 +997,7 @@ async function publishPendingPackages( } await publishPackageWithRetry(pkg, config); + publishedNow.set(pkg.name, pkg.version); return { result: isUpdate ? "updated" : "created", pkg }; } catch (error) { const message = error instanceof Error ? error.message : "Unknown error"; @@ -1029,12 +1045,459 @@ async function publishPendingPackages( } } - return { alreadyPublished, processed, errors }; + return { alreadyPublished, processed, errors, publishedNow }; +} + +// --------------------------------------------------------------------------- +// Phase 3: the set that was published must actually install +// --------------------------------------------------------------------------- +// +// THE HOLE THIS FILLS. Phase 2 reports on publishes, one package at a time, and +// nothing ever asked the question a consumer asks: does the set RESOLVE? On +// 2026-09-10 it did not. The 4.8.0 sweep publishes in directory order, ~10.8 s +// per package, 2.14 h end to end (measured on run 34509558730), and every +// package it publishes declares `^4.8.0` for its siblings. So for two hours the +// registry held packages demanding a version of `@girs/gjs` that was not there +// yet, and `npm install @gjsify/cli` answered: +// +// npm error code ETARGET +// npm error notarget No matching version found for @girs/gjs@^4.8.0. +// +// The run itself was green about this the whole time, and would have ended green. +// +// WHY RANGE-BY-RANGE AND NOT `npm install --dry-run`. The install is what a human +// reaches for, and it was how this was found, but as a gate it checks almost +// nothing: it only ever exercises the dependency CLOSURE of whatever root package +// it is pointed at. Measured on this tree โ€” 716 packages, 7637 dependency edges โ€” +// a root like `@gjsify/cli` pulls in a double-digit handful of them; the other +// ~690 would go unverified, and the incident would have been caught only because +// `@girs/gjs` happens to sit in that one closure. It also needs a root package +// from ANOTHER repository, published AFTER this one, which is a dependency +// pointing the wrong way down the release train. +// +// Asking the registry per range has none of that. It covers every edge, it names +// the exact dependent, and it is cheaper: the 7637 edges carry only 281 distinct +// dependency names (280 `@girs/*` siblings plus `typescript`), so the whole +// question costs 281 packument reads. +const CONSUMER_DEPENDENCY_FIELDS = ["dependencies", "peerDependencies", "optionalDependencies"] as const; + +/** + * How long a MISSING-BECAUSE-JUST-PUBLISHED version may stay missing. + * + * npm is read-after-write inconsistent and not even monotonic about it. Measured + * on the incident, comparing what attempt 1 published against what attempt 2's + * status check could see 2 minutes later: + * + * @girs/glib-2.0 published 18:31:07Z -> INVISIBLE at 18:33:09Z (>122 s) + * @girs/glibunix-2.0 published 18:31:17Z -> INVISIBLE at 18:33:09Z (>112 s) + * @girs/glibwin32-2.0 published 18:31:28Z -> visible + * @girs/gly-2 published 18:31:49Z -> visible + * + * Two packages published EARLIER were still hidden while two published later were + * already served, so "wait until the last publish is N seconds old" is not a rule + * the registry honours. The budget is therefore a per-name poll with a ceiling, + * set well above the 122 s actually observed โ€” and it is a CEILING: when it runs + * out the run fails. + */ +const VERIFY_LAG_BUDGET_SEC = Math.max(0, getEnvInt("NPM_VERIFY_LAG_BUDGET_SEC", 600)); +const VERIFY_LAG_POLL_MS = Math.max(1000, getEnvInt("NPM_VERIFY_LAG_POLL_MS", 15_000)); + +interface SemVer { + major: number; + minor: number; + patch: number; + prerelease: string; +} + +class UnsupportedRangeError extends Error {} + +function parseVersion(raw: string): SemVer | null { + const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/.exec(raw.trim()); + if (!m) return null; + return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), prerelease: m[4] ?? "" }; +} + +function comparePrerelease(a: string, b: string): number { + // Per semver: a version WITHOUT a prerelease outranks one with. + if (a === b) return 0; + if (a === "") return 1; + if (b === "") return -1; + const as = a.split("."); + const bs = b.split("."); + for (let i = 0; i < Math.max(as.length, bs.length); i++) { + const x = as[i]; + const y = bs[i]; + if (x === undefined) return -1; + if (y === undefined) return 1; + const xn = /^\d+$/.test(x); + const yn = /^\d+$/.test(y); + if (xn && yn) { + if (Number(x) !== Number(y)) return Number(x) < Number(y) ? -1 : 1; + } else if (xn !== yn) { + return xn ? -1 : 1; + } else if (x !== y) { + return x < y ? -1 : 1; + } + } + return 0; +} + +function compareVersions(a: SemVer, b: SemVer): number { + if (a.major !== b.major) return a.major < b.major ? -1 : 1; + if (a.minor !== b.minor) return a.minor < b.minor ? -1 : 1; + if (a.patch !== b.patch) return a.patch < b.patch ? -1 : 1; + return comparePrerelease(a.prerelease, b.prerelease); +} + +/** + * Does `version` satisfy `range`? + * + * A deliberately SMALL subset โ€” `*`, an exact version, `^` and `~` โ€” because that + * is what this tree contains: measured across all 716 packages, the 7637 + * dependency entries use exactly two spellings, `^4.8.0` and `typescript: "*"`. + * A dependency-free implementation is worth more here than semver-the-package, + * for a reason that is not taste: `sdk-types.yml` runs this publisher on its + * dry-run path WITHOUT having run `npm ci` in the release-script directory, so a + * runtime import would be a crash on a path nobody exercises before a release. + * + * Anything outside the subset THROWS rather than returning false or true. A + * matcher that silently mishandles a spelling it does not know is a gate that + * reports on a question it never asked, and this one exists precisely to stop + * that. The generator changing its range spelling should break the release + * loudly, once, and be a one-line addition here. + */ +function satisfiesRange(version: string, range: string): boolean { + const v = parseVersion(version); + if (!v) return false; + + const r = range.trim(); + if (r === "" || r === "*" || r === "x" || r === "X") { + // `*` still does not match a prerelease unless one is asked for. + return v.prerelease === ""; + } + + const operator = r.startsWith("^") ? "^" : r.startsWith("~") ? "~" : "="; + const base = parseVersion(operator === "=" ? r.replace(/^=/, "") : r.slice(1)); + if (!base) { + throw new UnsupportedRangeError(`unsupported dependency range: ${JSON.stringify(range)}`); + } + + // A prerelease candidate only counts when the range names a prerelease on the + // same major.minor.patch โ€” npm's rule, and the one that keeps a stray + // `4.9.0-beta.1` from satisfying `^4.8.0`. + if (v.prerelease !== "") { + const sameTuple = v.major === base.major && v.minor === base.minor && v.patch === base.patch; + if (base.prerelease === "" || !sameTuple) return false; + } + + if (operator === "=") return compareVersions(v, base) === 0; + if (compareVersions(v, base) < 0) return false; + + if (operator === "~") { + return v.major === base.major && v.minor === base.minor; + } + // Caret, including the 0.x and 0.0.x narrowings. + if (base.major > 0) return v.major === base.major; + if (base.minor > 0) return v.major === 0 && v.minor === base.minor; + return v.major === 0 && v.minor === 0 && v.patch === base.patch; +} + +/** One `dependent -> dependency@range` edge, kept whole so a failure can name both ends. */ +interface DependencyEdge { + from: string; + fromVersion: string; + dep: string; + range: string; + field: string; +} + +/** + * Why an edge is unsatisfied โ€” the whole point of the check, so it is a pure + * function the self-test can drive rather than a branch buried in the polling loop. + * + * `lag` is the ONLY verdict that earns a retry, and it is earned narrowly: this run + * published that exact name at a version that does satisfy the range, so the only + * thing missing is the registry catching up with a write we watched succeed. + * Everything else is `defect` and fails at once. Without that split, the obvious + * "just retry a bit, the registry is slow" turns the one failure this gate exists + * to catch โ€” a sibling that is NOT coming, because nothing is going to publish it โ€” + * into a ten-minute wait followed by the same red, and teaches everyone reading the + * log that the check is flaky. + */ +function classifyUnsatisfiedEdge( + edge: DependencyEdge, + registryVersions: string[], + publishedNow: Map, +): "lag" | "defect" { + const justPublished = publishedNow.get(edge.dep); + if (justPublished === undefined) return "defect"; + if (registryVersions.includes(justPublished)) return "defect"; + return satisfiesRange(justPublished, edge.range) ? "lag" : "defect"; +} + +/** Ask the registry which versions of `name` it currently serves. `null` = no such package. */ +async function fetchRegistryVersions(name: string, registry: string): Promise { + return await withRetry( + async () => { + const response = await fetch(getApiUrl(registry, name), { + headers: { Accept: "application/json", "User-Agent": "ts-for-gir-release-script/1.0.0" }, + signal: AbortSignal.timeout(API_TIMEOUT_MS), + }); + if (response.status === 404) return null; + if (!response.ok) throw new HttpStatusError(response.status, `Registry responded with ${response.status} for ${name}`); + const data = (await response.json()) as { versions?: Record }; + return Object.keys(data.versions ?? {}); + }, + { + label: `verify:${name}`, + maxRetries: MAX_RETRIES_STATUS, + baseDelayMs: RETRY_BASE_MS, + maxDelayMs: RETRY_MAX_MS, + shouldRetry: (err) => isHttpStatusError(err) && isRetryableHttpStatus(err.status), + }, + ); +} + +function collectDependencyEdges(packages: Package[]): DependencyEdge[] { + const edges: DependencyEdge[] = []; + for (const pkg of packages) { + for (const field of CONSUMER_DEPENDENCY_FIELDS) { + // devDependencies are deliberately absent: npm does not install a + // published package's devDependencies, so a range there cannot break a + // consumer's install. Measured, the only one in this tree is + // `typescript: "*"` on all 716 packages โ€” 716 questions nobody asks. + for (const [dep, range] of Object.entries(pkg.dependencies[field] ?? {})) { + edges.push({ from: pkg.name, fromVersion: pkg.version, dep, range, field }); + } + } + } + return edges; +} + +/** + * Phase 3 proper: every dependency range of every package in the set must resolve + * against the registry, or this run is red. + */ +async function verifyPublishedSetResolves( + packages: Package[], + publishedNow: Map, + config: Config, +): Promise { + console.log("\n๐Ÿ”Ž Phase 3: Verifying the published set resolves..."); + + const edges = collectDependencyEdges(packages); + const depNames = [...new Set(edges.map((e) => e.dep))].sort(); + + // ANTI-VACUITY. A check that iterates an empty list is green and has proved + // nothing, and this one is downstream of a directory scan and a JSON parse โ€” + // both of which can legitimately produce nothing and neither of which would + // complain. So the positive facts are asserted, then PRINTED on every run, so + // that "it passed" is never separable from "and here is what it looked at". + if (packages.length === 0) { + throw new Error("Phase 3 has no packages to verify โ€” the sweep found nothing, which cannot be a successful release"); + } + if (edges.length === 0) { + throw new Error( + `Phase 3 found 0 dependency ranges across ${packages.length} package(s). Every @girs package depends on ` + + "its siblings, so zero edges means the manifests were not read, not that there is nothing to check.", + ); + } + + console.log(` ${packages.length} package(s), ${edges.length} dependency range(s), ${depNames.length} distinct dependencies`); + + // Every distinct range spelling is probed BEFORE the registry is asked anything, + // so an unknown one fails here โ€” naming a package that uses it โ€” instead of + // deep inside the comparison loop where it would read like a resolution failure. + for (const range of new Set(edges.map((e) => e.range))) { + const example = edges.find((e) => e.range === range) as DependencyEdge; + try { + satisfiesRange("0.0.0", range); + } catch (error) { + if (!(error instanceof UnsupportedRangeError)) throw error; + throw new UnsupportedRangeError( + `${example.from}@${example.fromVersion} declares ${example.dep}@${range} (${example.field}) โ€” this verifier ` + + "cannot evaluate that range spelling. Teach satisfiesRange about it; do not let the release skip the check.", + ); + } + } + + const versions = new Map(); + let fetched = 0; + await pMap( + depNames, + async (name) => { + versions.set(name, await fetchRegistryVersions(name, config.registry)); + fetched++; + if (fetched % 50 === 0 || fetched === depNames.length) { + console.log(` Resolved ${fetched}/${depNames.length} dependencies...`); + } + }, + STATUS_CONCURRENCY, + ); + + const unsatisfied = (): DependencyEdge[] => + edges.filter((e) => !(versions.get(e.dep) ?? []).some((v) => satisfiesRange(v, e.range))); + + let broken = unsatisfied(); + + // Give the registry the measured propagation lag, but ONLY for names this run + // published at a version that satisfies the range. Anything else is already the + // answer. + const lagDeadline = Date.now() + VERIFY_LAG_BUDGET_SEC * 1000; + while (broken.length > 0) { + const lagging = new Set( + broken.filter((e) => classifyUnsatisfiedEdge(e, versions.get(e.dep) ?? [], publishedNow) === "lag").map((e) => e.dep), + ); + const defects = broken.filter((e) => classifyUnsatisfiedEdge(e, versions.get(e.dep) ?? [], publishedNow) === "defect"); + if (defects.length > 0 || lagging.size === 0) break; + if (Date.now() >= lagDeadline) { + console.log(` โฑ๏ธ lag budget of ${VERIFY_LAG_BUDGET_SEC}s exhausted with ${lagging.size} name(s) still not served`); + break; + } + console.log( + ` โณ ${lagging.size} just-published name(s) not served yet (${[...lagging].slice(0, 5).join(", ")}` + + `${lagging.size > 5 ? ", โ€ฆ" : ""}) โ€” re-asking in ${Math.round(VERIFY_LAG_POLL_MS / 1000)}s`, + ); + await sleep(VERIFY_LAG_POLL_MS); + await pMap( + [...lagging], + async (name) => { + versions.set(name, await fetchRegistryVersions(name, config.registry)); + }, + STATUS_CONCURRENCY, + ); + broken = unsatisfied(); + } + + if (broken.length > 0) { + const shown = broken.slice(0, 15); + const lines = shown.map((e) => { + const have = versions.get(e.dep); + const latest = + have === null || have === undefined ? "NO SUCH PACKAGE" : have.length === 0 ? "no versions" : have.slice(-5).join(", "); + return ` ${e.from}@${e.fromVersion} needs ${e.dep}@${e.range} (${e.field}); registry has: ${latest}`; + }); + throw new Error( + `the published set does not resolve: ${broken.length} of ${edges.length} dependency range(s) ` + + `across ${new Set(broken.map((e) => e.from)).size} package(s) cannot be satisfied.\n` + + `${lines.join("\n")}${broken.length > shown.length ? `\n โ€ฆ and ${broken.length - shown.length} more` : ""}\n` + + "A consumer installing this release gets npm ETARGET.", + ); + } + + console.log(`โœ… Phase 3: ${edges.length} dependency range(s) over ${packages.length} package(s) all resolve on ${config.registry}\n`); +} + +/** + * Always-on self-test of the range matcher and the lag/defect split. + * + * Same reasoning as the retry classifier above: there is no test runner in this + * repository, and a test nothing runs is worse than none. These two decide + * whether a release is allowed to be green, so they run every time the script + * does and they fail the process rather than warn. + * + * Vector "the incident" is the 4.8.0 state, verbatim in shape: the tree demands + * `^4.8.0` and the registry is still serving 4.7.0. + */ +const RANGE_VECTORS: { name: string; version: string; range: string; satisfied: boolean }[] = [ + { name: "the incident: ^4.8.0 is not satisfied by 4.7.0", version: "4.7.0", range: "^4.8.0", satisfied: false }, + { name: "^4.8.0 is satisfied by 4.8.0", version: "4.8.0", range: "^4.8.0", satisfied: true }, + { name: "^4.8.0 is satisfied by a later minor", version: "4.9.1", range: "^4.8.0", satisfied: true }, + { name: "^4.8.0 is not satisfied by an earlier patch on the same minor", version: "4.8.0", range: "^4.8.1", satisfied: false }, + { name: "^4.8.0 is not satisfied across a major", version: "5.0.0", range: "^4.8.0", satisfied: false }, + { name: "* is satisfied by anything released", version: "5.9.3", range: "*", satisfied: true }, + { name: "* is not satisfied by a prerelease", version: "5.9.3-beta.1", range: "*", satisfied: false }, + { name: "a prerelease does not sneak into a caret range", version: "4.9.0-beta.1", range: "^4.8.0", satisfied: false }, + { name: "a prerelease counts when the range names one on the same tuple", version: "4.9.0-beta.2", range: "^4.9.0-beta.1", satisfied: true }, + { name: "~ pins the minor", version: "4.9.0", range: "~4.8.0", satisfied: false }, + { name: "~ allows a patch", version: "4.8.7", range: "~4.8.0", satisfied: true }, + { name: "an exact range is exact", version: "4.8.1", range: "4.8.0", satisfied: false }, + { name: "caret on 0.x pins the minor", version: "0.3.0", range: "^0.2.9", satisfied: false }, + { name: "caret on 0.0.x pins the patch", version: "0.0.4", range: "^0.0.3", satisfied: false }, + { name: "a garbage version satisfies nothing", version: "not-a-version", range: "^4.8.0", satisfied: false }, +]; + +/** Ranges this matcher must REFUSE rather than guess at. */ +const UNSUPPORTED_RANGES = [">=4.8.0 <5.0.0", "^4.8.0 || ^5.0.0", "workspace:^", "npm:@girs/gjs@^4.8.0", "latest"]; + +const CLASSIFY_VECTORS: { + name: string; + edge: DependencyEdge; + registryVersions: string[]; + publishedNow: [string, string][]; + verdict: "lag" | "defect"; +}[] = [ + { + name: "we published it and the registry has not caught up โ€” lag", + edge: { from: "@girs/adw-1", fromVersion: "4.8.0", dep: "@girs/gjs", range: "^4.8.0", field: "dependencies" }, + registryVersions: ["4.7.0"], + publishedNow: [["@girs/gjs", "4.8.0"]], + verdict: "lag", + }, + { + name: "nobody published it this run โ€” defect, and it must NOT be waited on", + edge: { from: "@girs/adw-1", fromVersion: "4.8.0", dep: "@girs/gjs", range: "^4.8.0", field: "dependencies" }, + registryVersions: ["4.7.0"], + publishedNow: [], + verdict: "defect", + }, + { + name: "what we published does not satisfy the range either โ€” defect", + edge: { from: "@girs/adw-1", fromVersion: "4.8.0", dep: "@girs/gjs", range: "^4.8.0", field: "dependencies" }, + registryVersions: ["4.7.0"], + publishedNow: [["@girs/gjs", "4.7.1"]], + verdict: "defect", + }, + { + name: "the registry already serves what we published, so waiting cannot help โ€” defect", + edge: { from: "@girs/adw-1", fromVersion: "4.8.0", dep: "@girs/gjs", range: "^4.9.0", field: "dependencies" }, + registryVersions: ["4.7.0", "4.8.0"], + publishedNow: [["@girs/gjs", "4.8.0"]], + verdict: "defect", + }, +]; + +function selfTestResolution(): void { + const failures: string[] = []; + + for (const v of RANGE_VECTORS) { + let got: boolean | string; + try { + got = satisfiesRange(v.version, v.range); + } catch (error) { + got = `threw ${error instanceof Error ? error.message : String(error)}`; + } + if (got !== v.satisfied) failures.push(`${v.name}: expected ${v.satisfied}, got ${got}`); + } + + for (const range of UNSUPPORTED_RANGES) { + let threw = false; + try { + satisfiesRange("4.8.0", range); + } catch (error) { + threw = error instanceof UnsupportedRangeError; + } + if (!threw) failures.push(`unsupported range ${JSON.stringify(range)} was answered instead of refused`); + } + + for (const v of CLASSIFY_VECTORS) { + const got = classifyUnsatisfiedEdge(v.edge, v.registryVersions, new Map(v.publishedNow)); + if (got !== v.verdict) failures.push(`${v.name}: expected ${v.verdict}, got ${got}`); + } + + if (failures.length > 0) { + throw new Error(`resolution self-test FAILED:\n ${failures.join("\n ")}`); + } + console.log( + `๐Ÿงช resolution self-test green โ€” ${RANGE_VECTORS.length} range vector(s), ` + + `${UNSUPPORTED_RANGES.length} refusal(s), ${CLASSIFY_VECTORS.length} lag/defect vector(s)`, + ); } async function main(): Promise { try { selfTestClassifier(); + selfTestResolution(); const config = createConfig(); assertRunningInCi(config); @@ -1061,7 +1524,7 @@ async function main(): Promise { const statuses = await checkAllStatuses(packages, config.registry); // Phase 2: Publish only what's needed - const { alreadyPublished, processed, errors } = await publishPendingPackages(packages, statuses, config); + const { alreadyPublished, processed, errors, publishedNow } = await publishPendingPackages(packages, statuses, config); // Final summary console.log("๐Ÿ“Š Final Summary:"); @@ -1084,6 +1547,15 @@ async function main(): Promise { ); } + // Phase 3 runs only once the sweep is otherwise clean, and it is the LAST word: + // "every publish succeeded" and "the result installs" are different claims, and + // only the second one is what a release is for. + if (config.dryRun) { + console.log("\nโญ๏ธ Phase 3 skipped: --dry-run published nothing, so there is no set to verify"); + } else { + await verifyPublishedSetResolves(packages, publishedNow, config); + } + console.log(`โœ… ${config.dryRun ? "DRY RUN" : "Processing"} completed successfully`); } catch (error) { console.error(`โŒ Fatal error: ${error instanceof Error ? error.message : error}`); From 64959453894aa9500afe93894cfdfa96d328c801 Mon Sep 17 00:00:00 2001 From: Pascal Garber Date: Thu, 10 Sep 2026 22:33:14 +0200 Subject: [PATCH 4/5] fix(release): let a dependency-free set pass, provably MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The anti-vacuity guard added with Phase 3 would have taken down every SDK channel publish. `sdk-types.yml` runs this same publisher with `--root sdk` over a single self-contained channel bundle, and those declare no dependencies at all โ€” measured on @girs/sdk-gnome-50@4.8.0: 458 export subpaths, not one dependency field. `edges === 0 -> throw` is correct for the 716-package namespace tree and wrong for that one. Zero edges is still exactly what a broken collector looks like, so it is not simply waved through. It is CONFIRMED, against the manifests on disk, by a different path than the one that produced the claim: `assertNoDependenciesOnDisk` re-reads every package.json and counts entries across all four dependency fields, devDependencies included, because the question there is not "what would a consumer install" but "did we actually read these files". A claim of emptiness checked by the code that produced it confirms nothing. Both halves measured. The SDK shape โ€” one bundle, no dependencies โ€” passes: "1 package(s) declare no dependencies at all โ€” confirmed against the manifests on disk". With the collector sabotaged to return [] while the fixtures carry dependencies, the same path fails: Phase 3 collected 0 dependency ranges, but the manifests on disk disagree: @girs/fixture-a has 1 dependencies; @girs/fixture-gjs has 1 dependencies. The edge collector is broken โ€” a release must not pass by having looked at nothing. Claude-Session: https://claude.ai/code/session_01GtCcG3LLsz9voXAG9DVjhD --- .github/release-script/src/index.ts | 52 +++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/release-script/src/index.ts b/.github/release-script/src/index.ts index 4c1aca4055..d3362189b9 100644 --- a/.github/release-script/src/index.ts +++ b/.github/release-script/src/index.ts @@ -1274,6 +1274,38 @@ function collectDependencyEdges(packages: Package[]): DependencyEdge[] { return edges; } +/** + * Confirm "this set has no dependencies" against the raw manifests. + * + * Deliberately does NOT reuse `Package.dependencies` or `collectDependencyEdges`: + * a claim of emptiness checked by the code that produced it confirms nothing. This + * re-reads each package.json and counts entries across all four dependency fields, + * devDependencies included, because the question here is not "what would a consumer + * install" but "did we actually read these files". + */ +async function assertNoDependenciesOnDisk(packages: Package[]): Promise { + const fields = ["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"]; + const offenders: string[] = []; + + for (const pkg of packages) { + const raw = JSON.parse(await readFile(join(pkg.rootFolder, "package.json"), "utf-8")) as Record; + for (const field of fields) { + const value = raw[field]; + if (value && typeof value === "object" && Object.keys(value).length > 0) { + offenders.push(`${pkg.name} has ${Object.keys(value).length} ${field}`); + } + } + } + + if (offenders.length > 0) { + throw new Error( + `Phase 3 collected 0 dependency ranges, but the manifests on disk disagree: ${offenders.slice(0, 5).join("; ")}` + + `${offenders.length > 5 ? ` (and ${offenders.length - 5} more)` : ""}. The edge collector is broken โ€” ` + + "a release must not pass by having looked at nothing.", + ); + } +} + /** * Phase 3 proper: every dependency range of every package in the set must resolve * against the registry, or this run is red. @@ -1297,10 +1329,24 @@ async function verifyPublishedSetResolves( throw new Error("Phase 3 has no packages to verify โ€” the sweep found nothing, which cannot be a successful release"); } if (edges.length === 0) { - throw new Error( - `Phase 3 found 0 dependency ranges across ${packages.length} package(s). Every @girs package depends on ` + - "its siblings, so zero edges means the manifests were not read, not that there is nothing to check.", + // Zero edges is a LEGITIMATE shape here, and nearly was a self-inflicted + // outage: `sdk-types.yml` runs this same publisher with `--root sdk` over a + // single self-contained channel bundle, and those declare no dependencies at + // all โ€” measured on @girs/sdk-gnome-50@4.8.0, which has 458 export subpaths + // and not one dependency field. A flat `edges === 0 -> throw` would have + // turned every SDK channel publish red. + // + // But zero edges is ALSO what a broken collector looks like, and that is the + // vacuity this phase exists to refuse. So the claim is confirmed against the + // manifests on disk, by a different path than the one that produced it: if any + // manifest actually carries a dependency entry, the collector is wrong and the + // release stops. + await assertNoDependenciesOnDisk(packages); + console.log( + `โœ… Phase 3: ${packages.length} package(s) declare no dependencies at all โ€” confirmed against the manifests ` + + "on disk. Nothing to resolve.\n", ); + return; } console.log(` ${packages.length} package(s), ${edges.length} dependency range(s), ${depNames.length} distinct dependencies`); From 3125857d357e08e887d8fe727b604b0ae6a3dc11 Mon Sep 17 00:00:00 2001 From: Pascal Garber Date: Thu, 10 Sep 2026 22:33:35 +0200 Subject: [PATCH 5/5] chore(release): give the sweep a deadline under the job NPM_DEADLINE_MIN sits at 300, under this job's `timeout-minutes: 360`, so the publisher gets to name its own failure. A job killed by `timeout-minutes` reports that the runner exceeded its maximum execution time, which names the runner and not the package the sweep was on. The Publish step is renamed to say that it also verifies. A step called "Publish" that is green means the publish calls returned; it does not mean the result installs, and on 2026-09-10 those two came apart for two hours. Claude-Session: https://claude.ai/code/session_01GtCcG3LLsz9voXAG9DVjhD --- .github/workflows/release.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe27d45285..3b9a476732 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,12 @@ env: NPM_RETRY_BASE_MS: 5000 NPM_RETRY_MAX_MS: 300000 NPM_TIMEOUT_SEC: 600 + # The sweep's own wall-clock budget, deliberately UNDER `timeout-minutes: 360` + # below. A job killed by `timeout-minutes` reports that the runner exceeded its + # maximum execution time โ€” it names the runner, not the package the sweep was + # on. Crossing this first means the failure is the publisher's to explain. + # Measured for scale: the 4.8.0 sweep did 716 packages at 10.78 s each, 2.14 h. + NPM_DEADLINE_MIN: 300 jobs: release: @@ -79,7 +85,12 @@ jobs: # Without one: OIDC, via `id-token: write` above and the `registry-url` # npmrc setup-node wrote. An unset secret makes this an EMPTY string, which # both npm and the script read as no token at all. - - name: Publish + # Publishes, and then VERIFIES: the script's Phase 3 asks the registry whether + # every dependency range of every package it just published resolves. Without + # it this step reports on publish calls only, which is how the 4.8.0 release + # (run 34509558730) was green for two hours over a set that answered + # `npm error code ETARGET / No matching version found for @girs/gjs@^4.8.0`. + - name: Publish and verify the set resolves run: node --experimental-specifier-resolution=node --experimental-strip-types --experimental-transform-types --no-warnings ./.github/release-script/src/index.ts --continue-on-error env: NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }}