Repository navigation
pack install: release the lock only if this run owns it (flaky 'rmdir ... No such file or directory') #192
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,172 @@ | ||
| import { spawn, spawnSync } from 'node:child_process'; | ||
| import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; | ||
| import { tmpdir } from 'node:os'; | ||
| import { join } from 'node:path'; | ||
|
|
||
| /** | ||
| * The lock check-pack-install.sh takes around `npm pack`, run as real concurrent bash processes. | ||
| * | ||
| * It used to release with a bare `rmdir "$lock"` under `set -e`, which failed a run with "rmdir: | ||
| * failed to remove '/tmp/bugsee-pack-install.lock.d': No such file or directory" whenever the lock | ||
| * was already gone (the `pack install (0.82-0.86)` job failed that way on roughly one run in four), | ||
| * and removed another run's lock if one had taken it. Release now checks ownership and is | ||
| * idempotent. | ||
| */ | ||
| const lib = join(__dirname, '..', 'pack-lock.sh'); | ||
| const script = join(__dirname, '..', 'check-pack-install.sh'); | ||
|
|
||
| interface Result { | ||
| status: number | null; | ||
| stdout: string; | ||
| stderr: string; | ||
| } | ||
|
|
||
| /** Runs `body` in a bash that has sourced the library, with `set -euo pipefail` as the real script has. */ | ||
| function run(body: string, env: Record<string, string> = {}): Result { | ||
| const result = spawnSync('bash', ['-c', `set -euo pipefail\nsource "${lib}"\n${body}`], { | ||
| encoding: 'utf8', | ||
| env: { ...process.env, PACK_LOCK_POLL: '0.02', ...env }, | ||
| }); | ||
| return { status: result.status, stdout: result.stdout, stderr: result.stderr }; | ||
| } | ||
|
|
||
| function runAsync(body: string): Promise<Result> { | ||
| return new Promise((resolve) => { | ||
| const child = spawn('bash', ['-c', `set -euo pipefail\nsource "${lib}"\n${body}`], { | ||
| env: { ...process.env, PACK_LOCK_POLL: '0.01' }, | ||
| }); | ||
| let stdout = ''; | ||
| let stderr = ''; | ||
| child.stdout.on('data', (chunk) => (stdout += chunk)); | ||
| child.stderr.on('data', (chunk) => (stderr += chunk)); | ||
| child.on('close', (status) => resolve({ status, stdout, stderr })); | ||
| }); | ||
| } | ||
|
|
||
| describe('pack-lock.sh', () => { | ||
| let dir: string; | ||
| let lock: string; | ||
|
|
||
| beforeEach(() => { | ||
| dir = mkdtempSync(join(tmpdir(), 'pack-lock-')); | ||
| lock = join(dir, 'lock.d'); | ||
| }); | ||
| afterEach(() => rmSync(dir, { recursive: true, force: true })); | ||
|
|
||
| it('takes the lock, records its owner and removes it on release', () => { | ||
| const result = run(`pack_lock_acquire "${lock}" 5 | ||
| test -d "${lock}" | ||
| test -s "${lock}/owner" | ||
| pack_lock_release | ||
| test ! -e "${lock}"`); | ||
| expect(result).toMatchObject({ status: 0, stderr: '' }); | ||
| }); | ||
|
|
||
| it('serializes real concurrent processes: no two are in the critical section together', async () => { | ||
| const guard = join(dir, 'inside.d'); | ||
| const log = join(dir, 'log'); | ||
| writeFileSync(log, ''); | ||
| const worker = (id: number) => ` | ||
| pack_lock_acquire "${lock}" 60 | ||
| if ! mkdir "${guard}" 2>/dev/null; then echo "OVERLAP ${id}" >> "${log}"; fi | ||
| sleep 0.03 | ||
| rmdir "${guard}" 2>/dev/null || true | ||
| echo "done ${id}" >> "${log}" | ||
| pack_lock_release`; | ||
| const results = await Promise.all(Array.from({ length: 12 }, (_, id) => runAsync(worker(id)))); | ||
| expect(results.map((r) => r.status)).toEqual(results.map(() => 0)); | ||
| const lines = readFileSync(log, 'utf8').trim().split('\n'); | ||
| expect(lines.filter((line) => line.startsWith('OVERLAP'))).toEqual([]); | ||
| expect(lines.filter((line) => line.startsWith('done'))).toHaveLength(12); | ||
| expect(existsSync(lock)).toBe(false); | ||
| }); | ||
|
|
||
| it('does not fail the run when the lock has been removed while held', () => { | ||
| // What the old `rmdir "$lock"` died on, under set -e. | ||
| const result = run(`pack_lock_acquire "${lock}" 5 | ||
| rm -rf "${lock}" | ||
| pack_lock_release | ||
| echo survived`); | ||
| expect(result.status).toBe(0); | ||
| expect(result.stdout).toContain('survived'); | ||
| expect(result.stderr).toContain('was removed while it was held'); | ||
| }); | ||
|
|
||
| it('never removes a lock another run took after ours was lost', () => { | ||
| const result = run(`pack_lock_acquire "${lock}" 5 | ||
| rm -rf "${lock}" | ||
| mkdir "${lock}" | ||
| echo "someone-else" > "${lock}/owner" | ||
| pack_lock_release | ||
| test -d "${lock}" | ||
| test "$(cat "${lock}/owner")" = someone-else | ||
| echo intact`); | ||
| expect(result.status).toBe(0); | ||
| expect(result.stdout).toContain('intact'); | ||
| expect(result.stderr).toContain('held by another run now'); | ||
| }); | ||
|
|
||
| it('is idempotent: a trap and an explicit release can both run', () => { | ||
| const result = run(`trap 'pack_lock_release' EXIT | ||
| pack_lock_acquire "${lock}" 5 | ||
| pack_lock_release | ||
| pack_lock_release | ||
| echo ok`); | ||
| expect(result).toMatchObject({ status: 0, stderr: '' }); | ||
| expect(existsSync(lock)).toBe(false); | ||
| }); | ||
|
|
||
| it('does not release a lock it never took', () => { | ||
| // Another run holds it; this one only released, never acquired. | ||
| run(`mkdir "${lock}"; echo other > "${lock}/owner"`); | ||
| const result = run('pack_lock_release; echo ok'); | ||
| expect(result).toMatchObject({ status: 0, stderr: '' }); | ||
| expect(existsSync(lock)).toBe(true); | ||
| }); | ||
|
|
||
| it('releases from the EXIT trap when the run dies while holding it', () => { | ||
| const result = run(`trap 'pack_lock_release' EXIT | ||
| pack_lock_acquire "${lock}" 5 | ||
| exit 3`); | ||
| expect(result.status).toBe(3); | ||
| expect(existsSync(lock)).toBe(false); | ||
| }); | ||
|
|
||
| it('gives up after the wait, with the stale-lock message, and leaves the holder alone', () => { | ||
| run(`mkdir "${lock}"; echo holder > "${lock}/owner"`); | ||
| const result = run(`pack_lock_acquire "${lock}" 0.1 || echo "returned $?"`); | ||
| expect(result.stdout).toContain('returned 1'); | ||
| expect(result.stderr).toContain(`FAIL: could not take ${lock} (stale lock? remove it)`); | ||
| expect(readFileSync(join(lock, 'owner'), 'utf8').trim()).toBe('holder'); | ||
| }); | ||
|
|
||
| it('still tries once when the wait is shorter than one poll', () => { | ||
| expect(run(`pack_lock_acquire "${lock}" 0 && echo got`).stdout).toContain('got'); | ||
| }); | ||
|
|
||
| it('does not write its token anywhere but the lock', () => { | ||
| const result = run(`pack_lock_acquire "${lock}" 5 | ||
| echo "$PACK_LOCK_TOKEN" | ||
| pack_lock_release`); | ||
| expect(result.stdout.trim()).toMatch(/^\d+\.\d+$/); | ||
| }); | ||
| }); | ||
|
|
||
| describe('check-pack-install.sh takes the lock through the library', () => { | ||
| const source = readFileSync(script, 'utf8'); | ||
|
|
||
| it('releases only through pack_lock_release', () => { | ||
| // The bare rmdir is what failed the run when the lock was already gone. | ||
| expect(source).not.toMatch(/rmdir\s+"\$PACK_LOCK"/); | ||
| expect(source).toContain('pack_lock_release'); | ||
| }); | ||
|
|
||
| it('releases from the EXIT trap as well as explicitly', () => { | ||
| expect(source).toMatch(/trap 'pack_lock_release;[^']*' EXIT/); | ||
| expect(source.match(/^pack_lock_release$/gm)).toHaveLength(1); | ||
| }); | ||
|
|
||
| it('sources the library from the repo, not the working directory', () => { | ||
| expect(source).toContain('source "$REPO/scripts/pack-lock.sh"'); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| #!/usr/bin/env bash | ||
| # | ||
| # A mkdir lock (flock is not on macOS) that is only ever released by the process that took it. | ||
| # Sourced by check-pack-install.sh, whose parallel runs (scripts/ci-each-version.sh) share one lock | ||
| # around `npm pack`: prepack/postpack stage and unstage native-versions.json inside the package | ||
| # directory, so one run's postpack would delete the file under another's pack. | ||
| # | ||
| # pack_lock_acquire <lock dir> [seconds] waits (default 300 s); returns 1 on timeout | ||
| # pack_lock_release idempotent; safe to call from an EXIT trap AND explicitly | ||
| # | ||
| # Why release checks ownership. The original release was a bare `rmdir "$lock"` under `set -e`. It | ||
| # removed whatever sat at that path, and failed the whole run with "rmdir: failed to remove ...: No | ||
| # such file or directory" when the directory was already gone, although the pack it guarded had | ||
| # finished. A holder now writes a token into the lock, and release removes the lock only if the | ||
| # token is still its own. A lock that is gone or has been taken by someone else is reported, not | ||
| # removed and not fatal: serialization already failed by then, and failing the run as well, or | ||
| # deleting another run's lock, only spreads it. | ||
| # | ||
| # PACK_LOCK_POLL is the wait between attempts, in seconds (default 1; tests use less). | ||
|
|
||
| PACK_LOCK_DIR="" | ||
| PACK_LOCK_TOKEN="" | ||
|
|
||
| pack_lock_acquire() { | ||
| local lock="$1" seconds="${2:-300}" poll="${PACK_LOCK_POLL:-1}" waited=0 | ||
| # Attempts, not seconds, so a fractional poll still gives the caller's total wait. | ||
| local attempts | ||
| attempts=$(awk -v s="$seconds" -v p="$poll" 'BEGIN { n = int(s / p); print (n < 1 ? 1 : n) }') | ||
| while [[ "$waited" -lt "$attempts" ]]; do | ||
| if mkdir "$lock" 2>/dev/null; then | ||
| PACK_LOCK_DIR="$lock" | ||
| PACK_LOCK_TOKEN="$$.$RANDOM$RANDOM" | ||
| printf '%s\n' "$PACK_LOCK_TOKEN" > "$lock/owner" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Severity: P2 Medium Problem: A failed token write is swallowed, so the run reports success and the lock directory is leaked. Impact: The old Scenario: I reproduced it on this checkout with
Fix: Treat an unwritable token as a failed acquire and undo it: if ! printf '%s\n' "$PACK_LOCK_TOKEN" > "$lock/owner"; then
rmdir "$lock" 2>/dev/null || true
PACK_LOCK_DIR="" PACK_LOCK_TOKEN=""
echo "FAIL: could not record the owner in $lock" >&2
return 1
fiAdd a test for it. None of the 13 covers a failed write.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in the latest push, as suggested: a failed token write removes the (empty or partial) |
||
| return 0 | ||
| fi | ||
| waited=$((waited + 1)) | ||
| sleep "$poll" | ||
| done | ||
| echo "FAIL: could not take $lock (stale lock? remove it)" >&2 | ||
| return 1 | ||
| } | ||
|
|
||
| pack_lock_release() { | ||
| [[ -n "$PACK_LOCK_TOKEN" ]] || return 0 | ||
| local lock="$PACK_LOCK_DIR" token="$PACK_LOCK_TOKEN" owner="" | ||
| # Cleared first: a trap that fires while this runs must not release twice. | ||
| PACK_LOCK_DIR="" | ||
| PACK_LOCK_TOKEN="" | ||
| owner="$(cat "$lock/owner" 2>/dev/null || true)" | ||
| if [[ "$owner" == "$token" ]]; then | ||
| rm -f "$lock/owner" | ||
| rmdir "$lock" 2>/dev/null || true | ||
| return 0 | ||
| fi | ||
| if [[ -e "$lock" ]]; then | ||
| echo "warning: $lock is held by another run now; left it alone" >&2 | ||
| else | ||
| echo "warning: $lock was removed while it was held, so the pack was not serialized" >&2 | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Severity: P3 Low Problem: The only signal that serialization failed is a plain stderr line. Impact: The fix turns a loud failure into a silent pass. The overlapping Fix: Emit it as an annotation, for example
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed, this was the weak point: the fix turned a loud failure into a quiet pass and buried the only clue to the real cause. |
||
| fi | ||
| return 0 | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Severity: P3 Low
Problem: The EXIT trap is installed after
pack_lock_acquire. A SIGINT or SIGTERM betweenmkdirand this line leaves the lock behind, and the new token write widens that window slightly. The pre-existing code had the same gap.Impact: The leaked lock makes every other run wait 300 s and then fail. This matters mainly on a developer machine or a cancelled job on a reused host. Hosted runners are ephemeral.
Fix:
pack_lock_releaseis now idempotent and does nothing without a token. The trap can therefore be set before the acquire, with the same body:This is cheap. The pin test
trap 'pack_lock_release;[^']*' EXITstill matches.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed: the EXIT trap is now installed before
pack_lock_acquire, with the same body (the release does nothing without a token, so it is safe before one exists). A test pins that the trap comes before the acquire, and fails against the previous order.