From 68ab2ca4aff6830e160f4ab3437e67630bc71510 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Mon, 16 Nov 2026 17:01:57 +0500 Subject: [PATCH] pack install: release the lock only if this run owns it `pack install (0.82-0.86)` fails on roughly one run in four with rmdir: failed to remove '/tmp/bugsee-pack-install.lock.d': No such file or directory after the pack it guarded had finished. The lock (added in #163, first failure the next day) was released with a bare `rmdir "$lock"` under `set -e`, which removes whatever sits at that path and fails the whole run when nothing does. The release is now ownership-checked and idempotent (scripts/pack-lock.sh): a holder writes a token into the lock, and release removes the lock only if the token is still its own. A lock that has gone, or that another run now holds, is reported on stderr and left alone instead of failing this run or deleting the other run's lock. The EXIT trap and the explicit release are the same function and can both run. scripts/__tests__/pack-lock.test.ts runs the library as real concurrent bash processes: mutual exclusion over 12 contenders, a lock removed while held (the old rmdir died here), a lock taken by someone else, idempotent release, release from the EXIT trap, the timeout message. It also pins check-pack-install.sh to the library. What is NOT established: what removes the lock directory in CI. It did not reproduce in 25 cold-cache iterations of the real lock section on bash 5.2 / node 22.18, so this makes the run survive that event and stops it spreading, but does not stop the event. Human guided, AI implemented --- scripts/__tests__/pack-lock.test.ts | 172 ++++++++++++++++++++++++++++ scripts/check-pack-install.sh | 20 ++-- scripts/pack-lock.sh | 61 ++++++++++ 3 files changed, 242 insertions(+), 11 deletions(-) create mode 100644 scripts/__tests__/pack-lock.test.ts create mode 100755 scripts/pack-lock.sh diff --git a/scripts/__tests__/pack-lock.test.ts b/scripts/__tests__/pack-lock.test.ts new file mode 100644 index 00000000..af03a7c3 --- /dev/null +++ b/scripts/__tests__/pack-lock.test.ts @@ -0,0 +1,172 @@ +import { spawn, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** + * The lock check-pack-install.sh takes around `npm pack`, run as real concurrent bash processes. + * + * It used to release with a bare `rmdir "$lock"` under `set -e`, which failed a run with "rmdir: + * failed to remove '/tmp/bugsee-pack-install.lock.d': No such file or directory" whenever the lock + * was already gone (the `pack install (0.82-0.86)` job failed that way on roughly one run in four), + * and removed another run's lock if one had taken it. Release now checks ownership and is + * idempotent. + */ +const lib = join(__dirname, '..', 'pack-lock.sh'); +const script = join(__dirname, '..', 'check-pack-install.sh'); + +interface Result { + status: number | null; + stdout: string; + stderr: string; +} + +/** Runs `body` in a bash that has sourced the library, with `set -euo pipefail` as the real script has. */ +function run(body: string, env: Record = {}): Result { + const result = spawnSync('bash', ['-c', `set -euo pipefail\nsource "${lib}"\n${body}`], { + encoding: 'utf8', + env: { ...process.env, PACK_LOCK_POLL: '0.02', ...env }, + }); + return { status: result.status, stdout: result.stdout, stderr: result.stderr }; +} + +function runAsync(body: string): Promise { + return new Promise((resolve) => { + const child = spawn('bash', ['-c', `set -euo pipefail\nsource "${lib}"\n${body}`], { + env: { ...process.env, PACK_LOCK_POLL: '0.01' }, + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => (stdout += chunk)); + child.stderr.on('data', (chunk) => (stderr += chunk)); + child.on('close', (status) => resolve({ status, stdout, stderr })); + }); +} + +describe('pack-lock.sh', () => { + let dir: string; + let lock: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'pack-lock-')); + lock = join(dir, 'lock.d'); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + it('takes the lock, records its owner and removes it on release', () => { + const result = run(`pack_lock_acquire "${lock}" 5 +test -d "${lock}" +test -s "${lock}/owner" +pack_lock_release +test ! -e "${lock}"`); + expect(result).toMatchObject({ status: 0, stderr: '' }); + }); + + it('serializes real concurrent processes: no two are in the critical section together', async () => { + const guard = join(dir, 'inside.d'); + const log = join(dir, 'log'); + writeFileSync(log, ''); + const worker = (id: number) => ` + pack_lock_acquire "${lock}" 60 + if ! mkdir "${guard}" 2>/dev/null; then echo "OVERLAP ${id}" >> "${log}"; fi + sleep 0.03 + rmdir "${guard}" 2>/dev/null || true + echo "done ${id}" >> "${log}" + pack_lock_release`; + const results = await Promise.all(Array.from({ length: 12 }, (_, id) => runAsync(worker(id)))); + expect(results.map((r) => r.status)).toEqual(results.map(() => 0)); + const lines = readFileSync(log, 'utf8').trim().split('\n'); + expect(lines.filter((line) => line.startsWith('OVERLAP'))).toEqual([]); + expect(lines.filter((line) => line.startsWith('done'))).toHaveLength(12); + expect(existsSync(lock)).toBe(false); + }); + + it('does not fail the run when the lock has been removed while held', () => { + // What the old `rmdir "$lock"` died on, under set -e. + const result = run(`pack_lock_acquire "${lock}" 5 +rm -rf "${lock}" +pack_lock_release +echo survived`); + expect(result.status).toBe(0); + expect(result.stdout).toContain('survived'); + expect(result.stderr).toContain('was removed while it was held'); + }); + + it('never removes a lock another run took after ours was lost', () => { + const result = run(`pack_lock_acquire "${lock}" 5 +rm -rf "${lock}" +mkdir "${lock}" +echo "someone-else" > "${lock}/owner" +pack_lock_release +test -d "${lock}" +test "$(cat "${lock}/owner")" = someone-else +echo intact`); + expect(result.status).toBe(0); + expect(result.stdout).toContain('intact'); + expect(result.stderr).toContain('held by another run now'); + }); + + it('is idempotent: a trap and an explicit release can both run', () => { + const result = run(`trap 'pack_lock_release' EXIT +pack_lock_acquire "${lock}" 5 +pack_lock_release +pack_lock_release +echo ok`); + expect(result).toMatchObject({ status: 0, stderr: '' }); + expect(existsSync(lock)).toBe(false); + }); + + it('does not release a lock it never took', () => { + // Another run holds it; this one only released, never acquired. + run(`mkdir "${lock}"; echo other > "${lock}/owner"`); + const result = run('pack_lock_release; echo ok'); + expect(result).toMatchObject({ status: 0, stderr: '' }); + expect(existsSync(lock)).toBe(true); + }); + + it('releases from the EXIT trap when the run dies while holding it', () => { + const result = run(`trap 'pack_lock_release' EXIT +pack_lock_acquire "${lock}" 5 +exit 3`); + expect(result.status).toBe(3); + expect(existsSync(lock)).toBe(false); + }); + + it('gives up after the wait, with the stale-lock message, and leaves the holder alone', () => { + run(`mkdir "${lock}"; echo holder > "${lock}/owner"`); + const result = run(`pack_lock_acquire "${lock}" 0.1 || echo "returned $?"`); + expect(result.stdout).toContain('returned 1'); + expect(result.stderr).toContain(`FAIL: could not take ${lock} (stale lock? remove it)`); + expect(readFileSync(join(lock, 'owner'), 'utf8').trim()).toBe('holder'); + }); + + it('still tries once when the wait is shorter than one poll', () => { + expect(run(`pack_lock_acquire "${lock}" 0 && echo got`).stdout).toContain('got'); + }); + + it('does not write its token anywhere but the lock', () => { + const result = run(`pack_lock_acquire "${lock}" 5 +echo "$PACK_LOCK_TOKEN" +pack_lock_release`); + expect(result.stdout.trim()).toMatch(/^\d+\.\d+$/); + }); +}); + +describe('check-pack-install.sh takes the lock through the library', () => { + const source = readFileSync(script, 'utf8'); + + it('releases only through pack_lock_release', () => { + // The bare rmdir is what failed the run when the lock was already gone. + expect(source).not.toMatch(/rmdir\s+"\$PACK_LOCK"/); + expect(source).toContain('pack_lock_release'); + }); + + it('releases from the EXIT trap as well as explicitly', () => { + expect(source).toMatch(/trap 'pack_lock_release;[^']*' EXIT/); + expect(source.match(/^pack_lock_release$/gm)).toHaveLength(1); + }); + + it('sources the library from the repo, not the working directory', () => { + expect(source).toContain('source "$REPO/scripts/pack-lock.sh"'); + }); +}); diff --git a/scripts/check-pack-install.sh b/scripts/check-pack-install.sh index a69a0bbd..2a7649cb 100755 --- a/scripts/check-pack-install.sh +++ b/scripts/check-pack-install.sh @@ -49,16 +49,14 @@ mkdir -p "$WORK/tarballs" # prepack/postpack stage and unstage native-versions.json inside the package # directory, which parallel runs of this script (scripts/ci-each-version.sh) # share: one run's postpack would delete the file under another's pack. A -# mkdir lock (flock is not on macOS) serializes just this step. -PACK_LOCK="${TMPDIR:-/tmp}/bugsee-pack-install.lock.d" -PACK_LOCK_HELD=0 -for _ in $(seq 1 300); do - if mkdir "$PACK_LOCK" 2>/dev/null; then PACK_LOCK_HELD=1; break; fi - sleep 1 -done -[[ "$PACK_LOCK_HELD" -eq 1 ]] || { echo "FAIL: could not take $PACK_LOCK (stale lock? remove it)" >&2; exit 1; } -# A kill between mkdir and rmdir must not leave the lock behind. -trap 'rmdir "$PACK_LOCK" 2>/dev/null; [[ -n "${PACK_INSTALL_DIR:-}" ]] || rm -rf "$WORK"' EXIT +# mkdir lock (scripts/pack-lock.sh; flock is not on macOS) serializes just this +# step, and is released only by the run that took it. +# shellcheck source=scripts/pack-lock.sh +source "$REPO/scripts/pack-lock.sh" +pack_lock_acquire "${TMPDIR:-/tmp}/bugsee-pack-install.lock.d" 300 || exit 1 +# A kill between taking the lock and releasing it must not leave it behind. The +# release is idempotent, so this and the explicit one below can both run. +trap 'pack_lock_release; [[ -n "${PACK_INSTALL_DIR:-}" ]] || rm -rf "$WORK"' EXIT PACK_RC=0 for pkg in react-native react-native-feedback; do (cd "$REPO/packages/$pkg" && npm pack --silent --pack-destination "$WORK/tarballs" >/dev/null) || { PACK_RC=1; break; } @@ -67,7 +65,7 @@ done # runs. Parallel first-time `npx --yes` installs into ~/.npm/_npx corrupt each # other (TAR_ENTRY_ERROR ENOENT); once warm, later runs only read it. [[ "$PACK_RC" -eq 0 ]] && { npx --yes "@react-native-community/cli@${RN_CLI_VERSION:-20.2.0}" --version >/dev/null || { echo "FAIL: npx warm-up" >&2; PACK_RC=1; }; } -rmdir "$PACK_LOCK" +pack_lock_release if [[ -n "${PACK_INSTALL_DIR:-}" ]]; then trap - EXIT; else trap 'rm -rf "$WORK"' EXIT; fi [[ "$PACK_RC" -eq 0 ]] || { echo "FAIL: npm pack or npx warm-up (see above)" >&2; exit 1; } CORE_TGZ="$(ls "$WORK"/tarballs/bugsee-react-native-[0-9]*.tgz)" diff --git a/scripts/pack-lock.sh b/scripts/pack-lock.sh new file mode 100755 index 00000000..cd3e6305 --- /dev/null +++ b/scripts/pack-lock.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# A mkdir lock (flock is not on macOS) that is only ever released by the process that took it. +# Sourced by check-pack-install.sh, whose parallel runs (scripts/ci-each-version.sh) share one lock +# around `npm pack`: prepack/postpack stage and unstage native-versions.json inside the package +# directory, so one run's postpack would delete the file under another's pack. +# +# pack_lock_acquire [seconds] waits (default 300 s); returns 1 on timeout +# pack_lock_release idempotent; safe to call from an EXIT trap AND explicitly +# +# Why release checks ownership. The original release was a bare `rmdir "$lock"` under `set -e`. It +# removed whatever sat at that path, and failed the whole run with "rmdir: failed to remove ...: No +# such file or directory" when the directory was already gone, although the pack it guarded had +# finished. A holder now writes a token into the lock, and release removes the lock only if the +# token is still its own. A lock that is gone or has been taken by someone else is reported, not +# removed and not fatal: serialization already failed by then, and failing the run as well, or +# deleting another run's lock, only spreads it. +# +# PACK_LOCK_POLL is the wait between attempts, in seconds (default 1; tests use less). + +PACK_LOCK_DIR="" +PACK_LOCK_TOKEN="" + +pack_lock_acquire() { + local lock="$1" seconds="${2:-300}" poll="${PACK_LOCK_POLL:-1}" waited=0 + # Attempts, not seconds, so a fractional poll still gives the caller's total wait. + local attempts + attempts=$(awk -v s="$seconds" -v p="$poll" 'BEGIN { n = int(s / p); print (n < 1 ? 1 : n) }') + while [[ "$waited" -lt "$attempts" ]]; do + if mkdir "$lock" 2>/dev/null; then + PACK_LOCK_DIR="$lock" + PACK_LOCK_TOKEN="$$.$RANDOM$RANDOM" + printf '%s\n' "$PACK_LOCK_TOKEN" > "$lock/owner" + return 0 + fi + waited=$((waited + 1)) + sleep "$poll" + done + echo "FAIL: could not take $lock (stale lock? remove it)" >&2 + return 1 +} + +pack_lock_release() { + [[ -n "$PACK_LOCK_TOKEN" ]] || return 0 + local lock="$PACK_LOCK_DIR" token="$PACK_LOCK_TOKEN" owner="" + # Cleared first: a trap that fires while this runs must not release twice. + PACK_LOCK_DIR="" + PACK_LOCK_TOKEN="" + owner="$(cat "$lock/owner" 2>/dev/null || true)" + if [[ "$owner" == "$token" ]]; then + rm -f "$lock/owner" + rmdir "$lock" 2>/dev/null || true + return 0 + fi + if [[ -e "$lock" ]]; then + echo "warning: $lock is held by another run now; left it alone" >&2 + else + echo "warning: $lock was removed while it was held, so the pack was not serialized" >&2 + fi + return 0 +}