diff --git a/.gitignore b/.gitignore index b57a3177f..e245e1b6f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,8 @@ examples/express/node_modules yarn-error.log tsconfig.tsbuildinfo prelude/sea-bootstrap.bundle.js + +# Built by test-99-#295/main.js at test time, removed again afterwards +test/test-99-#295/lib +test/test-99-#295/reallib/inner.js +test/test-99-#295/linkinfo.json diff --git a/.prettierignore b/.prettierignore index 1a9082079..1ac800497 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,3 +3,7 @@ lib-es5/ prelude/sea-bootstrap.bundle.js # Symlink needed for test test/test-99-#108/lib/log.js +# Built by test-99-#295/main.js at test time, removed again afterwards +test/test-99-#295/lib +test/test-99-#295/reallib/inner.js +test/test-99-#295/linkinfo.json diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cdacdda65..d69a2710c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -178,7 +178,7 @@ Each file is stored with one or more store types: ### Runtime Bootstrap -`prelude/bootstrap.js` (1970 lines) executes before user code. It: +`prelude/bootstrap.js` (2174 lines) executes before user code. It: 1. **Sets up entrypoint** — Reads `DEFAULT_ENTRYPOINT` from injected parameters, sets `process.argv[1]` 2. **Initializes VFS** — Builds in-memory lookup from `VIRTUAL_FILESYSTEM` dictionary with optional path compression via `DICT` @@ -394,14 +394,16 @@ ASCII version: The `SEAProvider` (in `prelude/sea-vfs-setup.js`) implements lazy loading from a single archive blob: -| Method | Behavior | -| -------------------------- | ----------------------------------------------------------------------------- | -| `readFileSync(path)` | Resolve symlinks, `subarray()` from archive via `offsets` map, cache in `Map` | -| `statSync(path)` | Return metadata from manifest `stats` | -| `internalModuleStat(path)` | Fast path for module resolution: returns 0 (file), 1 (dir), or -2 (not found) | -| `readdirSync(path)` | Return directory entries from manifest `directories` | -| `existsSync(path)` | O(1) check against manifest `stats` | -| `readlinkSync(path)` | Return symlink target from manifest, fall back to `super.readlinkSync()` | +| Method | Behavior | +| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `readFileSync(path)` | Resolve symlinks, `subarray()` from archive via `offsets` map, cache in `Map` | +| `statSync(path)` | Return metadata from manifest `stats` | +| `internalModuleStat(path)` | Fast path for module resolution: returns 0 (file), 1 (dir), or -2 (not found) | +| `readdirSync(path)` | Return manifest directory entries directly; with `withFileTypes` builds shared `Dirent`s, marking links from `manifest.symlinks` | +| `existsSync(path)` | O(1) check against manifest `stats` | +| `readlinkSync(path)` | Return the symlink target from `manifest.symlinks`, resolving a symlinked parent first; `EINVAL` for a path that is present but not a link, `ENOENT` otherwise. Reached through `fs.readlinkSync` since #296 — `sea-vfs-setup.js` re-points the patch here, because the VFS's own one answers through `realpathSync` (yao-pkg/pkg#299) | +| `lstatSync(path)` | Describe the link itself: the target's stat with link semantics and `S_IFLNK` type bits. Reached through `fs.lstatSync` since #296, for the same reason as `readlinkSync` above | +| `realpathSync(path)` | Follow the symlink chain, then return the path if the manifest has it. Load-bearing: without it every archive path raises `ENOENT`, which also breaks `fs.readlinkSync`. The VFS mounts under a POSIX `/snapshot` prefix, so on Windows the result is converted back to `C:\snapshot\...` before it leaves `fs` (yao-pkg/pkg#305) | The entire archive is loaded once via `sea.getRawAsset('__pkg_archive__')` which returns a zero-copy `ArrayBuffer` reference to the executable's memory-mapped region. Individual files are extracted via `Buffer.subarray(offset, offset + length)` using the manifest's `offsets` map, then cached in a `Map` on first access. String results (when `encoding` is specified) are derived directly from the archive view; Buffer results are copied to prevent callers from corrupting the shared archive memory. @@ -467,7 +469,7 @@ This keeps the VFS setup, shared patches, worker interception, and diagnostics a ## Shared Runtime Code -`prelude/bootstrap-shared.js` (~438 lines) contains runtime patches used by both bootstraps: +`prelude/bootstrap-shared.js` (~969 lines) contains runtime patches used by both bootstraps: ### Injection Mechanisms @@ -502,6 +504,32 @@ This keeps the VFS setup, shared patches, worker interception, and diagnostics a - Set `PKG_EXECPATH` env var so child processes can detect they were spawned from a packaged app - Replace references to `node`, `process.argv[0]`, or the entrypoint with `process.execPath` (the actual executable) +**`makeSymlinkResolver(symlinks, sep)`** — Builds the symlink resolver used by **both** modes: the traditional bootstrap (`findVirtualFileSystemKeyAndFollowLinks`) and the SEA provider (`SEAProvider._resolveSymlink`). It returns a function mapping a virtual path onto what its symlinks point at, walking parent components the way POSIX does — so a link at `node_modules/@scope/lib` also resolves `node_modules/@scope/lib/package.json` (#295). + +An empty `symlinks` record yields the identity function, so a symlink-free binary pays nothing. Otherwise the resolver precomputes which path depths can host a symlink key and memoises each key's fully resolved target — the memo is keyed by manifest entry, not by the caller's path, so it stays bounded by the manifest however many paths are looked up. A manifest cycle raises `ELOOP` rather than hanging startup, with libuv's platform errno (`-4067` on Windows, `-40` elsewhere) and the caller's syscall name. + +Matching is **longest-prefix-wins**, not first-match-in-insertion-order: when both `/lib` and `/lib/sub` are keys, the deeper one describes the whole chain while the shallower one would strand the walk on a path the archive has no entry for. That differs from POSIX's leftmost-first walk, and the two agree only because every target the walker records is already a full realpath (`toNormalizedRealPath`), so no component of a target can itself be a key. + +### Symlink semantics in packaged binaries + +Both bootstraps resolve symlinks on parent path components, so `require`, `fs.readFile` and friends reach files under a linked directory. Where they differ: + +| | Traditional | Enhanced SEA | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `readdir({ withFileTypes: true })` | Reports a snapshot symlink as a link (`isSymbolicLink()` true, `isDirectory()`/`isFile()` false), matching real `readdir` | Same — the provider builds `Dirent`s from `manifest.symlinks`, using the shared `Dirent` in `bootstrap-shared.js` | +| `lstat` | Describes the link itself, agreeing with the dirent above | Same — `SEAProvider.lstatSync` gives the target's stat link semantics | +| `readlink` | Returns the target from `SYMLINKS`; `EINVAL` for a path that exists but is not a link | Same, from `manifest.symlinks`, including the shared parent-resolution step | +| `realpath` | Follows the chain | Follows the chain | + +> **Breaking change (traditional mode, since #296).** `readdir({ withFileTypes: true })` previously reported every snapshot entry as a plain file or directory — `Dirent.isSymbolicLink()` took an argument it is never called with, so it always returned `false`. It now reports links as links, which is what Node does outside a packaged binary. Two consequences for packaged apps whose snapshot contains symlinks (pnpm and workspace trees most of all, plus `node_modules/.bin`): +> +> - Recursive walkers that gate descent on `isDirectory()` and skip links by default (glob, fast-glob, readdirp, `fs.cp` with `recursive`) no longer descend into a symlinked directory unless told to follow links. +> - A filter like `entries.filter((e) => e.isFile())` no longer matches a symlinked **file** — `node_modules/.bin/*` is the common case. +> +> Both match unpackaged Node. `fs.readlink` and `fs.lstat` were patched in the same change so that code taking the `isSymbolicLink()` branch is served rather than falling through to the host filesystem. + +> **How SEA reports links (since #296).** `@roberts_lando/vfs` used to route `fs.lstat` through `findVFSForFsStat`, which calls `statSync` and therefore follows the link, and `fs.readlink` through `findVFSForRealpath`, which never reached the provider — so neither could report a symlink, and identical application code saw `lstatSync(link).isSymbolicLink() === true` in a traditional binary and `false` in a SEA one. Both now reach `SEAProvider`, which answers from `manifest.symlinks` — the same record the resolver walks, so reporting and resolution cannot disagree. The routing fix, and the `probeSync` change that stops the module hooks' `existsSync` gate flattening a provider's `ELOOP` into `ENOENT`, are upstream in [robertsLando/vfs#4](https://github.com/robertsLando/vfs/pull/4); **pkg needs a vfs release containing them**. + **`setupProcessPkg(entrypoint)`** — Creates the `process.pkg` compatibility object with `entrypoint`, `defaultEntrypoint`, and `path.resolve()`. **`installDiagnostic(snapshotPrefix)`** — Installs runtime diagnostics triggered by the `DEBUG_PKG` environment variable. Available in both traditional and SEA modes. The implementation lives in `prelude/bootstrap-shared.js` and is always present in the runtime bootstrap, but it is **only invoked when the binary was built with `--debug` / `-d`** — release builds omit the entrypoint call, so the diagnostic handler never runs and cannot expose the VFS tree contents. @@ -619,11 +647,11 @@ With `node:vfs` and `"useVfs": true` in the SEA config, assets will be auto-moun | File | Lines | Purpose | | -------------------------------- | ----- | -------------------------------------------------------------------------------------------- | -| `prelude/bootstrap.js` | ~1970 | Traditional runtime bootstrap (fs/module/process patching) | -| `prelude/bootstrap-shared.js` | ~486 | Shared runtime patches (dlopen, child_process, process.pkg, diagnostics) | +| `prelude/bootstrap.js` | ~2174 | Traditional runtime bootstrap (fs/module/process patching) | +| `prelude/bootstrap-shared.js` | ~969 | Shared runtime patches (dlopen, child_process, process.pkg, diagnostics, symlink resolution) | | `prelude/sea-bootstrap.js` | ~74 | CJS wrapper: Module.runMain() (CJS) or vm.Script + USE_MAIN_CONTEXT_DEFAULT_LOADER (ESM/TLA) | | `prelude/sea-bootstrap-core.js` | ~121 | Shared setup: VFS, patches, worker interception, diagnostics, perf start | -| `prelude/sea-vfs-setup.js` | ~469 | SEA VFS core: SEAProvider, archive loading, VFS mount, Windows patches | +| `prelude/sea-vfs-setup.js` | ~754 | SEA VFS core: SEAProvider, archive loading, VFS mount, Windows patches | | `prelude/sea-worker-entry.js` | ~11 | Worker thread entry: requires sea-vfs-setup.js for VFS in workers | | `scripts/build-sea-bootstrap.js` | ~50 | Build script: 2-step esbuild bundling (worker string + CJS main) | | `lib/index.ts` | ~704 | CLI entry point, mode routing | diff --git a/package.json b/package.json index a660992e0..d30c6866e 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "@babel/parser": "^7.23.0", "@babel/traverse": "^7.23.0", "@babel/types": "^7.23.0", - "@roberts_lando/vfs": "^0.3.3", + "@roberts_lando/vfs": "robertsLando/vfs#19a694fc4c8524009aa3e597e456e6e20fe30c31", "@yao-pkg/pkg-fetch": "3.6.6", "esbuild": "^0.28.1", "into-stream": "^9.1.0", diff --git a/prelude/bootstrap-shared.js b/prelude/bootstrap-shared.js index 044b490ef..b381aa17c 100644 --- a/prelude/bootstrap-shared.js +++ b/prelude/bootstrap-shared.js @@ -3,8 +3,9 @@ // Shared runtime utilities used by both the traditional bootstrap and // the SEA bootstrap. Each consumer require()s or inlines this module. // -// Traditional bootstrap: inlined via REQUIRE_COMMON (already has its -// own common.ts path helpers) — only calls the functions exported here. +// Traditional bootstrap: inlined via REQUIRE_SHARED (already has its +// own common.ts path helpers via REQUIRE_COMMON) — only calls the +// functions exported here. // SEA bootstrap: bundled by esbuild via require('./bootstrap-shared'). var childProcess = require('child_process'); @@ -596,6 +597,8 @@ function installDiagnostic(snapshotPrefix) { wrap(fs, 'readdir'); wrap(fs, 'realpathSync'); wrap(fs, 'realpath'); + wrap(fs, 'readlinkSync'); + wrap(fs, 'readlink'); wrap(fs, 'statSync'); wrap(fs, 'stat'); wrap(fs, 'lstatSync'); @@ -614,6 +617,7 @@ function installDiagnostic(snapshotPrefix) { wrap(fs.promises, 'write'); wrap(fs.promises, 'readdir'); wrap(fs.promises, 'realpath'); + wrap(fs.promises, 'readlink'); wrap(fs.promises, 'stat'); wrap(fs.promises, 'lstat'); wrap(fs.promises, 'access'); @@ -622,6 +626,331 @@ function installDiagnostic(snapshotPrefix) { } } +// ///////////////////////////////////////////////////////////////// +// SYMLINK PROCESSING ////////////////////////////////////////////// +// ///////////////////////////////////////////////////////////////// + +// Matches the typical Linux SYMLOOP_MAX. Bounds symlink resolution so a +// manifest cycle (or a corrupt manifest) cannot hang startup. +var MAX_SYMLINK_DEPTH = 40; + +// libuv numbers errnos differently on Windows (uv/errno.h). One table for both +// preludes: they were drifting apart, and sea-vfs-setup.js's ENOENT was not +// Windows-aware at all. Positive constants, negated at use. +var ERRNO = (function () { + var windows = process.platform === 'win32'; + return { + ENOTDIR: windows ? 4052 : 20, + ENOENT: windows ? 4058 : 2, + EISDIR: windows ? 4068 : 21, + EINVAL: windows ? 4071 : 22, + ELOOP: windows ? 4067 : 40, + }; +})(); + +/** + * One shape for every error the two preludes raise from a snapshot path. + * + * The *message* deliberately stays the caller's: traditional mode's ENOENT + * carries pkg's "recompile adding it as asset" guidance (asserted by + * test-50-not-found-wording), while the SEA provider uses Node's own wording. + * What has to match is the shape — code, errno, syscall, path, and the `pkg` + * marker bootstrap.js's module wrapper reads. + */ +function makeFsError(message, code, syscall, path_) { + var err = new Error(message); + err.code = code; + err.errno = -ERRNO[code]; + err.syscall = syscall; + err.path = path_; + err.pkg = true; + return err; +} + +// Marks a symlink key whose resolution is still on the stack, so a cycle +// (/a -> /b -> /a, or /a -> /a/b) is caught instead of recursing forever. +var RESOLVING = {}; + +// libuv dirent types, as readdir({ withFileTypes: true }) reports them. +var UV_DIRENT_FILE = 1; +var UV_DIRENT_DIR = 2; +var UV_DIRENT_LINK = 3; + +// POSIX file-type bits, for stats that have to agree with the predicate above. +var S_IFMT = 0o170000; +var S_IFLNK = 0o120000; + +/** + * The Dirent both bootstraps hand back from readdir({ withFileTypes: true }). + * + * fs.Dirent.isSymbolicLink() takes no argument, so link status has to be baked + * in at construction — which is why the type is passed rather than derived + * from a later lookup. + */ +function Dirent(name, type, parentPath) { + this.name = name; + this.type = type; + // Node's Dirent has carried parentPath since 20.12, and `path` is its + // deprecated alias. path.join(d.parentPath, d.name) is the documented way to + // use withFileTypes, so leaving it undefined throws ERR_INVALID_ARG_TYPE + // inside a packaged binary. + this.parentPath = parentPath; + this.path = parentPath; +} + +Dirent.prototype.isDirectory = function isDirectory() { + return this.type === UV_DIRENT_DIR; +}; + +Dirent.prototype.isFile = function isFile() { + return this.type === UV_DIRENT_FILE; +}; + +Dirent.prototype.isSymbolicLink = function isSymbolicLink() { + return this.type === UV_DIRENT_LINK; +}; + +function direntNoop() { + return false; +} + +/** + * readlink takes its options as a string encoding or an { encoding } object, + * and answers a Buffer for 'buffer'. Shared so the two modes cannot disagree. + */ +function readlinkEncoding(options) { + var encoding = + typeof options === 'string' ? options : options && options.encoding; + if (encoding && encoding !== 'buffer' && !Buffer.isEncoding(encoding)) { + var err = new TypeError('Unknown encoding: ' + encoding); + err.code = 'ERR_INVALID_ARG_VALUE'; + throw err; + } + return encoding; +} + +function applyReadlinkEncoding(target, encoding) { + if (encoding === 'buffer') return Buffer.from(target); + if (encoding && encoding !== 'utf8' && encoding !== 'utf-8') { + return Buffer.from(target).toString(encoding); + } + return target; +} + +Dirent.prototype.isBlockDevice = direntNoop; +Dirent.prototype.isCharacterDevice = direntNoop; +Dirent.prototype.isSocket = direntNoop; +Dirent.prototype.isFIFO = direntNoop; + +/** + * Give a stat object symlink semantics. + * + * Both walkers stat *through* the link, so what arrives describes the target. + * The mode's type bits are rewritten too: consumers that sniff + * `mode & S_IFMT` (tar, archiver, fs.cp) read those rather than the predicate. + */ +function asSymlinkStat(s, target) { + s.isSymbolicLink = function () { + return true; + }; + s.isFile = direntNoop; + s.isDirectory = direntNoop; + if (typeof s.mode === 'number') { + s.mode = (s.mode & ~S_IFMT) | S_IFLNK; + } + // POSIX lstat reports a link's size as the length of its target string, and + // a link occupies no blocks. Without the target we leave the through-the-link + // numbers alone rather than invent one. + if (typeof target === 'string') { + s.size = Buffer.byteLength(target); + s.blocks = 0; + s.nlink = 1; + } + return s; +} + +/** + * Build a symlink resolver over a manifest's symlinks record. + * + * The returned function maps a virtual path onto what its symlinks point at, + * following parent components the way POSIX does: `node_modules/@x/y` being a + * link makes `node_modules/@x/y/package.json` resolve too (#295). + * + * This runs before every fs operation inside a packaged binary (~30K times at + * startup on a large project). The empty-manifest case is allocation-free; the + * no-match case costs one `slice` per depth that actually hosts a key, not one + * per path component (see `depthHasKey` below). + * + * The returned resolver keeps hop-accounting state in its closure, so it is not + * reentrant — never call it from inside its own resolution. + */ +function makeSymlinkResolver(symlinks, sep) { + var keys = Object.keys(symlinks || {}); + + // Nothing to resolve: hand back identity, so no caller needs a guard of its + // own and a symlink-free binary pays nothing. It still has to carry .parent, + // or readlink and lstat break on every binary without symlinks — which is + // most of them. + if (keys.length === 0) { + var identity = function (p) { + return p; + }; + identity.parent = identity; + return identity; + } + + // Symlink keys sit at a handful of depths — a package manager's links all + // live at the same level of node_modules. Recording which separator counts + // can host a key lets the walk below slice only at those depths and stop + // past the deepest one: for a 15-segment path in a tree whose links live at + // depth 4, that is one probe instead of fifteen. + var depthHasKey = []; + var maxDepth = 0; + for (var i = 0; i < keys.length; i++) { + var depth = 0; + var at = keys[i].indexOf(sep, 1); + while (at > 0) { + depth++; + at = keys[i].indexOf(sep, at + 1); + } + depthHasKey[depth] = true; + if (depth > maxDepth) maxDepth = depth; + } + + // Symlink key -> { target, cost }: where the key fully resolves to, and how + // many hops that took. Keyed by manifest entry rather than by the caller's + // path, so the map stays bounded by the manifest no matter how many distinct + // paths are looked up — including ones an application derives from untrusted + // input. It also amortizes across siblings: every file under one linked + // directory reuses a single entry. + var resolved = new Map(); + + // High-water hop count of the resolution currently in flight. follow() reads + // it to record each key's `cost`, so a cache hit can charge the hops the + // collapsed chain stands for instead of getting them for free — otherwise + // MAX_SYMLINK_DEPTH would depend on which path happened to be looked up + // first. + var deepest = 0; + + // Syscall reported by any ELOOP raised by the resolution in flight. Held in + // the closure rather than threaded through resolve()/follow(), which are on + // the startup hot path. + var syscall = 'stat'; + + function eloop(origin) { + return makeFsError( + 'ELOOP: too many symbolic links encountered, ' + + syscall + + " '" + + origin + + "'", + 'ELOOP', + syscall, + origin, + ); + } + + function follow(key, origin, hops) { + var cached = resolved.get(key); + if (cached !== undefined) { + if (cached === RESOLVING) throw eloop(origin); + var reached = hops + cached.cost; + if (reached > MAX_SYMLINK_DEPTH) throw eloop(origin); + if (reached > deepest) deepest = reached; + return cached.target; + } + resolved.set(key, RESOLVING); + // Restart the high-water mark at this key's depth so `cost` measures this + // subtree alone, then fold it back into the caller's mark on the way out. + var outer = deepest; + deepest = hops; + var target; + try { + target = resolve(symlinks[key], origin, hops + 1); + } catch (e) { + // Don't leave the sentinel behind, or a caught ELOOP would poison this + // key for every later lookup. + resolved.delete(key); + throw e; + } + resolved.set(key, { target: target, cost: deepest - hops }); + if (outer > deepest) deepest = outer; + return target; + } + + function resolve(p, origin, hops) { + if (hops > MAX_SYMLINK_DEPTH) throw eloop(origin); + if (hops > deepest) deepest = hops; + + // Longest prefix wins. The walker keys every entry on the *unresolved* + // path it walked (`appendSymlink` in lib/walker.ts) and each target is + // already fully realpath'd, so the deepest key describes the whole chain + // while a shallower one would strand the walk on a path the archive has no + // entry for. `/lib` and `/lib/sub` can both be keys. An exact + // match is just the deepest case, so it short-circuits the scan below. + // + // Deepest-prefix-first is not POSIX's leftmost-first, and the two agree + // only because every target the walker records is already a full realpath + // (`toNormalizedRealPath` in lib/walker.ts), so no component of a target + // can itself be a key. A hand-written manifest that breaks that invariant + // would resolve differently here than on disk. + if (typeof symlinks[p] === 'string') return follow(p, origin, hops); + + var bestPos = -1; + var bestKey = null; + var pos = p.indexOf(sep, 1); + var depth = 0; + while (pos > 0 && depth <= maxDepth) { + if (depthHasKey[depth]) { + var prefix = p.slice(0, pos); + // typeof, not truthiness: the record is JSON-derived and read with a + // bracket index, so `__proto__`/`constructor`/`toString` would + // otherwise match on an inherited, non-string value. + if (typeof symlinks[prefix] === 'string') { + bestPos = pos; + bestKey = prefix; + } + } + pos = p.indexOf(sep, pos + 1); + depth++; + } + + if (bestKey === null) return p; + + var target = follow(bestKey, origin, hops); + // Drop the remainder's leading separator when the target already ends in + // one, so the join cannot double up. + var rest = target.endsWith(sep) ? p.slice(bestPos + 1) : p.slice(bestPos); + // The remainder may hold links of its own, so walk the result. + return resolve(target + rest, origin, hops + 1); + } + + function resolveKey(p, forSyscall, forPath) { + deepest = 0; + syscall = forSyscall || 'stat'; + // forPath is what an ELOOP reports. Callers pass the user's path, because + // `p` is a vfs key — base36 under DOCOMPRESS, and never what was asked for. + return resolve(p, forPath === undefined ? p : forPath, 0); + } + + // The key a path has once its *parents* are followed but its own last + // component is not — what POSIX resolves before reading a link, so readlink + // and lstat can find an entry the walker recorded under a followed parent. + // Shared so the two modes cannot grow their own join rules. + resolveKey.parent = function (p, forSyscall, forPath) { + var slash = p.lastIndexOf(sep); + if (slash <= 0) return p; + var parent = resolveKey(p.slice(0, slash), forSyscall, forPath); + // Drop the remainder's leading separator when the resolved parent already + // ends in one, so the join cannot produce `//name` and silently miss. + return ( + parent + (parent.endsWith(sep) ? p.slice(slash + 1) : p.slice(slash)) + ); + }; + + return resolveKey; +} + module.exports = { patchDlopen: patchDlopen, patchChildProcess: patchChildProcess, @@ -631,4 +960,14 @@ module.exports = { COMPRESS_NONE: COMPRESS_NONE, pickDecompressorSync: pickDecompressorSync, pickDecompressorAsync: pickDecompressorAsync, + makeSymlinkResolver: makeSymlinkResolver, + ERRNO: ERRNO, + makeFsError: makeFsError, + Dirent: Dirent, + asSymlinkStat: asSymlinkStat, + readlinkEncoding: readlinkEncoding, + applyReadlinkEncoding: applyReadlinkEncoding, + UV_DIRENT_FILE: UV_DIRENT_FILE, + UV_DIRENT_DIR: UV_DIRENT_DIR, + UV_DIRENT_LINK: UV_DIRENT_LINK, }; diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index e8e5ad8fe..9d85eed21 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -231,34 +231,33 @@ function toOriginal(fShort) { .join(path.sep); } -const symlinksEntries = Object.entries(SYMLINKS); - // separator for substitution depends on platform; const sepsep = DOCOMPRESS ? separator : path.sep; -function findVirtualFileSystemKeyAndFollowLinks(path_) { - let vfsKey = findVirtualFileSystemKey(path_, path.sep); - let needToSubstitute = true; - while (needToSubstitute) { - needToSubstitute = false; - for (const [k, v] of symlinksEntries) { - if (vfsKey.startsWith(`${k}${sepsep}`) || vfsKey === k) { - vfsKey = vfsKey.replace(k, v); - needToSubstitute = true; - break; - } - } - } - return vfsKey; +// The resolver owns the no-symlink fast path and its own memoisation, so +// there is nothing to guard here. +const resolveSymlink = REQUIRE_SHARED.makeSymlinkResolver(SYMLINKS, sepsep); + +// syscall names the fs call in flight, so an ELOOP out of the resolver says +// which one hit it rather than always saying `stat`. path_ is passed through as +// the error's path because the resolver only ever sees the vfs key. +function findVirtualFileSystemKeyAndFollowLinks(path_, syscall) { + return resolveSymlink( + findVirtualFileSystemKey(path_, path.sep), + syscall, + path_, + ); } function realpathFromSnapshot(path_) { - const realPath = toOriginal(findVirtualFileSystemKeyAndFollowLinks(path_)); + const realPath = toOriginal( + findVirtualFileSystemKeyAndFollowLinks(path_, 'realpath'), + ); return realPath; } -function findVirtualFileSystemEntry(path_) { - const vfsKey = findVirtualFileSystemKeyAndFollowLinks(path_); +function findVirtualFileSystemEntry(path_, syscall) { + const vfsKey = findVirtualFileSystemKeyAndFollowLinks(path_, syscall); return VIRTUAL_FILESYSTEM[vfsKey]; } @@ -498,6 +497,8 @@ function payloadFileSync(pointer) { readdir: fs.readdir, realpathSync: fs.realpathSync, realpath: fs.realpath, + readlinkSync: fs.readlinkSync, + readlink: fs.readlink, statSync: fs.statSync, stat: fs.stat, lstatSync: fs.lstatSync, @@ -521,9 +522,9 @@ function payloadFileSync(pointer) { const windows = process.platform === 'win32'; const docks = {}; - const ENOTDIR = windows ? 4052 : 20; - const ENOENT = windows ? 4058 : 2; - const EISDIR = windows ? 4068 : 21; + // One errno table for both preludes — see bootstrap-shared.js. + const makeFsError = REQUIRE_SHARED.makeFsError; + const ENOENT = REQUIRE_SHARED.ERRNO.ENOENT; function assertEncoding(encoding) { if (encoding && !Buffer.isEncoding(encoding)) { @@ -536,35 +537,63 @@ function payloadFileSync(pointer) { return typeof cb === 'function' ? cb : rethrow; } + // The messages stay traditional-mode's own — this one carries pkg's + // "recompile adding it as asset" guidance and test-50-not-found-wording + // asserts it. Only the shape is shared. + // Every callback-style snapshot shim needs the same guard: the resolver can + // throw ELOOP on a cyclic manifest, and Node's async fs never throws + // synchronously. cb2 is rethrow for sync callers, so one helper serves both + // and the next shim added cannot forget it. + // + // MISSED means the error was already delivered: the caller must stop, and + // must not also report its own ENOENT. + const MISSED = Symbol('eloop-reported'); + + function findEntryOr(cb2, path_, syscall) { + try { + return findVirtualFileSystemEntry(path_, syscall); + } catch (error) { + cb2(error); + return MISSED; + } + } + function error_ENOENT(fileOrDirectory, path_) { - const error = new Error( + return makeFsError( `${fileOrDirectory} '${stripSnapshot(path_)}' ` + `was not included into executable at compilation stage. ` + `Please recompile adding it as asset or script.`, + 'ENOENT', + undefined, + path_, ); - error.errno = -ENOENT; - error.code = 'ENOENT'; - error.path = path_; - error.pkg = true; - return error; } function error_EISDIR(path_) { - const error = new Error('EISDIR: illegal operation on a directory, read'); - error.errno = -EISDIR; - error.code = 'EISDIR'; - error.path = path_; - error.pkg = true; - return error; + return makeFsError( + 'EISDIR: illegal operation on a directory, read', + 'EISDIR', + 'read', + path_, + ); + } + + function error_EINVAL(syscall, path_) { + return makeFsError( + `EINVAL: invalid argument, ${syscall} '${stripSnapshot(path_)}'`, + 'EINVAL', + syscall, + path_, + ); } function error_ENOTDIR(path_) { - const error = new Error(`ENOTDIR: not a directory, scandir '${path_}'`); - error.errno = -ENOTDIR; - error.code = 'ENOTDIR'; - error.path = path_; - error.pkg = true; - return error; + return makeFsError( + `ENOTDIR: not a directory, scandir '${path_}'`, + 'ENOTDIR', + 'scandir', + path_, + ); } // /////////////////////////////////////////////////////////////// @@ -621,7 +650,7 @@ function payloadFileSync(pointer) { }; function uncompressExternallyPath(path_) { - const entity = findVirtualFileSystemEntry(path_); + const entity = findVirtualFileSystemEntry(path_, 'open'); const dock = { path: path_, entity, position: 0 }; return uncompressExternally(dock); } @@ -634,7 +663,8 @@ function payloadFileSync(pointer) { function openFromSnapshot(path_, uncompress, cb) { const cb2 = cb || rethrow; - const entity = findVirtualFileSystemEntry(path_); + const entity = findEntryOr(cb2, path_, 'open'); + if (entity === MISSED) return; if (!entity) return cb2(error_ENOENT('File or directory', path_)); const dock = { path: path_, entity, position: 0 }; @@ -925,7 +955,8 @@ function payloadFileSync(pointer) { function readFileFromSnapshot(path_, cb) { const cb2 = cb || rethrow; - const entity = findVirtualFileSystemEntry(path_); + const entity = findEntryOr(cb2, path_, 'open'); + if (entity === MISSED) return; if (!entity) return cb2(error_ENOENT('File', path_)); const entityLinks = entity[STORE_LINKS]; @@ -1085,36 +1116,67 @@ function payloadFileSync(pointer) { return null; } - function Dirent(name, type) { - this.name = name; - this.type = type; - } - - Dirent.prototype.isDirectory = function isDirectory() { - return this.type === 2; - }; - - Dirent.prototype.isFile = function isFile() { - return this.type === 1; - }; - + // Shared with the SEA provider so both modes report the same dirent shape — + // real readdir lstats, so a link is a link rather than what it points at. + const Dirent = REQUIRE_SHARED.Dirent; + const UV_DIRENT_FILE = REQUIRE_SHARED.UV_DIRENT_FILE; + const UV_DIRENT_DIR = REQUIRE_SHARED.UV_DIRENT_DIR; + const UV_DIRENT_LINK = REQUIRE_SHARED.UV_DIRENT_LINK; const noop = () => false; - Dirent.prototype.isBlockDevice = noop; - Dirent.prototype.isCharacterDevice = noop; - Dirent.prototype.isSocket = noop; - Dirent.prototype.isFIFO = noop; - Dirent.prototype.isSymbolicLink = (fileOrFolderName) => - Boolean(SYMLINKS[fileOrFolderName]); + // "Is this path a link, and to what?" — SYMLINKS is keyed by the *unresolved* + // vfs key the walker walked, so that spelling wins; the resolved one is a + // fallback for an entry recorded under an already-followed parent. Mirrors + // SEAProvider._linkTarget, so the modes cannot grow different answers. + function snapshotLinkTarget(vfsKey, resolvedKey) { + // typeof, not truthiness: the record is read with a bracket index. + if (typeof SYMLINKS[vfsKey] === 'string') return SYMLINKS[vfsKey]; + if (resolvedKey !== undefined && resolvedKey !== vfsKey) { + if (typeof SYMLINKS[resolvedKey] === 'string') { + return SYMLINKS[resolvedKey]; + } + } + return undefined; + } function getFileTypes(path_, entries) { + // Node's Dirent.parentPath is the directory readdir was called on, so it + // has to stay the caller's path — stripSnapshot() is for error text and + // yields something path.join() cannot use. + const parentPath = path_; return entries.map((entry) => { const ff = path.join(path_, entry); - const entity = findVirtualFileSystemEntry(ff); + // SYMLINKS is keyed by the *unresolved* vfs key, so this asks whether + // this entry is itself a link — not whether its target is one. It runs + // before the entity lookup, which follows links: an entry whose target + // is missing from the snapshot is still a link, and answering + // `undefined` there would put a hole in the readdir array. + // typeof, not truthiness: the record is read with a bracket index, so a + // key like `constructor` would otherwise match an inherited value. + const vfsKey = findVirtualFileSystemKey(ff, path.sep); + if (snapshotLinkTarget(vfsKey, undefined) !== undefined) + return new Dirent(entry, UV_DIRENT_LINK, parentPath); + // Same lookup findVirtualFileSystemEntry() does, reusing the key above + // rather than rebuilding it — in DOCOMPRESS mode that is a full + // normalize+split+map+join per directory entry. + // + // A cycle reachable only through this entry's parents (the entry itself + // need not be a key) would otherwise throw from inside fs.readdir's + // callback. A dirent that cannot be resolved is a hole, same as a + // missing entity. + let resolved; + try { + resolved = resolveSymlink(vfsKey, 'scandir', ff); + } catch (error) { + if (error.code !== 'ELOOP') throw error; + return undefined; + } + const entity = VIRTUAL_FILESYSTEM[resolved]; if (!entity) return undefined; if (entity[STORE_BLOB] || entity[STORE_CONTENT]) - return new Dirent(entry, 1); - if (entity[STORE_LINKS]) return new Dirent(entry, 2); + return new Dirent(entry, UV_DIRENT_FILE, parentPath); + if (entity[STORE_LINKS]) + return new Dirent(entry, UV_DIRENT_DIR, parentPath); throw new Error('UNEXPECTED-24'); }); } @@ -1122,7 +1184,7 @@ function payloadFileSync(pointer) { function readdirRoot(path_, options, cb) { function addSnapshot(entries) { if (options && options.withFileTypes) { - entries.push(new Dirent('snapshot', 2)); + entries.push(new Dirent('snapshot', UV_DIRENT_DIR, path_)); } else { entries.push('snapshot'); } @@ -1155,7 +1217,8 @@ function payloadFileSync(pointer) { function readdirFromSnapshot(path_, cb) { const cb2 = cb || rethrow; - const entity = findVirtualFileSystemEntry(path_); + const entity = findEntryOr(cb2, path_, 'scandir'); + if (entity === MISSED) return; if (!entity) { return cb2(error_ENOENT('Directory', path_)); @@ -1227,8 +1290,17 @@ function payloadFileSync(pointer) { readdirFromSnapshot(path_, (error, entries) => { if (error) return callback(error); - if (options.withFileTypes) entries = getFileTypes(path_, entries); - callback(null, entries); + // This runs inside payloadFile's real I/O completion handler, so a throw + // from getFileTypes has nothing above it to catch. + let dirents; + try { + dirents = options.withFileTypes + ? getFileTypes(path_, entries) + : entries; + } catch (err) { + return callback(err); + } + callback(null, dirents); }); }; @@ -1259,12 +1331,85 @@ function payloadFileSync(pointer) { } const callback = dezalgo(maybeCallback(arguments)); - callback(null, realpathFromSnapshot(path_)); + let realPath; + try { + realPath = realpathFromSnapshot(path_); + } catch (error) { + // Async fs never throws synchronously — an ELOOP belongs in the + // callback, like every sibling here. + return callback(error); + } + callback(null, realPath); }; fs.realpathSync.native = fs.realpathSync; fs.realpath.native = fs.realpath; + // /////////////////////////////////////////////////////////////// + // readlink ////////////////////////////////////////////////////// + // /////////////////////////////////////////////////////////////// + + // readdir({ withFileTypes: true }) reports snapshot symlinks as links, so + // the usual `if (d.isSymbolicLink()) fs.readlinkSync(p)` pairing has to be + // answerable here — unpatched it would fall through to the host fs and + // ENOENT on a /snapshot path. + const readlinkEncoding = REQUIRE_SHARED.readlinkEncoding; + const applyReadlinkEncoding = REQUIRE_SHARED.applyReadlinkEncoding; + + function readlinkFromSnapshot(path_) { + const vfsKey = findVirtualFileSystemKey(path_, path.sep); + // POSIX readlink resolves the parents and reads only the final component, + // so an entry the walker recorded under an already-followed parent is + // reachable too. Same step the SEA provider takes, from the same helper. + const viaParent = resolveSymlink.parent(vfsKey, 'readlink', path_); + const target = snapshotLinkTarget(vfsKey, viaParent); + if (target !== undefined) return toOriginal(target); + // Node answers EINVAL for a path that exists but is not a link, and + // ENOENT for one that does not exist at all. + // typeof, not truthiness — the record is read with a bracket index, same + // as every sibling lookup. + if ( + typeof VIRTUAL_FILESYSTEM[resolveSymlink(vfsKey, 'readlink', path_)] === + 'object' + ) { + throw error_EINVAL('readlink', path_); + } + throw error_ENOENT('File or directory', path_); + } + + fs.readlinkSync = function readlinkSync(path_, options) { + if (!insideSnapshot(path_)) { + return ancestor.readlinkSync.apply(fs, arguments); + } + if (insideMountpoint(path_)) { + return ancestor.readlinkSync.apply(fs, translateNth(arguments, 0, path_)); + } + + const encoding = readlinkEncoding(options); + return applyReadlinkEncoding(readlinkFromSnapshot(path_), encoding); + }; + + fs.readlink = function readlink(path_, options) { + if (!insideSnapshot(path_)) { + return ancestor.readlink.apply(fs, arguments); + } + if (insideMountpoint(path_)) { + return ancestor.readlink.apply(fs, translateNth(arguments, 0, path_)); + } + + const encoding = readlinkEncoding( + typeof options === 'function' ? undefined : options, + ); + const callback = dezalgo(maybeCallback(arguments)); + let target; + try { + target = readlinkFromSnapshot(path_); + } catch (error) { + return callback(error); + } + callback(null, applyReadlinkEncoding(target, encoding)); + }; + // /////////////////////////////////////////////////////////////// // stat ////////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// @@ -1343,7 +1488,8 @@ function payloadFileSync(pointer) { function statFromSnapshot(path_, cb) { const cb2 = cb || rethrow; - const entity = findVirtualFileSystemEntry(path_); + const entity = findEntryOr(cb2, path_, 'stat'); + if (entity === MISSED) return; if (!entity) return findNativeAddonForStat(path_, cb); const entityStat = entity[STORE_STAT]; if (entityStat) return statFromSnapshotSub(entityStat, cb); @@ -1377,6 +1523,46 @@ function payloadFileSync(pointer) { // lstat ///////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// + // lstat must describe the link itself rather than what it points at. The + // walker records every stat with fs.stat, so the stored isSymbolicLinkValue + // is false even for a link; SYMLINKS — keyed by the *unresolved* vfs key — + // is the only source of truth, and it is the same one readdir uses, so the + // two cannot disagree about an entry. + const asLink = REQUIRE_SHARED.asSymlinkStat; + + function lstatFromSnapshot(path_, cb) { + const cb2 = cb || rethrow; + const vfsKey = findVirtualFileSystemKey(path_, path.sep); + // Resolve the parents first, so lstat agrees with readdir and readlink + // about which entries are links — the same pair of keys all three use. + let target = snapshotLinkTarget(vfsKey, undefined); + if (target === undefined) { + let viaParent; + try { + viaParent = resolveSymlink.parent(vfsKey, 'lstat', path_); + } catch (error) { + return cb2(error); + } + target = snapshotLinkTarget(vfsKey, viaParent); + } + if (target === undefined) { + return statFromSnapshot(path_, cb); + } + const entity = VIRTUAL_FILESYSTEM[vfsKey]; + const entityStat = entity && entity[STORE_STAT]; + // A link the walker recorded without its own stat entry: fall back rather + // than invent one. + if (!entityStat) return statFromSnapshot(path_, cb); + const linkTarget = toOriginal(target); + if (cb) { + return statFromSnapshotSub(entityStat, (error, s) => { + if (error) return cb(error); + cb(null, asLink(s, linkTarget)); + }); + } + return asLink(statFromSnapshotSub(entityStat), linkTarget); + } + fs.lstatSync = function lstatSync(path_) { if (!insideSnapshot(path_)) { return ancestor.lstatSync.apply(fs, arguments); @@ -1385,7 +1571,7 @@ function payloadFileSync(pointer) { return ancestor.lstatSync.apply(fs, translateNth(arguments, 0, path_)); } - return statFromSnapshot(path_); + return lstatFromSnapshot(path_); }; fs.lstat = function lstat(path_) { @@ -1397,7 +1583,7 @@ function payloadFileSync(pointer) { } const callback = dezalgo(maybeCallback(arguments)); - statFromSnapshot(path_, callback); + lstatFromSnapshot(path_, callback); }; // /////////////////////////////////////////////////////////////// @@ -1440,7 +1626,16 @@ function payloadFileSync(pointer) { } function existsFromSnapshot(path_) { - const entity = findVirtualFileSystemEntry(path_); + let entity; + try { + entity = findVirtualFileSystemEntry(path_); + } catch (error) { + // fs.existsSync never throws — libuv swallows every errno and answers + // false — so a symlink cycle in the manifest has to read as "no" here + // rather than escaping into a caller that has no catch. + if (error.code === 'ELOOP') return false; + throw error; + } if (!entity) return findNativeAddonForExists(path_); return true; } @@ -1474,7 +1669,8 @@ function payloadFileSync(pointer) { function accessFromSnapshot(path_, cb) { const cb2 = cb || rethrow; - const entity = findVirtualFileSystemEntry(path_); + const entity = findEntryOr(cb2, path_, 'access'); + if (entity === MISSED) return; if (!entity) return cb2(error_ENOENT('File or directory', path_)); return cb2(null, undefined); } @@ -1549,6 +1745,7 @@ function payloadFileSync(pointer) { realpath: fs.promises.realpath, stat: fs.promises.stat, lstat: fs.promises.lstat, + readlink: fs.promises.readlink, fstat: fs.promises.fstat, access: fs.promises.access, copyFile: fs.promises.copyFile, @@ -1604,13 +1801,13 @@ function payloadFileSync(pointer) { fs.promises.read = util.promisify(fs.read); fs.promises.realpath = util.promisify(fs.realpath); + fs.promises.readlink = util.promisify(fs.readlink); fs.promises.fstat = util.promisify(fs.fstat); fs.promises.statfs = util.promisify(fs.statfs); fs.promises.access = util.promisify(fs.access); // TODO: all promises methods that try to edit files in snapshot should throw // TODO implement missing methods - // fs.promises.readlink ? // fs.promises.opendir ? } @@ -1650,7 +1847,15 @@ function payloadFileSync(pointer) { .internalModuleStat(makeLong(translate(path_))); } - const entity = findVirtualFileSystemEntry(path_); + let entity; + try { + entity = findVirtualFileSystemEntry(path_); + } catch (error) { + // Contract is a negative errno, not a throw: module resolution calls + // this and would turn a cyclic manifest into an uncaught exception. + if (error.code === 'ELOOP') return error.errno; + throw error; + } if (!entity) { return findNativeAddonForInternalModuleStat(path_); @@ -1699,7 +1904,15 @@ function payloadFileSync(pointer) { return readFile(makeLong(translate(path_))); } - const entity = findVirtualFileSystemEntry(path_); + let entity; + try { + entity = findVirtualFileSystemEntry(path_, 'open'); + } catch (error) { + // Same reason as internalModuleStat: require() probes package.json + // through here, so a cyclic manifest must read as a miss, not a throw. + if (error.code !== 'ELOOP') throw error; + return returnArray ? [undefined, false] : undefined; + } if (!entity) { return returnArray ? [undefined, false] : undefined; @@ -1781,7 +1994,7 @@ function payloadFileSync(pointer) { return ancestor._compile.apply(this, arguments); } - const entity = findVirtualFileSystemEntry(filename_); + const entity = findVirtualFileSystemEntry(filename_, 'open'); if (!entity) { // let user try to "_compile" a packaged file diff --git a/prelude/sea-vfs-setup.js b/prelude/sea-vfs-setup.js index 94a34ea58..6973a6c03 100644 --- a/prelude/sea-vfs-setup.js +++ b/prelude/sea-vfs-setup.js @@ -24,10 +24,6 @@ try { var VirtualFileSystem = vfsModule.VirtualFileSystem; var MemoryProvider = vfsModule.MemoryProvider; -// Matches the typical Linux SYMLOOP_MAX. Bounds the symlink resolution -// loop so a manifest cycle (or a corrupt manifest) cannot hang startup. -var MAX_SYMLINK_DEPTH = 40; - // ///////////////////////////////////////////////////////////////// // PERFORMANCE INSTRUMENTATION ///////////////////////////////////// // ///////////////////////////////////////////////////////////////// @@ -151,6 +147,7 @@ var perf = { 'statSync calls', 'existsSync calls', 'readdirSync calls', + 'symlink resolutions', ]; counterOrder.forEach(function (label) { var v = self._counters[label]; @@ -212,15 +209,33 @@ var toManifestKey = return _stripTrailingSeps(p); }; +// The VFS strips the mount prefix before calling the provider, so an error +// built from what arrives here names a path the caller never used. Put the +// prefix and the platform form back, the way classic mode's stripSnapshot does. +function toCallerPath(providerPath) { + if (typeof providerPath !== 'string') return providerPath; + if (providerPath.startsWith(SNAPSHOT_PREFIX)) return providerPath; + return toPlatformPath(SNAPSHOT_PREFIX + providerPath); +} + +function _einval(syscall, filePath) { + var shown = toCallerPath(filePath); + return shared.makeFsError( + 'EINVAL: invalid argument, ' + syscall + " '" + shown + "'", + 'EINVAL', + syscall, + shown, + ); +} + function _enoent(syscall, filePath) { - var err = new Error( - 'ENOENT: no such file or directory, ' + syscall + " '" + filePath + "'", + var shown = toCallerPath(filePath); + return shared.makeFsError( + 'ENOENT: no such file or directory, ' + syscall + " '" + shown + "'", + 'ENOENT', + syscall, + shown, ); - err.code = 'ENOENT'; - err.errno = -2; - err.syscall = syscall; - err.path = filePath; - return err; } // ///////////////////////////////////////////////////////////////// @@ -283,13 +298,22 @@ function _makeStats(meta) { * * Performance design: * - * - internalModuleStat() O(1) manifest hash lookup (no tree walk). + * - internalModuleStat() Symlink resolution (no-op O(1) if the manifest has + * no symlinks; O(path depth) for any path that isn't itself symlinked, + * paid on every call — not memoised, since most lookups are one-off + * candidate paths and caching them would grow the cache unboundedly for + * no benefit; O(1) amortized only for paths that actually traverse a + * symlink, via a Map keyed by the manifest entry rather than by the + * caller's path — see makeSymlinkResolver() in bootstrap-shared.js) + * + O(1) manifest lookup. * This is the hottest path (~30K calls for large projects). * - * - statSync() O(1) manifest lookup + lightweight stat allocation. + * - statSync() Same symlink resolution as above + O(1) manifest + * lookup + lightweight stat allocation. * Not on the module resolution hot path. Returns a fresh object each call. * - * - existsSync() O(1) manifest lookup. + * - existsSync() Same symlink resolution as above + O(1) manifest + * lookup. * * - readFileSync() Zero-copy subarray from the archive with a Map * cache. Bypasses the MemoryProvider tree entirely. Returns a Buffer @@ -307,6 +331,11 @@ class SEAProvider extends MemoryProvider { this._manifest = seaManifest; this._fileCache = new Map(); + // One normalised symlinks record for every consumer below, so the + // resolver and readlinkSync cannot disagree about whether it may be absent. + this._symlinks = seaManifest.symlinks || {}; + this._resolve = shared.makeSymlinkResolver(this._symlinks, '/'); + // Pick the per-file decompressor once at construction time. Absent or 0 = // uncompressed archive (backward compat with pre-#250 SEA binaries). The // shared helper raises a uniformly-worded error when the host Node.js is @@ -336,27 +365,27 @@ class SEAProvider extends MemoryProvider { perf.end('directory tree init'); } - _resolveSymlink(p) { - // Fast path: the vast majority of lookups (~30K per startup on large - // projects) are not symlinks. A single object-has-key check avoids - // entering the loop and the i++/target fetch overhead for the common - // case. - var symlinks = this._manifest.symlinks; - if (symlinks[p] === undefined) return p; - var original = p; - for (var i = 0; i < MAX_SYMLINK_DEPTH; i++) { - var target = symlinks[p]; - if (!target) return p; - p = target; + _resolveSymlink(p, syscall, forPath) { + // The resolver owns the no-symlink fast path, so there is nothing to guard + // here. Counting only the calls that actually moved the path keeps the + // counter meaningful on symlink-free binaries, where it used to be skipped + // by a separate guard. + // toCallerPath() is only ever read by the error, so it is built in the + // catch rather than on every call — this is the hot path (~30K calls on a + // large project) and a try block costs nothing until something throws. + var resolved; + try { + resolved = this._resolve(p, syscall, forPath); + } catch (error) { + if (error.code === 'ELOOP' && error.path === forPath) { + var shown = toCallerPath(forPath); + error.message = error.message.split("'")[0] + "'" + shown + "'"; + error.path = shown; + } + throw error; } - var err = new Error( - "ELOOP: too many symbolic links encountered, '" + original + "'", - ); - err.code = 'ELOOP'; - err.errno = -40; - err.syscall = 'stat'; - err.path = original; - throw err; + if (resolved !== p) perf.count('symlink resolutions'); + return resolved; } get fileCacheSize() { @@ -364,7 +393,7 @@ class SEAProvider extends MemoryProvider { } readFileSync(filePath, options) { - var p = this._resolveSymlink(toManifestKey(filePath)); + var p = this._resolveSymlink(toManifestKey(filePath), 'open', filePath); // Fast path: for compressed archives, a per-file decompressed Buffer is // memoised in _fileCache (decompression is expensive and most prelude // modules are read once during module resolution, twice for the compile @@ -443,16 +472,56 @@ class SEAProvider extends MemoryProvider { return copy; } - readlinkSync(filePath) { - var p = toManifestKey(filePath); - var target = this._manifest.symlinks[p]; - if (target) return target; - return super.readlinkSync(p); + readlinkSync(filePath, options) { + // Reached through fs.readlinkSync since #296 — sea-vfs-setup.js re-points + // the patch here, because @roberts_lando/vfs answers readlink by way of + // realpathSync (findVFSForRealpath) and so can never raise EINVAL. + // Manifest targets are full realpaths (toNormalizedRealPath in + // lib/walker.ts), not the raw link body POSIX readlink would return, so + // what comes back is a resolved path. + var encoding = shared.readlinkEncoding(options); + var key = toManifestKey(filePath); + // POSIX readlink resolves the parents and reads only the final component, + // so a link keyed under an already-followed parent is reachable too. + var resolvedKey = this._resolveParentKey(key, 'readlink', filePath); + var target = this._linkTarget(key, resolvedKey); + if (target !== undefined) { + return shared.applyReadlinkEncoding(target, encoding); + } + // Same contract as classic mode: EINVAL for a path that is there but is + // not a link, ENOENT for one that is not there at all. The base class + // only knows the directory tree, so it cannot tell those apart. + if (typeof this._manifest.stats[resolvedKey] === 'object') { + throw _einval('readlink', filePath); + } + throw _enoent('readlink', filePath); + } + + realpathSync(filePath, options) { + // The base class only knows the directory tree built in the constructor, + // so without this every archive file resolves to ENOENT — which also + // breaks fs.readlinkSync, since the VFS answers readlink by way of + // realpath. Following the symlink chain here is the whole point. + var p = this._resolveSymlink(toManifestKey(filePath), 'realpath', filePath); + // typeof, not `in` or truthiness: the manifest is JSON-derived and read + // with a bracket index, so both would report `constructor`/`toString` as + // existing files. This does not block `__proto__` — typeof gives 'object' + // for that one — but a lookup key can only ever be `/`-prefixed + // (toManifestKey, and producer.ts snapshotifies every manifest key), so a + // bare inherited name cannot be formed. Same idiom as the resolver's + // `typeof === 'string'`. + // The base signature takes options and the VFS passes them through, so + // honour the encoding here the way readlinkSync does. + var encoding = shared.readlinkEncoding(options); + if (typeof this._manifest.stats[p] === 'object') { + return shared.applyReadlinkEncoding(p, encoding); + } + return shared.applyReadlinkEncoding(super.realpathSync(p), encoding); } statSync(filePath) { perf.count('statSync calls'); - var p = this._resolveSymlink(toManifestKey(filePath)); + var p = this._resolveSymlink(toManifestKey(filePath), 'stat', filePath); var meta = this._manifest.stats[p]; if (meta) { // Return a fresh stat object — matches Node.js fs.statSync contract. @@ -467,26 +536,118 @@ class SEAProvider extends MemoryProvider { * startup (~30K calls for large projects) so it must be as lean as possible. */ internalModuleStat(filePath) { - var p = this._resolveSymlink(toManifestKey(filePath)); + var p; + try { + p = this._resolveSymlink(toManifestKey(filePath), 'stat', filePath); + } catch (error) { + // Negative errno, not a throw: module resolution calls this and a cyclic + // manifest must not become an uncaught exception during require(). + if (error.code === 'ELOOP') return error.errno; + throw error; + } var meta = this._manifest.stats[p]; if (meta) { return meta.isDirectory ? 1 : 0; } - return -2; + return -shared.ERRNO.ENOENT; } - readdirSync(dirPath) { + readdirSync(dirPath, options) { perf.count('readdirSync calls'); - var p = this._resolveSymlink(toManifestKey(dirPath)); + var key = toManifestKey(dirPath); + var p = this._resolveSymlink(key, 'scandir', dirPath); var entries = this._manifest.directories[p]; - if (entries) return entries.slice(); - return super.readdirSync(p); + if (!entries) return super.readdirSync(p, options); + if (!options || !options.withFileTypes) return entries.slice(); + // Two bases, and both are needed. manifest.symlinks is keyed by the + // *unresolved* path the walker walked (appendSymlink in lib/walker.ts), so + // a link under a symlinked directory is only found under the caller's key; + // stats and directories are keyed by the resolved one. + var unresolvedBase = key.endsWith('/') ? key : key + '/'; + var resolvedBase = p.endsWith('/') ? p : p + '/'; + var parentPath = toPlatformPath(SNAPSHOT_PREFIX + key); + var self = this; + return entries.map(function (name) { + var type = self._direntType(unresolvedBase + name, resolvedBase + name); + return new shared.Dirent(name, type, parentPath); + }); + } + + // The type readdir reports for one entry, from the same records classic mode + // reads: an entry is a link when the manifest keys it as one, and what it + // points at is deliberately not consulted. + // "Is this path a link, and to what?" — one policy for lstatSync, readdir and + // readlinkSync, which otherwise each grew their own. manifest.symlinks is + // keyed by the unresolved path the walker walked (appendSymlink in + // lib/walker.ts), so that spelling wins; the resolved one is a fallback for a + // manifest that keyed it under an already-followed parent. + // The key a path has once its *parents* are followed but its own last + // component is not — what POSIX resolves before reading a link. Returns the + // key unchanged when nothing moved. + // Shared with classic mode, so the join rule lives in one place. + _resolveParentKey(key, syscall, forPath) { + return this._resolve.parent(key, syscall, forPath); + } + + _linkTarget(unresolvedKey, resolvedKey) { + var target = this._symlinks[unresolvedKey]; + if (typeof target === 'string') return target; + if (resolvedKey !== undefined && resolvedKey !== unresolvedKey) { + target = this._symlinks[resolvedKey]; + if (typeof target === 'string') return target; + } + return undefined; + } + + _direntType(unresolvedKey, resolvedKey) { + if (this._linkTarget(unresolvedKey, resolvedKey) !== undefined) { + return shared.UV_DIRENT_LINK; + } + if (Array.isArray(this._manifest.directories[resolvedKey])) { + return shared.UV_DIRENT_DIR; + } + var meta = this._manifest.stats[resolvedKey]; + if (typeof meta === 'object' && meta.isDirectory) { + return shared.UV_DIRENT_DIR; + } + return shared.UV_DIRENT_FILE; + } + + // lstat describes the link itself. The base class has no override, and the + // manifest stats are recorded through the link, so the target's stat is + // fetched and then given link semantics — same shape classic mode returns. + lstatSync(filePath) { + var key = toManifestKey(filePath); + // Resolve the parents before asking, so lstat agrees with readdir and + // readlink about which entries are links — all three go through + // _linkTarget with the same pair of keys. + var resolvedKey = this._resolveParentKey(key, 'lstat', filePath); + if (this._linkTarget(key, resolvedKey) === undefined) { + return this.statSync(filePath); + } + var target = this._linkTarget(key, resolvedKey); + var p = this._resolveSymlink(key, 'lstat', filePath); + var meta = this._manifest.stats[p]; + if (typeof meta !== 'object') throw _enoent('lstat', filePath); + return shared.asSymlinkStat(_makeStats(meta), target); } existsSync(filePath) { perf.count('existsSync calls'); - var p = this._resolveSymlink(toManifestKey(filePath)); - return p in this._manifest.stats; + var p; + try { + p = this._resolveSymlink(toManifestKey(filePath), 'access', filePath); + } catch (error) { + // fs.existsSync never throws — libuv swallows every errno and answers + // false. The VFS probes with this method before the real call, so it + // keeps the reason and re-raises it there (probeSync in + // @roberts_lando/vfs); a cycle still surfaces as ELOOP from stat, open, + // readdir and realpath rather than as a bare ENOENT. + if (error.code === 'ELOOP') return false; + throw error; + } + // typeof, not truthiness — see realpathSync. + return typeof this._manifest.stats[p] === 'object'; } } @@ -524,9 +685,26 @@ if (process.platform === 'win32') { VirtualFileSystem.prototype.resolvePath = function (inputPath) { return _origResolvePath.call(this, _winToVFS(inputPath)); }; + // ...and convert back on the way out. realpathSync is the only method that + // returns a path, and it rejoins the POSIX mount prefix, so without this it + // answers `/snapshot/app/x.js` where __filename is `C:\snapshot\app\x.js` + // — the two stop comparing equal and path.relative() against either breaks. + var _origRealpathSync = VirtualFileSystem.prototype.realpathSync; + VirtualFileSystem.prototype.realpathSync = function (filePath, options) { + return toPlatformPath(_origRealpathSync.call(this, filePath, options)); + }; } virtualFs.mount(SNAPSHOT_PREFIX, { overlay: true }); + +// fs.lstat and fs.readlink reach SEAProvider through @roberts_lando/vfs's own +// patches. They did not before: lstat went through findVFSForFsStat, which +// calls statSync and so follows the link, and readlink through +// findVFSForRealpath, which never reached the provider — so neither could +// report a symlink, and a SEA binary disagreed with a traditional one about +// lstatSync(link).isSymbolicLink() for the same source. Fixed upstream in +// robertsLando/vfs#4; this file carried a local re-patch until that landed. + perf.end('vfs mount + hooks'); // ///////////////////////////////////////////////////////////////// diff --git a/test/test-80-compression-node-opcua/main.js b/test/test-80-compression-node-opcua/main.js index 304beea70..7d2d7f1ae 100644 --- a/test/test-80-compression-node-opcua/main.js +++ b/test/test-80-compression-node-opcua/main.js @@ -6,6 +6,10 @@ * A test with a large number of modules with symlinks * (installed with npm) and compress * + * node-opcua is pinned to 2.181.0 in package.json: 2.184.0 made the packages + * ESM ("type": "module", import.meta.dirname), which esbuild's CJS bundle + * resolves to undefined — node-opcua-nodesets then calls path.join(undefined). + * Unpin only together with an esbuild config that emits ESM. */ const fs = require('fs'); diff --git a/test/test-80-compression-node-opcua/package.json b/test/test-80-compression-node-opcua/package.json index 6a60ac5fb..463968bb4 100644 --- a/test/test-80-compression-node-opcua/package.json +++ b/test/test-80-compression-node-opcua/package.json @@ -12,9 +12,9 @@ "author": "", "license": "ISC", "dependencies": { - "node-opcua-address-space": "^2.36.0", + "node-opcua-address-space": "2.181.0", "node-opcua-crypto": "^1.7.1", - "node-opcua-nodesets": "^2.36.0" + "node-opcua-nodesets": "2.181.0" }, "devDependencies": { "esbuild": "^0.20.0" diff --git a/test/test-99-#295/index.js b/test/test-99-#295/index.js new file mode 100644 index 000000000..bce9d00e9 --- /dev/null +++ b/test/test-99-#295/index.js @@ -0,0 +1,227 @@ +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const log = require('./lib/log'); + +// `lib` links to `reallib`, and `reallib/inner.js` links to `log.js` inside +// it. The walker records both, so this path has its own manifest entry that +// must win over its symlinked parent. +require('./lib/inner.js'); + +// Windows can refuse to create the nested *file* link, in which case main.js +// leaves a plain copy in its place. The directory link is a real junction +// there, so the parent walk is covered either way. +const { nestedIsLink } = require('./linkinfo.json'); + +const nested = path.join(__dirname, 'lib', 'inner.js'); + +// realpath must follow the chain rather than throwing ENOENT. +assert.strictEqual( + path.basename(fs.realpathSync(nested)), + nestedIsLink ? 'log.js' : 'inner.js', +); +assert.strictEqual( + path.basename(fs.realpathSync(path.join(__dirname, 'lib', 'log.js'))), + 'log.js', +); + +// ...and it must answer in the platform's own path form. SEA mounts the VFS +// under a POSIX '/snapshot' prefix, so on Windows the resolved path has to be +// converted back to `C:\snapshot\...` before it leaves fs (yao-pkg/pkg#305). +// A non-link round-trips to itself, which makes this a no-op off Windows. +assert.strictEqual( + fs.realpathSync(__filename), + __filename, + 'realpath must round-trip a non-link in the platform path form', +); + +// Both modes answer readlink now: SEA by way of realpath through the VFS +// polyfill, classic from the SYMLINKS record (#296). +if (nestedIsLink) { + assert.strictEqual(path.basename(fs.readlinkSync(nested)), 'log.js'); +} + +// readlink on a path that exists but is not a link is EINVAL, not ENOENT — +// in both modes now (#296). +const notALink = path.join(__dirname, 'index.js'); +assert.throws(() => fs.readlinkSync(notALink), { code: 'EINVAL' }); + +// ...and the error names the path the caller asked about, not the internal +// mount-relative key the SEA provider works in. +try { + fs.readlinkSync(notALink); +} catch (err) { + assert.strictEqual(err.path, notALink, 'EINVAL must name the caller path'); +} + +// readlink honours its encoding option in both modes. +if (nestedIsLink) { + const asBuffer = fs.readlinkSync(nested, 'buffer'); + assert.ok(Buffer.isBuffer(asBuffer), "readlink 'buffer' must give a Buffer"); + assert.strictEqual( + asBuffer.toString(), + fs.readlinkSync(nested), + 'the buffer and string forms must agree', + ); + assert.ok( + Buffer.isBuffer(fs.readlinkSync(nested, { encoding: 'buffer' })), + 'the { encoding } form must work too', + ); +} + +// readdir must return a usable listing in both modes. SEA builds its listing +// from manifest.directories, which holds only the paths the walker recorded, +// so which entries appear there is not asserted — only that it works at all. +const dirents = fs.readdirSync(__dirname, { withFileTypes: true }); +assert.ok( + Array.isArray(dirents) && dirents.length > 0, + 'readdir returned nothing', +); + +// readdir is lstat-based, so a link reports as a link rather than as the +// directory it points at — same as outside a packaged binary — and lstat must +// agree with the dirent. Both modes answer from their own symlink record +// since #296, so this is asserted for both. +{ + const libEntry = dirents.find((e) => e.name === 'lib'); + assert.ok(libEntry, 'lib missing from readdir'); + assert.strictEqual(libEntry.isSymbolicLink(), true); + assert.strictEqual(libEntry.isDirectory(), false); + const reallibEntry = dirents.find((e) => e.name === 'reallib'); + assert.ok(reallibEntry, 'reallib missing from readdir'); + assert.strictEqual(reallibEntry.isSymbolicLink(), false); + assert.strictEqual(reallibEntry.isDirectory(), true); + + // lstat describes the link itself; stat follows it. readdir and lstat must + // not contradict each other about the same entry. + const libPath = path.join(__dirname, 'lib'); + assert.strictEqual(fs.lstatSync(libPath).isSymbolicLink(), true); + assert.strictEqual(fs.lstatSync(libPath).isDirectory(), false); + assert.strictEqual(fs.statSync(libPath).isDirectory(), true); + assert.strictEqual(fs.statSync(libPath).isSymbolicLink(), false); + + // readlink round-trips the directory link too. + assert.strictEqual(path.basename(fs.readlinkSync(libPath)), 'reallib'); + + // The type bits have to agree with the predicate: consumers that sniff + // `mode & S_IFMT` (tar, archiver, fs.cp) read those, not isSymbolicLink(). + const S_IFMT = 0o170000; + const S_IFLNK = 0o120000; + assert.strictEqual(fs.lstatSync(libPath).mode & S_IFMT, S_IFLNK); +} + +// path.join(d.parentPath, d.name) is the documented way to use withFileTypes, +// and what `recursive: true` consumers do. Undefined here throws. +for (const d of dirents) { + assert.strictEqual( + typeof d.parentPath, + 'string', + `dirent ${d.name} is missing parentPath`, + ); + assert.ok(fs.existsSync(path.join(d.parentPath, d.name))); +} + +// readdir *of the linked directory*: manifest.symlinks is keyed by the +// unresolved path, so resolving `lib` first and then looking entries up under +// `reallib` would silently report the nested link as a plain file. +const libDirents = fs.readdirSync(path.join(__dirname, 'lib'), { + withFileTypes: true, +}); +const innerEntry = libDirents.find((e) => e.name === 'inner.js'); +assert.ok(innerEntry, 'inner.js missing from readdir of the linked directory'); +assert.strictEqual(innerEntry.isSymbolicLink(), nestedIsLink); + +// The manifest records are read with a bracket index, so a key inherited from +// Object.prototype must not read as a packaged file in either mode. +for (const inherited of ['constructor', 'toString', '__proto__']) { + assert.strictEqual( + fs.existsSync(path.join(__dirname, inherited)), + false, + `${inherited} must not report as an existing snapshot file`, + ); +} + +// The callback and promise forms go through separate patches in both modes, +// and nothing exercised them before — which is how two ELOOP bugs reached the +// third review round. Run them before handing back to the harness. +const pending = []; + +if (nestedIsLink) { + pending.push( + new Promise((resolve, reject) => { + fs.readlink(nested, (err, target) => { + if (err) return reject(err); + try { + assert.strictEqual(path.basename(target), 'log.js'); + resolve(); + } catch (e) { + reject(e); + } + }); + }), + fs.promises + .readlink(nested) + .then((t) => assert.strictEqual(path.basename(t), 'log.js')), + ); +} + +pending.push( + new Promise((resolve, reject) => { + fs.lstat(path.join(__dirname, 'lib'), (err, st) => { + if (err) return reject(err); + try { + assert.strictEqual(st.isSymbolicLink(), true); + assert.strictEqual(st.isDirectory(), false); + resolve(); + } catch (e) { + reject(e); + } + }); + }), + fs.promises + .lstat(path.join(__dirname, 'lib')) + .then((st) => assert.strictEqual(st.isSymbolicLink(), true)), + new Promise((resolve, reject) => { + fs.readdir(__dirname, { withFileTypes: true }, (err, list) => { + if (err) return reject(err); + try { + const lib = list.find((e) => e.name === 'lib'); + assert.ok(lib, 'lib missing from async readdir'); + assert.strictEqual(lib.isSymbolicLink(), true); + assert.strictEqual(typeof lib.parentPath, 'string'); + resolve(); + } catch (e) { + reject(e); + } + }); + }), + // readlink on a non-link must reach the callback as an error, never throw. + new Promise((resolve, reject) => { + let threw = false; + try { + fs.readlink(notALink, (err) => { + try { + assert.ok(err, 'async readlink on a non-link must report an error'); + assert.strictEqual(err.code, 'EINVAL'); + resolve(); + } catch (e) { + reject(e); + } + }); + } catch { + threw = true; + } + if (threw) reject(new Error('fs.readlink threw synchronously')); + }), +); + +Promise.all(pending).then( + () => log(42), + (err) => { + console.error(err); + process.exit(1); + }, +); diff --git a/test/test-99-#295/main.js b/test/test-99-#295/main.js new file mode 100644 index 000000000..b46a4b1c5 --- /dev/null +++ b/test/test-99-#295/main.js @@ -0,0 +1,80 @@ +#!/usr/bin/env node + +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const utils = require('../utils.js'); + +// Enhanced SEA requires Node.js >= 22 +if (utils.getNodeMajorVersion() < 22) { + return; +} + +assert(__dirname === process.cwd()); + +// The links are built here instead of being committed: git on Windows checks +// a committed symlink out as a text file holding its target, which would make +// pkg bytecode-compile `./log.js` as if it were source and fail the *build*. +// Building them at test time also lets Windows use a junction — the shape npm +// actually creates for workspace links, which is what #295 was reported on. +const libLink = path.join(__dirname, 'lib'); +const innerLink = path.join(__dirname, 'reallib', 'inner.js'); +const linkInfo = path.join(__dirname, 'linkinfo.json'); +const generated = [libLink, innerLink, linkInfo]; + +function removeGenerated() { + for (const p of generated) utils.vacuum.sync(p); +} + +removeGenerated(); + +fs.symlinkSync( + path.join(__dirname, 'reallib'), + libLink, + process.platform === 'win32' ? 'junction' : 'dir', +); + +// A *file* symlink needs Developer Mode or elevation on Windows. Fall back to +// a plain copy there: the directory junction still exercises the parent walk, +// and index.js relaxes the nested assertions to match. +let nestedIsLink = true; +try { + fs.symlinkSync('log.js', innerLink, 'file'); +} catch (error) { + if (process.platform !== 'win32') throw error; + fs.copyFileSync(path.join(__dirname, 'reallib', 'log.js'), innerLink); + nestedIsLink = false; +} +fs.writeFileSync(linkInfo, `${JSON.stringify({ nestedIsLink }, null, 2)}\n`); + +try { + const input = './package.json'; + const testName = 'test-99-#295'; + const standardOutput = 'test-output.exe'; + + const expectedOutput = '42\n'; + + const newcomers = utils.seaHostOutputs(testName).concat(standardOutput); + + const before = utils.filesBefore(newcomers); + + // SEA mode — the mode #295 was reported against. + utils.runSeaHostOnly(input, testName); + utils.assertSeaOutput(testName, expectedOutput); + + // Standard mode resolves symlinks through the same shared helper, so it + // needs the same fixture: bootstrap.js was rewritten onto that helper in + // #296 and would otherwise have no end-to-end coverage of the parent-symlink + // walk. + utils.pkg.sync(['--target', 'host', '--output', standardOutput, input]); + assert.strictEqual( + utils.spawn.sync(`./${standardOutput}`, []), + expectedOutput, + ); + + utils.filesAfter(before, newcomers, { tolerateWindowsEbusy: true }); +} finally { + removeGenerated(); +} diff --git a/test/test-99-#295/package.json b/test/test-99-#295/package.json new file mode 100644 index 000000000..33fd9e6eb --- /dev/null +++ b/test/test-99-#295/package.json @@ -0,0 +1,6 @@ +{ + "name": "test-99-295", + "version": "1.0.0", + "main": "index.js", + "bin": "index.js" +} diff --git a/test/test-99-#295/reallib/log.js b/test/test-99-#295/reallib/log.js new file mode 100644 index 000000000..2e92b2d93 --- /dev/null +++ b/test/test-99-#295/reallib/log.js @@ -0,0 +1,3 @@ +'use strict'; + +module.exports = console.log; diff --git a/test/test.js b/test/test.js index 87e6dad8a..3541f2e78 100644 --- a/test/test.js +++ b/test/test.js @@ -85,6 +85,7 @@ const npmTests = [ 'test-91-sea-esm-entry', 'test-92-sea-tla', 'test-94-sea-esm-import-meta', + 'test-99-#295', ]; if (testFilter) { diff --git a/test/unit/dirent-shared.test.ts b/test/unit/dirent-shared.test.ts new file mode 100644 index 000000000..a471feb7a --- /dev/null +++ b/test/unit/dirent-shared.test.ts @@ -0,0 +1,100 @@ +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { describe, it } from 'node:test'; + +const shared = createRequire(__filename)('../../prelude/bootstrap-shared.js'); + +const { Dirent, asSymlinkStat, UV_DIRENT_FILE, UV_DIRENT_DIR, UV_DIRENT_LINK } = + shared as { + Dirent: new ( + _name: string, + _type: number, + ) => { + name: string; + isFile(): boolean; + isDirectory(): boolean; + isSymbolicLink(): boolean; + isBlockDevice(): boolean; + isCharacterDevice(): boolean; + isSocket(): boolean; + isFIFO(): boolean; + }; + asSymlinkStat: (_s: T) => T; + UV_DIRENT_FILE: number; + UV_DIRENT_DIR: number; + UV_DIRENT_LINK: number; + }; + +const S_IFMT = 0o170000; +const S_IFLNK = 0o120000; +const S_IFREG = 0o100000; +const S_IFDIR = 0o040000; + +// Dirent and asSymlinkStat back readdir({ withFileTypes: true }) and lstat in +// *both* bootstraps (#296), so a change here moves classic and SEA together. +describe('shared Dirent', () => { + it('reports exactly one type per libuv constant', () => { + const cases: [number, 'isFile' | 'isDirectory' | 'isSymbolicLink'][] = [ + [UV_DIRENT_FILE, 'isFile'], + [UV_DIRENT_DIR, 'isDirectory'], + [UV_DIRENT_LINK, 'isSymbolicLink'], + ]; + for (const [type, predicate] of cases) { + const d = new Dirent('entry', type); + for (const p of ['isFile', 'isDirectory', 'isSymbolicLink'] as const) { + assert.equal(d[p](), p === predicate, `${p} for type ${type}`); + } + } + }); + + it('uses the libuv numbering real readdir reports', () => { + assert.deepEqual( + [UV_DIRENT_FILE, UV_DIRENT_DIR, UV_DIRENT_LINK], + [1, 2, 3], + ); + }); + + it('keeps the name it was built with and answers false for device types', () => { + const d = new Dirent('node_modules', UV_DIRENT_LINK, '/snapshot/app'); + assert.equal(d.name, 'node_modules'); + assert.equal(d.isBlockDevice(), false); + assert.equal(d.isCharacterDevice(), false); + assert.equal(d.isSocket(), false); + assert.equal(d.isFIFO(), false); + }); + + it('carries parentPath and its deprecated `path` alias', () => { + // path.join(d.parentPath, d.name) is the documented way to use + // withFileTypes, so an undefined parentPath throws ERR_INVALID_ARG_TYPE. + const d = new Dirent('lib', UV_DIRENT_LINK, '/snapshot/app'); + assert.equal(d.parentPath, '/snapshot/app'); + assert.equal(d.path, '/snapshot/app', "`path` is Node's alias for it"); + }); +}); + +describe('asSymlinkStat', () => { + it('flips the predicates a stat taken through the link got wrong', () => { + const s = asSymlinkStat({ + mode: S_IFDIR | 0o755, + isFile: () => false, + isDirectory: () => true, + isSymbolicLink: () => false, + }); + assert.equal(s.isSymbolicLink(), true); + assert.equal(s.isDirectory(), false); + assert.equal(s.isFile(), false); + }); + + it('rewrites the mode type bits without disturbing the permissions', () => { + for (const base of [S_IFREG | 0o644, S_IFDIR | 0o755]) { + const s = asSymlinkStat({ mode: base }) as { mode: number }; + assert.equal(s.mode & S_IFMT, S_IFLNK, 'type bits must say S_IFLNK'); + assert.equal(s.mode & 0o777, base & 0o777, 'permissions must survive'); + } + }); + + it('leaves a stat with no numeric mode alone', () => { + const s = asSymlinkStat({}) as { mode?: number }; + assert.equal(s.mode, undefined); + }); +}); diff --git a/test/unit/resolve-symlink.test.ts b/test/unit/resolve-symlink.test.ts new file mode 100644 index 000000000..8f045cb5c --- /dev/null +++ b/test/unit/resolve-symlink.test.ts @@ -0,0 +1,428 @@ +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { describe, it } from 'node:test'; + +const shared = createRequire(__filename)('../../prelude/bootstrap-shared.js'); +const makeSymlinkResolver = shared.makeSymlinkResolver as ( + _symlinks: Record, + _sep: string, +) => ((_p: string, _syscall?: string, _forPath?: string) => string) & { + parent: (_p: string, _syscall?: string, _forPath?: string) => string; +}; + +// makeSymlinkResolver() backs both the classic bootstrap (prelude/bootstrap.js) +// and the SEA VFS provider (prelude/sea-vfs-setup.js) — see #295/#296. These +// are table-driven pure-logic tests against the shared implementation +// directly, requested during PR review as a complement to the e2e +// test-99-#295 (which only covers one level of symlink nesting end to end). +describe('makeSymlinkResolver', () => { + it('returns non-symlinked paths unchanged', () => { + const resolve = makeSymlinkResolver( + { '/snapshot/linked': '/snapshot/real' }, + '/', + ); + assert.equal(resolve('/snapshot/other/file.js'), '/snapshot/other/file.js'); + }); + + it('resolves an exact match (the path itself is the symlink)', () => { + const resolve = makeSymlinkResolver( + { '/snapshot/linked': '/snapshot/real' }, + '/', + ); + assert.equal(resolve('/snapshot/linked'), '/snapshot/real'); + }); + + it('does not match a key that is only a string prefix of the path', () => { + // The scan slices at separator offsets, so /snapshot/foo must not swallow + // /snapshot/foobar. A naive startsWith() would pass every other case here. + const resolve = makeSymlinkResolver( + { '/snapshot/foo': '/snapshot/real' }, + '/', + ); + assert.equal( + resolve('/snapshot/foobar/x.js'), + '/snapshot/foobar/x.js', + 'sibling with a longer name must be left alone', + ); + assert.equal(resolve('/snapshot/foo/x.js'), '/snapshot/real/x.js'); + }); + + it('resolves a nested path under a symlinked directory', () => { + const resolve = makeSymlinkResolver( + { '/snapshot/linked': '/snapshot/real' }, + '/', + ); + assert.equal( + resolve('/snapshot/linked/lib/deep/file.js'), + '/snapshot/real/lib/deep/file.js', + ); + }); + + it('resolves the deepest matching symlink (longest prefix wins)', () => { + // Two symlinks where one key is a literal prefix of the other. The walker + // keys entries on the path it walked, *before* resolution, and every + // target is already fully realpath'd — so the deeper key is the complete + // answer and taking the shallower one would strand the walk on a path the + // archive has no entry for. + const resolve = makeSymlinkResolver( + { + '/a': '/shallow-target', + '/a/b': '/deep-target', + }, + '/', + ); + assert.equal(resolve('/a/b/c'), '/deep-target/c'); + }); + + it('follows a directory symlink nested inside another one', () => { + // The real manifest shape behind the case above: `walker.appendSymlink` + // records `/sub` under the unresolved path because it descended + // through the `` link to reach it. Resolving the parent first would + // yield /app/reallib/sub/file.js, which the archive has no entry for. + const resolve = makeSymlinkResolver( + { + '/app/lib': '/app/reallib', + '/app/lib/sub': '/app/reallib/realsub', + }, + '/', + ); + assert.equal( + resolve('/app/lib/sub/file.js'), + '/app/reallib/realsub/file.js', + ); + // A sibling with no entry of its own still follows the parent link. + assert.equal(resolve('/app/lib/other.js'), '/app/reallib/other.js'); + }); + + it('prefers an exact entry over its symlinked parent', () => { + // The real manifest shape: the walker descends through a symlinked + // directory, so a link inside one gets its own key under the unresolved + // path. Both keys exist, and the exact (more specific) one must win — + // resolving through the parent instead would land on a path the archive + // has no entry for. Regression guard for test-99-#295/reallib/inner.js. + const resolve = makeSymlinkResolver( + { + '/app/lib': '/app/reallib', + '/app/lib/inner.js': '/app/reallib/log.js', + }, + '/', + ); + assert.equal(resolve('/app/lib/inner.js'), '/app/reallib/log.js'); + // A path with no exact entry still follows the symlinked parent. + assert.equal(resolve('/app/lib/sub/deep.js'), '/app/reallib/sub/deep.js'); + }); + + it('chains through multiple independent symlinks', () => { + const resolve = makeSymlinkResolver( + { + '/a': '/b', + '/b/c': '/d', + }, + '/', + ); + // /a/c/file.js -> (hop 1: /a -> /b) /b/c/file.js + // -> (hop 2: /b/c -> /d) /d/file.js + assert.equal(resolve('/a/c/file.js'), '/d/file.js'); + }); + + it('avoids a double separator when the target ends with one', () => { + // Regression case from review: a symlink whose target is the bare root. + const resolve = makeSymlinkResolver({ '/node_modules/@t/root': '/' }, '/'); + assert.equal( + resolve('/node_modules/@t/root/package.json'), + '/package.json', + ); + }); + + it('avoids a double separator for any target ending with a separator, not just root', () => { + const resolve = makeSymlinkResolver( + { '/snapshot/linked': '/snapshot/real/' }, + '/', + ); + assert.equal(resolve('/snapshot/linked/file.js'), '/snapshot/real/file.js'); + }); + + it('throws ELOOP on a cyclic manifest instead of hanging', () => { + const resolve = makeSymlinkResolver({ '/a': '/a/b' }, '/'); + assert.throws( + () => resolve('/a/x'), + (err: NodeJS.ErrnoException) => { + assert.equal(err.code, 'ELOOP'); + return true; + }, + ); + }); + + it('throws ELOOP on a cycle spanning two entries', () => { + const resolve = makeSymlinkResolver({ '/a': '/b/x', '/b': '/a' }, '/'); + assert.throws( + () => resolve('/a/f.js'), + (err: NodeJS.ErrnoException) => { + assert.equal(err.code, 'ELOOP'); + return true; + }, + ); + }); + + it('gives ELOOP the errno shape Node uses for the platform', () => { + // libuv numbers ELOOP differently on Windows (uv/errno.h: -4067 vs -40). + const expected = process.platform === 'win32' ? -4067 : -40; + const resolve = makeSymlinkResolver({ '/a': '/a/b' }, '/'); + assert.throws( + () => resolve('/a/x'), + (err: NodeJS.ErrnoException) => { + assert.equal(err.code, 'ELOOP'); + assert.equal(err.errno, expected); + assert.equal(err.path, '/a/x'); + return true; + }, + ); + }); + + it("reports the caller's syscall, defaulting to stat", () => { + const resolve = makeSymlinkResolver({ '/a': '/a/b' }, '/'); + assert.throws( + () => resolve('/a/x'), + (err: NodeJS.ErrnoException) => { + assert.equal(err.syscall, 'stat'); + assert.match(err.message, /^ELOOP: .*, stat '\/a\/x'$/); + return true; + }, + ); + assert.throws( + () => resolve('/a/x', 'realpath'), + (err: NodeJS.ErrnoException) => { + assert.equal(err.syscall, 'realpath'); + assert.match(err.message, /realpath '\/a\/x'$/); + return true; + }, + ); + }); + + it("reports the caller's path, not the vfs key, when given one", () => { + // Under DOCOMPRESS the key is base36, so the bare key means nothing to the + // user reading the error. + const resolve = makeSymlinkResolver({ '/1': '/1/2' }, '/'); + assert.throws( + () => resolve('/1/2', 'stat', '/snapshot/app/lib/index.js'), + (err: NodeJS.ErrnoException) => { + assert.equal(err.code, 'ELOOP'); + assert.equal(err.path, '/snapshot/app/lib/index.js'); + assert.match(err.message, /'\/snapshot\/app\/lib\/index\.js'$/); + return true; + }, + ); + }); + + it('marks ELOOP as pkg-originated, like the other snapshot errors', () => { + const resolve = makeSymlinkResolver({ '/a': '/a/b' }, '/'); + assert.throws( + () => resolve('/a/x'), + ( + err: NodeJS.ErrnoException & { + pkg?: boolean; + }, + ) => { + assert.equal(err.pkg, true); + return true; + }, + ); + }); + + describe('resolveKey.parent — the readlink/lstat step', () => { + // POSIX resolves a path's parents before reading its last component, so an + // entry recorded under an already-followed parent stays reachable. Both + // bootstraps drive this, which is why it lives on the resolver. + it('resolves the parents and keeps the last component', () => { + const resolve = makeSymlinkResolver( + { '/snapshot/lib': '/snapshot/reallib' }, + '/', + ); + assert.equal( + resolve.parent('/snapshot/lib/inner.js'), + '/snapshot/reallib/inner.js', + ); + }); + + it('does not follow the last component itself', () => { + // The whole point: resolve() would answer /snapshot/reallib here. + const resolve = makeSymlinkResolver( + { '/snapshot/lib': '/snapshot/reallib' }, + '/', + ); + assert.equal(resolve.parent('/snapshot/lib'), '/snapshot/lib'); + }); + + it('does not double the separator when the target ends in one', () => { + const resolve = makeSymlinkResolver({ '/snapshot/lib': '/real/' }, '/'); + assert.equal(resolve.parent('/snapshot/lib/x.js'), '/real/x.js'); + }); + + it('is present on the empty-manifest fast path too', () => { + // The identity resolver is what a symlink-free binary gets, which is + // most of them — readlink and lstat call .parent unconditionally. + const resolve = makeSymlinkResolver({}, '/'); + assert.equal(typeof resolve.parent, 'function'); + assert.equal( + resolve.parent('/snapshot/app/lib/inner.js'), + '/snapshot/app/lib/inner.js', + ); + }); + + it('leaves a key with no parent alone', () => { + const resolve = makeSymlinkResolver({ '/a': '/b' }, '/'); + assert.equal(resolve.parent('/a'), '/a'); + assert.equal(resolve.parent('a'), 'a'); + }); + + it('raises ELOOP from the parent walk, naming the caller', () => { + const resolve = makeSymlinkResolver({ '/a': '/a/b' }, '/'); + assert.throws( + () => resolve.parent('/a/x/y.js', 'readlink', '/snapshot/a/x/y.js'), + (err: NodeJS.ErrnoException) => { + assert.equal(err.code, 'ELOOP'); + assert.equal(err.syscall, 'readlink'); + assert.equal(err.path, '/snapshot/a/x/y.js'); + return true; + }, + ); + }); + + it('walks win32 keys on their own separator', () => { + const resolve = makeSymlinkResolver( + { 'C:\\snapshot\\lib': 'C:\\snapshot\\reallib' }, + '\\', + ); + assert.equal( + resolve.parent('C:\\snapshot\\lib\\inner.js'), + 'C:\\snapshot\\reallib\\inner.js', + ); + }); + }); + + it("does not leak the previous call's syscall into the next", () => { + const resolve = makeSymlinkResolver({ '/a': '/a/b' }, '/'); + assert.throws(() => resolve('/a/x', 'readlink'), { syscall: 'readlink' }); + assert.throws(() => resolve('/a/x'), { syscall: 'stat' }); + }); + + it('keeps throwing ELOOP on a repeat lookup', () => { + // The in-progress sentinel must not be left behind in the memo, or a + // caught ELOOP would poison unrelated later lookups. + const resolve = makeSymlinkResolver({ '/a': '/a/b' }, '/'); + assert.throws(() => resolve('/a/x'), { code: 'ELOOP' }); + assert.throws(() => resolve('/a/y'), { code: 'ELOOP' }); + }); + + describe('the MAX_SYMLINK_DEPTH bound', () => { + // Chain of `n` links ending at '/end': /l0 -> /l1 -> ... -> /ln -> /end. + const chain = (n: number) => { + const m: Record = {}; + for (let i = 0; i < n; i += 1) m[`/l${i}`] = `/l${i + 1}`; + m[`/l${n}`] = '/end'; + return m; + }; + + it('does not depend on which path was resolved first', () => { + // A cache hit hands back a target that stands for many hops. Those hops + // have to be charged back, or warming the tail of an over-long chain + // would let the head through the bound that a cold lookup rejects. + const cold = makeSymlinkResolver(chain(41), '/'); + assert.throws(() => cold('/l0'), { code: 'ELOOP' }); + + const warm = makeSymlinkResolver(chain(41), '/'); + warm('/l20'); + assert.throws(() => warm('/l0'), { code: 'ELOOP' }); + }); + + it('still resolves a chain that fits, warm or cold', () => { + const cold = makeSymlinkResolver(chain(30), '/'); + assert.equal(cold('/l0'), '/end'); + + const warm = makeSymlinkResolver(chain(30), '/'); + warm('/l15'); + assert.equal(warm('/l0'), '/end'); + }); + + it('charges each key only for its own hops', () => { + // '/end/short' is one hop, but it is first reached at the tail of a long + // chain. Billing it the whole chain's depth would make a later, shallow + // lookup through it blow the bound for no reason. + const symlinks: Record = chain(38); + symlinks['/end/short'] = '/y'; + symlinks['/p'] = '/q'; + symlinks['/q'] = '/r'; + symlinks['/r'] = '/end/short'; + const resolve = makeSymlinkResolver(symlinks, '/'); + assert.equal(resolve('/l0/short/f.js'), '/y/f.js'); + assert.equal(resolve('/p/f.js'), '/y/f.js'); + }); + }); + + it('is separator-agnostic (works with a non-"/" separator)', () => { + const resolve = makeSymlinkResolver( + { '\\snapshot\\linked': '\\snapshot\\real' }, + '\\', + ); + assert.equal( + resolve('\\snapshot\\linked\\file.js'), + '\\snapshot\\real\\file.js', + ); + }); + + describe('empty manifest', () => { + it('returns every path unchanged', () => { + const resolve = makeSymlinkResolver({}, '/'); + assert.equal(resolve('/snapshot/app/index.js'), '/snapshot/app/index.js'); + }); + + it('tolerates an absent symlinks record', () => { + const resolve = makeSymlinkResolver( + undefined as unknown as Record, + '/', + ); + assert.equal(resolve('/snapshot/app/index.js'), '/snapshot/app/index.js'); + }); + }); + + describe('inherited Object properties', () => { + // The manifest record is JSON-derived and read with a bracket index, so + // a path component that names an Object.prototype key must not match. + for (const key of ['__proto__', 'constructor', 'toString', 'valueOf']) { + it(`does not treat "${key}" as a symlink`, () => { + const resolve = makeSymlinkResolver({ '/snapshot/x': '/y' }, '/'); + assert.equal(resolve(`/${key}/file.js`), `/${key}/file.js`); + assert.equal(resolve(`/${key}`), `/${key}`); + }); + } + }); + + describe('memoisation', () => { + it('memoises the symlink hop, not the caller path', () => { + // Resolve one file, then mutate the manifest and resolve a *different* + // file under the same link. The stale target proves the memo is keyed + // on the manifest entry — a cache keyed on the full caller path would + // re-walk here, and would grow without bound on caller-supplied paths. + const symlinks: Record = { + '/snapshot/linked': '/snapshot/real', + }; + const resolve = makeSymlinkResolver(symlinks, '/'); + assert.equal(resolve('/snapshot/linked/a.js'), '/snapshot/real/a.js'); + + symlinks['/snapshot/linked'] = '/snapshot/changed'; + assert.equal(resolve('/snapshot/linked/b.js'), '/snapshot/real/b.js'); + }); + + it('gives each resolver its own memo', () => { + const symlinks: Record = { + '/snapshot/linked': '/snapshot/real', + }; + const first = makeSymlinkResolver(symlinks, '/'); + assert.equal(first('/snapshot/linked/a.js'), '/snapshot/real/a.js'); + + symlinks['/snapshot/linked'] = '/snapshot/changed'; + const second = makeSymlinkResolver(symlinks, '/'); + assert.equal(second('/snapshot/linked/a.js'), '/snapshot/changed/a.js'); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index e599886ef..bc675f81e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -636,10 +636,9 @@ conventional-recommended-bump "^11.2.0" semver "^7.7.3" -"@roberts_lando/vfs@^0.3.3": +"@roberts_lando/vfs@robertsLando/vfs#19a694fc4c8524009aa3e597e456e6e20fe30c31": version "0.3.3" - resolved "https://registry.yarnpkg.com/@roberts_lando/vfs/-/vfs-0.3.3.tgz#6359feed30e3773041279cb11be82ca60f4ff1f1" - integrity sha512-YjkxVSLw5WMZQoARaryRAjcxA+GbBzWMJdwYZX5oLUt9cC/gew9as4Dn7tcLzPp7BPoR221VpTZ+78TRPawnjg== + resolved "https://codeload.github.com/robertsLando/vfs/tar.gz/19a694fc4c8524009aa3e597e456e6e20fe30c31" "@rtsao/scc@^1.1.0": version "1.1.0"