diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fea73c77..1b756adf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,38 @@ jobs: exit 1 fi + # The napi addon is only ever loaded by Node, so cargo's own tests never + # exercise its JS loader or the engine/provider boundary. + node-addon: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Rust + run: rustup show + + - name: Cache cargo + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20.20.2' + + - name: Build addon + env: + SOCKET_PATCH_NODE_CARGO_PROFILE: dev + run: node crates/socket-patch-node/npm/scripts/build-addon.mjs + + - name: Smoke-test addon + run: node --test crates/socket-patch-node/npm/test/smoke.mjs + # Lint the out-of-workspace packaging artifacts: the RubyGems CLI launcher # gem + the Bundler plugin gem (Ruby), and the curl|sh installer. Ruby is # pre-installed on the ubuntu-latest runner. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ce21c1e3..7c2abcb4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -175,11 +175,11 @@ jobs: - name: Build (cargo) if: matrix.build-tool == 'cargo' - run: cargo build --release --target ${{ matrix.target }} + run: cargo build --release -p socket-patch-cli --target ${{ matrix.target }} - name: Build (cross) if: matrix.build-tool == 'cross' - run: cross build --release --target ${{ matrix.target }} + run: cross build --release -p socket-patch-cli --target ${{ matrix.target }} - name: Package (unix) if: matrix.archive == 'tar.gz' diff --git a/Cargo.lock b/Cargo.lock index 63d88559..7b687168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,6 +257,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "convert_case" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -319,6 +328,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" + [[package]] name = "deadpool" version = "0.12.3" @@ -955,6 +970,16 @@ version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1015,6 +1040,65 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "napi" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b12638855cec016784a7e17cc4ecbdee7ca8b0a3456aa50640d0ee5ed1a42f8" +dependencies = [ + "bitflags 2.11.0", + "ctor", + "futures", + "libc", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash", + "tokio", +] + +[[package]] +name = "napi-build" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0941983559cd62bfa7f04646a16428967a847365adeef27711e4e3927046bbed" + +[[package]] +name = "napi-derive" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1d232c24dc6754979b27efdf2047a5849efc7c92c1245de87e6bedbb9f5c3c" +dependencies = [ + "convert_case", + "ctor", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "napi-derive-backend" +version = "6.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6856fe1686867da90ed0dc292d7c0c87a695eb7c578b913360f33619ee01e3" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "semver", + "syn", +] + +[[package]] +name = "napi-sys" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6801fc65ba4d7035636e93635e281113f00410401a237dcd4f55b1565ead2115" +dependencies = [ + "libloading", +] + [[package]] name = "nix" version = "0.28.0" @@ -1027,6 +1111,12 @@ dependencies = [ "libc", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "num-traits" version = "0.2.19" @@ -1691,6 +1781,7 @@ dependencies = [ "tar", "tempfile", "tokio", + "tokio-util", "uuid", "windows-sys 0.59.0", "wiremock", @@ -1732,6 +1823,21 @@ dependencies = [ "zip", ] +[[package]] +name = "socket-patch-node" +version = "4.0.0" +dependencies = [ + "napi", + "napi-build", + "napi-derive", + "serde", + "serde_json", + "socket-patch-cli", + "socket-patch-core", + "tokio", + "tokio-util", +] + [[package]] name = "socket2" version = "0.6.2" @@ -2065,6 +2171,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index ce052f34..b0e6e7ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,13 @@ members = [ "crates/socket-patch-core", "crates/socket-patch-cli", + "crates/socket-patch-node", +] +# Bare `cargo build` (release.yml's per-target builds) skips the Node addon, +# which no release artifact ships; `--workspace` and `-p` still build it. +default-members = [ + "crates/socket-patch-core", + "crates/socket-patch-cli", ] resolver = "2" @@ -21,6 +28,7 @@ sha1 = "=0.10.6" hex = "=0.4.3" reqwest = { version = "=0.12.28", features = ["rustls-tls", "json"], default-features = false } tokio = { version = "=1.50.0", features = ["full"] } +tokio-util = "=0.7.18" futures-util = { version = "=0.3.32", default-features = false, features = ["std"] } thiserror = "=2.0.18" walkdir = "=2.5.0" @@ -48,6 +56,9 @@ wiremock = "=0.6.5" portable-pty = "=0.9.0" base64 = "=0.22.1" serial_test = "=3.4.0" +napi = { version = "=3.13.0", features = ["napi8", "tokio_rt"] } +napi-derive = "=3.6.9" +napi-build = "=2.5.0" [profile.release] strip = true @@ -65,6 +76,12 @@ inherits = "release" lto = false strip = "none" # unstripped test binaries => usable backtraces on failure +# The Node addon (crates/socket-patch-node) for hosts that load it +# in-process: release semantics without the slow full-LTO link. +[profile.addon] +inherits = "release" +lto = "thin" + # Test-execution speed: `cargo test` builds dependencies with the dev # profile; at opt-level 0 the hash/compression/bsdiff hot loops are 10-100x # slower (the self_update fixture family measured 403s debug vs ~2s release diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 594ec5f0..74ac23df 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -27,6 +27,8 @@ This document defines the **public surface** of the `socket-patch` binary. Anyth **Root `--update` flag.** `socket-patch --update [VERSION]` updates the binary itself from GitHub Releases. It is a root flag, not a subcommand: argv is rewritten (the same mechanism as the bare-UUID fallback) onto an internal hidden subcommand whose name carries no stability guarantee — script the flag, never the internal name. Combining the flag with a subcommand (`socket-patch --update scan`) is a usage error (exit 2). Full contract: [Self-update contract](#self-update-contract-socket-patch---update). +**Internal `hosted-bundle` subcommand.** `socket-patch hosted-bundle` is a hidden, INTERNAL parity/debug harness for the in-memory hosted engine (`src/hosted_memory/`, the engine the Node addon embeds): it reads a JSON bundle `{"files": {path: text}, "binaryFiles"?: {path: base64}, "presentOnly"?: [path], "symlinks"?: [path], "projectRoots"?: [dir], "pipenvMajor"?: n, "batchSize"?: n}` on stdin, queries the authenticated org API built from `--api-url` / `--api-token` / `--org` only (both of the latter are required; no public-proxy fallback), and prints the engine result — or `{"status":"error","error":{"code","message"}}` with exit 1 (exit 2 for unusable input or missing credentials). It never touches the filesystem. Its name, input and output carry NO stability guarantee; do not script it. + ## Global arguments In v3.0 every subcommand accepts the same set of "global" flags via a single shared `GlobalArgs` struct that's `#[command(flatten)]`-ed into each per-command struct (`crates/socket-patch-cli/src/args.rs`). Subcommands that don't actually consume a given flag accept it silently — e.g. `list --global` parses fine and is a no-op. Every flag also has an environment-variable binding; precedence is **CLI arg > env var > default** — and for exactly three keys (`--api-token`, `--org`, `--api-url`) the JS socket-cli's persisted login sits between env var and default: **CLI arg > env var (canonical, then `SOCKET_CLI_*` alias) > socket-cli `config.json` > default**. See "Persisted configuration" under Environment variables. diff --git a/crates/socket-patch-cli/Cargo.toml b/crates/socket-patch-cli/Cargo.toml index 9c8dc1e9..8429e56b 100644 --- a/crates/socket-patch-cli/Cargo.toml +++ b/crates/socket-patch-cli/Cargo.toml @@ -22,6 +22,8 @@ clap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } +base64 = { workspace = true } futures-util = { workspace = true } console = { workspace = true } dialoguer = { workspace = true } diff --git a/crates/socket-patch-cli/src/commands/hosted_bundle.rs b/crates/socket-patch-cli/src/commands/hosted_bundle.rs new file mode 100644 index 00000000..67fe2bb1 --- /dev/null +++ b/crates/socket-patch-cli/src/commands/hosted_bundle.rs @@ -0,0 +1,158 @@ +//! `socket-patch hosted-bundle` (hidden, internal): run the in-memory +//! hosted engine over a JSON bundle read from stdin and print its result. +//! A parity/debugging harness for [`crate::hosted_memory`]: the patch API +//! is the authenticated org API built from the global `--api-url` / +//! `--api-token` / `--org` arguments, which (like every command's) fall back +//! to `SOCKET_API_URL` / `SOCKET_API_TOKEN` / `SOCKET_ORG_SLUG` — so an +//! exported production token is used, reference grants included. It never +//! uses the public proxy, and the engine itself reads no environment. +//! +//! Stdin: `{"files": {path: text}, "binaryFiles"?: {path: base64}, +//! "presentOnly"?: [path], "symlinks"?: [path], "projectRoots"?: [dir], +//! "pipenvMajor"?: n, "batchSize"?: n}`. Stdout: the engine result +//! (`HostedScanResult`, binary contents base64), or +//! `{"status":"error","error":{"code","message"}}` with exit 1. + +use std::collections::BTreeMap; +use std::io::Read; +use std::sync::Arc; + +use base64::Engine; +use clap::Args; +use serde::Deserialize; +use socket_patch_core::api::client::{ApiClient, ApiClientOptions}; +use socket_patch_core::constants::DEFAULT_SOCKET_API_URL; +use tokio_util::sync::CancellationToken; + +use crate::args::GlobalArgs; +use crate::hosted_memory::{ + run_in_memory, EngineError, HostedScanOptions, MarkKind, PresentKind, SessionBuilder, +}; + +#[derive(Args)] +pub struct HostedBundleArgs { + #[command(flatten)] + pub common: GlobalArgs, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Bundle { + #[serde(default)] + files: BTreeMap, + #[serde(default)] + binary_files: BTreeMap, + #[serde(default)] + present_only: Vec, + #[serde(default)] + symlinks: Vec, + #[serde(default)] + project_roots: Option>, + #[serde(default)] + pipenv_major: Option, + #[serde(default)] + batch_size: Option, +} + +fn print_error(code: &str, message: &str) { + println!( + "{}", + serde_json::json!({ "status": "error", "error": { "code": code, "message": message } }) + ); +} + +fn build_input( + bundle: Bundle, + options: HostedScanOptions, +) -> Result { + let mut builder = SessionBuilder::new(options)?; + for (path, text) in &bundle.files { + builder.add_text(path, text)?; + } + for (path, encoded) in &bundle.binary_files { + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| EngineError::InvalidInput { + code: "invalid_base64", + message: format!("binaryFiles[{path}]: {e}"), + })?; + builder.add_binary(path, &bytes)?; + } + for path in &bundle.present_only { + builder.mark_present(path, MarkKind::Present(PresentKind::Present))?; + } + for path in &bundle.symlinks { + builder.mark_present(path, MarkKind::Symlink)?; + } + builder.finish() +} + +pub async fn run(args: HostedBundleArgs) -> i32 { + let common = &args.common; + let (Some(token), Some(org)) = ( + common.api_token.clone().filter(|t| !t.is_empty()), + common.org.clone().filter(|o| !o.is_empty()), + ) else { + print_error( + "missing_credentials", + "hosted-bundle requires --api-token and --org", + ); + return 2; + }; + let mut raw = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut raw) { + print_error("invalid_bundle", &format!("cannot read stdin: {e}")); + return 2; + } + let bundle: Bundle = match serde_json::from_str(&raw) { + Ok(bundle) => bundle, + Err(e) => { + print_error("invalid_bundle", &e.to_string()); + return 2; + } + }; + let options = HostedScanOptions { + org_slug: org.clone(), + ecosystems: common.ecosystems.clone(), + batch_size: bundle.batch_size, + dry_run: common.dry_run, + pipenv_major: bundle.pipenv_major, + trust_lockfile_config: Some(!common.no_trust_lockfile_config), + npm_allow_remote_config: Some(!common.no_npm_allow_remote_config), + project_roots: bundle.project_roots.clone(), + ..HostedScanOptions::default() + }; + let input = match build_input(bundle, options) { + Ok(input) => input, + Err(e) => { + print_error(e.code(), &e.to_string()); + return 2; + } + }; + let client = ApiClient::new(ApiClientOptions { + api_url: common + .api_url + .clone() + .filter(|u| !u.is_empty()) + .unwrap_or_else(|| DEFAULT_SOCKET_API_URL.to_string()), + api_token: Some(token), + use_public_proxy: false, + org_slug: Some(org), + }); + match run_in_memory(input, Arc::new(client), CancellationToken::new()).await { + Ok(output) => match serde_json::to_string_pretty(&output) { + Ok(text) => { + println!("{text}"); + 0 + } + Err(e) => { + print_error("engine_internal", &e.to_string()); + 1 + } + }, + Err(e) => { + print_error(e.code(), &e.to_string()); + 1 + } + } +} diff --git a/crates/socket-patch-cli/src/commands/mod.rs b/crates/socket-patch-cli/src/commands/mod.rs index 11ce2634..60dba585 100644 --- a/crates/socket-patch-cli/src/commands/mod.rs +++ b/crates/socket-patch-cli/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod apply; pub(crate) mod bun_preflight; pub(crate) mod fetch_stage; pub mod get; +pub mod hosted_bundle; pub mod list; pub(crate) mod lock_cli; pub mod remove; diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 799a89c3..e2031dbb 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -27,9 +27,10 @@ mod python; /// Fragment-edit kinds whose lockfile the package manager re-lays in place /// (keeping the Socket source) — a re-scan REBASES their ledger edits instead /// of appending; see the ledger merge below. -const REBASE_KINDS: &[&str] = &["redirect_poetry_lock_package", "redirect_pdm_lock_package"]; +pub(crate) const REBASE_KINDS: &[&str] = + &["redirect_poetry_lock_package", "redirect_pdm_lock_package"]; -const REDIRECT_CANDIDATE_FILES: &[&str] = &[ +pub(crate) const REDIRECT_CANDIDATE_FILES: &[&str] = &[ "package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", @@ -119,7 +120,7 @@ fn wheel_metadata_concurrency(use_public_proxy: bool) -> usize { /// dependency (reqwest is dev-only here); per RFC 3986 a raw `@` in the /// authority can ONLY be the userinfo terminator (it is percent-encoded /// everywhere else), so the tail after the LAST `@` is exactly host[:port]. -fn url_host(url: &str) -> Option<&str> { +pub(crate) fn url_host(url: &str) -> Option<&str> { let rest = url.split_once("://").map_or(url, |(_, r)| r); let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest); let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h); @@ -129,7 +130,7 @@ fn url_host(url: &str) -> Option<&str> { /// Repo-relative path of the pnpm workspace manifest the trustLockfile /// auto-config edits (the same file the vendor backend's override surface /// uses). -const PNPM_WORKSPACE_REL: &str = "pnpm-workspace.yaml"; +pub(crate) const PNPM_WORKSPACE_REL: &str = "pnpm-workspace.yaml"; /// `FileEdit.kind` recorded when the hosted flow ensures `trustLockfile: /// true` in pnpm-workspace.yaml. `action: "created"` — the workspace file @@ -137,14 +138,14 @@ const PNPM_WORKSPACE_REL: &str = "pnpm-workspace.yaml"; /// `trustLockfile: true` line was appended to an existing file (a revert /// removes exactly that line). Additive ledger vocabulary: older ledgers /// without it load unchanged (kind is an opaque string to the loader). -const REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND: &str = "redirect_pnpm_workspace_trust"; +pub(crate) const REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND: &str = "redirect_pnpm_workspace_trust"; /// The honest-tradeoff + don't-rebuild tail shared by every trustLockfile /// warning variant. The tradeoff sentence is a security disclosure, not /// prose garnish: `trustLockfile: true` disables pnpm's lockfile /// re-verification for the WHOLE lock, so it must be stated wherever the /// setting is written or recommended. -const PNPM_TRUST_TRADEOFF_AND_CAUTION: &str = +pub(crate) const PNPM_TRUST_TRADEOFF_AND_CAUTION: &str = "Note: trustLockfile makes pnpm skip its lockfile re-verification \ (minimumReleaseAge / trustPolicy re-checks) for ALL lockfile entries, \ not just the patched ones — the per-entry sha512 integrity pins are \ @@ -155,7 +156,7 @@ const PNPM_TRUST_TRADEOFF_AND_CAUTION: &str = /// The policy preamble shared by every trustLockfile warning variant: /// what was repointed, and how pnpm >=11 fails without trust. -fn pnpm_trust_policy_preamble(server: &str) -> String { +pub(crate) fn pnpm_trust_policy_preamble(server: &str) -> String { format!( "pnpm-lock.yaml was repointed at {server}; pnpm >=11 rejects the \ rewritten lock (pnpm 11: ERR_PNPM_TARBALL_URL_MISMATCH, pnpm 12: \ @@ -166,7 +167,7 @@ fn pnpm_trust_policy_preamble(server: &str) -> String { /// The pre-auto-config guidance, kept verbatim for the runs where the /// auto-config does not apply (legacy 5.x/6.0 locks, Rush nested locks, /// `--no-trust-lockfile-config`): both verified recoveries, spelled exactly. -fn pnpm_trust_manual_guidance(server: &str) -> String { +pub(crate) fn pnpm_trust_manual_guidance(server: &str) -> String { format!( "{}. Install with `pnpm install --trust-lockfile`, or commit \ `trustLockfile: true` in pnpm-workspace.yaml so every install \ @@ -184,7 +185,7 @@ fn pnpm_trust_manual_guidance(server: &str) -> String { /// no trust step exists or is needed. Deliberately NEVER mentions /// `pnpm install --trust-lockfile`: pnpm 7/8 reject the flag as an unknown /// option, so headlining it here would hand users a command that errors. -fn pnpm_trust_legacy_detail(server: &str) -> String { +pub(crate) fn pnpm_trust_legacy_detail(server: &str) -> String { format!( "The pnpm lockfile was repointed at {server}. This is a legacy \ lock read by pnpm 1–8, which have no \ @@ -202,7 +203,7 @@ fn pnpm_trust_legacy_detail(server: &str) -> String { /// here would OVERWRITE the user's file with the root-only scaffold — /// destroying their `packages:` globs — so the auto-config stands down and /// the warning names the file, the error, and both manual recoveries. -fn pnpm_trust_workspace_unreadable_detail(server: &str, err: &std::io::Error) -> String { +pub(crate) fn pnpm_trust_workspace_unreadable_detail(server: &str, err: &std::io::Error) -> String { format!( "{}. {PNPM_WORKSPACE_REL} exists but could not be read ({err}); it \ was left untouched — auto-configuring trust would risk overwriting \ @@ -277,7 +278,7 @@ fn npm_lock_url_needles(artifact_url: &str) -> Vec { /// block engages anyway. Legacy (<9) and unparseable-version locks stay /// `None` (fail closed: never write config for a lock era we can't read), /// as does a root lock this run DID splice (the splice path covers it). -fn pnpm_heal_root<'a>( +pub(crate) fn pnpm_heal_root<'a>( root_spliced: bool, disk_root: Option<&'a String>, overrides: &[socket_patch_core::patch::redirect::DepOverride], @@ -293,7 +294,7 @@ fn pnpm_heal_root<'a>( /// The auto-config variant: trust was (or, on `--dry-run`, would be) /// configured in pnpm-workspace.yaml, so installs need no flags. -fn pnpm_trust_configured_detail(server: &str, created: bool, dry_run: bool) -> String { +pub(crate) fn pnpm_trust_configured_detail(server: &str, created: bool, dry_run: bool) -> String { let how = match (created, dry_run) { (true, false) => "`trustLockfile: true` was written to a new", (false, false) => "`trustLockfile: true` was merged into the existing", @@ -313,7 +314,7 @@ fn pnpm_trust_configured_detail(server: &str, created: bool, dry_run: bool) -> S /// `'6.0'`, pnpm 7 an unquoted `5.4`. `None` when no parseable version line /// exists — callers treat that as "not trust-policy era" and stay /// hands-off (fail closed: never write config for a lock we can't read). -fn pnpm_lock_version_major(lock_text: &str) -> Option { +pub(crate) fn pnpm_lock_version_major(lock_text: &str) -> Option { lock_text.lines().find_map(|line| { let rest = line.strip_prefix("lockfileVersion:")?; let value = rest.trim().trim_matches(|c| c == '\'' || c == '"'); @@ -325,7 +326,7 @@ fn pnpm_lock_version_major(lock_text: &str) -> Option { /// `--store` (pnpm 1–3 can silently ignore `--store-dir`; early pnpm 4 /// rejects it): a `shrinkwrapVersion` lock (pnpm 1–2) or lockfileVersion /// 5.0–5.2 (pnpm 3–5). Later locks never get the `--store` note. -fn pnpm_lock_may_need_store_flag(lock_text: &str) -> bool { +pub(crate) fn pnpm_lock_may_need_store_flag(lock_text: &str) -> bool { lock_text.lines().any(|line| { if line.starts_with("shrinkwrapVersion:") { return true; @@ -345,7 +346,7 @@ fn pnpm_lock_may_need_store_flag(lock_text: &str) -> bool { } /// The planned pnpm-workspace.yaml `trustLockfile: true` edit. -enum TrustPlan { +pub(crate) enum TrustPlan { /// No workspace file: create it (root-only `packages` scaffold — pnpm 9 /// refuses a workspace file with no `packages` field — plus the trust /// key; the same scaffold shape the vendor backend creates). @@ -365,7 +366,7 @@ enum TrustPlan { /// Line splices only (never a YAML library), mirroring the vendor backend's /// workspace surgery: untouched lines stay byte-identical, so a revert can /// remove exactly what was added. -fn plan_workspace_trust(existing: Option<&str>) -> TrustPlan { +pub(crate) fn plan_workspace_trust(existing: Option<&str>) -> TrustPlan { let Some(text) = existing else { return TrustPlan::Create("packages:\n - '.'\ntrustLockfile: true\n".to_string()); }; @@ -395,7 +396,7 @@ fn plan_workspace_trust(existing: Option<&str>) -> TrustPlan { /// The root npm locks the hosted rewriter edits (`rewrite_npm_lock` rewrites /// every one present — npm 12 installs from package-lock.json beside a /// committed shrinkwrap). -const NPM_LOCKS: [&str; 2] = ["npm-shrinkwrap.json", "package-lock.json"]; +pub(crate) const NPM_LOCKS: [&str; 2] = ["npm-shrinkwrap.json", "package-lock.json"]; /// The honest-tradeoff + opt-out tail shared by every `allow-remote` /// warning variant. The tradeoff sentence is a security disclosure, not @@ -422,7 +423,11 @@ fn npm_allow_remote_preamble(hosts: &[&str]) -> String { /// The auto-config variant: `allow-remote=all` was (or, on `--dry-run`, /// would be) written to the project `.npmrc`, so installs need no flags. -fn npm_allow_remote_configured_detail(hosts: &[&str], created: bool, dry_run: bool) -> String { +pub(crate) fn npm_allow_remote_configured_detail( + hosts: &[&str], + created: bool, + dry_run: bool, +) -> String { let how = match (created, dry_run) { (true, false) => "`allow-remote=all` was written to a new", (false, false) => "`allow-remote=all` was appended to the existing", @@ -439,7 +444,7 @@ fn npm_allow_remote_configured_detail(hosts: &[&str], created: bool, dry_run: bo } /// The project `.npmrc` already resolves to `allow-remote=all`. -fn npm_allow_remote_already_detail(hosts: &[&str]) -> String { +pub(crate) fn npm_allow_remote_already_detail(hosts: &[&str]) -> String { format!( "{}, and the project .npmrc already sets `allow-remote=all` — keep it committed \ alongside the lock; `npm ci` needs no extra flags. {NPM_ALLOW_REMOTE_TRADEOFF}", @@ -450,7 +455,7 @@ fn npm_allow_remote_already_detail(hosts: &[&str]) -> String { /// The user explicitly set another value: respected, never flipped (the /// pnpm `trustLockfile: false` precedent) — the warning names the manual /// recoveries instead. -fn npm_allow_remote_user_set_detail(hosts: &[&str], value: &str) -> String { +pub(crate) fn npm_allow_remote_user_set_detail(hosts: &[&str], value: &str) -> String { format!( "{}. The project .npmrc explicitly sets `allow-remote={value}`, which was respected \ and left untouched — set `allow-remote=all` there yourself (or install with \ @@ -463,7 +468,7 @@ fn npm_allow_remote_user_set_detail(hosts: &[&str], value: &str) -> String { /// An `npm_config_allow_remote` environment variable sets another value. /// npm's env layer beats every `.npmrc`, so a project write could not take /// effect in this environment — and an explicit setting is respected. -fn npm_allow_remote_env_set_detail(hosts: &[&str], var: &str, value: &str) -> String { +pub(crate) fn npm_allow_remote_env_set_detail(hosts: &[&str], var: &str, value: &str) -> String { format!( "{}. The environment variable {var}={value} explicitly sets `allow-remote`, which \ was respected: npm's environment layer overrides every .npmrc, so a project \ @@ -478,7 +483,7 @@ fn npm_allow_remote_env_set_detail(hosts: &[&str], var: &str, value: &str) -> St /// another value. A committed project `allow-remote=all` would silently /// override that machine / org policy on every checkout, so it is /// respected like a project value and the override is left to the user. -fn npm_allow_remote_outer_set_detail( +pub(crate) fn npm_allow_remote_outer_set_detail( hosts: &[&str], layer: &str, path: &std::path::Path, @@ -498,7 +503,7 @@ fn npm_allow_remote_outer_set_detail( /// The opt-out (`--no-npm-allow-remote-config`) variant: nothing written, /// both manual recoveries spelled out. -fn npm_allow_remote_manual_detail(hosts: &[&str]) -> String { +pub(crate) fn npm_allow_remote_manual_detail(hosts: &[&str]) -> String { format!( "{}. Commit `allow-remote=all` in the project .npmrc (or install with \ `npm ci --allow-remote=all`) so npm >=12 installs the patched artifacts. \ @@ -511,7 +516,7 @@ fn npm_allow_remote_manual_detail(hosts: &[&str]) -> String { /// be read, or is a symlink / non-regular file the atomic writer would /// replace. Planning a Create here would OVERWRITE the user's registry / /// auth config, so the auto-config stands down and names the problem. -fn npm_allow_remote_unreadable_detail(hosts: &[&str], why: &str) -> String { +pub(crate) fn npm_allow_remote_unreadable_detail(hosts: &[&str], why: &str) -> String { format!( "{}. The project .npmrc exists but {why}; it was left untouched. Add \ `allow-remote=all` to it yourself (or install with `npm ci --allow-remote=all`) \ @@ -601,7 +606,7 @@ fn build_redirect_json_envelope( /// `mode` is `"hosted"` (the final mode name for `--redirect`): an additive /// key so consumers dispatch on the mode without inferring it from which /// sub-object is present. -pub(super) fn redirect_json_block( +pub(crate) fn redirect_json_block( redirected: usize, rewritten: Vec, skipped: Vec, diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 6ab75db7..e32f3e04 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -37,7 +37,7 @@ use super::get::{download_and_apply_patches_with, select_patches, DownloadParams mod discovery; mod gc; -mod hosted; +pub(crate) mod hosted; pub(crate) mod render; pub(crate) mod vendor_flow; diff --git a/crates/socket-patch-cli/src/hosted_memory/discover.rs b/crates/socket-patch-cli/src/hosted_memory/discover.rs new file mode 100644 index 00000000..43572ece --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/discover.rs @@ -0,0 +1,452 @@ +//! Patch discovery across every project root with one provider lookup per +//! purl / uuid / url: the union of the roots' purls is batch-searched in +//! sorted, deterministic chunks; packages with patches get their details +//! fetched with bounded concurrency; each root then takes the disk JSON +//! flow's selection (accessible patches only, top-ranked per purl through +//! [`cmp_search_results`]). + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::Poll; +use std::time::Duration; + +use socket_patch_core::api::client::{ApiError, ApiFuture, PatchApi}; +use socket_patch_core::api::ranking::cmp_search_results; +use socket_patch_core::api::types::{ + BatchPackagePatches, PackageVendorResult, PatchResponse, PatchSearchResult, SearchResponse, +}; +use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; + +use super::types::MAX_REFERENCE_BATCH; + +/// A boxed future the bounded joiner drives. +pub(crate) type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// Drive `futures` with at most `limit` in flight, returning their outputs +/// in input order. Futures are polled in place (no task spawning), so +/// dropping the returned future cancels every one of them. +pub(crate) async fn join_bounded<'a, T>(futures: Vec>, limit: usize) -> Vec { + let limit = limit.max(1); + let total = futures.len(); + let mut queued: VecDeque<(usize, BoxFuture<'a, T>)> = futures.into_iter().enumerate().collect(); + let mut active: Vec<(usize, BoxFuture<'a, T>)> = Vec::new(); + let mut out: Vec> = (0..total).map(|_| None).collect(); + let mut done = 0usize; + std::future::poll_fn(|cx| loop { + while active.len() < limit { + match queued.pop_front() { + Some(next) => active.push(next), + None => break, + } + } + let mut progressed = false; + let mut i = 0; + while i < active.len() { + if let Poll::Ready(value) = active[i].1.as_mut().poll(cx) { + let (index, _) = active.swap_remove(i); + out[index] = Some(value); + done += 1; + progressed = true; + } else { + i += 1; + } + } + if done == total { + return Poll::Ready(()); + } + if !progressed || queued.is_empty() { + return Poll::Pending; + } + }) + .await; + out.into_iter().flatten().collect() +} + +/// The host's [`PatchApi`] behind per-call timeouts and call counting. +pub(crate) struct Provider { + api: Arc, + timeout: Duration, + pub(crate) concurrency: usize, + calls: Mutex>, +} + +impl Provider { + pub(crate) fn new(api: Arc, timeout: Duration, concurrency: usize) -> Self { + Self { + api, + timeout, + concurrency, + calls: Mutex::new(BTreeMap::new()), + } + } + + pub(crate) fn calls(&self) -> BTreeMap { + self.calls.lock().map(|c| c.clone()).unwrap_or_default() + } + + async fn call(&self, method: &'static str, fut: ApiFuture<'_, T>) -> Result { + if let Ok(mut calls) = self.calls.lock() { + *calls.entry(method.to_string()).or_insert(0) += 1; + } + match tokio::time::timeout(self.timeout, fut).await { + Ok(result) => result, + Err(_) => Err(ApiError::Network(format!( + "{method} timed out after {} ms", + self.timeout.as_millis() + ))), + } + } + + pub(crate) async fn search_patches_batch( + &self, + purls: &[String], + ) -> Result { + let mut response = self + .call("searchPatchesBatch", self.api.search_patches_batch(purls)) + .await?; + socket_patch_core::api::client::sort_batch_response(&mut response); + Ok(response) + } + + pub(crate) async fn search_patches_by_package( + &self, + purl: &str, + ) -> Result { + let mut response = self + .call( + "searchPatchesByPackage", + self.api.search_patches_by_package(purl), + ) + .await?; + response.patches.sort_by(cmp_search_results); + Ok(response) + } + + pub(crate) async fn fetch_registry_references( + &self, + uuids: &[String], + ) -> Result, ApiError> { + self.call( + "fetchRegistryReferences", + self.api.fetch_registry_references(uuids), + ) + .await + } + + pub(crate) async fn fetch_patch(&self, uuid: &str) -> Result, ApiError> { + self.call("fetchPatch", self.api.fetch_patch(uuid)).await + } + + pub(crate) async fn download_artifact( + &self, + url: &str, + max_bytes: u64, + ) -> Result, ApiError> { + let bytes = self + .call( + "downloadArtifact", + self.api.download_artifact(url, max_bytes), + ) + .await?; + if bytes.len() as u64 > max_bytes { + return Err(ApiError::Other(format!( + "artifact exceeds the {max_bytes}-byte limit" + ))); + } + Ok(bytes) + } +} + +/// The literal, qualifier-free purl the disk flow's lockfile supplement +/// queries for an inventory entry (`crawled_from_purl`'s shape rule), or +/// `None` for a purl it drops. +pub(crate) fn supplement_purl(purl: &str) -> Option { + let decoded = normalize_purl(strip_purl_qualifiers(purl)).into_owned(); + let rest = decoded.strip_prefix("pkg:")?; + let (_eco, rest) = rest.split_once('/')?; + rest.rfind('@').filter(|&i| i > 0)?; + Some(decoded) +} + +/// One root's batch-search outcome. +#[derive(Debug, Default)] +pub(crate) struct RootBatch { + /// Packages with at least one patch, purl-sorted. + pub(crate) packages: Vec, + /// How many of the root's purls sat in a failed chunk. + pub(crate) failed_purls: usize, + pub(crate) last_error: Option, +} + +/// The run-level batch outcome. +#[derive(Debug, Default)] +pub(crate) struct BatchOutcome { + pub(crate) roots: BTreeMap, + pub(crate) can_access_paid_patches: bool, +} + +/// Batch-search the union of every root's purls (`root_purls` values are +/// sorted, deduplicated supplement purls). A response package is credited +/// to every root that asked for it; one whose purl matches no request +/// spelling goes to every root that had a purl in its chunk. +pub(crate) async fn batch_search( + provider: &Provider, + root_purls: &BTreeMap>, + batch_size: usize, +) -> BatchOutcome { + let mut owners: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (root, purls) in root_purls { + for purl in purls { + owners.entry(purl.as_str()).or_default().push(root.as_str()); + } + } + let union: Vec = owners.keys().map(|p| p.to_string()).collect(); + let chunks: Vec> = union + .chunks(batch_size.max(1)) + .map(<[String]>::to_vec) + .collect(); + let futures: Vec> = chunks + .iter() + .map(|chunk| -> BoxFuture<'_, _> { Box::pin(provider.search_patches_batch(chunk)) }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + + let mut outcome = BatchOutcome::default(); + for root in root_purls.keys() { + outcome.roots.insert(root.clone(), RootBatch::default()); + } + for (chunk, result) in chunks.iter().zip(results) { + let chunk_roots: BTreeSet<&str> = chunk + .iter() + .flat_map(|p| owners.get(p.as_str()).into_iter().flatten().copied()) + .collect(); + match result { + Ok(response) => { + if response.can_access_paid_patches { + outcome.can_access_paid_patches = true; + } + for pkg in response.packages { + if pkg.patches.is_empty() { + continue; + } + let key = normalize_purl(strip_purl_qualifiers(&pkg.purl)).into_owned(); + let targets: Vec<&str> = match owners.get(key.as_str()) { + Some(roots) => roots.clone(), + None => chunk_roots.iter().copied().collect(), + }; + for root in targets { + if let Some(entry) = outcome.roots.get_mut(root) { + entry.packages.push(pkg.clone()); + } + } + } + } + Err(error) => { + let message = error.to_string(); + for purl in chunk { + for root in owners.get(purl.as_str()).into_iter().flatten() { + if let Some(entry) = outcome.roots.get_mut(*root) { + entry.failed_purls += 1; + entry.last_error = Some(message.clone()); + } + } + } + } + } + } + for entry in outcome.roots.values_mut() { + entry.packages.sort_by(|a, b| a.purl.cmp(&b.purl)); + } + outcome +} + +/// `search_patches_by_package` once per distinct purl. +pub(crate) async fn fetch_details( + provider: &Provider, + purls: &BTreeSet, +) -> BTreeMap> { + let ordered: Vec<&String> = purls.iter().collect(); + let futures: Vec> = ordered + .iter() + .map(|purl| -> BoxFuture<'_, _> { Box::pin(provider.search_patches_by_package(purl)) }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + ordered + .into_iter() + .cloned() + .zip(results.into_iter().map(|r| r.map_err(|e| e.to_string()))) + .collect() +} + +/// The disk `--json` selection over one root's merged detail results: +/// accessible patches only, then the top-ranked patch per purl, sorted by +/// purl, as `(purl, uuid)`. +pub(crate) fn select_top_ranked( + results: &[PatchSearchResult], + can_access_paid: bool, +) -> Vec<(String, String)> { + let mut by_purl: BTreeMap<&str, Vec<&PatchSearchResult>> = BTreeMap::new(); + for patch in results + .iter() + .filter(|p| can_access_paid || p.tier == "free") + { + by_purl.entry(patch.purl.as_str()).or_default().push(patch); + } + by_purl + .into_iter() + .filter_map(|(purl, mut group)| { + group.sort_by(|a, b| cmp_search_results(a, b)); + group.first().map(|p| (purl.to_string(), p.uuid.clone())) + }) + .collect() +} + +/// Reference grants for every distinct uuid (`MAX_REFERENCE_BATCH` per +/// request): the merged results, plus the uuids whose request failed. +pub(crate) async fn fetch_references( + provider: &Provider, + uuids: &BTreeSet, +) -> ( + HashMap, + BTreeMap, +) { + let ordered: Vec = uuids.iter().cloned().collect(); + let chunks: Vec> = ordered + .chunks(MAX_REFERENCE_BATCH) + .map(<[String]>::to_vec) + .collect(); + let futures: Vec> = chunks + .iter() + .map(|chunk| -> BoxFuture<'_, _> { Box::pin(provider.fetch_registry_references(chunk)) }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + let mut merged: HashMap = HashMap::new(); + let mut failed: BTreeMap = BTreeMap::new(); + for (chunk, result) in chunks.iter().zip(results) { + match result { + Ok(map) => merged.extend(map), + Err(error) => { + let message = error.to_string(); + for uuid in chunk { + failed.insert(uuid.clone(), message.clone()); + } + } + } + } + (merged, failed) +} + +/// Hosted wheel metadata once per distinct `(url, sha256)`: the disk +/// flow's `fetch_hosted_wheel_metadata` over the provider. +pub(crate) async fn fetch_wheel_metadata( + provider: &Provider, + wanted: &BTreeSet<(String, String)>, + max_bytes: u64, +) -> BTreeMap, String>> { + let ordered: Vec<&(String, String)> = wanted.iter().collect(); + let futures: Vec> = ordered + .iter() + .map( + |(url, sha256)| -> BoxFuture<'_, Result, String>> { + Box::pin(async move { + let bytes = provider + .download_artifact(url, max_bytes) + .await + .map_err(|error| format!("cannot fetch hosted wheel metadata: {error}"))?; + socket_patch_core::vendor::pypi::decode_hosted_wheel_metadata(&bytes, sha256) + }) + }, + ) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + ordered + .into_iter() + .map(|(url, _)| url.clone()) + .zip(results) + .collect() +} + +/// Patch views for every distinct confirmed uuid (wet runs only). +pub(crate) async fn fetch_records( + provider: &Provider, + uuids: &BTreeSet, +) -> BTreeMap> { + let ordered: Vec<&String> = uuids.iter().collect(); + let futures: Vec> = ordered + .iter() + .map(|uuid| -> BoxFuture<'_, _> { Box::pin(provider.fetch_patch(uuid)) }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + ordered + .into_iter() + .cloned() + .zip(results.into_iter().map(|r| r.ok().flatten())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[tokio::test] + async fn join_bounded_keeps_order_and_caps_concurrency() { + let in_flight = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let futures: Vec> = (0..20usize) + .map(|i| -> BoxFuture<'static, usize> { + let in_flight = in_flight.clone(); + let peak = peak.clone(); + Box::pin(async move { + let now = in_flight.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis((20 - i as u64) % 7)).await; + in_flight.fetch_sub(1, Ordering::SeqCst); + i + }) + }) + .collect(); + let out = join_bounded(futures, 3).await; + assert_eq!(out, (0..20).collect::>()); + assert!(peak.load(Ordering::SeqCst) <= 3); + } + + #[test] + fn supplement_purl_matches_the_disk_shape_rule() { + assert_eq!( + supplement_purl("pkg:npm/%40scope/x@1.0.0?arch=x").as_deref(), + Some("pkg:npm/@scope/x@1.0.0") + ); + assert_eq!(supplement_purl("pkg:npm/x"), None); + assert_eq!(supplement_purl("npm/x@1"), None); + } + + fn result(purl: &str, uuid: &str, tier: &str, severity: &str) -> PatchSearchResult { + serde_json::from_value(serde_json::json!({ + "uuid": uuid, "purl": purl, "publishedAt": "2024-01-01T00:00:00Z", + "description": "", "license": "MIT", "tier": tier, + "vulnerabilities": {"GHSA-x": {"cves": [], "summary": "", "severity": severity, "description": ""}} + })) + .unwrap() + } + + #[test] + fn selection_filters_paid_and_takes_the_top_ranked() { + let results = vec![ + result("pkg:npm/b@1", "b-low", "free", "low"), + result("pkg:npm/b@1", "b-crit", "free", "critical"), + result("pkg:npm/a@1", "a-paid", "paid", "critical"), + result("pkg:npm/a@1", "a-free", "free", "low"), + ]; + assert_eq!( + select_top_ranked(&results, false), + vec![ + ("pkg:npm/a@1".to_string(), "a-free".to_string()), + ("pkg:npm/b@1".to_string(), "b-crit".to_string()) + ] + ); + assert_eq!(select_top_ranked(&results, true)[0].1, "a-paid"); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/ledger.rs b/crates/socket-patch-cli/src/hosted_memory/ledger.rs new file mode 100644 index 00000000..9a876a0c --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/ledger.rs @@ -0,0 +1,177 @@ +//! The redirect ledger (`.socket/vendor/redirect-state.json`) in memory: +//! loaded strictly (a malformed ledger is a project error, never a fresh +//! start), merged exactly like the disk flow (edits appended unless already +//! recorded, `REBASE_KINDS` rebased, records extended newest-wins), and +//! serialized with the disk writer's bytes (`to_vec_pretty` + `\n`). + +use std::collections::BTreeMap; + +use socket_patch_core::manifest::schema::PatchRecord; +use socket_patch_core::patch::redirect::{ + CorruptRedirectState, FileEdit, RedirectState, REDIRECT_STATE_REL, +}; +use socket_patch_core::vendor::lock_inventory::{MemoryEntry, MemoryProject}; + +use crate::commands::scan::hosted::REBASE_KINDS; + +/// Load the project's ledger: `Ok(None)` when absent, `Err` (the disk +/// message) when present but unreadable or malformed. +pub(crate) fn load(project: &MemoryProject, root: &str) -> Result, String> { + let path = super::roots::join_root(root, REDIRECT_STATE_REL); + let corrupt = |detail: String, unreadable: bool| { + CorruptRedirectState { + path: path.clone().into(), + detail, + quarantined_to: None, + unreadable, + } + .to_string() + }; + let bytes: &[u8] = match project.get(REDIRECT_STATE_REL) { + None => return Ok(None), + Some(MemoryEntry::Text(text)) => text.as_bytes(), + Some(MemoryEntry::Binary(bytes)) => bytes, + Some(MemoryEntry::Present) => { + return Err(corrupt("file content was not provided".into(), true)) + } + Some(MemoryEntry::Symlink) => return Err(corrupt("is a symbolic link".into(), true)), + }; + serde_json::from_slice(bytes) + .map(Some) + .map_err(|e| corrupt(format!("invalid JSON: {e}"), false)) +} + +/// Merge this run's `edits` and `records` into `ledger` (the disk flow's +/// merge, verbatim). `files` are the pre-rewrite candidate contents the +/// rebase drift check reads. +pub(crate) fn merge( + ledger: &mut RedirectState, + edits: &[FileEdit], + records: BTreeMap, + files: &BTreeMap, +) { + ledger.mode = "hosted".to_string(); + let mut rebased: Vec = Vec::new(); + for edit in edits + .iter() + .filter(|e| REBASE_KINDS.contains(&e.kind.as_str())) + { + let siblings: Vec = ledger + .edits + .iter() + .enumerate() + .filter(|(_, old)| { + old.path == edit.path && old.kind == edit.kind && old.key == edit.key + }) + .map(|(i, _)| i) + .collect(); + let before = files.get(&edit.path).map(String::as_str).unwrap_or(""); + let drifted = !siblings.is_empty() + && siblings.iter().all(|&i| { + ledger.edits[i] + .new + .as_ref() + .and_then(serde_json::Value::as_str) + .is_none_or(|new| !before.contains(new)) + }); + if !drifted { + continue; + } + let nth = edits + .iter() + .filter(|e| e.path == edit.path && e.kind == edit.kind && e.key == edit.key) + .position(|e| std::ptr::eq(e, edit)) + .unwrap_or(0); + if let Some(&target) = siblings.get(nth) { + if !rebased.contains(&target) { + if edit.kind == "redirect_pdm_lock_package" { + ledger.edits[target].original = edit.original.clone(); + } + ledger.edits[target].new = edit.new.clone(); + ledger.edits[target].action = edit.action.clone(); + rebased.push(target); + } + } + } + let recorded = ledger.edits.len(); + for edit in edits { + let is_rebased = REBASE_KINDS.contains(&edit.kind.as_str()) + && rebased.iter().any(|&t| { + let old = &ledger.edits[t]; + old.path == edit.path + && old.kind == edit.kind + && old.key == edit.key + && old.new == edit.new + }); + if !is_rebased && !ledger.edits[..recorded].contains(edit) { + ledger.edits.push(edit.clone()); + } + } + ledger.records.extend(records); +} + +/// The ledger's on-disk bytes. +pub(crate) fn serialize(ledger: &RedirectState) -> Result { + let mut bytes = serde_json::to_vec_pretty(ledger).map_err(|e| e.to_string())?; + bytes.push(b'\n'); + String::from_utf8(bytes).map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn edit(kind: &str, new: &str) -> FileEdit { + FileEdit { + path: "poetry.lock".into(), + kind: kind.into(), + action: "replaced".into(), + key: Some("k".into()), + original: Some(serde_json::json!("orig")), + new: Some(serde_json::json!(new)), + } + } + + #[test] + fn corrupt_and_absent_ledgers() { + let mut p = MemoryProject::new(); + assert!(load(&p, "").unwrap().is_none()); + p.insert_text(REDIRECT_STATE_REL, "{not json"); + let err = load(&p, "sub").unwrap_err(); + assert!( + err.contains("sub/.socket/vendor/redirect-state.json"), + "{err}" + ); + assert!(err.contains("malformed"), "{err}"); + p.insert_symlink(REDIRECT_STATE_REL); + assert!(load(&p, "").unwrap_err().contains("cannot be read")); + } + + #[test] + fn merge_appends_new_edits_and_rebases_drifted_fragments() { + let mut ledger = RedirectState::new(); + ledger.edits.push(edit("redirect_npm_lock_entry", "a")); + ledger + .edits + .push(edit("redirect_poetry_lock_package", "old-new")); + let files = BTreeMap::from([("poetry.lock".to_string(), "relocked".to_string())]); + merge( + &mut ledger, + &[ + edit("redirect_npm_lock_entry", "a"), + edit("redirect_npm_lock_entry", "b"), + edit("redirect_poetry_lock_package", "fresh"), + ], + BTreeMap::new(), + &files, + ); + let news: Vec<&str> = ledger + .edits + .iter() + .map(|e| e.new.as_ref().and_then(|v| v.as_str()).unwrap()) + .collect(); + assert_eq!(news, vec!["a", "fresh", "b"]); + let text = serialize(&ledger).unwrap(); + assert!(text.ends_with("}\n")); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/limits.rs b/crates/socket-patch-cli/src/hosted_memory/limits.rs new file mode 100644 index 00000000..4bcdb635 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/limits.rs @@ -0,0 +1,409 @@ +//! Option validation and the streaming session builder: every size and +//! count limit on what a host may hand the engine is enforced here, before +//! any parser sees a byte. + +use std::collections::BTreeMap; + +use super::select::{is_binary_candidate, safe_repo_path, safe_root_path}; +use super::types::{ + EngineError, EngineWarning, HostedScanInput, HostedScanOptions, InputFile, MarkKind, + PresentKind, ResolvedLimits, DEFAULT_BATCH_SIZE, DEFAULT_PROVIDER_CONCURRENCY, + DEFAULT_REQUEST_TIMEOUT_MS, ECOSYSTEMS, MAX_BATCH_SIZE, +}; + +/// Upper bound on concurrent provider calls, whatever the host asks for. +const MAX_PROVIDER_CONCURRENCY: u32 = 64; + +/// The options every engine entry point validates, with defaults applied. +#[derive(Debug, Clone)] +pub(crate) struct ResolvedOptions { + pub(crate) ecosystems: Option>, + pub(crate) batch_size: usize, + pub(crate) dry_run: bool, + pub(crate) pipenv_major: Option, + pub(crate) trust_lockfile_config: bool, + pub(crate) npm_allow_remote_config: bool, + pub(crate) project_roots: Option>, + pub(crate) provider_concurrency: usize, + pub(crate) request_timeout: std::time::Duration, + pub(crate) limits: ResolvedLimits, +} + +pub(crate) fn resolve_options(options: &HostedScanOptions) -> Result { + if options.org_slug.trim().is_empty() { + return Err(EngineError::invalid( + "invalid_org_slug", + "orgSlug must be a non-empty string", + )); + } + let batch_size = options.batch_size.unwrap_or(DEFAULT_BATCH_SIZE); + if !(1..=MAX_BATCH_SIZE).contains(&batch_size) { + return Err(EngineError::invalid( + "invalid_batch_size", + format!("batchSize must be between 1 and {MAX_BATCH_SIZE}"), + )); + } + if let Some(ecosystems) = &options.ecosystems { + if let Some(bad) = ecosystems + .iter() + .find(|e| !ECOSYSTEMS.contains(&e.as_str())) + { + return Err(EngineError::invalid( + "invalid_ecosystem", + format!("unknown ecosystem `{bad}`"), + )); + } + } + let project_roots = match &options.project_roots { + Some(roots) => { + let mut out: Vec = Vec::with_capacity(roots.len()); + for root in roots { + let normalized = safe_root_path(root).ok_or_else(|| { + EngineError::invalid( + "invalid_project_root", + format!( + "projectRoots entry `{root}` is not a safe repo-relative directory" + ), + ) + })?; + out.push(normalized); + } + out.sort(); + out.dedup(); + Some(out) + } + None => None, + }; + let provider_concurrency = options + .provider_concurrency + .unwrap_or(DEFAULT_PROVIDER_CONCURRENCY); + if provider_concurrency == 0 { + return Err(EngineError::invalid( + "invalid_provider_concurrency", + "providerConcurrency must be at least 1", + )); + } + let timeout_ms = options + .request_timeout_ms + .unwrap_or(DEFAULT_REQUEST_TIMEOUT_MS); + if timeout_ms == 0 { + return Err(EngineError::invalid( + "invalid_request_timeout", + "requestTimeoutMs must be at least 1", + )); + } + Ok(ResolvedOptions { + ecosystems: options.ecosystems.clone(), + batch_size: batch_size as usize, + dry_run: options.dry_run, + pipenv_major: options.pipenv_major, + trust_lockfile_config: options.trust_lockfile_config.unwrap_or(true), + npm_allow_remote_config: options.npm_allow_remote_config.unwrap_or(true), + project_roots, + provider_concurrency: provider_concurrency.min(MAX_PROVIDER_CONCURRENCY) as usize, + request_timeout: std::time::Duration::from_millis(timeout_ms), + limits: options.limits.clone().unwrap_or_default().resolve(), + }) +} + +/// Accumulates a host's streamed files into a [`HostedScanInput`], +/// enforcing `maxFiles`, `maxFileBytes` and `maxTotalBytes` on every chunk. +/// `bun.lockb` is kept as bytes; every other file must be UTF-8 (a file +/// that is not is kept as present-but-unreadable with an `invalid_utf8` +/// warning, exactly like an unreadable file on disk). +#[derive(Debug)] +pub struct SessionBuilder { + options: HostedScanOptions, + limits: ResolvedLimits, + open: BTreeMap>, + files: BTreeMap, + total_bytes: u64, + warnings: Vec, +} + +impl SessionBuilder { + pub fn new(options: HostedScanOptions) -> Result { + let resolved = resolve_options(&options)?; + Ok(Self { + options, + limits: resolved.limits, + open: BTreeMap::new(), + files: BTreeMap::new(), + total_bytes: 0, + warnings: Vec::new(), + }) + } + + pub fn options(&self) -> &HostedScanOptions { + &self.options + } + + pub fn total_bytes(&self) -> u64 { + self.total_bytes + } + + fn checked_path(&self, path: &str) -> Result { + let normalized = safe_repo_path(path).ok_or_else(|| { + EngineError::invalid( + "invalid_path", + format!("`{path}` is not a safe repo-relative path"), + ) + })?; + if self.files.contains_key(&normalized) { + return Err(EngineError::invalid( + "duplicate_path", + format!("`{normalized}` was already provided"), + )); + } + Ok(normalized) + } + + fn check_file_count(&self, adding_new: bool) -> Result<(), EngineError> { + let count = (self.files.len() + self.open.len() + usize::from(adding_new)) as u64; + if count > self.limits.max_files { + return Err(EngineError::limit( + "max_files", + format!("more than {} files were provided", self.limits.max_files), + )); + } + Ok(()) + } + + /// Append `chunk` to `path`'s content. + pub fn push_chunk(&mut self, path: &str, chunk: &[u8]) -> Result<(), EngineError> { + let path = self.checked_path(path)?; + let is_new = !self.open.contains_key(&path); + self.check_file_count(is_new)?; + let current = self.open.get(&path).map_or(0, Vec::len) as u64; + let len = chunk.len() as u64; + if current + len > self.limits.max_file_bytes { + return Err(EngineError::limit( + "max_file_bytes", + format!( + "`{path}` exceeds the {}-byte per-file limit", + self.limits.max_file_bytes + ), + )); + } + if self.total_bytes + len > self.limits.max_total_bytes { + return Err(EngineError::limit( + "max_total_bytes", + format!( + "the session exceeds the {}-byte total limit", + self.limits.max_total_bytes + ), + )); + } + self.total_bytes += len; + self.open.entry(path).or_default().extend_from_slice(chunk); + Ok(()) + } + + /// Complete `path` (a path with no chunks is an empty file). + pub fn end_file(&mut self, path: &str) -> Result<(), EngineError> { + let path = self.checked_path(path)?; + let bytes = match self.open.remove(&path) { + Some(bytes) => bytes, + None => { + self.check_file_count(true)?; + Vec::new() + } + }; + let file = if is_binary_candidate(&path) { + InputFile::Binary(bytes) + } else { + match String::from_utf8(bytes) { + Ok(text) => InputFile::Text(text), + Err(_) => { + self.warnings.push(EngineWarning::new( + "invalid_utf8", + format!("{path} is not valid UTF-8 and was treated as unreadable"), + None, + )); + InputFile::Present(PresentKind::BinarySkipped) + } + } + }; + self.files.insert(path, file); + Ok(()) + } + + /// Record that `path` exists without providing its content. + pub fn mark_present(&mut self, path: &str, kind: MarkKind) -> Result<(), EngineError> { + let path = self.checked_path(path)?; + if self.open.contains_key(&path) { + return Err(EngineError::invalid( + "duplicate_path", + format!("`{path}` already has streamed content"), + )); + } + self.check_file_count(true)?; + let file = match kind { + MarkKind::Symlink => InputFile::Symlink, + MarkKind::Present(kind) => { + let code = match kind { + PresentKind::Present => None, + PresentKind::BinarySkipped => Some("binary_skipped"), + PresentKind::Oversize => Some("file_oversize"), + PresentKind::LfsPointer => Some("lfs_pointer"), + }; + if let Some(code) = code { + self.warnings.push(EngineWarning::new( + code, + format!("{path} was not provided ({code}) and was treated as unreadable"), + None, + )); + } + InputFile::Present(kind) + } + }; + self.files.insert(path, file); + Ok(()) + } + + /// Whole-file convenience over [`Self::push_chunk`] + [`Self::end_file`]. + pub fn add_text(&mut self, path: &str, text: &str) -> Result<(), EngineError> { + self.push_chunk(path, text.as_bytes())?; + self.end_file(path) + } + + /// Whole-file convenience for binary content. + pub fn add_binary(&mut self, path: &str, bytes: &[u8]) -> Result<(), EngineError> { + let normalized = self.checked_path(path)?; + self.push_chunk(&normalized, bytes)?; + let bytes = self.open.remove(&normalized).unwrap_or_default(); + self.files.insert(normalized, InputFile::Binary(bytes)); + Ok(()) + } + + /// The finished input. Fails when a streamed file was never ended. + pub fn finish(self) -> Result { + if let Some(path) = self.open.keys().next() { + return Err(EngineError::invalid( + "unterminated_file", + format!("`{path}` received chunks but endFile was never called"), + )); + } + Ok(HostedScanInput { + options: self.options, + files: self.files, + warnings: self.warnings, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hosted_memory::types::HostedScanLimits; + + fn options(limits: HostedScanLimits) -> HostedScanOptions { + HostedScanOptions { + org_slug: "org".into(), + limits: Some(limits), + ..HostedScanOptions::default() + } + } + + #[test] + fn rejects_bad_options() { + let mut o = options(HostedScanLimits::default()); + o.batch_size = Some(0); + assert_eq!( + resolve_options(&o).unwrap_err().code(), + "invalid_batch_size" + ); + o.batch_size = Some(501); + assert_eq!( + resolve_options(&o).unwrap_err().code(), + "invalid_batch_size" + ); + o.batch_size = None; + o.ecosystems = Some(vec!["deno".into()]); + assert_eq!(resolve_options(&o).unwrap_err().code(), "invalid_ecosystem"); + o.ecosystems = None; + o.project_roots = Some(vec!["../x".into()]); + assert_eq!( + resolve_options(&o).unwrap_err().code(), + "invalid_project_root" + ); + o.project_roots = None; + o.org_slug = " ".into(); + assert_eq!(resolve_options(&o).unwrap_err().code(), "invalid_org_slug"); + } + + #[test] + fn enforces_session_limits() { + let mut b = SessionBuilder::new(options(HostedScanLimits { + max_file_bytes: Some(4), + max_total_bytes: Some(6), + max_files: Some(2), + ..HostedScanLimits::default() + })) + .unwrap(); + b.push_chunk("a", b"12").unwrap(); + b.push_chunk("a", b"34").unwrap(); + assert_eq!( + b.push_chunk("a", b"5").unwrap_err().code(), + "max_file_bytes" + ); + b.end_file("a").unwrap(); + b.push_chunk("b", b"12").unwrap(); + assert_eq!( + b.push_chunk("b", b"3").unwrap_err().code(), + "max_total_bytes" + ); + b.end_file("b").unwrap(); + assert_eq!(b.end_file("c").unwrap_err().code(), "max_files"); + assert_eq!( + b.push_chunk("a", b"x").unwrap_err().code(), + "duplicate_path" + ); + assert_eq!( + b.push_chunk("../evil", b"x").unwrap_err().code(), + "invalid_path" + ); + } + + #[test] + fn classifies_finished_files() { + let mut b = SessionBuilder::new(options(HostedScanLimits::default())).unwrap(); + b.add_text("package-lock.json", "{}").unwrap(); + b.push_chunk("bun.lockb", &[0xff, 0x00]).unwrap(); + b.end_file("bun.lockb").unwrap(); + b.push_chunk("yarn.lock", &[0xff, 0xfe]).unwrap(); + b.end_file("yarn.lock").unwrap(); + b.mark_present(".pnp.cjs", MarkKind::Present(PresentKind::Present)) + .unwrap(); + b.mark_present(".npmrc", MarkKind::Symlink).unwrap(); + b.push_chunk("open.txt", b"x").unwrap(); + assert_eq!(b.finish().unwrap_err().code(), "unterminated_file"); + } + + #[test] + fn finished_input_keeps_every_kind() { + let mut b = SessionBuilder::new(options(HostedScanLimits::default())).unwrap(); + b.add_text("package-lock.json", "{}").unwrap(); + b.push_chunk("bun.lockb", &[0xff, 0x00]).unwrap(); + b.end_file("bun.lockb").unwrap(); + b.push_chunk("yarn.lock", &[0xff, 0xfe]).unwrap(); + b.end_file("yarn.lock").unwrap(); + b.mark_present(".npmrc", MarkKind::Symlink).unwrap(); + let input = b.finish().unwrap(); + assert_eq!( + input.files.get("package-lock.json"), + Some(&InputFile::Text("{}".into())) + ); + assert_eq!( + input.files.get("bun.lockb"), + Some(&InputFile::Binary(vec![0xff, 0x00])) + ); + assert_eq!( + input.files.get("yarn.lock"), + Some(&InputFile::Present(PresentKind::BinarySkipped)) + ); + assert_eq!(input.files.get(".npmrc"), Some(&InputFile::Symlink)); + assert_eq!(input.warnings.len(), 1); + assert_eq!(input.warnings[0].code, "invalid_utf8"); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/mod.rs b/crates/socket-patch-cli/src/hosted_memory/mod.rs new file mode 100644 index 00000000..b5f8b0f3 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/mod.rs @@ -0,0 +1,1113 @@ +//! The hosted-mode redirect engine over an in-memory repository: no +//! filesystem, no subprocesses, no environment reads, no telemetry. Every +//! patch lookup goes through the caller's [`PatchApi`]; the caller hands +//! in the repository's candidate files (chosen by [`select_paths`]) and +//! gets back the changed files, ledger included. +//! +//! Per project root the result matches `scan --mode hosted --json` over a +//! checkout holding the same files (the parity tests hold the two paths to +//! it), with these differences by design: patch lookups are shared across +//! roots (one call per purl / uuid / url), a 401/403 is a project error +//! instead of a public-proxy fallback, the Pipenv major is an option +//! instead of a `pipenv --version` probe, the npm allow-remote planner sees +//! no user/global npm config, and a vendored→hosted takeover is refused +//! (`vendored_takeover_unsupported_in_memory`). Maven and NuGet have no +//! lockfile inventory (disk discovers them only through installed-tree +//! crawlers), so their files raise `ecosystem_unsupported_in_memory`. A dry +//! run still requests reference grants, exactly like the disk dry run. +//! +//! Cargo member manifests are chosen by content on disk (`members`, +//! `exclude`, path dependencies and `[patch]` paths, anywhere but +//! `target/`), which a tree listing cannot evaluate. So [`select_paths`] +//! asks for every `Cargo.toml` under a Cargo root outside `target/` +//! segments (`vendor/`, `node_modules/` and dot-directories included) and +//! the engine repeats the disk walk over them; manifests the walk never +//! reaches are read and ignored. The one exception is `cargo vendor` +//! output: a manifest in or below a directory holding +//! `.cargo-checksum.json` is registry source the walk never reads, so it +//! is not fetched; a path dependency or `[patch]` path into such a crate +//! then fails closed (`redirect_cargo_transitive_dependents`). The cost is +//! over-fetch: a Rust repo with many committed fixture or example crates +//! streams all of them, and each counts toward `maxFiles` / +//! `maxTotalBytes`, so such a repo can fail with a `limit` error where the +//! disk run would succeed. Fetching fewer would instead silently drop +//! manifests the disk run pins. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; +use std::time::Instant; + +use socket_patch_core::api::client::PatchApi; +use socket_patch_core::api::types::{PatchResponse, PatchSearchResult}; +use socket_patch_core::crawlers::Ecosystem; +use socket_patch_core::manifest::schema::PatchRecord; +use socket_patch_core::patch::redirect::{RedirectState, REDIRECT_STATE_REL}; +use socket_patch_core::utils::cargo_workspace::member_manifests_in; +use socket_patch_core::vendor::lock_inventory::{ + inventory_project_diagnosed_in, MemoryEntry, MemoryProject, ProjectView, +}; +use tokio_util::sync::CancellationToken; + +pub(crate) mod discover; +pub(crate) mod ledger; +pub mod limits; +pub(crate) mod redirect; +pub(crate) mod roots; +pub mod select; +pub mod types; + +pub use limits::SessionBuilder; +pub use select::{candidate_files, safe_repo_path, select_paths}; +pub use types::*; + +use discover::Provider; +use redirect::{Planned, Refused, Rewritten, StageOptions}; + +/// `"+"`; the sha comes from the +/// `SOCKET_PATCH_GIT_SHA` build-time variable. +pub fn engine_version() -> String { + format!( + "{}+{}", + env!("CARGO_PKG_VERSION"), + option_env!("SOCKET_PATCH_GIT_SHA").unwrap_or("unknown") + ) +} + +/// Run the engine. Resolves with the result, or rejects with an +/// [`EngineError`]: a limit breach, unusable input, cancellation through +/// `cancel` (checked around every await and between roots in the CPU-bound +/// inventory / plan / rewrite phases — one root's running parser is not +/// interrupted), or an engine bug. Provider failures never reject; they +/// become project errors and warnings. +pub async fn run_in_memory( + input: HostedScanInput, + provider: Arc, + cancel: CancellationToken, +) -> Result { + tokio::select! { + biased; + _ = cancel.cancelled() => Err(EngineError::Cancelled), + result = engine(input, provider, cancel.clone()) => result, + } +} + +/// Yield to the runtime between roots of a CPU-bound phase (memory reads +/// never pend, so without it neither `cancel` nor other tasks on this +/// worker would run until the phase ends). +async fn checkpoint(cancel: &CancellationToken) -> Result<(), EngineError> { + tokio::task::yield_now().await; + if cancel.is_cancelled() { + return Err(EngineError::Cancelled); + } + Ok(()) +} + +/// One root's working state across the phases. +struct RootState { + root: String, + project: Option, + /// Root-relative paths that exist but whose content was not provided + /// (oversize, LFS pointers, presence-only): the disk flow would read + /// them, so a rewrite that depends on one is refused. + unreadable: BTreeSet, + ledger: Option, + purls: Vec, + summary: ProjectSummary, + packages: Vec, + selected: Vec<(String, String)>, + skipped: Vec, + error: Option, +} + +impl RootState { + fn fail(&mut self, code: &str, message: String) { + if self.error.is_none() { + self.error = Some(ProjectError { + code: code.to_string(), + message, + }); + } + } +} + +struct Phases { + at: Instant, + ms: BTreeMap, +} + +impl Phases { + fn mark(&mut self, name: &str) { + let now = Instant::now(); + self.ms.insert( + name.to_string(), + now.duration_since(self.at).as_millis() as u64, + ); + self.at = now; + } +} + +fn validate_input(input: &HostedScanInput, limits: &ResolvedLimits) -> Result { + if input.files.len() as u64 > limits.max_files { + return Err(EngineError::limit( + "max_files", + format!("more than {} files were provided", limits.max_files), + )); + } + let mut total = 0u64; + for (path, file) in &input.files { + if safe_repo_path(path).as_deref() != Some(path.as_str()) { + return Err(EngineError::invalid( + "invalid_path", + format!("`{path}` is not a safe repo-relative path"), + )); + } + let len = match file { + InputFile::Text(text) => text.len() as u64, + InputFile::Binary(bytes) => bytes.len() as u64, + InputFile::Present(_) | InputFile::Symlink => 0, + }; + if len > limits.max_file_bytes { + return Err(EngineError::limit( + "max_file_bytes", + format!( + "`{path}` exceeds the {}-byte per-file limit", + limits.max_file_bytes + ), + )); + } + total += len; + } + if total > limits.max_total_bytes { + return Err(EngineError::limit( + "max_total_bytes", + format!( + "the input exceeds the {}-byte total limit", + limits.max_total_bytes + ), + )); + } + Ok(total) +} + +/// One input file, stored once: every root's project shares its bytes. +struct SharedFile { + entry: MemoryEntry, + /// Exists, but its content (which disk would read) was not provided. + /// A non-UTF-8 text file is not: disk cannot read it either. + unreadable: bool, +} + +fn share(file: InputFile) -> SharedFile { + let (entry, unreadable) = match file { + InputFile::Text(text) => (MemoryEntry::Text(Arc::from(text)), false), + InputFile::Binary(bytes) => (MemoryEntry::Binary(Arc::from(bytes)), false), + InputFile::Present(kind) => (MemoryEntry::Present, kind != PresentKind::BinarySkipped), + InputFile::Symlink => (MemoryEntry::Symlink, false), + }; + SharedFile { entry, unreadable } +} + +/// The root-relative in-memory project for `root` and its unreadable +/// paths. +fn project_for( + root: &str, + files: &BTreeMap, +) -> (MemoryProject, BTreeSet) { + let mut project = MemoryProject::new(); + let mut unreadable = BTreeSet::new(); + for (path, file) in files { + let Some(rel) = roots::strip_root(root, path) else { + continue; + }; + if file.unreadable { + unreadable.insert(rel.to_string()); + } + project.insert(rel, file.entry.clone()); + } + (project, unreadable) +} + +/// A root that is a member of an enclosing root's Cargo workspace builds +/// through that workspace's Cargo.lock, which the enclosing root already +/// pins its manifest against; the member's own Cargo.lock (which cargo +/// ignores) is dropped so only the workspace root redirects cargo packages. +fn demote_cargo_members(states: &mut [RootState], warnings: &mut Vec) { + let mut member_of: BTreeMap = BTreeMap::new(); + for state in states.iter() { + let Some(project) = state.project.as_ref() else { + continue; + }; + if !project.contains("Cargo.lock") { + continue; + } + for rel in member_manifests_in(&ProjectView::Memory(project)) { + if let Some(dir) = rel.strip_suffix("/Cargo.toml") { + member_of + .entry(roots::join_root(&state.root, dir)) + .or_insert_with(|| state.root.clone()); + } + } + } + for state in states.iter_mut() { + let Some(owner) = member_of.get(&state.root) else { + continue; + }; + let Some(project) = state.project.as_mut() else { + continue; + }; + if project.remove("Cargo.lock").is_some() { + state.unreadable.remove("Cargo.lock"); + warnings.push(EngineWarning::new( + "cargo_member_lock_ignored", + format!( + "{} is a member of the Cargo workspace at `{owner}`, whose Cargo.lock \ + cargo builds it from; its own Cargo.lock was not scanned", + if state.root.is_empty() { + "." + } else { + state.root.as_str() + } + ), + Some(&state.root), + )); + } + } +} + +fn ecosystem_allowed(ecosystems: Option<&[String]>, purl: &str) -> bool { + match ecosystems { + None => true, + Some(list) => { + Ecosystem::from_purl(purl).is_some_and(|eco| list.iter().any(|a| a == eco.cli_name())) + } + } +} + +fn unsupported_ecosystem_warnings( + root: &str, + project: &MemoryProject, + ecosystems: Option<&[String]>, + out: &mut Vec, +) { + let allowed = |eco: &str| ecosystems.is_none_or(|list| list.iter().any(|e| e == eco)); + for (eco, markers) in roots::UNSUPPORTED_MARKERS { + if !allowed(eco) { + continue; + } + if let Some(found) = markers.iter().find(|m| project.contains(m)) { + out.push(EngineWarning::new( + "ecosystem_unsupported_in_memory", + format!( + "{found} is present, but {eco} dependencies are discovered only from an \ + installed tree, which the in-memory hosted scan does not have; {eco} \ + dependencies were not scanned" + ), + Some(root), + )); + } + } +} + +/// Maven / NuGet marker files outside every root (a repo with only those +/// has no root at all): one run-level warning per ecosystem. +fn unrooted_unsupported_warnings<'a>( + paths: impl Iterator, + roots: &[String], + ecosystems: Option<&[String]>, + out: &mut Vec, +) { + let root_set: BTreeSet<&str> = roots.iter().map(String::as_str).collect(); + let mut found: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for path in paths { + let (dir, base) = roots::split_path(path); + let Some(&(eco, _)) = roots::UNSUPPORTED_MARKERS + .iter() + .find(|(_, markers)| markers.contains(&base)) + else { + continue; + }; + if root_set.contains(dir) + || !ecosystems.is_none_or(|list| list.iter().any(|e| e == eco)) + || dir + .split('/') + .any(|seg| roots::EXCLUDED_ROOT_SEGMENTS.contains(&seg)) + { + continue; + } + found.entry(eco).or_default().push(path); + } + for (eco, paths) in found { + let more = match paths.len() { + 1 => String::new(), + n => format!(" (and {} more)", n - 1), + }; + out.push(EngineWarning::new( + "ecosystem_unsupported_in_memory", + format!( + "{}{more} is present, but {eco} dependencies are discovered only from an \ + installed tree, which the in-memory hosted scan does not have; {eco} \ + dependencies were not scanned", + paths[0] + ), + None, + )); + } +} + +async fn engine( + input: HostedScanInput, + api: Arc, + cancel: CancellationToken, +) -> Result { + let options = limits::resolve_options(&input.options)?; + let bytes_input = validate_input(&input, &options.limits)?; + let files_input = input.files.len() as u64; + let files: BTreeMap = input + .files + .into_iter() + .map(|(path, file)| (path, share(file))) + .collect(); + let mut phases = Phases { + at: Instant::now(), + ms: BTreeMap::new(), + }; + let mut warnings: Vec = input.warnings; + let ecosystems = options.ecosystems.as_deref(); + let provider = Provider::new(api, options.request_timeout, options.provider_concurrency); + + let root_list: Vec = match &options.project_roots { + Some(roots) => roots.clone(), + None => roots::detect_roots(files.keys().map(String::as_str), ecosystems).0, + }; + if root_list.len() as u64 > options.limits.max_projects { + return Err(EngineError::limit( + "max_projects", + format!( + "{} project roots exceed the {}-project limit", + root_list.len(), + options.limits.max_projects + ), + )); + } + unrooted_unsupported_warnings( + files.keys().map(String::as_str), + &root_list, + ecosystems, + &mut warnings, + ); + let mut states: Vec = root_list + .iter() + .map(|root| { + let (project, unreadable) = project_for(root, &files); + (root, project, unreadable) + }) + .map(|(root, project, unreadable)| RootState { + root: root.clone(), + project: Some(project), + unreadable, + ledger: None, + purls: Vec::new(), + summary: ProjectSummary::default(), + packages: Vec::new(), + selected: Vec::new(), + skipped: Vec::new(), + error: None, + }) + .collect(); + drop(files); + demote_cargo_members(&mut states, &mut warnings); + phases.mark("roots"); + + for state in &mut states { + checkpoint(&cancel).await?; + let Some(project) = state.project.as_ref() else { + continue; + }; + match ledger::load(project, &state.root) { + Ok(loaded) => state.ledger = loaded, + Err(message) => { + state.fail("corrupt_ledger", message); + continue; + } + } + let (entries, unsupported) = + inventory_project_diagnosed_in(&ProjectView::Memory(project)).await; + for (code, detail) in crate::commands::scan::unsupported_layout_warnings(&unsupported) { + warnings.push(EngineWarning::new(code, detail, Some(&state.root))); + } + unsupported_ecosystem_warnings(&state.root, project, ecosystems, &mut warnings); + let purls: BTreeSet = entries + .iter() + .filter_map(|e| discover::supplement_purl(&e.purl)) + .filter(|p| ecosystem_allowed(ecosystems, p)) + .collect(); + state.purls = purls.into_iter().collect(); + state.summary.scanned_packages = state.purls.len() as u64; + } + let union_purls: BTreeSet<&str> = states + .iter() + .filter(|s| s.error.is_none()) + .flat_map(|s| s.purls.iter().map(String::as_str)) + .collect(); + if union_purls.len() as u64 > options.limits.max_purls { + return Err(EngineError::limit( + "max_purls", + format!( + "{} distinct packages exceed the {}-package limit", + union_purls.len(), + options.limits.max_purls + ), + )); + } + phases.mark("inventory"); + + let root_purls: BTreeMap> = states + .iter() + .filter(|s| s.error.is_none() && !s.purls.is_empty()) + .map(|s| (s.root.clone(), s.purls.clone())) + .collect(); + let batch = discover::batch_search(&provider, &root_purls, options.batch_size).await; + let can_access_paid = batch.can_access_paid_patches; + for state in states.iter_mut().filter(|s| s.error.is_none()) { + state.summary.can_access_paid_patches = can_access_paid; + let Some(outcome) = batch.roots.get(&state.root) else { + continue; + }; + let error = outcome + .last_error + .clone() + .unwrap_or_else(|| "all batches failed".to_string()); + if outcome.failed_purls > 0 && outcome.failed_purls >= state.purls.len() { + state.fail("patch_lookup_failed", error); + continue; + } + if outcome.failed_purls > 0 { + warnings.push(EngineWarning::new( + "batch_failed", + format!( + "{} of {} packages could not be queried for patches: {error}", + outcome.failed_purls, + state.purls.len() + ), + Some(&state.root), + )); + } + state.packages = outcome.packages.clone(); + state.summary.packages_with_patches = state.packages.len() as u64; + for pkg in &state.packages { + for patch in &pkg.patches { + if patch.tier == "free" { + state.summary.free_patches += 1; + } else { + state.summary.paid_patches += 1; + } + } + } + state.summary.total_patches = state.summary.free_patches + state.summary.paid_patches; + } + phases.mark("batch"); + + let detail_purls: BTreeSet = states + .iter() + .filter(|s| s.error.is_none()) + .flat_map(|s| s.packages.iter().map(|p| p.purl.clone())) + .collect(); + let details = discover::fetch_details(&provider, &detail_purls).await; + for state in states.iter_mut().filter(|s| s.error.is_none()) { + if state.packages.is_empty() { + continue; + } + let mut results: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + for pkg in &state.packages { + match details.get(&pkg.purl) { + Some(Ok(response)) => results.extend(response.patches.iter().cloned()), + Some(Err(error)) => failures.push(error.clone()), + None => failures.push("patch details were not fetched".to_string()), + } + } + if !failures.is_empty() && failures.len() == state.packages.len() { + let last = failures.last().cloned().unwrap_or_default(); + state.fail( + "patch_lookup_failed", + format!("all {} patch-detail queries failed: {last}", failures.len()), + ); + continue; + } + if !failures.is_empty() { + warnings.push(EngineWarning::new( + "detail_lookup_failed", + format!( + "patch details could not be fetched for {} of {} packages", + failures.len(), + state.packages.len() + ), + Some(&state.root), + )); + } + state.selected = discover::select_top_ranked(&results, can_access_paid); + } + phases.mark("details"); + + let uuids: BTreeSet = states + .iter() + .filter(|s| s.error.is_none()) + .flat_map(|s| s.selected.iter().map(|(_, u)| u.clone())) + .collect(); + let (references, failed_refs) = if uuids.is_empty() { + (HashMap::new(), BTreeMap::new()) + } else { + discover::fetch_references(&provider, &uuids).await + }; + for state in states.iter_mut().filter(|s| s.error.is_none()) { + if let Some(error) = state + .selected + .iter() + .find_map(|(_, uuid)| failed_refs.get(uuid)) + { + state.fail( + "reference_lookup_failed", + format!("failed to resolve patch references: {error}"), + ); + } + } + phases.mark("references"); + + let mut planned: Vec<(usize, Planned)> = Vec::new(); + for (index, state) in states.iter_mut().enumerate() { + if state.error.is_some() { + continue; + } + checkpoint(&cancel).await?; + let Some(project) = state.project.take() else { + continue; + }; + let unreadable = std::mem::take(&mut state.unreadable); + match redirect::plan(project, unreadable, &state.selected, &references) { + Ok(plan) => planned.push((index, plan)), + Err(Refused { error }) => state.error = Some(error), + } + } + let wheels: BTreeSet<(String, String)> = planned + .iter() + .flat_map(|(_, p)| p.wheels.iter().cloned()) + .collect(); + let wheel_metadata = if wheels.is_empty() { + BTreeMap::new() + } else { + discover::fetch_wheel_metadata(&provider, &wheels, options.limits.max_artifact_bytes).await + }; + phases.mark("plan"); + + let stage = StageOptions { + dry_run: options.dry_run, + pipenv_major: options.pipenv_major, + trust_lockfile_config: options.trust_lockfile_config, + npm_allow_remote_config: options.npm_allow_remote_config, + }; + let mut rewritten: Vec<(usize, Rewritten)> = Vec::new(); + for (index, plan) in planned { + checkpoint(&cancel).await?; + let skipped_before = plan.skipped.clone(); + match redirect::rewrite(plan, &wheel_metadata, stage) { + Ok(done) => rewritten.push((index, done)), + Err(Refused { error }) => { + states[index].skipped = skipped_before; + states[index].error = Some(error); + } + } + } + phases.mark("rewrite"); + + let record_uuids: BTreeSet = if options.dry_run { + BTreeSet::new() + } else { + rewritten + .iter() + .flat_map(|(_, r)| r.confirmed.iter().map(|(_, u)| u.clone())) + .collect() + }; + let records: BTreeMap> = if record_uuids.is_empty() { + BTreeMap::new() + } else { + discover::fetch_records(&provider, &record_uuids).await + }; + phases.mark("records"); + + let mut changed: BTreeMap = BTreeMap::new(); + let mut changed_binary: BTreeMap)> = BTreeMap::new(); + let mut results: BTreeMap = BTreeMap::new(); + for (index, done) in rewritten { + let state = &mut states[index]; + let result = finish_root( + state, + done, + &records, + options.dry_run, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + results.insert(index, result); + } + let mut projects: Vec = Vec::with_capacity(states.len()); + for (index, state) in states.iter().enumerate() { + if let Some(result) = results.remove(&index) { + projects.push(result); + continue; + } + let redirect = match &state.error { + Some(_) => serde_json::json!({ "mode": "hosted" }), + None => crate::commands::scan::hosted::redirect_json_block( + 0, + Vec::new(), + Vec::new(), + Vec::new(), + options.dry_run, + ), + }; + projects.push(ProjectResult { + root: state.root.clone(), + redirect, + summary: state.summary.clone(), + redirected: Vec::new(), + skipped: state.skipped.clone(), + error: state.error.clone(), + }); + } + phases.mark("finish"); + + let changed_files: Vec = changed + .into_iter() + .map(|(path, (_, content))| ChangedFile { path, content }) + .collect(); + let changed_binary_files: Vec = changed_binary + .into_iter() + .map(|(path, (_, content))| ChangedBinaryFile { path, content }) + .collect(); + let stats = EngineStats { + projects: projects.len() as u64, + files_input, + bytes_input, + packages_scanned: projects.iter().map(|p| p.summary.scanned_packages).sum(), + packages_with_patches: projects + .iter() + .map(|p| p.summary.packages_with_patches) + .sum(), + patches_selected: states.iter().map(|s| s.selected.len() as u64).sum(), + patches_redirected: projects.iter().map(|p| p.redirected.len() as u64).sum(), + files_changed: (changed_files.len() + changed_binary_files.len()) as u64, + provider_calls: provider.calls(), + phase_ms: phases.ms, + }; + Ok(HostedScanOutput { + projects, + changed_files, + changed_binary_files, + deleted_files: Vec::new(), + warnings, + stats, + engine_version: engine_version(), + }) +} + +/// Records → ledger merge → the project's result and changed files. +fn finish_root( + state: &mut RootState, + done: Rewritten, + records: &BTreeMap>, + dry_run: bool, + changed: &mut BTreeMap, + changed_binary: &mut BTreeMap)>, + warnings: &mut Vec, +) -> ProjectResult { + let Rewritten { + planned, + rewrite, + rewritten, + confirmed, + rush_warnings, + pnpm_warnings, + npm_warnings, + } = done; + let root = state.root.clone(); + let mut record_map: BTreeMap = BTreeMap::new(); + let mut record_warnings: Vec = Vec::new(); + if !dry_run { + for (purl, uuid) in &confirmed { + match records.get(uuid) { + Some(Some(response)) => { + let (rec_purl, record) = + crate::commands::get::record_from_patch_response(response); + record_map.insert(rec_purl, record); + } + _ => record_warnings.push(serde_json::json!({ + "code": "record_fetch_failed", + "detail": format!( + "{purl} redirected, but its patch record could not be fetched; \ + it will be missing from VEX until `socket-patch scan --mode \ + hosted` is re-run" + ), + })), + } + } + } + + let mut project_changes: Vec<(String, String)> = Vec::new(); + let mut ledger_error: Option = None; + if !dry_run && (!rewrite.edits.is_empty() || !record_map.is_empty()) { + let mut ledger = state.ledger.take().unwrap_or_default(); + ledger::merge(&mut ledger, &rewrite.edits, record_map, &planned.files); + match ledger::serialize(&ledger) { + Ok(text) => { + if planned.project.text(REDIRECT_STATE_REL) != Some(text.as_str()) { + project_changes.push((REDIRECT_STATE_REL.to_string(), text)); + } + } + Err(message) => { + ledger_error = Some(ProjectError { + code: "ledger_serialize_failed".into(), + message, + }) + } + } + } + if let Some(error) = ledger_error { + return ProjectResult { + root, + redirect: serde_json::json!({ "mode": "hosted" }), + summary: state.summary.clone(), + redirected: Vec::new(), + skipped: planned.skipped, + error: Some(error), + }; + } + for (rel, content) in &rewrite.files { + if planned.project.text(rel) != Some(content.as_str()) { + project_changes.push((rel.clone(), content.clone())); + } + } + let text_writes: Vec<(String, String)> = project_changes + .into_iter() + .map(|(rel, content)| (roots::join_root(&root, &rel), content)) + .collect(); + let binary_writes: Vec<(String, Vec)> = rewrite + .binary_files + .iter() + .filter(|(rel, bytes)| { + !matches!( + planned.project.get(rel.as_str()), + Some(MemoryEntry::Binary(existing)) if existing.as_ref() == bytes.as_slice() + ) + }) + .map(|(rel, bytes)| (roots::join_root(&root, rel), bytes.clone())) + .collect(); + let conflict = text_writes + .iter() + .find_map(|(path, content)| match changed.get(path) { + Some((owner, existing)) if existing != content => Some((path, owner)), + _ => None, + }) + .or_else(|| { + binary_writes + .iter() + .find_map(|(path, bytes)| match changed_binary.get(path) { + Some((owner, existing)) if existing != bytes => Some((path, owner)), + _ => None, + }) + }); + if let Some((path, owner)) = conflict { + let message = format!( + "{path} is rewritten differently by project `{owner}`; none of this project's \ + changes were written" + ); + warnings.push(EngineWarning::new( + "conflicting_write", + message.clone(), + Some(&root), + )); + return ProjectResult { + root, + redirect: serde_json::json!({ "mode": "hosted" }), + summary: state.summary.clone(), + redirected: Vec::new(), + skipped: planned.skipped, + error: Some(ProjectError { + code: "conflicting_write".into(), + message, + }), + }; + } + for (path, content) in text_writes { + changed + .entry(path) + .or_insert_with(|| (root.clone(), content)); + } + for (path, bytes) in binary_writes { + changed_binary + .entry(path) + .or_insert_with(|| (root.clone(), bytes)); + } + + let mut redirect_warnings: Vec = rewrite + .warnings + .iter() + .map(|w| serde_json::json!({ "code": w.code, "detail": w.detail })) + .collect(); + redirect_warnings.extend(record_warnings); + redirect_warnings.extend(rush_warnings); + redirect_warnings.extend(pnpm_warnings); + redirect_warnings.extend(npm_warnings); + redirect_warnings.extend(planned.pre_warnings.iter().cloned()); + let skipped_values: Vec = planned + .skipped + .iter() + .map(|s| serde_json::to_value(s).unwrap_or(serde_json::Value::Null)) + .collect(); + let redirect = crate::commands::scan::hosted::redirect_json_block( + confirmed.len(), + rewritten, + skipped_values, + redirect_warnings, + dry_run, + ); + ProjectResult { + root, + redirect, + summary: state.summary.clone(), + redirected: confirmed + .into_iter() + .map(|(purl, uuid)| RedirectedPatch { purl, uuid }) + .collect(), + skipped: planned.skipped, + error: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use socket_patch_core::patch::redirect::{FileEdit, RewriteResult}; + + fn state(root: &str, project: MemoryProject) -> RootState { + RootState { + root: root.to_string(), + project: Some(project), + unreadable: BTreeSet::new(), + ledger: None, + purls: Vec::new(), + summary: ProjectSummary::default(), + packages: Vec::new(), + selected: Vec::new(), + skipped: Vec::new(), + error: None, + } + } + + fn rewritten(files: &[(&str, &str)]) -> Rewritten { + let planned = redirect::plan(MemoryProject::new(), BTreeSet::new(), &[], &HashMap::new()) + .unwrap_or_else(|r| panic!("{:?}", r.error)); + let mut rewrite = RewriteResult::default(); + for (rel, content) in files { + rewrite + .files + .insert((*rel).to_string(), (*content).to_string()); + rewrite.edits.push(FileEdit { + path: (*rel).to_string(), + kind: "cargo_registry_redirect".into(), + action: "set".into(), + key: None, + original: None, + new: None, + }); + } + Rewritten { + planned, + rewrite, + rewritten: files.iter().map(|(rel, _)| (*rel).to_string()).collect(), + confirmed: vec![("pkg:cargo/serde@1.0.190".into(), "u".into())], + rush_warnings: Vec::new(), + pnpm_warnings: Vec::new(), + npm_warnings: Vec::new(), + } + } + + #[test] + fn a_conflicting_project_writes_nothing() { + let mut changed = BTreeMap::new(); + let mut changed_binary = BTreeMap::new(); + let mut warnings = Vec::new(); + let records = BTreeMap::new(); + let first = finish_root( + &mut state("", MemoryProject::new()), + rewritten(&[("crates/b/Cargo.toml", "A")]), + &records, + false, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + assert!(first.error.is_none()); + let before: Vec = changed.keys().cloned().collect(); + let second = finish_root( + &mut state("crates/b", MemoryProject::new()), + rewritten(&[("Cargo.toml", "B"), ("Cargo.lock", "L")]), + &records, + false, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + assert_eq!(second.error.as_ref().unwrap().code, "conflicting_write"); + assert!(second.redirected.is_empty()); + assert_eq!(second.redirect, serde_json::json!({ "mode": "hosted" })); + assert_eq!(changed.keys().cloned().collect::>(), before); + assert_eq!(changed["crates/b/Cargo.toml"].1, "A"); + assert!(warnings.iter().any( + |w| w.code == "conflicting_write" && w.project_root.as_deref() == Some("crates/b") + )); + + let same = finish_root( + &mut state("crates/b", MemoryProject::new()), + rewritten(&[("Cargo.toml", "A")]), + &records, + true, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + assert!(same.error.is_none()); + } + + #[test] + fn a_conflicting_binary_write_refuses_the_project() { + let mut changed = BTreeMap::new(); + let mut changed_binary = BTreeMap::new(); + changed_binary.insert("web/bun.lockb".to_string(), ("".to_string(), vec![1u8])); + let mut warnings = Vec::new(); + let mut done = rewritten(&[]); + done.rewrite + .binary_files + .insert("bun.lockb".to_string(), vec![2u8]); + let result = finish_root( + &mut state("web", MemoryProject::new()), + done, + &BTreeMap::new(), + true, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + assert_eq!(result.error.unwrap().code, "conflicting_write"); + assert_eq!(changed_binary["web/bun.lockb"].1, vec![1u8]); + assert!(changed.is_empty()); + } + + #[test] + fn workspace_member_roots_lose_their_own_cargo_lock() { + let mut ws = MemoryProject::new(); + ws.insert_text( + "Cargo.toml", + "[workspace]\nmembers = [\"crates/*\"]\nexclude = [\"crates/fuzz\"]\n", + ); + ws.insert_text("Cargo.lock", "version = 3\n"); + ws.insert_text("crates/b/Cargo.toml", "[package]\nname = \"b\"\n"); + ws.insert_text("crates/b/Cargo.lock", "version = 3\n"); + ws.insert_text("crates/b/package-lock.json", "{}"); + ws.insert_text("crates/fuzz/Cargo.toml", "[package]\nname = \"fuzz\"\n"); + ws.insert_text("crates/fuzz/Cargo.lock", "version = 3\n"); + ws.insert_text("crates/fuzz/package-lock.json", "{}"); + let sub = |dir: &str| { + let mut p = MemoryProject::new(); + for (path, entry) in ws.entries() { + if let Some(rel) = roots::strip_root(dir, path) { + p.insert(rel, entry.clone()); + } + } + p + }; + let mut states = vec![ + state("", ws.clone()), + state("crates/b", sub("crates/b")), + state("crates/fuzz", sub("crates/fuzz")), + ]; + let mut warnings = Vec::new(); + demote_cargo_members(&mut states, &mut warnings); + let has_lock = |i: usize| states[i].project.as_ref().unwrap().contains("Cargo.lock"); + assert!(has_lock(0)); + assert!(!has_lock(1), "a member's lock is the workspace's"); + assert!(has_lock(2), "an excluded crate is its own workspace"); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "cargo_member_lock_ignored"); + assert_eq!(warnings[0].project_root.as_deref(), Some("crates/b")); + } + + #[test] + fn nested_roots_share_one_copy_of_each_file() { + let mut files: BTreeMap = BTreeMap::new(); + files.insert( + "a/b/package-lock.json".into(), + share(InputFile::Text("{}".repeat(1024))), + ); + files.insert( + "a/big.lock".into(), + share(InputFile::Present(PresentKind::Oversize)), + ); + files.insert( + "a/odd.txt".into(), + share(InputFile::Present(PresentKind::BinarySkipped)), + ); + let (outer, outer_unreadable) = project_for("", &files); + let (inner, _) = project_for("a/b", &files); + match ( + outer.get("a/b/package-lock.json"), + inner.get("package-lock.json"), + ) { + (Some(MemoryEntry::Text(x)), Some(MemoryEntry::Text(y))) => { + assert!(Arc::ptr_eq(x, y), "each root must share the input's bytes") + } + other => panic!("{other:?}"), + } + assert_eq!( + outer_unreadable, + BTreeSet::from(["a/big.lock".to_string()]), + "a non-UTF-8 file is absent to disk too" + ); + } + + #[test] + fn unsupported_markers_outside_roots_warn_once_per_ecosystem() { + let paths = [ + "java/a/pom.xml", + "java/b/pom.xml", + "web/pom.xml", + "dotnet/nuget.config", + "tests/pom.xml", + "src/Main.java", + ]; + let mut out = Vec::new(); + unrooted_unsupported_warnings(paths.into_iter(), &["web".to_string()], None, &mut out); + assert_eq!(out.len(), 2); + assert!(out + .iter() + .all(|w| w.code == "ecosystem_unsupported_in_memory" && w.project_root.is_none())); + assert!( + out[0].detail.starts_with("java/a/pom.xml (and 1 more)"), + "{}", + out[0].detail + ); + assert!( + out[1].detail.starts_with("dotnet/nuget.config is"), + "{}", + out[1].detail + ); + let mut filtered = Vec::new(); + unrooted_unsupported_warnings( + paths.into_iter(), + &[], + Some(&["npm".to_string()]), + &mut filtered, + ); + assert!(filtered.is_empty()); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/redirect.rs b/crates/socket-patch-cli/src/hosted_memory/redirect.rs new file mode 100644 index 00000000..e514be09 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/redirect.rs @@ -0,0 +1,1151 @@ +//! One project root's hosted redirect over an in-memory file set: the +//! disk flow's `run_redirect_selected` stages as pure functions +//! (reference → `DepOverride` candidates, candidate-file reads, the +//! rewrite, the pnpm `trustLockfile` and npm `allow-remote` auto-configs, +//! per-ecosystem confirmation, the symlink guard). Everything that needs +//! the host machine (the apply lock, vendored takeover reverts, stale +//! install probes, VEX, telemetry, subprocesses) is left out; a vendored +//! takeover is refused instead of performed. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use socket_patch_core::api::types::PackageVendorResult; +use socket_patch_core::constants::npm_family::{RUSH_COMMON_LOCK_REL, RUSH_SUBSPACES_DIR}; +use socket_patch_core::patch::redirect::npmrc::{ + plan_npmrc_allow_remote_with, NpmrcPlan, OuterAllowRemote, NPMRC_ALLOW_REMOTE_EDIT_KIND, + NPMRC_REL, +}; +use socket_patch_core::patch::redirect::{ + rewrite_registry_redirect_with_pipenv_version, DepOverride, FileEdit, RewriteResult, + RewriteWarning, +}; +use socket_patch_core::utils::purl::{purl_parts, strip_purl_qualifiers}; +use socket_patch_core::vendor::lock_inventory::{MemoryEntry, MemoryProject}; +use socket_patch_core::vendor::VendorState; + +use super::select::{RUSH_REPO_STATE_REL, VENDOR_STATE_REL}; +use super::types::{ProjectError, SkippedPatch}; +use crate::commands::scan::hosted::{ + npm_allow_remote_already_detail, npm_allow_remote_configured_detail, + npm_allow_remote_env_set_detail, npm_allow_remote_manual_detail, + npm_allow_remote_outer_set_detail, npm_allow_remote_unreadable_detail, + npm_allow_remote_user_set_detail, plan_workspace_trust, pnpm_heal_root, + pnpm_lock_may_need_store_flag, pnpm_lock_version_major, pnpm_trust_configured_detail, + pnpm_trust_legacy_detail, pnpm_trust_manual_guidance, pnpm_trust_policy_preamble, + pnpm_trust_workspace_unreadable_detail, url_host, TrustPlan, NPM_LOCKS, + PNPM_TRUST_TRADEOFF_AND_CAUTION, PNPM_WORKSPACE_REL, REDIRECT_CANDIDATE_FILES, + REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND, +}; + +/// Skip reason / warning code for a candidate the disk flow would migrate +/// from vendored to hosted (the migration reverts committed wiring, which +/// the in-memory engine does not do). +pub(crate) const VENDORED_TAKEOVER_UNSUPPORTED: &str = "vendored_takeover_unsupported_in_memory"; + +/// The disk flow's symlink refusal code. +pub(crate) const SYMLINK_REFUSAL: &str = "redirect_symlinked_file_unsupported"; + +/// A candidate file exists but its content was not provided (oversize, an +/// LFS pointer, presence-only); disk would read and rewrite it. +pub(crate) const UNREADABLE_REFUSAL: &str = "candidate_file_unreadable"; + +/// One granted reference: the purl it was granted for plus its override. +#[derive(Debug, Clone)] +pub(crate) struct Candidate { + pub(crate) purl: String, + pub(crate) dep: DepOverride, +} + +/// The engine-level options the per-root stages read. +#[derive(Debug, Clone, Copy)] +pub(crate) struct StageOptions { + pub(crate) dry_run: bool, + pub(crate) pipenv_major: Option, + pub(crate) trust_lockfile_config: bool, + pub(crate) npm_allow_remote_config: bool, +} + +/// Reference grants → candidates (disk: the loop over `selected` after +/// `fetch_registry_references`). +pub(crate) fn build_candidates( + selected: &[(String, String)], + references: &HashMap, + skipped: &mut Vec, +) -> Vec { + let skip = |purl: &str, uuid: &str, reason: &str| SkippedPatch { + purl: purl.to_string(), + uuid: uuid.to_string(), + reason: reason.to_string(), + detail: None, + }; + let mut candidates = Vec::new(); + for (sel_purl, sel_uuid) in selected { + let Some(reference) = references.get(sel_uuid) else { + skipped.push(skip(sel_purl, sel_uuid, "not_found")); + continue; + }; + if reference.status != "granted" && reference.status != "reused" { + skipped.push(skip(sel_purl, sel_uuid, &reference.status)); + continue; + } + let purl = reference.purl.as_deref().unwrap_or(sel_purl); + let Some((ecosystem, name, version)) = purl_parts(purl) else { + skipped.push(skip(purl, sel_uuid, "bad_purl")); + continue; + }; + let Some(url) = reference.url.clone() else { + skipped.push(skip(purl, sel_uuid, "no_url")); + continue; + }; + let mut integrity = reference + .artifacts + .iter() + .flatten() + .find(|a| a.kind == "tarball") + .map(|a| a.integrity.clone()) + .unwrap_or_default(); + let berry_zip = reference + .artifacts + .iter() + .flatten() + .find(|a| a.kind == "yarn-berry-zip"); + if let Some(c) = berry_zip.and_then(|a| a.integrity.yarn_berry10c0.clone()) { + integrity.yarn_berry10c0 = Some(c); + } + if let Some(ov) = reference + .registry_override + .as_ref() + .filter(|o| o.kind == "goproxy") + { + if let (Some(zip_h1), Some(gomod_h1)) = ( + ov.identifiers.go_zip_dirhash_h1.clone(), + ov.identifiers.go_mod_h1.clone(), + ) { + integrity.dirhash_h1 = Some(zip_h1); + integrity.go_mod_h1 = Some(gomod_h1); + } + } + let token = reference + .registry_override + .as_ref() + .and_then(|o| { + socket_patch_core::patch::redirect::grant_token_path_segment(&o.index_url, sel_uuid) + }) + .or_else(|| { + socket_patch_core::patch::redirect::grant_token_path_segment(&url, sel_uuid) + }) + .unwrap_or_default(); + candidates.push(Candidate { + purl: purl.to_string(), + dep: DepOverride { + ecosystem, + name, + namespace: None, + version, + token, + patch_uuid: sel_uuid.clone(), + artifact_url: url, + berry_zip_url: berry_zip.and_then(|a| a.url.clone()), + registry_override: reference.registry_override.clone(), + integrity, + }, + }); + } + candidates +} + +/// The ecosystem a candidate file's rewriter belongs to (`None` for files +/// no rewriter edits), for the symlinked-read refusal. +fn file_ecosystem(rel: &str) -> Option<&'static str> { + let base = rel.rsplit('/').next().unwrap_or(rel); + Some(match base { + "package-lock.json" + | "npm-shrinkwrap.json" + | "pnpm-lock.yaml" + | "shrinkwrap.yaml" + | ".modules.yaml" + | "yarn.lock" + | ".yarnrc.yml" + | "bun.lock" + | "bun.lockb" => "npm", + "requirements.txt" | "uv.lock" | "poetry.lock" | "pdm.lock" | "Pipfile.lock" + | "pyproject.toml" | "hatch.toml" => "pypi", + "Cargo.toml" | "Cargo.lock" | "config.toml" | "config" => "cargo", + "composer.lock" => "composer", + "nuget.config" | "packages.lock.json" => "nuget", + "Gemfile" | "Gemfile.lock" | "gems.rb" | "gems.locked" => "gem", + "go.mod" | "go.sum" => "golang", + "pom.xml" | "maven.config" | "checksums.sha256" => "maven", + _ if socket_patch_core::utils::python_lock::is_python_lock_name(base) + || base.ends_with(".py") => + { + "pypi" + } + _ => return None, + }) +} + +/// A project's state between the reference grants and the wheel-metadata +/// fetch. +#[derive(Debug)] +pub(crate) struct Planned { + pub(crate) project: MemoryProject, + pub(crate) candidates: Vec, + pub(crate) skipped: Vec, + pub(crate) pre_warnings: Vec, + pub(crate) files: BTreeMap, + pub(crate) rush_lock_keys: Vec, + pub(crate) bun_lock_present: bool, + /// Candidate files the disk flow reads through a symbolic link: their + /// bytes are unknown here, so a project whose candidates could rewrite + /// one is refused like the disk symlink guard refuses the write. + pub(crate) symlinked_reads: Vec, + /// Candidate files that exist without content: a project whose + /// candidates could rewrite (or whose rewrite depends on) one is + /// refused, since the rewriters would treat it as absent. + pub(crate) unreadable_reads: Vec, + /// `(artifact url, sha256)` of every pypi wheel whose metadata a + /// native lock rewrite needs. + pub(crate) wheels: Vec<(String, String)>, +} + +/// A refused project: its error and whatever was skipped before it. +#[derive(Debug)] +pub(crate) struct Refused { + pub(crate) error: ProjectError, +} + +fn refusal(code: &str, message: String) -> Refused { + Refused { + error: ProjectError { + code: code.to_string(), + message, + }, + } +} + +fn unreadable_refusal(rel: &str) -> Refused { + refusal( + UNREADABLE_REFUSAL, + format!( + "{rel} exists but its content was not provided (too large, an LFS pointer, or \ + not fetched), so it cannot be rewritten alongside the other lockfiles; nothing \ + was written" + ), + ) +} + +fn symlink_refusal(linked: &str) -> Refused { + refusal( + SYMLINK_REFUSAL, + format!( + "{linked} is a symbolic link; socket-patch rewrites files in place with an atomic \ + rename, which would replace the link — replace the link with a regular file (or \ + run socket-patch in the directory it points to) and re-run; nothing was written" + ), + ) +} + +/// The vendored ledger's entries (the disk `vendor::load_state` parse, +/// including its legacy `{mode}`-only shape); `None` when absent or +/// unreadable. +fn vendored_entries(project: &MemoryProject) -> Option { + let bytes: Vec = match project.get(VENDOR_STATE_REL)? { + MemoryEntry::Text(text) => text.as_bytes().to_vec(), + MemoryEntry::Binary(bytes) => bytes.to_vec(), + _ => return None, + }; + match serde_json::from_slice::(&bytes) { + Ok(state) => Some(state), + Err(_) => { + let value: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + (value.get("mode").is_some() && value.get("entries").is_none()).then(VendorState::new) + } + } +} + +/// Whether Socket-owned vendored `[patch.crates-io]` wiring for exactly +/// `name@version` is committed in the root manifest or (legacy) the +/// project's cargo config — the disk `socket_wiring_present` probe over +/// the in-memory files. The config cargo reads is `.cargo/config` when it +/// exists, else `.cargo/config.toml`. +fn cargo_vendored_wiring(files: &MemoryProject, name: &str, version: &str) -> bool { + use socket_patch_core::vendor::cargo_manifest::{ + crates_io_patch_entries, entry_wires, parse_manifest, + }; + let manifest_wired = files + .text("Cargo.toml") + .and_then(|text| parse_manifest(text).ok()) + .is_some_and(|doc| { + crates_io_patch_entries(&doc) + .iter() + .any(|e| entry_wires(e, name, version)) + }); + let config_rel = if files.contains(".cargo/config") { + ".cargo/config" + } else { + ".cargo/config.toml" + }; + let config_wired = files + .text(config_rel) + .and_then(|text| parse_manifest(text).ok()) + .is_some_and(|doc| { + crates_io_patch_entries(&doc) + .iter() + .any(|e| e.source == "crates-io" && entry_wires(e, name, version)) + }); + manifest_wired || config_wired +} + +/// Everything up to the wheel-metadata fetch. +pub(crate) fn plan( + project: MemoryProject, + unreadable: BTreeSet, + selected: &[(String, String)], + references: &HashMap, +) -> Result { + let mut skipped: Vec = Vec::new(); + let mut candidates = if selected.is_empty() { + Vec::new() + } else { + build_candidates(selected, references, &mut skipped) + }; + + let bun_lock_present = project.contains("bun.lock"); + if candidates.iter().any(|c| c.dep.ecosystem == "npm") + && !bun_lock_present + && project.is_symlink("bun.lockb") + { + return Err(refusal( + SYMLINK_REFUSAL, + "bun.lockb is a symbolic link; replace it with a regular file (or run \ + socket-patch in the directory it points to) before patching; nothing was written" + .to_string(), + )); + } + + let mut pre_warnings: Vec = Vec::new(); + let takeover_capable = |p: &str| { + p.starts_with("pkg:cargo/") || p.starts_with("pkg:npm/") || p.starts_with("pkg:golang/") + }; + if candidates.iter().any(|c| takeover_capable(&c.purl)) { + let vendored = vendored_entries(&project); + let mut refused: BTreeSet = BTreeSet::new(); + for candidate in candidates.iter().filter(|c| takeover_capable(&c.purl)) { + let has_entry = vendored.as_ref().is_some_and(|s| { + socket_patch_core::vendor::lookup_entry( + &s.entries, + strip_purl_qualifiers(&candidate.purl), + ) + .is_some() + }); + let cargo_wired = !has_entry + && candidate.purl.starts_with("pkg:cargo/") + && cargo_vendored_wiring(&project, &candidate.dep.name, &candidate.dep.version); + if has_entry || cargo_wired { + refused.insert(candidate.purl.clone()); + } + } + if !refused.is_empty() { + pre_warnings.push(serde_json::json!({ + "code": VENDORED_TAKEOVER_UNSUPPORTED, + "detail": format!( + "{} currently vendored ({}); migrating a vendored package to hosted \ + reverts its committed vendored wiring, which the in-memory hosted scan \ + does not do — run `socket-patch scan --mode hosted` in a checkout to \ + migrate, then re-run", + if refused.len() == 1 { "1 package is" } else { "packages are" }, + refused.iter().cloned().collect::>().join(", ") + ), + })); + for c in candidates.iter().filter(|c| refused.contains(&c.purl)) { + skipped.push(SkippedPatch { + purl: c.purl.clone(), + uuid: c.dep.patch_uuid.clone(), + reason: VENDORED_TAKEOVER_UNSUPPORTED.to_string(), + detail: None, + }); + } + candidates.retain(|c| !refused.contains(&c.purl)); + } + } + + let mut files: BTreeMap = BTreeMap::new(); + let mut rush_lock_keys: Vec = Vec::new(); + let mut symlinked_reads: Vec = Vec::new(); + let mut unreadable_reads: Vec = Vec::new(); + if !candidates.is_empty() { + let mut read = |rel: &str, files: &mut BTreeMap| -> bool { + if project.is_symlink(rel) { + symlinked_reads.push(rel.to_string()); + return false; + } + if unreadable.contains(rel) { + unreadable_reads.push(rel.to_string()); + return false; + } + // Disk reads any UTF-8 regular file; a non-UTF-8 one is absent + // to it as well. + let text = match project.get(rel) { + Some(MemoryEntry::Text(text)) => Some(text.to_string()), + Some(MemoryEntry::Binary(bytes)) => { + std::str::from_utf8(bytes).ok().map(str::to_string) + } + _ => None, + }; + match text { + Some(text) => { + files.insert(rel.to_string(), text); + true + } + None => false, + } + }; + for name in REDIRECT_CANDIDATE_FILES { + if *name == "bun.lockb" { + continue; + } + read(name, &mut files); + } + if files.contains_key("Cargo.toml") && candidates.iter().any(|c| c.dep.ecosystem == "cargo") + { + let view = socket_patch_core::vendor::lock_inventory::ProjectView::Memory(&project); + for rel in socket_patch_core::utils::cargo_workspace::member_manifests_in(&view) { + read(&rel, &mut files); + } + } + let python_locks: Vec = project + .children("") + .into_iter() + .filter(|(name, is_dir)| { + !is_dir && socket_patch_core::utils::python_lock::is_python_lock_name(name) + }) + .map(|(name, _)| name) + .collect(); + for path in python_locks { + if let Some(script) = socket_patch_core::utils::python_lock::script_of_lock(&path) { + read(script, &mut files); + } + read(&path, &mut files); + } + if project.contains("rush.json") { + if read(RUSH_COMMON_LOCK_REL, &mut files) { + rush_lock_keys.push(RUSH_COMMON_LOCK_REL.to_string()); + } + for (name, is_dir) in project.children(RUSH_SUBSPACES_DIR) { + if !is_dir { + continue; + } + let key = format!("{RUSH_SUBSPACES_DIR}/{name}/pnpm-lock.yaml"); + if read(&key, &mut files) { + rush_lock_keys.push(key); + } + } + } + } + symlinked_reads.sort(); + symlinked_reads.dedup(); + unreadable_reads.sort(); + unreadable_reads.dedup(); + + let mut wheels: Vec<(String, String)> = Vec::new(); + for dep in candidates + .iter() + .map(|c| &c.dep) + .filter(|dep| dep.ecosystem == "pypi") + { + let Some(sha256) = dep.integrity.sha256.as_deref() else { + continue; + }; + if !dep + .artifact_url + .split(['?', '#']) + .next() + .is_some_and(|path| path.ends_with(".whl")) + { + continue; + } + let native_target = files + .iter() + .filter(|(path, _)| { + *path == "uv.lock" + || socket_patch_core::utils::python_lock::is_script_lock_name(path) + }) + .any(|(_, text)| { + socket_patch_core::utils::python_lock::rewrite_python_lock( + text, + &dep.name, + &dep.version, + socket_patch_core::utils::python_lock::ArtifactSource::Url(&dep.artifact_url), + sha256, + ) + .ok() + .flatten() + .is_some() + }); + if native_target { + wheels.push((dep.artifact_url.clone(), sha256.to_string())); + } + } + + Ok(Planned { + project, + candidates, + skipped, + pre_warnings, + files, + rush_lock_keys, + bun_lock_present, + symlinked_reads, + unreadable_reads, + wheels, + }) +} + +/// A project's rewrite, ready for the record fetch and the ledger merge. +#[derive(Debug)] +pub(crate) struct Rewritten { + pub(crate) planned: Planned, + pub(crate) rewrite: RewriteResult, + pub(crate) rewritten: Vec, + pub(crate) confirmed: Vec<(String, String)>, + pub(crate) rush_warnings: Vec, + pub(crate) pnpm_warnings: Vec, + pub(crate) npm_warnings: Vec, +} + +/// The `.npmrc` read the allow-remote planner classifies (disk: +/// `read_npmrc_for_allow_remote`). +fn read_npmrc(project: &MemoryProject) -> Result, String> { + match project.get(NPMRC_REL) { + None => Ok(None), + Some(MemoryEntry::Symlink) => { + Err("is a symbolic link (socket-patch never writes through one)".into()) + } + Some(MemoryEntry::Text(text)) => Ok(Some(text.to_string())), + Some(MemoryEntry::Binary(_)) => { + Err("could not be read (stream did not contain valid UTF-8)".into()) + } + Some(MemoryEntry::Present) => { + Err("could not be read (file content was not provided)".into()) + } + } +} + +/// Wheel metadata → rewrite → install-policy configs → confirmation → +/// symlink guard. +pub(crate) fn rewrite( + mut planned: Planned, + wheel_metadata: &BTreeMap, String>>, + options: StageOptions, +) -> Result { + let project = &planned.project; + let files = &planned.files; + + let mut python_metadata: BTreeMap = BTreeMap::new(); + let mut unavailable: BTreeSet = BTreeSet::new(); + for (url, _) in &planned.wheels { + match wheel_metadata.get(url) { + Some(Ok(Some(metadata))) => { + python_metadata.insert(url.clone(), metadata.clone()); + } + Some(Ok(None)) => {} + Some(Err(detail)) => { + if unavailable.insert(url.clone()) { + for dep in planned + .candidates + .iter() + .map(|c| &c.dep) + .filter(|d| &d.artifact_url == url) + { + planned.skipped.push(SkippedPatch { + purl: format!("pkg:pypi/{}@{}", dep.name, dep.version), + uuid: dep.patch_uuid.clone(), + reason: "python_metadata_unavailable".to_string(), + detail: Some(detail.replace(&dep.artifact_url, "")), + }); + } + } + } + None => { + unavailable.insert(url.clone()); + } + } + } + planned + .candidates + .retain(|c| !unavailable.contains(&c.dep.artifact_url)); + let candidates = &planned.candidates; + let overrides: Vec = candidates.iter().map(|c| c.dep.clone()).collect(); + + let targets_pipenv_lock = + socket_patch_core::patch::redirect::pipenv_lock_targets(files, &overrides); + let pipenv_major = if targets_pipenv_lock { + options.pipenv_major + } else { + None + }; + let binary_bun = !planned.bun_lock_present && project.contains("bun.lockb"); + let binary_content: Option, RewriteWarning>> = + if binary_bun && overrides.iter().any(|o| o.ecosystem == "npm") { + let read = match project.get("bun.lockb") { + Some(MemoryEntry::Binary(bytes)) => Ok(bytes.to_vec()), + Some(MemoryEntry::Text(text)) => Ok(text.as_bytes().to_vec()), + _ => Err("file content was not provided".to_string()), + }; + Some( + read.map_err(|e| RewriteWarning { + code: "redirect_bun_lockb_invalid".into(), + detail: format!("cannot read bun.lockb: {e}"), + }) + .and_then(|bytes| { + socket_patch_core::patch::redirect::preflight_bun_binary(&bytes)?; + Ok(bytes) + }), + ) + } else { + None + }; + let rewrite_overrides: Vec = overrides + .iter() + .filter(|o| !(binary_content.as_ref().is_some_and(Result::is_err) && o.ecosystem == "npm")) + .cloned() + .collect(); + let mut rewrite = rewrite_registry_redirect_with_pipenv_version( + files, + &rewrite_overrides, + &python_metadata, + pipenv_major, + ); + if let Some(content) = binary_content { + rewrite + .warnings + .retain(|w| w.code != "redirect_npm_no_lockfile"); + match content { + Ok(bytes) => socket_patch_core::patch::redirect::rewrite_bun_binary( + &bytes, + &overrides, + &mut rewrite, + ), + Err(warning) => rewrite.warnings.push(warning), + } + } + + if targets_pipenv_lock && pipenv_major.is_none() && rewrite.files.contains_key("Pipfile.lock") { + rewrite.warnings.push(RewriteWarning { + code: "redirect_pipenv_installer_unknown".into(), + detail: "The scan did not set `pipenvMajor`, so the Pipfile.lock references use the modern `file` form (Pipenv 2018 and later). A project installed with Pipenv 7–11 needs `path` references instead: re-run the scan with `pipenvMajor` set to that Pipenv major version.".into(), + }); + } + + let mut rush_warnings: Vec = Vec::new(); + if planned + .rush_lock_keys + .iter() + .any(|key| rewrite.files.contains_key(key)) + && (project.contains(RUSH_REPO_STATE_REL)) + { + rush_warnings.push(serde_json::json!({ + "code": "redirect_rush_repo_state_stale", + "detail": + "pnpm-lock.yaml was edited outside `rush update`; if \ + preventManualShrinkwrapChanges is enabled, `rush install` fails until \ + `rush update` refreshes repo-state.json (the redirect survives `rush \ + update`)", + })); + } + + let mut pnpm_warnings: Vec = Vec::new(); + let mut trust_config_write: Option<(String, FileEdit)> = None; + let mut workspace_symlink_write = false; + { + let mut pnpm_lock_texts: Vec<&String> = rewrite + .files + .iter() + .filter(|(key, _)| { + std::path::Path::new(key) + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|name| matches!(name, "pnpm-lock.yaml" | "shrinkwrap.yaml")) + }) + .map(|(_, content)| content) + .collect(); + let heal_root: Option<&String> = pnpm_heal_root( + rewrite.files.contains_key("pnpm-lock.yaml"), + files.get("pnpm-lock.yaml"), + &overrides, + ); + if let Some(text) = heal_root { + pnpm_lock_texts.push(text); + } + if !pnpm_lock_texts.is_empty() { + let mut hosts: Vec<&str> = overrides + .iter() + .filter(|o| o.ecosystem == "npm") + .filter(|o| { + let encoded = + socket_patch_core::utils::uri::encode_uri_component(&o.artifact_url); + pnpm_lock_texts.iter().any(|text| { + socket_patch_core::patch::redirect::artifact_url_present( + text, + &o.artifact_url, + ) || text.contains(encoded.as_str()) + }) + }) + .filter_map(|o| url_host(&o.artifact_url)) + .collect(); + hosts.sort_unstable(); + hosts.dedup(); + let server = if hosts.is_empty() { + "the hosted patch server".to_string() + } else { + format!("the hosted patch server ({})", hosts.join(", ")) + }; + let root_lock_v9 = heal_root + .and_then(|text| pnpm_lock_version_major(text)) + .is_some_and(|major| major >= 9) + || rewrite + .files + .get("pnpm-lock.yaml") + .and_then(|text| pnpm_lock_version_major(text)) + .is_some_and(|major| major >= 9); + let all_locks_legacy = pnpm_lock_texts.iter().all(|text| { + pnpm_lock_version_major(text).is_some_and(|major| major < 9) + || text + .lines() + .any(|line| line.starts_with("shrinkwrapVersion:")) + }); + let workspace: Result, std::io::Error> = + match project.get(PNPM_WORKSPACE_REL) { + None => Ok(None), + Some(MemoryEntry::Text(text)) => Ok(Some(text.to_string())), + Some(MemoryEntry::Symlink) => { + workspace_symlink_write = true; + Ok(None) + } + Some(MemoryEntry::Binary(_)) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + )), + Some(MemoryEntry::Present) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "file content was not provided", + )), + }; + let detail = if all_locks_legacy { + workspace_symlink_write = false; + pnpm_trust_legacy_detail(&server) + } else if !root_lock_v9 || !options.trust_lockfile_config { + workspace_symlink_write = false; + pnpm_trust_manual_guidance(&server) + } else { + match workspace { + Err(e) => pnpm_trust_workspace_unreadable_detail(&server, &e), + Ok(ws_existing) => match plan_workspace_trust(ws_existing.as_deref()) { + TrustPlan::Create(text) => { + trust_config_write = Some(( + text, + FileEdit { + path: PNPM_WORKSPACE_REL.into(), + kind: REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND.into(), + action: "created".into(), + key: Some("trustLockfile".into()), + original: None, + new: Some(serde_json::json!("true")), + }, + )); + pnpm_trust_configured_detail(&server, true, options.dry_run) + } + TrustPlan::Append(text) => { + trust_config_write = Some(( + text, + FileEdit { + path: PNPM_WORKSPACE_REL.into(), + kind: REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND.into(), + action: "added".into(), + key: Some("trustLockfile".into()), + original: None, + new: Some(serde_json::json!("true")), + }, + )); + pnpm_trust_configured_detail(&server, false, options.dry_run) + } + TrustPlan::AlreadyTrue => format!( + "{}, and {PNPM_WORKSPACE_REL} already carries `trustLockfile: \ + true` — keep it committed alongside the lock; installs need \ + no extra flags. {PNPM_TRUST_TRADEOFF_AND_CAUTION}", + pnpm_trust_policy_preamble(&server), + ), + TrustPlan::UserSet(value) => format!( + "{}. {PNPM_WORKSPACE_REL} explicitly sets `trustLockfile: \ + {value}`, which was respected and left untouched — install \ + with `pnpm install --trust-lockfile`, or set `trustLockfile: \ + true` yourself so every install accepts the patched \ + artifacts. {PNPM_TRUST_TRADEOFF_AND_CAUTION}", + pnpm_trust_policy_preamble(&server), + ), + }, + } + }; + let store_note = if pnpm_lock_texts + .iter() + .any(|text| pnpm_lock_may_need_store_flag(text)) + { + " (pnpm 1–4 spell the option `--store`)" + } else { + "" + }; + pnpm_warnings.push(serde_json::json!({ + "code": "redirect_pnpm_trust_lockfile", + "detail": format!( + "{}. After a lock-only change, existing node_modules or a warm pnpm store \ + can still contain upstream files. For a reliable reinstall, use a clean \ + node_modules tree and an empty store with \ + `pnpm install --frozen-lockfile --store-dir `\ + {store_note}. Do not rely on `--force`: some versions re-resolve the \ + upstream artifact. Run `socket-patch vex` after installation to verify \ + the patched files.", + detail.trim_end_matches('.') + ), + })); + } + } + if workspace_symlink_write { + return Err(symlink_refusal(PNPM_WORKSPACE_REL)); + } + + let mut npm_warnings: Vec = Vec::new(); + let mut npmrc_config_write: Option<(String, FileEdit)> = None; + { + let npm_hosts: Vec<&str> = { + let mut hosts: Vec<&str> = overrides + .iter() + .filter(|o| o.ecosystem == "npm") + .filter(|o| { + NPM_LOCKS.iter().any(|lock| { + rewrite + .files + .get(*lock) + .or_else(|| files.get(*lock)) + .is_some_and(|text| { + socket_patch_core::patch::redirect::artifact_url_present( + text, + &o.artifact_url, + ) + }) + }) + }) + .filter_map(|o| url_host(&o.artifact_url)) + .collect(); + hosts.sort_unstable(); + hosts.dedup(); + hosts + }; + if !npm_hosts.is_empty() { + let edit = |action: &str| FileEdit { + path: NPMRC_REL.into(), + kind: NPMRC_ALLOW_REMOTE_EDIT_KIND.into(), + action: action.into(), + key: Some("allow-remote".into()), + original: None, + new: Some(serde_json::json!("all")), + }; + let outer = OuterAllowRemote::default(); + let detail = match read_npmrc(project) { + Ok(existing) => match plan_npmrc_allow_remote_with(existing.as_deref(), &outer) { + NpmrcPlan::AlreadyAll => npm_allow_remote_already_detail(&npm_hosts), + NpmrcPlan::UserSet(value) => { + npm_allow_remote_user_set_detail(&npm_hosts, &value) + } + NpmrcPlan::EnvSet { var, value } => { + npm_allow_remote_env_set_detail(&npm_hosts, &var, &value) + } + NpmrcPlan::OuterSet { layer, path, value } => { + npm_allow_remote_outer_set_detail(&npm_hosts, layer, &path, &value) + } + NpmrcPlan::Unsupported(why) => { + npm_allow_remote_unreadable_detail(&npm_hosts, &why) + } + _ if !options.npm_allow_remote_config => { + npm_allow_remote_manual_detail(&npm_hosts) + } + NpmrcPlan::Create(text) => { + npmrc_config_write = Some((text, edit("created"))); + npm_allow_remote_configured_detail(&npm_hosts, true, options.dry_run) + } + NpmrcPlan::Append(text) => { + npmrc_config_write = Some((text, edit("added"))); + npm_allow_remote_configured_detail(&npm_hosts, false, options.dry_run) + } + }, + Err(why) => npm_allow_remote_unreadable_detail(&npm_hosts, &why), + }; + npm_warnings.push(serde_json::json!({ + "code": "redirect_npm_allow_remote", + "detail": detail, + })); + } + } + if let Some((text, edit)) = trust_config_write { + rewrite.files.insert(PNPM_WORKSPACE_REL.to_string(), text); + rewrite.edits.push(edit); + } + if let Some((text, edit)) = npmrc_config_write { + rewrite.files.insert(NPMRC_REL.to_string(), text); + rewrite.edits.push(edit); + } + let rewritten: Vec = rewrite + .files + .keys() + .chain(rewrite.binary_files.keys()) + .cloned() + .collect(); + + let pdm_inactive = + files.contains_key("pdm.lock") && !socket_patch_core::patch::redirect::pdm_drives(files); + let final_texts: Vec<&String> = files + .iter() + .filter(|(name, _)| !(pdm_inactive && name.as_str() == "pdm.lock")) + .map(|(name, content)| rewrite.files.get(name).unwrap_or(content)) + .chain( + rewrite + .files + .iter() + .filter(|(name, _)| !files.contains_key(*name)) + .map(|(_, content)| content), + ) + .collect(); + let confirmed: Vec<(String, String)> = candidates + .iter() + .filter(|c| { + let purl = c.purl.as_str(); + let uuid = c.dep.patch_uuid.as_str(); + if binary_bun && purl.starts_with("pkg:npm/") { + return rewrite.confirmed_bun_binary_uuids.contains(uuid); + } + if rewrite.refused_pipenv_uuids.contains(uuid) { + return false; + } + if rewrite.refused_pdm_uuids.contains(uuid) { + return false; + } + if purl.starts_with("pkg:pypi/") + && socket_patch_core::patch::redirect::pdm_drives(files) + { + return rewrite.confirmed_pdm_uuids.contains(uuid); + } + if rewrite.python_lock_uuids.contains(uuid) { + return rewrite.confirmed_python_lock_uuids.contains(uuid) + && !rewrite.refused_python_lock_uuids.contains(uuid); + } + if rewrite.hatch_uuids.contains(uuid) { + return rewrite.confirmed_hatch_uuids.contains(uuid); + } + if purl.starts_with("pkg:pypi/") { + return rewrite.confirmed_pipenv_uuids.contains(uuid) + || rewrite.confirmed_requirements_uuids.contains(uuid); + } + if rewrite.refused_pnpm_uuids.contains(uuid) { + return false; + } + if purl.starts_with("pkg:cargo/") { + return rewrite.confirmed_cargo_uuids.contains(uuid); + } + if purl.starts_with("pkg:golang/") { + return rewrite.confirmed_golang_uuids.contains(uuid); + } + let artifact_url = c.dep.artifact_url.as_str(); + let registry = c.dep.registry_override.as_ref(); + let index_url = registry.map(|o| o.index_url.as_str()); + let suffixed_version = + registry.and_then(|o| o.identifiers.maven_suffixed_version.as_deref()); + let encoded = socket_patch_core::utils::uri::encode_uri_component(artifact_url); + final_texts.iter().any(|text| { + socket_patch_core::patch::redirect::artifact_url_present(text, artifact_url) + || text.contains(encoded.as_str()) + || index_url.is_some_and(|iu| text.contains(iu)) + || suffixed_version.is_some_and(|sv| text.contains(sv)) + }) + }) + .map(|c| (c.purl.clone(), c.dep.patch_uuid.clone())) + .collect(); + + if let Some(linked) = rewrite + .files + .keys() + .chain(rewrite.binary_files.keys()) + .find(|k| project.is_symlink(k)) + { + return Err(symlink_refusal(linked)); + } + let candidate_ecosystems: BTreeSet<&str> = candidates + .iter() + .map(|c| c.dep.ecosystem.as_str()) + .collect(); + if let Some(linked) = planned + .symlinked_reads + .iter() + .find(|rel| file_ecosystem(rel).is_some_and(|eco| candidate_ecosystems.contains(eco))) + { + return Err(symlink_refusal(linked)); + } + if let Some(rel) = planned + .unreadable_reads + .iter() + .find(|rel| { + rewrite.files.contains_key(rel.as_str()) + || file_ecosystem(rel).is_some_and(|eco| candidate_ecosystems.contains(eco)) + }) + .or_else(|| { + rewrite + .files + .keys() + .find(|k| matches!(project.get(k), Some(MemoryEntry::Present))) + }) + { + return Err(unreadable_refusal(rel)); + } + + Ok(Rewritten { + planned, + rewrite, + rewritten, + confirmed, + rush_warnings, + pnpm_warnings, + npm_warnings, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reference(value: serde_json::Value) -> PackageVendorResult { + serde_json::from_value(value).unwrap() + } + + #[test] + fn candidates_skip_every_unusable_reference() { + let mut refs: HashMap = HashMap::new(); + refs.insert( + "u-pending".into(), + reference(serde_json::json!({"status": "pending_build"})), + ); + refs.insert( + "u-nourl".into(), + reference(serde_json::json!({"status": "granted", "purl": "pkg:npm/b@1"})), + ); + refs.insert( + "u-ok".into(), + reference(serde_json::json!({ + "status": "reused", + "url": "https://patch.example/patch/npm/c/1/tok/u-ok/c-1.tgz", + "purl": "pkg:npm/c@1", + "artifacts": [{"kind": "tarball", "url": null, "integrity": {"sha512": "sha512-x"}}], + "registryOverride": null + })), + ); + let selected = vec![ + ("pkg:npm/a@1".to_string(), "u-missing".to_string()), + ("pkg:npm/p@1".to_string(), "u-pending".to_string()), + ("pkg:npm/b@1".to_string(), "u-nourl".to_string()), + ("pkg:npm/c@1".to_string(), "u-ok".to_string()), + ]; + let mut skipped = Vec::new(); + let candidates = build_candidates(&selected, &refs, &mut skipped); + let reasons: Vec<&str> = skipped.iter().map(|s| s.reason.as_str()).collect(); + assert_eq!(reasons, vec!["not_found", "pending_build", "no_url"]); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].dep.token, "tok"); + assert_eq!( + candidates[0].dep.integrity.sha512.as_deref(), + Some("sha512-x") + ); + } + + #[test] + fn cargo_wiring_probe_is_scoped_to_the_crate_and_version() { + let mut p = MemoryProject::new(); + p.insert_text( + "Cargo.toml", + "[package]\nname = \"app\"\n\n[dependencies]\nlog = \"0.4\"\ncc = \"1\"\n\n\ + [patch.crates-io]\nopenssl-socket-0123abcd = { package = \"openssl\", path = \ + \".socket/vendor/cargo/0123abcd-0000-4000-8000-000000000000/openssl-0.10.66\" }\n", + ); + assert!(cargo_vendored_wiring(&p, "openssl", "0.10.66")); + assert!(!cargo_vendored_wiring(&p, "openssl", "0.10.65")); + assert!(!cargo_vendored_wiring(&p, "log", "0.4.22")); + assert!(!cargo_vendored_wiring(&p, "cc", "1.1.0")); + + let mut legacy = MemoryProject::new(); + legacy.insert_text("Cargo.toml", "[package]\nname = \"app\"\n"); + let config = "[patch.crates-io]\ncc = { path = \ + \".socket/vendor/cargo/0123abcd-0000-4000-8000-000000000000/cc-1.1.0\" }\n"; + legacy.insert_text(".cargo/config.toml", config); + assert!(cargo_vendored_wiring(&legacy, "cc", "1.1.0")); + // cargo reads the legacy spelling when it exists. + legacy.insert_text(".cargo/config", ""); + assert!(!cargo_vendored_wiring(&legacy, "cc", "1.1.0")); + } + + fn cargo_reference( + uuid: &str, + ) -> (Vec<(String, String)>, HashMap) { + let purl = "pkg:cargo/serde@1.0.190"; + let mut refs = HashMap::new(); + refs.insert( + uuid.to_string(), + reference(serde_json::json!({ + "status": "granted", + "url": format!("https://patch.example/patch/cargo/serde/1.0.190/tok/{uuid}/serde-1.0.190.crate"), + "purl": purl, + "artifacts": [{"kind": "tarball", "url": null, "integrity": {"sha256": "ab"}}], + "registryOverride": null + })), + ); + (vec![(purl.to_string(), uuid.to_string())], refs) + } + + #[test] + fn an_unreadable_candidate_file_refuses_its_ecosystem() { + let (selected, refs) = cargo_reference("u-1"); + let mut p = MemoryProject::new(); + p.insert_text("Cargo.toml", "[dependencies]\nserde = \"1\"\n"); + p.insert_text( + "Cargo.lock", + "version = 3\n\n[[package]]\nname = \"serde\"\nversion = \"1.0.190\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n", + ); + p.insert_present(".cargo/config"); + let options = StageOptions { + dry_run: false, + pipenv_major: None, + trust_lockfile_config: true, + npm_allow_remote_config: true, + }; + let unreadable = BTreeSet::from([".cargo/config".to_string()]); + let planned = plan(p.clone(), unreadable, &selected, &refs) + .unwrap_or_else(|r| panic!("{:?}", r.error)); + assert_eq!(planned.unreadable_reads, vec![".cargo/config"]); + let err = rewrite(planned, &BTreeMap::new(), options).unwrap_err(); + assert_eq!(err.error.code, UNREADABLE_REFUSAL); + + // A non-UTF-8 file is absent to disk too: not a refusal. + let planned = + plan(p, BTreeSet::new(), &selected, &refs).unwrap_or_else(|r| panic!("{:?}", r.error)); + assert!(planned.unreadable_reads.is_empty()); + } + + #[test] + fn file_ecosystems_cover_the_rewrite_targets() { + assert_eq!(file_ecosystem("package-lock.json"), Some("npm")); + assert_eq!( + file_ecosystem("common/config/rush/pnpm-lock.yaml"), + Some("npm") + ); + assert_eq!(file_ecosystem("tool.py.lock"), Some("pypi")); + assert_eq!(file_ecosystem("crates/a/Cargo.toml"), Some("cargo")); + assert_eq!(file_ecosystem("build.gradle"), None); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/roots.rs b/crates/socket-patch-cli/src/hosted_memory/roots.rs new file mode 100644 index 00000000..da5a0ad8 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/roots.rs @@ -0,0 +1,246 @@ +//! Project-root detection over a repository path list. A root is a +//! directory holding a root LOCK marker (manifests alone never make one), +//! outside vendored / test-fixture trees, and not an internal directory of +//! an enclosing Rush monorepo. A nested Cargo.lock stays a root here: only +//! the enclosing workspace's `members`/`exclude` can say whether it is a +//! member, so the engine demotes members once manifests are readable. + +use std::collections::{BTreeMap, BTreeSet}; + +use socket_patch_core::utils::python_lock::is_python_lock_name; + +use super::types::IgnoredPath; + +/// Lock markers that make their directory a project root, with the +/// ecosystem each belongs to. +pub(crate) const ROOT_LOCK_MARKERS: [(&str, &str); 18] = [ + ("package-lock.json", "npm"), + ("npm-shrinkwrap.json", "npm"), + ("pnpm-lock.yaml", "npm"), + ("yarn.lock", "npm"), + ("bun.lock", "npm"), + ("bun.lockb", "npm"), + ("rush.json", "npm"), + ("uv.lock", "pypi"), + ("poetry.lock", "pypi"), + ("pdm.lock", "pypi"), + ("Pipfile.lock", "pypi"), + ("requirements.txt", "pypi"), + ("Cargo.lock", "cargo"), + ("go.mod", "golang"), + ("go.sum", "golang"), + ("composer.lock", "composer"), + ("Gemfile.lock", "gem"), + ("gems.locked", "gem"), +]; + +/// Marker files of the ecosystems the in-memory engine cannot inventory +/// (disk discovers them only through installed-tree crawlers). +pub(crate) const UNSUPPORTED_MARKERS: [(&str, &[&str]); 2] = [ + ( + "maven", + &[ + "pom.xml", + "build.gradle", + "build.gradle.kts", + "settings.gradle", + "settings.gradle.kts", + ], + ), + ("nuget", &["packages.lock.json", "nuget.config"]), +]; + +/// Directory names whose subtrees never hold a project root: installed +/// trees, VCS and tool state, vendored dependencies, and test fixtures. +pub(crate) const EXCLUDED_ROOT_SEGMENTS: [&str; 10] = [ + "node_modules", + ".git", + ".socket", + ".yarn", + "vendor", + "test", + "tests", + "fixtures", + "__fixtures__", + "testdata", +]; + +/// The ecosystem a root marker basename belongs to. +pub(crate) fn marker_ecosystem(base: &str) -> Option<&'static str> { + if let Some((_, eco)) = ROOT_LOCK_MARKERS.iter().find(|(name, _)| *name == base) { + return Some(eco); + } + is_python_lock_name(base).then_some("pypi") +} + +/// `(dir, basename)` of a `/`-separated path. +pub(crate) fn split_path(path: &str) -> (&str, &str) { + match path.rsplit_once('/') { + Some((dir, base)) => (dir, base), + None => ("", path), + } +} + +/// `root`-relative form of `path`, or `None` when `path` is not under it. +pub(crate) fn strip_root<'a>(root: &str, path: &'a str) -> Option<&'a str> { + if root.is_empty() { + return Some(path); + } + path.strip_prefix(root)?.strip_prefix('/') +} + +/// `root/rel` (`rel` alone for the repo root). +pub(crate) fn join_root(root: &str, rel: &str) -> String { + if root.is_empty() { + rel.to_string() + } else { + format!("{root}/{rel}") + } +} + +fn allowed(ecosystems: Option<&[String]>, eco: &str) -> bool { + ecosystems.is_none_or(|list| list.iter().any(|e| e == eco)) +} + +/// The detected roots (sorted) and the marker paths that did not make one. +pub(crate) fn detect_roots<'a>( + paths: impl IntoIterator, + ecosystems: Option<&[String]>, +) -> (Vec, Vec) { + let mut ignored: Vec = Vec::new(); + let mut markers: BTreeMap> = BTreeMap::new(); + let mut marker_paths: BTreeMap> = BTreeMap::new(); + for path in paths { + let (dir, base) = split_path(path); + let Some(eco) = marker_ecosystem(base) else { + continue; + }; + let ignore = |reason: &str, ignored: &mut Vec| { + ignored.push(IgnoredPath { + path: path.to_string(), + reason: reason.to_string(), + }); + }; + if dir + .split('/') + .any(|seg| EXCLUDED_ROOT_SEGMENTS.contains(&seg)) + { + ignore("excluded_dir", &mut ignored); + continue; + } + if !allowed(ecosystems, eco) { + ignore("ecosystem_filtered", &mut ignored); + continue; + } + let key: &'static str = ROOT_LOCK_MARKERS + .iter() + .find(|(name, _)| *name == base) + .map_or("python-lock", |(name, _)| name); + markers.entry(dir.to_string()).or_default().insert(key); + marker_paths + .entry(dir.to_string()) + .or_default() + .push(path.to_string()); + } + + let rush_roots: Vec = markers + .iter() + .filter(|(_, m)| m.contains("rush.json")) + .map(|(d, _)| d.clone()) + .collect(); + let mut roots: Vec = Vec::new(); + for dir in markers.keys() { + let rush_internal = rush_roots.iter().any(|r| { + let internal = |sub: &str| join_root(r, sub); + *dir == internal("common/config/rush") + || dir.starts_with(&format!("{}/", internal("common/config/subspaces"))) + || *dir == internal("common/temp") + || dir.starts_with(&format!("{}/", internal("common/temp"))) + }); + let reason = if rush_internal { + Some("rush_internal") + } else { + None + }; + match reason { + Some(reason) => { + for path in marker_paths.get(dir).into_iter().flatten() { + ignored.push(IgnoredPath { + path: path.clone(), + reason: reason.to_string(), + }); + } + } + None => roots.push(dir.clone()), + } + } + roots.sort(); + ignored.sort_by(|a, b| a.path.cmp(&b.path)); + (roots, ignored) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn roots(paths: &[&str]) -> Vec { + detect_roots(paths.iter().copied(), None).0 + } + + #[test] + fn lock_markers_make_roots_and_manifests_do_not() { + assert_eq!( + roots(&[ + "package-lock.json", + "apps/web/pnpm-lock.yaml", + "libs/a/package.json", + "svc/Cargo.toml", + "py/tool.py.lock", + "py2/pylock.toml" + ]), + vec!["", "apps/web", "py", "py2"] + ); + } + + #[test] + fn excluded_trees_never_hold_a_root() { + let (found, ignored) = detect_roots( + [ + "node_modules/x/package-lock.json", + "test/fixtures/yarn.lock", + "a/vendor/b/composer.lock", + ".socket/vendor/npm/package-lock.json", + "docs/requirements.txt", + ], + None, + ); + assert_eq!(found, vec!["docs"]); + assert_eq!(ignored.len(), 4); + assert!(ignored.iter().all(|i| i.reason == "excluded_dir")); + } + + #[test] + fn rush_internals_are_not_roots_but_nested_cargo_locks_are() { + assert_eq!( + roots(&[ + "rush.json", + "common/config/rush/pnpm-lock.yaml", + "common/config/subspaces/web/pnpm-lock.yaml", + "ws/Cargo.lock", + "ws/crates/a/Cargo.lock", + "ws/crates/b/Cargo.lock", + "ws/crates/b/package-lock.json", + ]), + vec!["", "ws", "ws/crates/a", "ws/crates/b"] + ); + } + + #[test] + fn ecosystem_filter_limits_markers() { + let only_npm = vec!["npm".to_string()]; + let (found, ignored) = + detect_roots(["a/package-lock.json", "b/Cargo.lock"], Some(&only_npm)); + assert_eq!(found, vec!["a"]); + assert_eq!(ignored[0].reason, "ecosystem_filtered"); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/select.rs b/crates/socket-patch-cli/src/hosted_memory/select.rs new file mode 100644 index 00000000..67a4b4b7 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/select.rs @@ -0,0 +1,478 @@ +//! Which repository files the in-memory engine needs: root detection plus, +//! per root, the same root-relative candidate set the disk hosted flow +//! reads (`REDIRECT_CANDIDATE_FILES`, Python lock / script pairs, Cargo +//! member manifests, Rush locks, the install-policy configs, the +//! Plug'n'Play markers and the two `.socket/vendor/` ledgers), plus one +//! presence-only Maven / NuGet marker per ecosystem so a repo holding only +//! those still gets its `ecosystem_unsupported_in_memory` warning. + +use std::collections::{BTreeMap, BTreeSet}; + +use socket_patch_core::constants::npm_family::{ + BUN_LOCKB, PNP_MARKERS, RUSH_COMMON_LOCK_REL, RUSH_SUBSPACES_DIR, +}; +use socket_patch_core::patch::redirect::npmrc::NPMRC_REL; +use socket_patch_core::patch::redirect::REDIRECT_STATE_REL; +use socket_patch_core::utils::python_lock::is_python_lock_name; + +use super::roots::{ + detect_roots, split_path, strip_root, EXCLUDED_ROOT_SEGMENTS, UNSUPPORTED_MARKERS, +}; +use super::types::{IgnoredPath, PathSelection, SelectOptions, TreeEntryInput}; +use crate::commands::scan::hosted::{PNPM_WORKSPACE_REL, REDIRECT_CANDIDATE_FILES}; + +/// Most entries [`PathSelection::ignored_sample`] carries. +pub const IGNORED_SAMPLE_MAX: usize = 100; + +/// Longest path accepted, in bytes. +const MAX_PATH_LEN: usize = 1024; + +/// Deepest path accepted, in segments. +const MAX_PATH_DEPTH: usize = 64; + +/// The vendored-mode ledger (its presence refuses a vendored takeover). +pub(crate) const VENDOR_STATE_REL: &str = ".socket/vendor/state.json"; + +/// Rush's repo-state file (presence feeds the stale-hash warning). +pub(crate) const RUSH_REPO_STATE_REL: &str = "common/config/rush/repo-state.json"; + +/// Root-relative text files read beyond `REDIRECT_CANDIDATE_FILES`. +const EXTRA_TEXT_FILES: [&str; 4] = [ + PNPM_WORKSPACE_REL, + NPMRC_REL, + VENDOR_STATE_REL, + REDIRECT_STATE_REL, +]; + +/// The one directory name the disk Cargo member walk never enters (it +/// follows `members`, `exclude`, path dependencies and `[patch]` paths +/// anywhere else, `vendor/` included); which manifests it reaches depends +/// on their content, so every other `Cargo.toml` under a Cargo root is +/// fetched and the engine repeats the walk over them. +const CARGO_SKIP_SEGMENT: &str = "target"; + +/// The file `cargo vendor` writes into every crate it copies. The disk walk +/// reads vendored crates only when a path dependency or `[patch]` path +/// names one; without their manifests the engine fails closed +/// (`redirect_cargo_transitive_dependents`), which beats streaming +/// thousands of registry manifests into the file limits. +const CARGO_VENDOR_CHECKSUM: &str = ".cargo-checksum.json"; + +/// Whether `manifest_dir` (root-relative) or a directory above it, below +/// the root, holds a `cargo vendor` checksum file. +fn is_cargo_vendored(manifest_dir: &str, root_files: &BTreeSet<&str>) -> bool { + let mut dir = manifest_dir; + loop { + if root_files.contains(format!("{dir}/{CARGO_VENDOR_CHECKSUM}").as_str()) { + return true; + } + match dir.rsplit_once('/') { + Some((parent, _)) => dir = parent, + None => return false, + } + } +} + +/// A normalized repo-relative file path, or `None` when it is unsafe: +/// absolute, `..`/`.`/empty segments, backslashes, control characters, +/// overlong or overdeep. +pub fn safe_repo_path(path: &str) -> Option { + if path.is_empty() + || path.len() > MAX_PATH_LEN + || path.starts_with('/') + || path.contains('\\') + || path.chars().any(char::is_control) + { + return None; + } + let segments: Vec<&str> = path.split('/').collect(); + if segments.len() > MAX_PATH_DEPTH + || segments + .iter() + .any(|s| s.is_empty() || *s == "." || *s == "..") + { + return None; + } + Some(path.to_string()) +} + +/// A normalized repo-relative directory (`""` is the repo root; a trailing +/// `/` is dropped), or `None` when unsafe. +pub fn safe_root_path(root: &str) -> Option { + let trimmed = root.strip_suffix('/').unwrap_or(root); + if trimmed.is_empty() || trimmed == "." { + return Some(String::new()); + } + safe_repo_path(trimmed) +} + +/// Whether the engine keeps `path` as raw bytes rather than UTF-8 text. +pub fn is_binary_candidate(path: &str) -> bool { + split_path(path).1 == BUN_LOCKB +} + +/// How a root-relative file is consumed, when it is a candidate at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Need { + Text, + Binary, + Present, +} + +fn is_rush_subspace_lock(rel: &str) -> bool { + let Some(rest) = rel + .strip_prefix(RUSH_SUBSPACES_DIR) + .and_then(|r| r.strip_prefix('/')) + else { + return false; + }; + matches!(rest.split_once('/'), Some((name, "pnpm-lock.yaml")) if !name.is_empty()) +} + +/// What `rel` (relative to a root whose files are `root_files`) is needed +/// for. +fn classify(rel: &str, root_files: &BTreeSet<&str>) -> Option { + if rel == BUN_LOCKB { + return Some(Need::Binary); + } + if REDIRECT_CANDIDATE_FILES.contains(&rel) || EXTRA_TEXT_FILES.contains(&rel) { + return Some(Need::Text); + } + if PNP_MARKERS.contains(&rel) || rel == "rush.json" { + return Some(Need::Present); + } + if !rel.contains('/') { + if is_python_lock_name(rel) { + return Some(Need::Text); + } + if rel.ends_with(".py") && root_files.contains(format!("{rel}.lock").as_str()) { + return Some(Need::Text); + } + return None; + } + let rush = root_files.contains("rush.json"); + if rush && (rel == RUSH_COMMON_LOCK_REL || is_rush_subspace_lock(rel)) { + return Some(Need::Text); + } + if rush && rel == RUSH_REPO_STATE_REL { + return Some(Need::Present); + } + if let Some(manifest_dir) = rel.strip_suffix("/Cargo.toml") { + if root_files.contains("Cargo.toml") + && !manifest_dir.split('/').any(|seg| seg == CARGO_SKIP_SEGMENT) + && !is_cargo_vendored(manifest_dir, root_files) + { + return Some(Need::Text); + } + } + None +} + +/// `selectHostedScanPaths`: roots (detected, or `options.projectRoots`) +/// plus the files to stream for them. Only `blob` entries are files; mode +/// `120000` is a symbolic link and is reported, never fetched. +pub fn select_paths(entries: &[TreeEntryInput], options: &SelectOptions) -> PathSelection { + let mut ignored: Vec = Vec::new(); + let mut blobs: BTreeMap = BTreeMap::new(); + for entry in entries { + if entry.kind != "blob" { + continue; + } + match safe_repo_path(&entry.path) { + Some(path) => { + blobs.insert(path, entry.mode == "120000"); + } + None => ignored.push(IgnoredPath { + path: entry.path.chars().take(MAX_PATH_LEN).collect(), + reason: "unsafe_path".to_string(), + }), + } + } + + let roots: Vec = match &options.project_roots { + Some(requested) => { + let mut out: BTreeSet = BTreeSet::new(); + for root in requested { + match safe_root_path(root) { + Some(r) => { + out.insert(r); + } + None => ignored.push(IgnoredPath { + path: root.clone(), + reason: "invalid_project_root".to_string(), + }), + } + } + out.into_iter().collect() + } + None => { + let (found, skipped) = detect_roots( + blobs.keys().map(String::as_str), + options.ecosystems.as_deref(), + ); + ignored.extend(skipped); + found + } + }; + let root_set: BTreeSet<&str> = roots.iter().map(String::as_str).collect(); + + let mut per_root: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for path in blobs.keys() { + let mut dir = path.as_str(); + loop { + dir = match dir.rsplit_once('/') { + Some((parent, _)) => parent, + None => "", + }; + if let Some(root) = root_set.get(dir) { + if let Some(rel) = strip_root(root, path) { + per_root.entry(root).or_default().insert(rel); + } + } + if dir.is_empty() { + break; + } + } + } + + let mut needs: BTreeMap = BTreeMap::new(); + for (root, files) in &per_root { + for rel in files { + let Some(need) = classify(rel, files) else { + continue; + }; + let full = super::roots::join_root(root, rel); + let slot = needs.entry(full).or_insert(need); + *slot = (*slot).min(need); + } + } + + for (eco, markers) in UNSUPPORTED_MARKERS { + if !options + .ecosystems + .as_deref() + .is_none_or(|list| list.iter().any(|e| e == eco)) + { + continue; + } + let first = blobs.keys().find(|path| { + let (dir, base) = split_path(path); + markers.contains(&base) + && !dir + .split('/') + .any(|seg| EXCLUDED_ROOT_SEGMENTS.contains(&seg)) + }); + if let Some(path) = first { + needs.entry(path.clone()).or_insert(Need::Present); + } + } + + let mut selection = PathSelection { + roots, + ..PathSelection::default() + }; + for (path, need) in needs { + if blobs.get(&path).copied().unwrap_or(false) { + selection.symlinks.push(path); + continue; + } + match need { + Need::Text => selection.fetch_text.push(path), + Need::Binary => selection.fetch_binary.push(path), + Need::Present => selection.present_only.push(path), + } + } + ignored.sort_by(|a, b| a.path.cmp(&b.path)); + selection.ignored_count = ignored.len() as u64; + ignored.truncate(IGNORED_SAMPLE_MAX); + selection.ignored_sample = ignored; + selection +} + +/// `hostedScanCandidateFiles`: the root-relative candidate names and +/// patterns, for debugging only (selection is [`select_paths`]). +pub fn candidate_files() -> Vec { + let mut out: BTreeSet = REDIRECT_CANDIDATE_FILES + .iter() + .chain(EXTRA_TEXT_FILES.iter()) + .chain(PNP_MARKERS.iter()) + .map(|s| s.to_string()) + .collect(); + for pattern in [ + "rush.json", + RUSH_COMMON_LOCK_REL, + RUSH_REPO_STATE_REL, + "common/config/subspaces/*/pnpm-lock.yaml", + "*.py.lock", + "*.py (beside *.py.lock)", + "pylock.toml", + "pylock.*.toml", + "**/Cargo.toml (Cargo workspaces)", + ] { + out.insert(pattern.to_string()); + } + out.into_iter().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn blob(path: &str) -> TreeEntryInput { + TreeEntryInput { + path: path.into(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(10), + } + } + + #[test] + fn selects_root_candidates_only() { + let mut entries = vec![ + blob("package-lock.json"), + blob("package.json"), + blob("src/index.js"), + blob(".npmrc"), + blob("bun.lockb"), + blob(".pnp.cjs"), + blob("tool.py.lock"), + blob("tool.py"), + blob("other.py"), + blob(".socket/vendor/redirect-state.json"), + blob("web/yarn.lock"), + blob("web/.yarnrc.yml"), + blob("web/node_modules/x/package-lock.json"), + ]; + entries.push(TreeEntryInput { + path: "pnpm-workspace.yaml".into(), + mode: "120000".into(), + kind: "blob".into(), + size: None, + }); + entries.push(TreeEntryInput { + path: "sub".into(), + mode: "160000".into(), + kind: "commit".into(), + size: None, + }); + entries.push(blob("../escape/package-lock.json")); + let s = select_paths(&entries, &SelectOptions::default()); + assert_eq!(s.roots, vec!["", "web"]); + assert_eq!( + s.fetch_text, + vec![ + ".npmrc", + ".socket/vendor/redirect-state.json", + "package-lock.json", + "tool.py", + "tool.py.lock", + "web/.yarnrc.yml", + "web/yarn.lock" + ] + ); + assert_eq!(s.fetch_binary, vec!["bun.lockb"]); + assert_eq!(s.present_only, vec![".pnp.cjs"]); + assert_eq!(s.symlinks, vec!["pnpm-workspace.yaml"]); + assert_eq!(s.ignored_count, 2); + } + + #[test] + fn rush_and_cargo_members_are_fetched_under_their_root() { + let entries = vec![ + blob("rush.json"), + blob("common/config/rush/pnpm-lock.yaml"), + blob("common/config/rush/repo-state.json"), + blob("common/config/subspaces/a/pnpm-lock.yaml"), + blob("rs/Cargo.toml"), + blob("rs/Cargo.lock"), + blob("rs/crates/x/Cargo.toml"), + blob("rs/vendor/foo/Cargo.toml"), + blob("rs/target/debug/Cargo.toml"), + blob("rs/vendor/serde/Cargo.toml"), + blob("rs/vendor/serde/.cargo-checksum.json"), + blob("rs/vendor/serde/tests/ui/Cargo.toml"), + blob("rs/third_party/.cargo-checksum.json"), + blob("rs/third_party/Cargo.toml"), + ]; + let s = select_paths(&entries, &SelectOptions::default()); + assert_eq!(s.roots, vec!["", "rs"]); + assert_eq!( + s.fetch_text, + vec![ + "common/config/rush/pnpm-lock.yaml", + "common/config/subspaces/a/pnpm-lock.yaml", + "rs/Cargo.lock", + "rs/Cargo.toml", + "rs/crates/x/Cargo.toml", + "rs/vendor/foo/Cargo.toml" + ] + ); + assert_eq!( + s.present_only, + vec!["common/config/rush/repo-state.json", "rush.json"] + ); + } + + #[test] + fn a_maven_or_nuget_only_repo_still_names_one_marker_per_ecosystem() { + let entries = vec![ + blob("svc/b/pom.xml"), + blob("svc/a/pom.xml"), + blob("test/pom.xml"), + blob("app/nuget.config"), + blob("src/Main.java"), + ]; + let s = select_paths(&entries, &SelectOptions::default()); + assert!(s.roots.is_empty()); + assert!(s.fetch_text.is_empty()); + assert_eq!(s.present_only, vec!["app/nuget.config", "svc/a/pom.xml"]); + let s = select_paths( + &entries, + &SelectOptions { + project_roots: None, + ecosystems: Some(vec!["npm".into()]), + }, + ); + assert!(s.present_only.is_empty()); + } + + #[test] + fn explicit_roots_override_detection() { + let entries = vec![blob("a/package-lock.json"), blob("b/yarn.lock")]; + let s = select_paths( + &entries, + &SelectOptions { + project_roots: Some(vec!["b/".into(), "../x".into()]), + ecosystems: None, + }, + ); + assert_eq!(s.roots, vec!["b"]); + assert_eq!(s.fetch_text, vec!["b/yarn.lock"]); + assert_eq!(s.ignored_sample[0].reason, "invalid_project_root"); + } + + #[test] + fn path_safety() { + for bad in [ + "", "/abs", "a/../b", "a//b", "./a", "a\\b", "a\u{0}b", "a\nb", + ] { + assert_eq!(safe_repo_path(bad), None, "{bad:?}"); + } + assert_eq!(safe_repo_path("a/b.json").as_deref(), Some("a/b.json")); + assert_eq!(safe_root_path("").as_deref(), Some("")); + assert_eq!(safe_root_path("a/").as_deref(), Some("a")); + } + + #[test] + fn candidate_listing_is_sorted_and_names_the_ledgers() { + let listed = candidate_files(); + let mut sorted = listed.clone(); + sorted.sort(); + assert_eq!(listed, sorted); + assert!(listed + .iter() + .any(|f| f == ".socket/vendor/redirect-state.json")); + assert!(listed.iter().any(|f| f == "package-lock.json")); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/types.rs b/crates/socket-patch-cli/src/hosted_memory/types.rs new file mode 100644 index 00000000..ed8e3a84 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/types.rs @@ -0,0 +1,403 @@ +//! The engine's input and output shapes. Every struct serializes camelCase +//! and maps 1:1 onto the addon's JS contract (`HostedScanSessionOptions`, +//! `HostedScanResult`, `ProjectResult`, `PathSelection`, …), so a binding +//! converts them field by field. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// Ecosystems the addon contract names (`Ecosystem` in index.d.ts). +pub const ECOSYSTEMS: [&str; 8] = [ + "npm", "pypi", "cargo", "golang", "gem", "composer", "maven", "nuget", +]; + +/// `HostedScanLimits`. `None` fields take the defaults in +/// [`ResolvedLimits::DEFAULT`]. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct HostedScanLimits { + #[serde(skip_serializing_if = "Option::is_none")] + pub max_file_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_total_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_files: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_purls: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_projects: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_artifact_bytes: Option, +} + +/// [`HostedScanLimits`] with every default applied. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ResolvedLimits { + pub max_file_bytes: u64, + pub max_total_bytes: u64, + pub max_files: u64, + pub max_purls: u64, + pub max_projects: u64, + pub max_artifact_bytes: u64, +} + +impl ResolvedLimits { + pub const DEFAULT: ResolvedLimits = ResolvedLimits { + max_file_bytes: 20 * 1024 * 1024, + max_total_bytes: 64 * 1024 * 1024, + max_files: 2000, + max_purls: 20_000, + max_projects: 200, + max_artifact_bytes: 32 * 1024 * 1024, + }; +} + +impl HostedScanLimits { + pub fn resolve(&self) -> ResolvedLimits { + let d = ResolvedLimits::DEFAULT; + ResolvedLimits { + max_file_bytes: self.max_file_bytes.unwrap_or(d.max_file_bytes), + max_total_bytes: self.max_total_bytes.unwrap_or(d.max_total_bytes), + max_files: self.max_files.unwrap_or(d.max_files), + max_purls: self.max_purls.unwrap_or(d.max_purls), + max_projects: self.max_projects.unwrap_or(d.max_projects), + max_artifact_bytes: self.max_artifact_bytes.unwrap_or(d.max_artifact_bytes), + } + } +} + +/// `HostedScanSessionOptions`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HostedScanOptions { + pub org_slug: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ecosystems: Option>, + /// 1..=500, default 100. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub batch_size: Option, + #[serde(default)] + pub dry_run: bool, + /// The installing Pipenv's major; `None` is the CLI's "Pipenv not on + /// PATH" default (modern `file` references). Never probed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pipenv_major: Option, + /// Default true. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trust_lockfile_config: Option, + /// Default true. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub npm_allow_remote_config: Option, + /// Must match the `projectRoots` given to path selection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project_roots: Option>, + /// Default 8. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_concurrency: Option, + /// Per provider call, default 60000. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_timeout_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limits: Option, +} + +pub const DEFAULT_BATCH_SIZE: u32 = 100; +pub const MAX_BATCH_SIZE: u32 = 500; +pub const DEFAULT_PROVIDER_CONCURRENCY: u32 = 8; +pub const DEFAULT_REQUEST_TIMEOUT_MS: u64 = 60_000; +/// The reference endpoint accepts at most this many uuids per request. +pub const MAX_REFERENCE_BATCH: usize = 500; + +/// Why a path was marked present without content (`markPresent` kinds +/// other than `symlink`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PresentKind { + Present, + BinarySkipped, + Oversize, + LfsPointer, +} + +impl PresentKind { + pub fn parse(kind: &str) -> Option { + Some(match kind { + "present" => MarkKind::Present(PresentKind::Present), + "binary_skipped" => MarkKind::Present(PresentKind::BinarySkipped), + "oversize" => MarkKind::Present(PresentKind::Oversize), + "lfs_pointer" => MarkKind::Present(PresentKind::LfsPointer), + "symlink" => MarkKind::Symlink, + _ => return None, + }) + } +} + +/// A `markPresent(path, kind)` kind. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MarkKind { + Present(PresentKind), + Symlink, +} + +/// One repo file handed to the engine. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InputFile { + Text(String), + Binary(Vec), + Present(PresentKind), + Symlink, +} + +/// Everything [`super::run_in_memory`] consumes. Build it with +/// [`super::SessionBuilder`] (which enforces the session limits) or +/// directly for tests. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HostedScanInput { + pub options: HostedScanOptions, + /// Repo-relative `/`-separated paths. + pub files: BTreeMap, + /// Session-level warnings (e.g. a text file that was not UTF-8). + pub warnings: Vec, +} + +/// `EngineWarning`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EngineWarning { + pub code: String, + pub detail: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project_root: Option, +} + +impl EngineWarning { + pub fn new(code: impl Into, detail: impl Into, root: Option<&str>) -> Self { + Self { + code: code.into(), + detail: detail.into(), + project_root: root.map(str::to_string), + } + } +} + +/// `ProjectResult.summary`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProjectSummary { + pub scanned_packages: u64, + pub packages_with_patches: u64, + pub total_patches: u64, + pub free_patches: u64, + pub paid_patches: u64, + pub can_access_paid_patches: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RedirectedPatch { + pub purl: String, + pub uuid: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SkippedPatch { + pub purl: String, + pub uuid: String, + pub reason: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProjectError { + pub code: String, + pub message: String, +} + +/// `ProjectResult`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProjectResult { + /// Repo-relative project root (`""` is the repo root). + pub root: String, + /// The CLI `--json` `redirect` block, byte-for-byte the same shape. + pub redirect: serde_json::Value, + pub summary: ProjectSummary, + pub redirected: Vec, + pub skipped: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChangedFile { + pub path: String, + pub content: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChangedBinaryFile { + pub path: String, + #[serde(with = "base64_bytes")] + pub content: Vec, +} + +/// `HostedScanResult.stats`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EngineStats { + pub projects: u64, + pub files_input: u64, + pub bytes_input: u64, + pub packages_scanned: u64, + pub packages_with_patches: u64, + pub patches_selected: u64, + pub patches_redirected: u64, + pub files_changed: u64, + /// Provider calls made, by method name (`searchPatchesBatch`, …). + pub provider_calls: BTreeMap, + /// Wall time per engine phase, milliseconds. + pub phase_ms: BTreeMap, +} + +/// `HostedScanResult`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HostedScanOutput { + pub projects: Vec, + /// Repo-relative, sorted, byte-changed only; ledgers included on wet + /// runs. + pub changed_files: Vec, + pub changed_binary_files: Vec, + pub deleted_files: Vec, + pub warnings: Vec, + pub stats: EngineStats, + pub engine_version: String, +} + +/// `TreeEntryInput`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TreeEntryInput { + pub path: String, + pub mode: String, + #[serde(rename = "type")] + pub kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, +} + +/// `selectHostedScanPaths` options. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SelectOptions { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project_roots: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ecosystems: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IgnoredPath { + pub path: String, + pub reason: String, +} + +/// `PathSelection`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PathSelection { + pub roots: Vec, + pub fetch_text: Vec, + pub fetch_binary: Vec, + pub present_only: Vec, + pub symlinks: Vec, + pub ignored_count: u64, + /// At most [`super::select::IGNORED_SAMPLE_MAX`] entries. + pub ignored_sample: Vec, +} + +/// Engine failure (`finish()` rejection codes). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EngineError { + /// A size/count limit was breached (`code` names the limit). + Limit { code: &'static str, message: String }, + /// The input or options are unusable. + InvalidInput { code: &'static str, message: String }, + /// Cancelled through the cancellation token. + Cancelled, + /// An engine bug. + Internal { message: String }, +} + +impl EngineError { + /// The stable machine code (`cancelled`, `limit_exceeded`'s limit name, + /// …). + pub fn code(&self) -> &str { + match self { + EngineError::Limit { code, .. } | EngineError::InvalidInput { code, .. } => code, + EngineError::Cancelled => "cancelled", + EngineError::Internal { .. } => "engine_internal", + } + } + + /// `limit` / `invalid_input` / `cancelled` / `internal`. + pub fn kind(&self) -> &'static str { + match self { + EngineError::Limit { .. } => "limit", + EngineError::InvalidInput { .. } => "invalid_input", + EngineError::Cancelled => "cancelled", + EngineError::Internal { .. } => "internal", + } + } + + pub(crate) fn limit(code: &'static str, message: impl Into) -> Self { + EngineError::Limit { + code, + message: message.into(), + } + } + + pub(crate) fn invalid(code: &'static str, message: impl Into) -> Self { + EngineError::InvalidInput { + code, + message: message.into(), + } + } +} + +impl std::fmt::Display for EngineError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EngineError::Limit { code, message } | EngineError::InvalidInput { code, message } => { + write!(f, "{code}: {message}") + } + EngineError::Cancelled => write!(f, "cancelled"), + EngineError::Internal { message } => write!(f, "engine_internal: {message}"), + } + } +} + +impl std::error::Error for EngineError {} + +mod base64_bytes { + use base64::Engine; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(bytes: &[u8], s: S) -> Result { + s.serialize_str(&base64::engine::general_purpose::STANDARD.encode(bytes)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let text = String::deserialize(d)?; + base64::engine::general_purpose::STANDARD + .decode(text) + .map_err(serde::de::Error::custom) + } +} diff --git a/crates/socket-patch-cli/src/lib.rs b/crates/socket-patch-cli/src/lib.rs index 6406d9ab..da5533f0 100644 --- a/crates/socket-patch-cli/src/lib.rs +++ b/crates/socket-patch-cli/src/lib.rs @@ -8,6 +8,7 @@ pub mod args; pub mod commands; pub(crate) mod ecosystem_dispatch; +pub mod hosted_memory; pub mod json_envelope; pub mod path_scope; pub mod ui; @@ -108,6 +109,12 @@ pub enum Commands { override_usage = "socket-patch --update [VERSION] [OPTIONS]" )] SelfUpdate(commands::update::UpdateArgs), + + // Internal parity/debug harness for the in-memory hosted engine: reads a + // JSON file bundle on stdin, prints the engine result. Hidden and + // documented as internal in CLI_CONTRACT.md (no stability guarantee). + #[command(hide = true, name = "hosted-bundle")] + HostedBundle(commands::hosted_bundle::HostedBundleArgs), } impl Commands { @@ -127,6 +134,7 @@ impl Commands { Commands::Remove(a) => &a.common, Commands::Repair(a) => &a.common, Commands::SelfUpdate(a) => &a.common, + Commands::HostedBundle(a) => &a.common, } } } diff --git a/crates/socket-patch-cli/src/main.rs b/crates/socket-patch-cli/src/main.rs index a1434d09..b50bff6b 100644 --- a/crates/socket-patch-cli/src/main.rs +++ b/crates/socket-patch-cli/src/main.rs @@ -107,6 +107,7 @@ async fn main() { Commands::Remove(args) => commands::remove::run(args).await, Commands::Repair(args) => commands::repair::run(args).await, Commands::SelfUpdate(args) => commands::update::run(args).await, + Commands::HostedBundle(args) => commands::hosted_bundle::run(args).await, }; // Never delays exit beyond its 500 ms grace budget; never changes the diff --git a/crates/socket-patch-cli/tests/cli_global_args.rs b/crates/socket-patch-cli/tests/cli_global_args.rs index b0ca583c..334e800a 100644 --- a/crates/socket-patch-cli/tests/cli_global_args.rs +++ b/crates/socket-patch-cli/tests/cli_global_args.rs @@ -42,6 +42,8 @@ const SUBCOMMANDS_NO_POSITIONAL: &[&str] = &[ // Hidden parse target of the root `--update` flag; its VERSION // positional is optional, so the no-positional variant covers it. "self-update", + // Hidden internal harness for the in-memory hosted engine. + "hosted-bundle", ]; /// Subcommands that require a positional identifier. @@ -137,6 +139,7 @@ fn common_of(cli: &Cli) -> &GlobalArgs { Vendor(a) => &a.common, Vex(a) => &a.common, SelfUpdate(a) => &a.common, + HostedBundle(a) => &a.common, } } diff --git a/crates/socket-patch-cli/tests/hosted_memory_common/mod.rs b/crates/socket-patch-cli/tests/hosted_memory_common/mod.rs new file mode 100644 index 00000000..c0e60e47 --- /dev/null +++ b/crates/socket-patch-cli/tests/hosted_memory_common/mod.rs @@ -0,0 +1,413 @@ +//! Shared harness for the in-memory hosted engine tests: a wiremock patch +//! API that serves every patch a fixture's `overrides.json` describes, the +//! engine run over that API, and a disk `scan --mode hosted --json` run of +//! the same files through the real binary under a scrubbed environment. + +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use serde_json::Value; +use socket_patch_cli::hosted_memory::{ + run_in_memory, HostedScanOptions, HostedScanOutput, MarkKind, PresentKind, SessionBuilder, +}; +use socket_patch_core::api::client::{ApiClient, ApiClientOptions, PatchApi}; +use tokio_util::sync::CancellationToken; +use wiremock::matchers::{method, path, path_regex}; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; + +pub const ORG: &str = "test-org"; + +pub fn fixtures_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../socket-patch-core/tests/fixtures") +} + +/// One patch the fake API serves. +#[derive(Debug, Clone)] +pub struct Patch { + pub purl: String, + pub uuid: String, + pub reference: Value, +} + +fn percent_decode(s: &str) -> String { + socket_patch_core::utils::purl::percent_decode_purl_component(s).into_owned() +} + +fn purl_key(purl: &str) -> String { + let decoded = socket_patch_core::utils::purl::normalize_purl( + socket_patch_core::utils::purl::strip_purl_qualifiers(purl), + ) + .into_owned(); + if decoded.starts_with("pkg:pypi/") { + decoded.to_ascii_lowercase().replace(['_', '.'], "-") + } else { + decoded + } +} + +/// Patches from a golden fixture's `overrides.json` (one `DepOverride` +/// each). `rewrite_host` replaces `https://patch.socket.dev` in the served +/// URLs (so a wheel-metadata download hits the mock, never the network). +pub fn patches_from_overrides(overrides: &Path, rewrite_host: Option<&str>) -> Vec { + let text = std::fs::read_to_string(overrides).expect("read overrides.json"); + let list: Vec = serde_json::from_str(&text).expect("overrides.json is a list"); + list.into_iter() + .map(|o| { + let eco = o["ecosystem"].as_str().unwrap().to_string(); + let name = o["name"].as_str().unwrap().to_string(); + let full = match o["namespace"].as_str() { + Some(ns) if !ns.is_empty() => format!("{ns}/{name}"), + _ => name, + }; + let version = o["version"].as_str().unwrap(); + let purl = format!("pkg:{eco}/{full}@{version}"); + let uuid = o["patchUuid"].as_str().unwrap().to_string(); + let fix = |v: &Value| -> Value { + match (v.as_str(), rewrite_host) { + (Some(s), Some(host)) => { + Value::String(s.replace("https://patch.socket.dev", host)) + } + _ => v.clone(), + } + }; + let url = fix(&o["artifactUrl"]); + let mut artifacts = vec![serde_json::json!({ + "kind": "tarball", + "url": url, + "integrity": o["integrity"].clone(), + })]; + if let Some(zip) = o["berryZipUrl"].as_str() { + artifacts.push(serde_json::json!({ + "kind": "yarn-berry-zip", + "url": fix(&Value::String(zip.to_string())), + "integrity": {"yarnBerry10c0": o["integrity"]["yarnBerry10c0"].clone()}, + })); + } + let mut registry_override = o.get("registryOverride").cloned().unwrap_or(Value::Null); + if let Some(index) = registry_override.get("indexUrl").cloned() { + registry_override["indexUrl"] = fix(&index); + } + Patch { + purl, + uuid, + reference: serde_json::json!({ + "status": "granted", + "url": url, + "purl": null, + "artifacts": artifacts, + "registryOverride": registry_override, + }), + } + }) + .collect() +} + +struct Batch(Vec); +impl Respond for Batch { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let mut packages = Vec::new(); + for component in body["components"].as_array().into_iter().flatten() { + let Some(purl) = component["purl"].as_str() else { + continue; + }; + let patches: Vec = self + .0 + .iter() + .filter(|p| purl_key(&p.purl) == purl_key(purl)) + .map(|p| { + serde_json::json!({ + "uuid": p.uuid, "purl": purl, "tier": "free", "cveIds": [], + "ghsaIds": ["GHSA-test-aaaa-bbbb"], "severity": "high", "title": "fixture" + }) + }) + .collect(); + if !patches.is_empty() { + packages.push(serde_json::json!({ "purl": purl, "patches": patches })); + } + } + ResponseTemplate::new(200).set_body_json( + serde_json::json!({ "packages": packages, "canAccessPaidPatches": false }), + ) + } +} + +struct ByPackage(Vec); +impl Respond for ByPackage { + fn respond(&self, request: &Request) -> ResponseTemplate { + let raw = request + .url + .path() + .rsplit_once("/by-package/") + .map(|(_, p)| p) + .unwrap_or(""); + let purl = percent_decode(raw); + let patches: Vec = self + .0 + .iter() + .filter(|p| purl_key(&p.purl) == purl_key(&purl)) + .map(|p| { + serde_json::json!({ + "uuid": p.uuid, "purl": purl, "publishedAt": "2024-01-01T00:00:00Z", + "description": "fixture", "license": "MIT", "tier": "free", + "vulnerabilities": {"GHSA-test-aaaa-bbbb": { + "cves": ["CVE-2024-0001"], "summary": "s", "severity": "high", "description": "d" + }} + }) + }) + .collect(); + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ "patches": patches, "canAccessPaidPatches": false })) + } +} + +struct References(Vec); +impl Respond for References { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let mut results = serde_json::Map::new(); + for uuid in body["uuids"].as_array().into_iter().flatten() { + let Some(uuid) = uuid.as_str() else { continue }; + if let Some(p) = self.0.iter().find(|p| p.uuid == uuid) { + results.insert(uuid.to_string(), p.reference.clone()); + } + } + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "results": results })) + } +} + +struct View(Vec); +impl Respond for View { + fn respond(&self, request: &Request) -> ResponseTemplate { + let uuid = request.url.path().rsplit('/').next().unwrap_or(""); + match self.0.iter().find(|p| p.uuid == uuid) { + Some(p) => ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "uuid": p.uuid, "purl": p.purl, "publishedAt": "2024-01-01T00:00:00Z", + "files": {"package/index.js": {"beforeHash": "a".repeat(64), "afterHash": "b".repeat(64)}}, + "vulnerabilities": {"GHSA-test-aaaa-bbbb": { + "cves": ["CVE-2024-0001"], "summary": "s", "severity": "high", "description": "d" + }}, + "description": "fixture", "license": "MIT", "tier": "free" + })), + None => ResponseTemplate::new(404), + } + } +} + +/// Mount the fake patch API for `patches`. +pub async fn mount_api(server: &MockServer, patches: &[Patch]) { + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(Batch(patches.to_vec())) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path_regex(format!( + "^/v0/orgs/{ORG}/patches/by-package/.+$" + ))) + .respond_with(ByPackage(patches.to_vec())) + .mount(server) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with(References(patches.to_vec())) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path_regex(format!("^/v0/orgs/{ORG}/patches/view/.+$"))) + .respond_with(View(patches.to_vec())) + .mount(server) + .await; +} + +pub fn client(server: &MockServer) -> Arc { + Arc::new(ApiClient::new(ApiClientOptions { + api_url: server.uri(), + api_token: Some("fake-token".to_string()), + use_public_proxy: false, + org_slug: Some(ORG.to_string()), + })) +} + +/// Every regular file under `dir`, keyed by `/`-separated relative path. +pub fn read_tree(dir: &Path) -> BTreeMap> { + fn walk(base: &Path, at: &Path, out: &mut BTreeMap>) { + let Ok(entries) = std::fs::read_dir(at) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + let meta = std::fs::symlink_metadata(&path).unwrap(); + if meta.is_dir() { + walk(base, &path, out); + } else if meta.is_file() { + let rel = path + .strip_prefix(base) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + out.insert(rel, std::fs::read(&path).unwrap()); + } + } + } + let mut out = BTreeMap::new(); + walk(dir, dir, &mut out); + out +} + +/// `files` added to a session (`bun.lockb` as bytes, the rest as text) +/// plus presence-only markers. +pub fn build_input( + files: &BTreeMap>, + present: &[&str], + options: HostedScanOptions, +) -> socket_patch_cli::hosted_memory::HostedScanInput { + let mut builder = SessionBuilder::new(options).expect("valid options"); + for (path, bytes) in files { + if path.ends_with("bun.lockb") { + builder.add_binary(path, bytes).unwrap(); + } else { + builder.push_chunk(path, bytes).unwrap(); + builder.end_file(path).unwrap(); + } + } + for path in present { + builder + .mark_present(path, MarkKind::Present(PresentKind::Present)) + .unwrap(); + } + builder.finish().unwrap() +} + +pub fn options(dry_run: bool) -> HostedScanOptions { + HostedScanOptions { + org_slug: ORG.to_string(), + dry_run, + ..HostedScanOptions::default() + } +} + +pub async fn run_engine( + server: &MockServer, + input: socket_patch_cli::hosted_memory::HostedScanInput, +) -> HostedScanOutput { + run_in_memory(input, client(server), CancellationToken::new()) + .await + .expect("engine run") +} + +/// The disk run's outcome: the `--json` envelope and the files it changed +/// (new or byte-changed, relative to the input). +pub struct DiskRun { + pub envelope: Value, + pub changed: BTreeMap>, + pub stderr: String, +} + +/// `socket-patch scan --mode hosted --json` over a copy of `files`, through +/// the real binary, with every ambient input scrubbed: an empty `PATH` (no +/// node / pipenv / gem subprocesses), `HOME` and the language caches at +/// empty directories, no socket-cli config, no telemetry. +pub fn run_disk(server: &MockServer, files: &BTreeMap>, dry_run: bool) -> DiskRun { + let project = tempfile::tempdir().unwrap(); + let home = tempfile::tempdir().unwrap(); + for (rel, bytes) in files { + let path = project.path().join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, bytes).unwrap(); + } + let mut cmd = std::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")); + cmd.env_clear(); + for keep in [ + "SYSTEMROOT", + "SystemRoot", + "windir", + "TMPDIR", + "TEMP", + "TMP", + ] { + if let Some(v) = std::env::var_os(keep) { + cmd.env(keep, v); + } + } + let empty = |name: &str| { + let dir = home.path().join(name); + std::fs::create_dir_all(&dir).unwrap(); + dir + }; + cmd.env("PATH", "") + .env("HOME", home.path()) + .env("USERPROFILE", home.path()) + .env("CARGO_HOME", empty("cargo")) + .env("GOPATH", empty("go")) + .env("GOMODCACHE", empty("gomodcache")) + .env("GEM_HOME", empty("gem")) + .env("XDG_CONFIG_HOME", empty("xdg")) + .env("SOCKET_NO_CONFIG", "1") + .env("SOCKET_TELEMETRY_DISABLED", "1") + .env("SOCKET_NO_UPDATE_CHECK", "1"); + cmd.args([ + "scan", + "--mode", + "hosted", + "--json", + "--yes", + "--cwd", + project.path().to_str().unwrap(), + "--org", + ORG, + "--api-token", + "fake-token", + "--api-url", + &server.uri(), + ]); + if dry_run { + cmd.arg("--dry-run"); + } + let output = cmd.output().expect("spawn socket-patch"); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + let envelope: Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("disk --json output is not JSON ({e}):\n{stdout}\n{stderr}")); + let after = read_tree(project.path()); + let changed = after + .into_iter() + .filter(|(rel, bytes)| files.get(rel) != Some(bytes)) + .collect(); + DiskRun { + envelope, + changed, + stderr, + } +} + +/// The engine's changed files (text and binary) as bytes. +pub fn engine_changed(output: &HostedScanOutput) -> BTreeMap> { + output + .changed_files + .iter() + .map(|f| (f.path.clone(), f.content.as_bytes().to_vec())) + .chain( + output + .changed_binary_files + .iter() + .map(|f| (f.path.clone(), f.content.clone())), + ) + .collect() +} + +/// Every file under `dir` (a golden fixture's `input/`). +pub fn fixture_files(dir: &Path) -> BTreeMap> { + read_tree(dir) +} + +/// Show a readable diff of two changed-file maps. +pub fn describe(map: &BTreeMap>) -> String { + map.iter() + .map(|(k, v)| format!("--- {k}\n{}", String::from_utf8_lossy(v))) + .collect::>() + .join("\n") +} diff --git a/crates/socket-patch-cli/tests/hosted_memory_engine.rs b/crates/socket-patch-cli/tests/hosted_memory_engine.rs new file mode 100644 index 00000000..9357c3e0 --- /dev/null +++ b/crates/socket-patch-cli/tests/hosted_memory_engine.rs @@ -0,0 +1,932 @@ +//! Behavior of the in-memory hosted engine beyond disk parity: lookup +//! dedup across roots, isolation from the process environment, hostile +//! and malformed input, limits, cancellation and timeouts, dry runs, and +//! the per-project refusals. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::Value; +use serial_test::serial; +use socket_patch_cli::hosted_memory::{ + run_in_memory, select_paths, EngineError, HostedScanLimits, HostedScanOptions, + HostedScanOutput, SelectOptions, SessionBuilder, TreeEntryInput, +}; +use socket_patch_core::api::client::{ApiError, ApiFuture, PatchApi}; +use socket_patch_core::api::types::{ + BatchSearchResponse, PackageVendorResult, PatchResponse, SearchResponse, +}; +use tokio_util::sync::CancellationToken; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +#[path = "hosted_memory_common/mod.rs"] +mod common; + +use common::*; + +const NPM_FIXTURE: &str = "redirect/npm/package-lock-v3/basic"; + +fn npm_files() -> BTreeMap> { + fixture_files(&fixtures_root().join(NPM_FIXTURE).join("input")) +} + +async fn npm_server() -> MockServer { + let server = MockServer::start().await; + let patches = patches_from_overrides( + &fixtures_root().join(NPM_FIXTURE).join("overrides.json"), + None, + ); + mount_api(&server, &patches).await; + server +} + +async fn count(server: &MockServer, method_name: &str, fragment: &str) -> usize { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|r| r.method.as_str() == method_name && r.url.path().contains(fragment)) + .count() +} + +fn prefixed(prefix: &str, files: &BTreeMap>) -> BTreeMap> { + files + .iter() + .map(|(k, v)| (format!("{prefix}/{k}"), v.clone())) + .collect() +} + +fn comparable(output: &HostedScanOutput) -> Value { + let mut value = serde_json::to_value(output).unwrap(); + value["stats"]["phaseMs"] = Value::Null; + value +} + +#[tokio::test] +async fn roots_sharing_a_purl_share_every_lookup() { + let server = npm_server().await; + let mut repo = prefixed("a", &npm_files()); + repo.extend(prefixed("b", &npm_files())); + let output = run_engine(&server, build_input(&repo, &[], options(false))).await; + assert_eq!(output.projects.len(), 2); + for project in &output.projects { + assert!(project.error.is_none(), "{:?}", project.error); + assert_eq!(project.redirected.len(), 1, "{}", project.root); + } + assert_eq!(count(&server, "POST", "/patches/batch").await, 1); + assert_eq!(count(&server, "GET", "/patches/by-package/").await, 1); + assert_eq!(count(&server, "POST", "/patches/package").await, 1); + assert_eq!(count(&server, "GET", "/patches/view/").await, 1); + let paths: Vec<&str> = output + .changed_files + .iter() + .map(|f| f.path.as_str()) + .collect(); + assert_eq!( + paths, + vec![ + "a/.npmrc", + "a/.socket/vendor/redirect-state.json", + "a/package-lock.json", + "b/.npmrc", + "b/.socket/vendor/redirect-state.json", + "b/package-lock.json" + ] + ); + assert_eq!(output.stats.projects, 2); + assert_eq!(output.stats.patches_redirected, 2); + assert_eq!( + output.stats.provider_calls.get("searchPatchesBatch"), + Some(&1) + ); +} + +#[tokio::test] +async fn output_is_deterministic() { + let server = npm_server().await; + let mut repo = prefixed("x", &npm_files()); + repo.extend(prefixed("y/z", &npm_files())); + let first = run_engine(&server, build_input(&repo, &[], options(false))).await; + let second = run_engine(&server, build_input(&repo, &[], options(false))).await; + assert_eq!(comparable(&first), comparable(&second)); +} + +const HOSTILE_ENV: &[(&str, &str)] = &[ + ("SOCKET_API_URL", "http://127.0.0.1:9"), + ("SOCKET_API_TOKEN", "sktsec_hostile_api"), + ("SOCKET_ORG_SLUG", "hostile-org"), + ("SOCKET_PROXY_URL", "http://127.0.0.1:9"), + ("SOCKET_OFFLINE", "1"), + ("SOCKET_DRY_RUN", "1"), + ("SOCKET_ECOSYSTEMS", "cargo"), + ("SOCKET_BATCH_SIZE", "1"), + ("SOCKET_PIPENV_MAJOR", "7"), + ("SOCKET_NO_TRUST_LOCKFILE_CONFIG", "1"), + ("SOCKET_NO_NPM_ALLOW_REMOTE_CONFIG", "1"), + ("SOCKET_TELEMETRY_DISABLED", "0"), + ("SOCKET_NPM_REGISTRY", "http://127.0.0.1:9"), + ("SOCKET_PYPI_JSON_API", "http://127.0.0.1:9"), + ("npm_config_allow_remote", "none"), + ("NPM_CONFIG_USERCONFIG", "/nonexistent/.npmrc"), +]; + +#[tokio::test] +#[serial] +async fn hostile_process_environment_changes_nothing() { + let server = npm_server().await; + let mut repo = npm_files(); + repo.insert( + "pnpm-lock.yaml".into(), + std::fs::read(fixtures_root().join("redirect/npm/pnpm/basic/input/pnpm-lock.yaml")) + .unwrap(), + ); + let baseline = run_engine(&server, build_input(&repo, &[], options(false))).await; + let saved: Vec<(&str, Option)> = HOSTILE_ENV + .iter() + .map(|(k, _)| (*k, std::env::var_os(k))) + .collect(); + for (k, v) in HOSTILE_ENV { + std::env::set_var(k, v); + } + let hostile = run_engine(&server, build_input(&repo, &[], options(false))).await; + for (k, v) in saved { + match v { + Some(v) => std::env::set_var(k, v), + None => std::env::remove_var(k), + } + } + assert_eq!(comparable(&baseline), comparable(&hostile)); + assert!(!baseline.projects[0].redirected.is_empty()); +} + +fn deep(open: &str, close: &str, depth: usize) -> Vec { + let mut s = String::with_capacity(depth * (open.len() + close.len())); + for _ in 0..depth { + s.push_str(open); + } + for _ in 0..depth { + s.push_str(close); + } + s.into_bytes() +} + +async fn run_on_small_stack( + server: &MockServer, + input: socket_patch_cli::hosted_memory::HostedScanInput, +) -> Result { + let api = client(server); + let handle = tokio::runtime::Handle::current(); + tokio::task::spawn_blocking(move || { + std::thread::Builder::new() + .stack_size(2 * 1024 * 1024) + .spawn(move || handle.block_on(run_in_memory(input, api, CancellationToken::new()))) + .unwrap() + .join() + .expect("the engine must not panic or overflow on hostile input") + }) + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn malformed_and_deeply_nested_inputs_never_panic() { + let server = MockServer::start().await; + let mut patches = patches_from_overrides( + &fixtures_root().join(NPM_FIXTURE).join("overrides.json"), + None, + ); + patches.extend(patches_from_overrides( + &fixtures_root().join("redirect/cargo/cargo/basic/overrides.json"), + None, + )); + patches.extend(patches_from_overrides( + &fixtures_root().join("redirect/pypi/requirements/basic/overrides.json"), + None, + )); + mount_api(&server, &patches).await; + let depth = 200_000; + let mut repo: BTreeMap> = BTreeMap::new(); + repo.extend(prefixed("npm", &npm_files())); + repo.insert("npm/.yarnrc.yml".into(), deep("- ", "", depth)); + repo.insert("npm/pnpm-workspace.yaml".into(), deep("[", "]", depth)); + repo.insert("npm/.npmrc".into(), deep("[", "]", depth)); + repo.extend(prefixed( + "rs", + &fixture_files(&fixtures_root().join("redirect/cargo/cargo/basic/input")), + )); + repo.insert("rs/.cargo/config.toml".into(), deep("a = [", "]", depth)); + repo.extend(prefixed( + "py", + &fixture_files(&fixtures_root().join("redirect/pypi/requirements/basic/input")), + )); + repo.insert("py/pyproject.toml".into(), deep("x = [", "]", depth)); + repo.insert("py/hatch.toml".into(), deep("{a=", "}", depth)); + repo.insert("deepjson/package-lock.json".into(), deep("[", "]", depth)); + repo.insert("deepjson/composer.lock".into(), deep("{\"a\":", "}", depth)); + repo.insert("deepjson/Pipfile.lock".into(), deep("[", "]", depth)); + repo.insert( + "deepjson/.socket/vendor/redirect-state.json".into(), + deep("[", "]", depth), + ); + repo.insert("deeptoml/uv.lock".into(), deep("a = [", "]", depth)); + repo.insert("deeptoml/Cargo.lock".into(), deep("a = {b=", "}", depth)); + repo.insert("deeptoml/Cargo.toml".into(), deep("[", "]", depth)); + repo.insert("deeptoml/poetry.lock".into(), deep("a = [", "]", depth)); + repo.insert( + "garbage/yarn.lock".into(), + b"\x00\x01 not a lock \"\"\"\n ::".to_vec(), + ); + repo.insert( + "garbage/go.mod".into(), + b"module \nrequire (\n(((\n".to_vec(), + ); + repo.insert("garbage/go.sum".into(), b"x y z\n\n h1:\n".to_vec()); + repo.insert( + "garbage/Gemfile.lock".into(), + b"GEM\n specs:\n (((\n".to_vec(), + ); + repo.insert( + "garbage/bun.lock".into(), + b"{\"lockfileVersion\": 99,".to_vec(), + ); + repo.insert("garbage2/bun.lockb".into(), vec![0xa5; 4096]); + repo.insert("garbage3/pnpm-lock.yaml".into(), deep("- ", "", depth)); + let mut opts = options(false); + opts.limits = Some(HostedScanLimits { + max_total_bytes: Some(64 * 1024 * 1024), + ..HostedScanLimits::default() + }); + let output = run_on_small_stack(&server, build_input(&repo, &[], opts)) + .await + .expect("engine result"); + let roots: Vec<&str> = output.projects.iter().map(|p| p.root.as_str()).collect(); + for expected in [ + "npm", "rs", "py", "deepjson", "deeptoml", "garbage", "garbage2", + ] { + assert!( + roots.contains(&expected), + "{expected} missing from {roots:?}" + ); + } + let deepjson = output + .projects + .iter() + .find(|p| p.root == "deepjson") + .unwrap(); + assert_eq!(deepjson.error.as_ref().unwrap().code, "corrupt_ledger"); +} + +#[test] +fn session_limits_reject_oversized_input() { + let mut opts = options(false); + opts.limits = Some(HostedScanLimits { + max_files: Some(1), + ..HostedScanLimits::default() + }); + let mut builder = SessionBuilder::new(opts).unwrap(); + builder.add_text("a/package-lock.json", "{}").unwrap(); + let err = builder.add_text("b/package-lock.json", "{}").unwrap_err(); + assert_eq!(err.code(), "max_files"); + assert_eq!(err.kind(), "limit"); +} + +#[tokio::test] +async fn project_and_purl_limits_reject_the_run() { + let server = npm_server().await; + let mut repo = prefixed("a", &npm_files()); + repo.extend(prefixed("b", &npm_files())); + let mut opts = options(false); + opts.limits = Some(HostedScanLimits { + max_projects: Some(1), + ..HostedScanLimits::default() + }); + let err = run_in_memory( + build_input(&repo, &[], opts), + client(&server), + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + EngineError::Limit { + code: "max_projects", + .. + } + ), + "{err}" + ); + + let mut opts = options(false); + opts.limits = Some(HostedScanLimits { + max_purls: Some(0), + ..HostedScanLimits::default() + }); + let err = run_in_memory( + build_input(&npm_files(), &[], opts), + client(&server), + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert_eq!(err.code(), "max_purls"); + assert_eq!(count(&server, "POST", "/patches/batch").await, 0); +} + +/// A provider whose every call pends forever. +struct Stalled; + +impl PatchApi for Stalled { + fn uses_public_proxy(&self) -> bool { + false + } + fn search_patches_batch<'a>(&'a self, _: &'a [String]) -> ApiFuture<'a, BatchSearchResponse> { + Box::pin(std::future::pending()) + } + fn search_patches_by_package<'a>(&'a self, _: &'a str) -> ApiFuture<'a, SearchResponse> { + Box::pin(std::future::pending()) + } + fn fetch_registry_references<'a>( + &'a self, + _: &'a [String], + ) -> ApiFuture<'a, std::collections::HashMap> { + Box::pin(std::future::pending()) + } + fn fetch_patch<'a>(&'a self, _: &'a str) -> ApiFuture<'a, Option> { + Box::pin(std::future::pending()) + } + fn download_artifact<'a>(&'a self, _: &'a str, _: u64) -> ApiFuture<'a, Vec> { + Box::pin(async { Err(ApiError::Other("unused".into())) }) + } +} + +#[tokio::test] +async fn cancellation_rejects_with_cancelled() { + let cancel = CancellationToken::new(); + let trigger = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + trigger.cancel(); + }); + let err = run_in_memory( + build_input(&npm_files(), &[], options(false)), + Arc::new(Stalled), + cancel, + ) + .await + .unwrap_err(); + assert_eq!(err, EngineError::Cancelled); + assert_eq!(err.code(), "cancelled"); + + let cancelled = CancellationToken::new(); + cancelled.cancel(); + let err = run_in_memory( + build_input(&npm_files(), &[], options(false)), + Arc::new(Stalled), + cancelled, + ) + .await + .unwrap_err(); + assert_eq!(err, EngineError::Cancelled); +} + +/// A provider that counts batch searches, each of which pends forever. +#[derive(Default)] +struct CountingStalled(std::sync::atomic::AtomicUsize); + +impl PatchApi for CountingStalled { + fn uses_public_proxy(&self) -> bool { + false + } + fn search_patches_batch<'a>(&'a self, _: &'a [String]) -> ApiFuture<'a, BatchSearchResponse> { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Box::pin(std::future::pending()) + } + fn search_patches_by_package<'a>(&'a self, _: &'a str) -> ApiFuture<'a, SearchResponse> { + Box::pin(std::future::pending()) + } + fn fetch_registry_references<'a>( + &'a self, + _: &'a [String], + ) -> ApiFuture<'a, std::collections::HashMap> { + Box::pin(std::future::pending()) + } + fn fetch_patch<'a>(&'a self, _: &'a str) -> ApiFuture<'a, Option> { + Box::pin(std::future::pending()) + } + fn download_artifact<'a>(&'a self, _: &'a str, _: u64) -> ApiFuture<'a, Vec> { + Box::pin(async { Err(ApiError::Other("unused".into())) }) + } +} + +/// The inventory phase never pends on memory reads; it must still yield +/// between roots so a cancel (and every other task on this single-threaded +/// runtime) runs before the whole phase — and the first provider call — +/// completes. +#[tokio::test(flavor = "current_thread")] +async fn cancellation_is_honored_between_roots_of_the_inventory() { + let mut repo: BTreeMap> = BTreeMap::new(); + for i in 0..20 { + repo.extend(prefixed(&format!("r{i:02}"), &npm_files())); + } + let cancel = CancellationToken::new(); + let trigger = cancel.clone(); + tokio::spawn(async move { trigger.cancel() }); + let provider = Arc::new(CountingStalled::default()); + let err = run_in_memory( + build_input(&repo, &[], options(false)), + provider.clone(), + cancel, + ) + .await + .unwrap_err(); + assert_eq!(err, EngineError::Cancelled); + assert_eq!(provider.0.load(std::sync::atomic::Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn provider_timeouts_become_project_errors() { + let mut opts = options(false); + opts.request_timeout_ms = Some(30); + let output = run_in_memory( + build_input(&npm_files(), &[], opts), + Arc::new(Stalled), + CancellationToken::new(), + ) + .await + .unwrap(); + let error = output.projects[0].error.as_ref().unwrap(); + assert_eq!(error.code, "patch_lookup_failed"); + assert!(error.message.contains("timed out"), "{}", error.message); + assert!(output.changed_files.is_empty()); +} + +#[tokio::test] +async fn unauthorized_is_a_project_error_without_proxy_fallback() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(ResponseTemplate::new(401).set_body_string("bad token")) + .mount(&server) + .await; + let output = run_engine(&server, build_input(&npm_files(), &[], options(false))).await; + let error = output.projects[0].error.as_ref().unwrap(); + assert_eq!(error.code, "patch_lookup_failed"); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests + .iter() + .all(|r| !r.url.path().starts_with("/patch/"))); +} + +#[tokio::test] +async fn dry_run_previews_without_records_or_ledger() { + let server = npm_server().await; + let output = run_engine(&server, build_input(&npm_files(), &[], options(true))).await; + let paths: Vec<&str> = output + .changed_files + .iter() + .map(|f| f.path.as_str()) + .collect(); + assert_eq!(paths, vec![".npmrc", "package-lock.json"]); + assert_eq!(output.projects[0].redirect["dryRun"], true); + assert_eq!(count(&server, "GET", "/patches/view/").await, 0); + assert_eq!(count(&server, "POST", "/patches/package").await, 1); +} + +#[tokio::test] +async fn symlinked_workspace_config_refuses_the_project() { + let dir = fixtures_root().join("redirect/npm/pnpm/basic"); + let server = MockServer::start().await; + mount_api( + &server, + &patches_from_overrides(&dir.join("overrides.json"), None), + ) + .await; + let files = fixture_files(&dir.join("input")); + let mut builder = SessionBuilder::new(options(false)).unwrap(); + for (path, bytes) in &files { + builder + .add_text(path, std::str::from_utf8(bytes).unwrap()) + .unwrap(); + } + builder + .mark_present( + "pnpm-workspace.yaml", + socket_patch_cli::hosted_memory::MarkKind::Symlink, + ) + .unwrap(); + let output = run_engine(&server, builder.finish().unwrap()).await; + let error = output.projects[0].error.as_ref().unwrap(); + assert_eq!(error.code, "redirect_symlinked_file_unsupported"); + assert!(output.changed_files.is_empty()); +} + +#[tokio::test] +async fn symlinked_npmrc_is_left_alone_with_a_warning() { + let server = npm_server().await; + let mut builder = SessionBuilder::new(options(false)).unwrap(); + for (path, bytes) in &npm_files() { + builder + .add_text(path, std::str::from_utf8(bytes).unwrap()) + .unwrap(); + } + builder + .mark_present(".npmrc", socket_patch_cli::hosted_memory::MarkKind::Symlink) + .unwrap(); + let output = run_engine(&server, builder.finish().unwrap()).await; + let project = &output.projects[0]; + assert!(project.error.is_none()); + let warnings = project.redirect["warnings"].as_array().unwrap(); + let npm = warnings + .iter() + .find(|w| w["code"] == "redirect_npm_allow_remote") + .unwrap(); + assert!(npm["detail"].as_str().unwrap().contains("symbolic link")); + assert!(output.changed_files.iter().all(|f| f.path != ".npmrc")); +} + +#[tokio::test] +async fn vendored_takeover_is_refused() { + let server = npm_server().await; + let mut files = npm_files(); + files.insert( + ".socket/vendor/state.json".into(), + serde_json::to_vec(&serde_json::json!({ + "version": 1, + "entries": { + "pkg:npm/left-pad@1.3.0": { + "ecosystem": "npm", + "basePurl": "pkg:npm/left-pad@1.3.0", + "uuid": "22222222-2222-2222-2222-222222222222", + "artifact": {"path": ".socket/vendor/npm/22222222-2222-2222-2222-222222222222/left-pad-1.3.0.tgz"}, + "wiring": [] + } + } + })) + .unwrap(), + ); + let output = run_engine(&server, build_input(&files, &[], options(false))).await; + let project = &output.projects[0]; + assert!(project + .skipped + .iter() + .any(|s| s.reason == "vendored_takeover_unsupported_in_memory")); + assert!(project.redirected.is_empty()); + let warnings = project.redirect["warnings"].as_array().unwrap(); + assert!(warnings + .iter() + .any(|w| w["code"] == "vendored_takeover_unsupported_in_memory")); + assert!(output.changed_files.is_empty()); +} + +#[tokio::test] +async fn corrupt_ledger_fails_only_its_project() { + let server = npm_server().await; + let mut repo = prefixed("good", &npm_files()); + repo.extend(prefixed("bad", &npm_files())); + repo.insert( + "bad/.socket/vendor/redirect-state.json".into(), + b"{ torn".to_vec(), + ); + let output = run_engine(&server, build_input(&repo, &[], options(false))).await; + let bad = output.projects.iter().find(|p| p.root == "bad").unwrap(); + let good = output.projects.iter().find(|p| p.root == "good").unwrap(); + assert_eq!(bad.error.as_ref().unwrap().code, "corrupt_ledger"); + assert_eq!(bad.redirect, serde_json::json!({ "mode": "hosted" })); + assert!(good.error.is_none()); + assert_eq!(good.redirected.len(), 1); + assert!(output + .changed_files + .iter() + .all(|f| !f.path.starts_with("bad/"))); +} + +#[tokio::test] +async fn maven_files_warn_that_the_ecosystem_is_unsupported() { + let server = npm_server().await; + let mut files = npm_files(); + files.insert("pom.xml".into(), b"".to_vec()); + let output = run_engine(&server, build_input(&files, &[], options(false))).await; + assert!(output + .warnings + .iter() + .any(|w| w.code == "ecosystem_unsupported_in_memory" + && w.project_root.as_deref() == Some(""))); +} + +#[tokio::test] +async fn a_maven_only_repo_warns_through_selection() { + let server = npm_server().await; + let entries: Vec = ["java/app/pom.xml", "java/app/src/Main.java"] + .iter() + .map(|p| TreeEntryInput { + path: (*p).to_string(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(1), + }) + .collect(); + let selection = select_paths(&entries, &SelectOptions::default()); + assert!(selection.roots.is_empty()); + let present: Vec<&str> = selection.present_only.iter().map(String::as_str).collect(); + assert_eq!(present, vec!["java/app/pom.xml"]); + let output = run_engine( + &server, + build_input(&BTreeMap::new(), &present, options(false)), + ) + .await; + assert!(output.projects.is_empty()); + let warning = output + .warnings + .iter() + .find(|w| w.code == "ecosystem_unsupported_in_memory") + .expect("the unsupported ecosystem is reported"); + assert!(warning.project_root.is_none()); + assert!( + warning.detail.contains("java/app/pom.xml"), + "{}", + warning.detail + ); +} + +#[tokio::test] +async fn an_unreadable_higher_precedence_lock_refuses_the_project() { + let server = npm_server().await; + let output = run_engine( + &server, + build_input(&npm_files(), &["npm-shrinkwrap.json"], options(false)), + ) + .await; + let project = &output.projects[0]; + assert_eq!( + project.error.as_ref().map(|e| e.code.as_str()), + Some("candidate_file_unreadable"), + "{:#}", + project.redirect + ); + assert!(project.redirected.is_empty()); + assert!(output.changed_files.is_empty()); +} + +#[tokio::test] +async fn selection_drives_the_engine_roots() { + let server = npm_server().await; + let mut repo = prefixed("apps/web", &npm_files()); + repo.extend(prefixed("apps/web/test/fixture", &npm_files())); + repo.insert("apps/web/src/index.js".into(), b"x".to_vec()); + let entries: Vec = repo + .keys() + .map(|p| TreeEntryInput { + path: p.clone(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(1), + }) + .collect(); + let selection = select_paths(&entries, &SelectOptions::default()); + assert_eq!(selection.roots, vec!["apps/web"]); + let fetched: BTreeMap> = selection + .fetch_text + .iter() + .map(|p| (p.clone(), repo[p].clone())) + .collect(); + let mut opts = options(false); + opts.project_roots = Some(selection.roots.clone()); + let output = run_engine(&server, build_input(&fetched, &[], opts)).await; + assert_eq!(output.projects.len(), 1); + assert_eq!(output.projects[0].root, "apps/web"); + assert_eq!(output.projects[0].redirected.len(), 1); +} + +#[tokio::test] +async fn ecosystem_filter_skips_other_ecosystems() { + let server = npm_server().await; + let mut opts = options(false); + opts.ecosystems = Some(vec!["cargo".to_string()]); + let output = run_engine(&server, build_input(&npm_files(), &[], opts)).await; + assert!(output.projects.is_empty() || output.projects[0].summary.scanned_packages == 0); + assert_eq!(count(&server, "POST", "/patches/batch").await, 0); +} + +#[tokio::test] +async fn invalid_options_reject() { + let server = npm_server().await; + let err = run_in_memory( + build_input(&npm_files(), &[], options(false)), + client(&server), + CancellationToken::new(), + ) + .await + .map(|_| ()) + .err(); + assert!(err.is_none()); + let err = SessionBuilder::new(HostedScanOptions { + org_slug: String::new(), + ..HostedScanOptions::default() + }) + .unwrap_err(); + assert_eq!(err.code(), "invalid_org_slug"); +} + +#[tokio::test] +async fn hosted_bundle_command_prints_the_engine_result() { + let server = npm_server().await; + let files: BTreeMap = npm_files() + .into_iter() + .map(|(k, v)| (k, String::from_utf8(v).unwrap())) + .collect(); + let bundle = serde_json::json!({ "files": files, "presentOnly": [".pnp.loader.mjs"] }); + let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")) + .args([ + "hosted-bundle", + "--org", + ORG, + "--api-token", + "fake-token", + "--api-url", + &server.uri(), + ]) + .env("SOCKET_NO_CONFIG", "1") + .env("SOCKET_TELEMETRY_DISABLED", "1") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + child + .stdin + .take() + .unwrap() + .write_all(bundle.to_string().as_bytes()) + .unwrap(); + } + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let result: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(result["projects"][0]["root"], ""); + let warnings = result["warnings"].as_array().unwrap(); + assert!( + warnings.iter().any(|w| w["code"] == "yarn_pnp_unsupported"), + "{warnings:?}" + ); + assert!(result["engineVersion"] + .as_str() + .unwrap() + .starts_with(env!("CARGO_PKG_VERSION"))); + + let missing = std::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")) + .args(["hosted-bundle"]) + .env_remove("SOCKET_API_TOKEN") + .env_remove("SOCKET_ORG_SLUG") + .env("SOCKET_NO_CONFIG", "1") + .stdin(std::process::Stdio::null()) + .output() + .unwrap(); + assert_eq!(missing.status.code(), Some(2)); +} + +#[tokio::test] +async fn warnings_are_scoped_to_their_project() { + let server = npm_server().await; + let output = run_engine( + &server, + build_input( + &prefixed("pkg", &npm_files()), + &["pkg/.pnp.cjs"], + options(false), + ), + ) + .await; + let pnp = output + .warnings + .iter() + .find(|w| w.code == "yarn_pnp_unsupported") + .expect("pnp refusal"); + assert_eq!(pnp.project_root.as_deref(), Some("pkg")); +} + +#[tokio::test] +async fn stats_count_inputs_and_calls() { + let server = npm_server().await; + let files = npm_files(); + let bytes: usize = files.values().map(Vec::len).sum(); + let output = run_engine(&server, build_input(&files, &[], options(false))).await; + assert_eq!(output.stats.files_input, files.len() as u64); + assert_eq!(output.stats.bytes_input, bytes as u64); + assert_eq!(output.stats.patches_selected, 1); + assert_eq!( + output.stats.files_changed, + output.changed_files.len() as u64 + ); + for method_name in [ + "searchPatchesBatch", + "searchPatchesByPackage", + "fetchRegistryReferences", + "fetchPatch", + ] { + assert_eq!( + output.stats.provider_calls.get(method_name), + Some(&1), + "{method_name}" + ); + } +} + +#[test] +fn engine_future_is_send_and_static() { + fn assert_send_static(_: &T) {} + let input = build_input(&npm_files(), &[], options(false)); + let future = run_in_memory(input, Arc::new(Stalled), CancellationToken::new()); + assert_send_static(&future); +} + +#[test] +fn contract_shapes_are_camel_case() { + let options: HostedScanOptions = serde_json::from_value(serde_json::json!({ + "orgSlug": "org", "ecosystems": ["npm"], "batchSize": 50, "dryRun": true, + "pipenvMajor": 11, "trustLockfileConfig": false, "npmAllowRemoteConfig": false, + "projectRoots": ["a"], "providerConcurrency": 4, "requestTimeoutMs": 1000, + "limits": {"maxFileBytes": 1, "maxTotalBytes": 2, "maxFiles": 3, "maxPurls": 4, + "maxProjects": 5, "maxArtifactBytes": 6} + })) + .unwrap(); + assert_eq!(options.batch_size, Some(50)); + assert_eq!(options.limits.unwrap().max_artifact_bytes, Some(6)); + let entry: TreeEntryInput = serde_json::from_value(serde_json::json!({ + "path": "a/package-lock.json", "mode": "100644", "type": "blob", "size": 10 + })) + .unwrap(); + let selection = + serde_json::to_value(select_paths(&[entry], &SelectOptions::default())).unwrap(); + for key in [ + "roots", + "fetchText", + "fetchBinary", + "presentOnly", + "symlinks", + "ignoredCount", + "ignoredSample", + ] { + assert!( + selection.get(key).is_some(), + "{key} missing from {selection}" + ); + } +} + +#[tokio::test] +async fn result_serializes_with_the_contract_keys() { + let server = npm_server().await; + let output = run_engine(&server, build_input(&npm_files(), &[], options(false))).await; + let value = serde_json::to_value(&output).unwrap(); + for key in [ + "projects", + "changedFiles", + "changedBinaryFiles", + "deletedFiles", + "warnings", + "stats", + "engineVersion", + ] { + assert!(value.get(key).is_some(), "{key}"); + } + let project = &value["projects"][0]; + for key in ["root", "redirect", "summary", "redirected", "skipped"] { + assert!(project.get(key).is_some(), "{key}"); + } + for key in [ + "scannedPackages", + "packagesWithPatches", + "totalPatches", + "freePatches", + "paidPatches", + "canAccessPaidPatches", + ] { + assert!(project["summary"].get(key).is_some(), "{key}"); + } + for key in [ + "projects", + "filesInput", + "bytesInput", + "packagesScanned", + "packagesWithPatches", + "patchesSelected", + "patchesRedirected", + "filesChanged", + "providerCalls", + "phaseMs", + ] { + assert!(value["stats"].get(key).is_some(), "{key}"); + } +} diff --git a/crates/socket-patch-cli/tests/hosted_memory_parity.rs b/crates/socket-patch-cli/tests/hosted_memory_parity.rs new file mode 100644 index 00000000..5876fe19 --- /dev/null +++ b/crates/socket-patch-cli/tests/hosted_memory_parity.rs @@ -0,0 +1,747 @@ +//! Parity: the in-memory hosted engine over a file set produces the same +//! changed bytes, ledger bytes and `redirect` block as `scan --mode hosted +//! --json` over a checkout of the same files (the disk run goes through +//! the real binary under a scrubbed environment, so crawlers, host caches +//! and subprocesses contribute nothing on either side). + +use std::collections::BTreeMap; + +use base64::Engine; +use serde_json::Value; +use wiremock::MockServer; + +#[path = "hosted_memory_common/mod.rs"] +mod common; + +use common::*; + +struct Case { + /// Fixture dir under `crates/socket-patch-core/tests/fixtures/redirect`. + fixture: &'static str, + /// Extra files layered over the fixture input. + extra: Vec<(&'static str, Vec)>, + /// Require at least one redirect (a parity of two no-ops proves + /// nothing for the formats the engine must rewrite). + expect_redirect: bool, + dry_run: bool, +} + +fn case(fixture: &'static str) -> Case { + Case { + fixture, + extra: Vec::new(), + expect_redirect: true, + dry_run: false, + } +} + +async fn assert_parity(case: Case) { + let dir = fixtures_root().join("redirect").join(case.fixture); + let server = MockServer::start().await; + let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let mut files = fixture_files(&dir.join("input")); + for (rel, bytes) in &case.extra { + files.insert((*rel).to_string(), bytes.clone()); + } + + let disk = run_disk(&server, &files, case.dry_run); + let input = build_input(&files, &[], options(case.dry_run)); + let memory = run_engine(&server, input).await; + + assert_eq!(memory.projects.len(), 1, "{}: one root", case.fixture); + let project = &memory.projects[0]; + assert_eq!(project.root, ""); + assert!( + project.error.is_none(), + "{}: {:?}", + case.fixture, + project.error + ); + let disk_redirect = disk + .envelope + .get("redirect") + .cloned() + .unwrap_or(Value::Null); + assert_eq!( + project.redirect, disk_redirect, + "{}: redirect block differs\nmemory: {:#}\ndisk: {:#}\nstderr: {}", + case.fixture, project.redirect, disk_redirect, disk.stderr + ); + let memory_changed = engine_changed(&memory); + let expected_changed = if case.dry_run { + assert!( + disk.changed.is_empty(), + "{}: a disk dry run wrote files", + case.fixture + ); + let mut wet = run_disk(&server, &files, false).changed; + wet.remove(".socket/vendor/redirect-state.json"); + wet + } else { + disk.changed.clone() + }; + assert_eq!( + memory_changed.keys().collect::>(), + expected_changed.keys().collect::>(), + "{}: changed file sets differ\nmemory:\n{}\ndisk:\n{}", + case.fixture, + describe(&memory_changed), + describe(&expected_changed) + ); + for (rel, bytes) in &expected_changed { + assert_eq!( + String::from_utf8_lossy(&memory_changed[rel]), + String::from_utf8_lossy(bytes), + "{}: {rel} differs", + case.fixture + ); + } + let redirected = disk_redirect["redirected"].as_u64().unwrap_or(0); + assert_eq!(redirected as usize, project.redirected.len()); + if case.expect_redirect { + assert!( + redirected > 0, + "{}: the fixture redirected nothing: {disk_redirect:#}\n{}", + case.fixture, + disk.stderr + ); + } + assert!( + project.summary.scanned_packages >= redirected, + "{}", + case.fixture + ); +} + +#[tokio::test] +async fn parity_package_lock() { + assert_parity(case("npm/package-lock-v3/basic")).await; +} + +#[tokio::test] +async fn parity_package_lock_dry_run() { + assert_parity(Case { + dry_run: true, + ..case("npm/package-lock-v3/basic") + }) + .await; +} + +#[tokio::test] +async fn parity_pnpm_v9_trust_lockfile() { + assert_parity(case("npm/pnpm/basic")).await; +} + +#[tokio::test] +async fn parity_pnpm_existing_workspace() { + assert_parity(Case { + extra: vec![("pnpm-workspace.yaml", b"packages:\n - 'apps/*'\n".to_vec())], + ..case("npm/pnpm/basic") + }) + .await; +} + +#[tokio::test] +async fn parity_yarn_classic() { + assert_parity(case("npm/yarn-classic/basic")).await; +} + +#[tokio::test] +async fn parity_yarn_berry() { + assert_parity(case("npm/yarn-berry/basic")).await; +} + +#[tokio::test] +async fn parity_bun_text_lock() { + assert_parity(case("npm/bun/basic")).await; +} + +#[tokio::test] +async fn parity_rush() { + assert_parity(Case { + extra: vec![ + ("rush.json", b"{}\n".to_vec()), + ("common/config/rush/repo-state.json", b"{}\n".to_vec()), + ], + ..case("npm/pnpm/nested-rush-lock") + }) + .await; +} + +#[tokio::test] +async fn parity_uv() { + assert_parity(Case { + expect_redirect: false, + ..case("pypi/uv/basic") + }) + .await; +} + +#[tokio::test] +async fn parity_requirements() { + assert_parity(case("pypi/requirements/basic")).await; +} + +#[tokio::test] +async fn parity_cargo() { + assert_parity(case("cargo/cargo/basic")).await; +} + +#[tokio::test] +async fn parity_cargo_workspace() { + assert_parity(case("cargo/cargo/workspace-member")).await; +} + +#[tokio::test] +async fn parity_composer() { + assert_parity(case("composer/composer-lock/basic")).await; +} + +#[tokio::test] +async fn parity_gemfile() { + assert_parity(case("gem/bundler/basic")).await; +} + +#[tokio::test] +async fn parity_golang() { + assert_parity(case("golang/gomod/basic")).await; +} + +/// Formats with no golden redirect fixture: a committed native lock plus a +/// synthetic override for one of its packages. +async fn assert_native_parity( + files: BTreeMap>, + overrides: Value, + expect_redirect: bool, +) { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("overrides.json"), overrides.to_string()).unwrap(); + let server = MockServer::start().await; + let patches = patches_from_overrides(&tmp.path().join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let disk = run_disk(&server, &files, false); + let memory = run_engine(&server, build_input(&files, &[], options(false))).await; + let project = &memory.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + assert_eq!( + without_pipenv_advice(&project.redirect), + without_pipenv_advice(&disk.envelope["redirect"]), + "{}", + disk.stderr + ); + let memory_changed = engine_changed(&memory); + assert_eq!( + memory_changed, + disk.changed, + "memory:\n{}\ndisk:\n{}", + describe(&memory_changed), + describe(&disk.changed) + ); + if expect_redirect { + assert!(!project.redirected.is_empty(), "{:#}", project.redirect); + } +} + +/// The in-memory engine cannot find Pipenv on PATH, so its +/// `redirect_pipenv_installer_unknown` advice names the `pipenvMajor` +/// option instead of the disk run's PATH/env remedy. +fn without_pipenv_advice(redirect: &Value) -> Value { + let mut redirect = redirect.clone(); + if let Some(warnings) = redirect.get_mut("warnings").and_then(Value::as_array_mut) { + for warning in warnings.iter_mut() { + if warning["code"] == "redirect_pipenv_installer_unknown" { + warning["detail"] = Value::Null; + } + } + } + redirect +} + +fn read_fixture(rel: &str) -> Vec { + std::fs::read(fixtures_root().join(rel)).unwrap() +} + +#[tokio::test] +async fn parity_poetry() { + let files = BTreeMap::from([ + ( + "poetry.lock".to_string(), + read_fixture("poetry/2.4.3/poetry.lock"), + ), + ( + "pyproject.toml".to_string(), + read_fixture("poetry/2.4.3/pyproject.toml"), + ), + ]); + let overrides = serde_json::json!([{ + "ecosystem": "pypi", "name": "urllib3", "version": "1.26.18", + "token": "22222222-2222-4222-8222-222222222222", + "patchUuid": "e828efa5-5c6d-43f3-9909-03f5ac232b98", + "artifactUrl": "https://patch.socket.dev/patch/pypi/urllib3/1.26.18/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/urllib3-1.26.18-py2.py3-none-any.whl", + "integrity": {"sha256": "c".repeat(64)} + }]); + assert_native_parity(files, overrides, true).await; +} + +#[tokio::test] +async fn parity_pipfile() { + let files = BTreeMap::from([ + ( + "Pipfile.lock".to_string(), + read_fixture("pipenv/2026.8.0/Pipfile.lock"), + ), + ( + "Pipfile".to_string(), + read_fixture("pipenv/2026.8.0/Pipfile"), + ), + ]); + let lock: Value = serde_json::from_slice(&files["Pipfile.lock"]).unwrap(); + let (name, entry) = lock["default"] + .as_object() + .and_then(|m| m.iter().next()) + .expect("a default package"); + let version = entry["version"].as_str().unwrap().trim_start_matches("=="); + let overrides = serde_json::json!([{ + "ecosystem": "pypi", "name": name, "version": version, + "token": "22222222-2222-4222-8222-222222222222", + "patchUuid": "e828efa5-5c6d-43f3-9909-03f5ac232b98", + "artifactUrl": format!("https://patch.socket.dev/patch/pypi/{name}/{version}/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/{name}-{version}-py3-none-any.whl"), + "integrity": {"sha256": "c".repeat(64)} + }]); + assert_native_parity(files, overrides, true).await; +} + +#[tokio::test] +async fn parity_bun_binary_lock() { + let files = BTreeMap::from([ + ( + "bun.lockb".to_string(), + read_fixture("bun-lockb/1.1.45/bun.lockb"), + ), + ( + "package.json".to_string(), + read_fixture("bun-lockb/1.1.45/package.json"), + ), + ]); + let overrides = serde_json::json!([{ + "ecosystem": "npm", "name": "minimist", "version": "1.2.2", + "token": "22222222-2222-4222-8222-222222222222", + "patchUuid": "33333333-3333-4333-8333-333333333333", + "artifactUrl": "https://patch.socket.dev/patch/npm/minimist/1.2.2/22222222-2222-4222-8222-222222222222/33333333-3333-4333-8333-333333333333/minimist-1.2.2.tgz", + "integrity": {"sha512": format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode([0x5au8; 64]) + )} + }]); + assert_native_parity(files, overrides, true).await; +} + +#[tokio::test] +async fn parity_nested_monorepo_roots_match_their_own_disk_runs() { + let npm = fixtures_root().join("redirect/npm/package-lock-v3/basic"); + let cargo = fixtures_root().join("redirect/cargo/cargo/basic"); + let server = MockServer::start().await; + let mut patches = patches_from_overrides(&npm.join("overrides.json"), None); + patches.extend(patches_from_overrides(&cargo.join("overrides.json"), None)); + mount_api(&server, &patches).await; + let web = fixture_files(&npm.join("input")); + let svc = fixture_files(&cargo.join("input")); + let mut repo: BTreeMap> = BTreeMap::new(); + for (rel, bytes) in &web { + repo.insert(format!("apps/web/{rel}"), bytes.clone()); + } + for (rel, bytes) in &svc { + repo.insert(format!("services/api/{rel}"), bytes.clone()); + } + let memory = run_engine(&server, build_input(&repo, &[], options(false))).await; + let roots: Vec<&str> = memory.projects.iter().map(|p| p.root.as_str()).collect(); + assert_eq!(roots, vec!["apps/web", "services/api"]); + let changed = engine_changed(&memory); + for (root, files) in [("apps/web", &web), ("services/api", &svc)] { + let disk = run_disk(&server, files, false); + let project = memory.projects.iter().find(|p| p.root == root).unwrap(); + assert_eq!(project.redirect, disk.envelope["redirect"], "{root}"); + let prefixed: BTreeMap> = changed + .iter() + .filter_map(|(k, v)| { + k.strip_prefix(&format!("{root}/")) + .map(|rel| (rel.to_string(), v.clone())) + }) + .collect(); + assert_eq!(prefixed, disk.changed, "{root}"); + } +} + +fn wheel(name: &str, version: &str) -> Vec { + use std::io::Write; + let mut buf = std::io::Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut buf); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file(format!("{name}-{version}.dist-info/METADATA"), options) + .unwrap(); + write!( + zip, + "Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n" + ) + .unwrap(); + zip.finish().unwrap(); + } + buf.into_inner() +} + +#[tokio::test] +async fn parity_uv_with_hosted_wheel_metadata() { + use sha2::Digest; + let dir = fixtures_root().join("redirect/pypi/uv/basic"); + let server = MockServer::start().await; + let bytes = wheel("click", "8.1.7"); + let sha = hex::encode(sha2::Sha256::digest(&bytes)); + let tmp = tempfile::tempdir().unwrap(); + let mut overrides: Value = + serde_json::from_str(&std::fs::read_to_string(dir.join("overrides.json")).unwrap()) + .unwrap(); + overrides[0]["integrity"]["sha256"] = Value::String(sha); + std::fs::write(tmp.path().join("overrides.json"), overrides.to_string()).unwrap(); + let patches = patches_from_overrides(&tmp.path().join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path_regex("^/patch/pypi/.+\\.whl$")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_bytes(bytes)) + .mount(&server) + .await; + let files = fixture_files(&dir.join("input")); + let disk = run_disk(&server, &files, false); + let memory = run_engine(&server, build_input(&files, &[], options(false))).await; + let project = &memory.projects[0]; + assert_eq!( + project.redirect, disk.envelope["redirect"], + "{}", + disk.stderr + ); + assert_eq!(engine_changed(&memory), disk.changed); + assert_eq!(project.redirected.len(), 1, "{:#}", project.redirect); + assert_eq!( + memory.stats.provider_calls.get("downloadArtifact"), + Some(&1) + ); +} + +/// `files` narrowed to what [`select_paths`] asks the host to stream +/// (presence-only paths marked present), so the parity covers selection. +fn selected_input( + files: &BTreeMap>, +) -> socket_patch_cli::hosted_memory::HostedScanInput { + use socket_patch_cli::hosted_memory::{select_paths, SelectOptions, TreeEntryInput}; + let entries: Vec = files + .iter() + .map(|(p, bytes)| TreeEntryInput { + path: p.clone(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(bytes.len() as u64), + }) + .collect(); + let selection = select_paths(&entries, &SelectOptions::default()); + let fetched: BTreeMap> = selection + .fetch_text + .iter() + .chain(selection.fetch_binary.iter()) + .map(|p| (p.clone(), files[p].clone())) + .collect(); + let present: Vec<&str> = selection.present_only.iter().map(String::as_str).collect(); + let mut opts = options(false); + opts.project_roots = Some(selection.roots.clone()); + build_input(&fetched, &present, opts) +} + +#[tokio::test] +async fn parity_cargo_patch_path_under_vendor_through_selection() { + let dir = fixtures_root().join("redirect/cargo/cargo/workspace-member"); + let server = MockServer::start().await; + let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let mut files = fixture_files(&dir.join("input")); + let mut manifest = String::from_utf8(files["Cargo.toml"].clone()).unwrap(); + manifest.push_str("\n[patch.crates-io]\nfoo = { path = \"vendor/foo\" }\n"); + files.insert("Cargo.toml".into(), manifest.into_bytes()); + let mut lock = String::from_utf8(files["Cargo.lock"].clone()).unwrap(); + lock.push_str( + "\n[[package]]\nname = \"foo\"\nversion = \"0.1.0\"\ndependencies = [\n \"serde\",\n]\n", + ); + files.insert("Cargo.lock".into(), lock.into_bytes()); + files.insert( + "vendor/foo/Cargo.toml".into(), + b"[package]\nname = \"foo\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = \"1.0.190\"\n" + .to_vec(), + ); + files.insert( + "tests/fixtures/other/Cargo.toml".into(), + b"[package]\nname = \"other\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = \"1\"\n" + .to_vec(), + ); + + let disk = run_disk(&server, &files, false); + assert!( + disk.changed.contains_key("vendor/foo/Cargo.toml"), + "disk pins the [patch] path crate: {}\n{}", + describe(&disk.changed), + disk.stderr + ); + let memory = run_engine(&server, selected_input(&files)).await; + let project = &memory.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + assert_eq!( + project.redirect, disk.envelope["redirect"], + "{}", + disk.stderr + ); + let memory_changed = engine_changed(&memory); + assert_eq!( + memory_changed, + disk.changed, + "memory:\n{}\ndisk:\n{}", + describe(&memory_changed), + describe(&disk.changed) + ); + assert_eq!(project.redirected.len(), 1, "{:#}", project.redirect); +} + +/// `cargo vendor` output (`.cargo-checksum.json` in every crate) sits under +/// `vendor/` with a `[patch]`-free workspace: disk never reads it, and +/// selection must not fetch it either. +fn with_cargo_vendor_tree(files: &mut BTreeMap>) { + for krate in ["serde", "itoa"] { + files.insert( + format!("vendor/{krate}/Cargo.toml"), + format!("[package]\nname = \"{krate}\"\nversion = \"1.0.190\"\n").into_bytes(), + ); + files.insert( + format!("vendor/{krate}/.cargo-checksum.json"), + b"{\"files\":{},\"package\":\"00\"}".to_vec(), + ); + files.insert( + format!("vendor/{krate}/tests/ui/Cargo.toml"), + b"[package]\nname = \"ui\"\nversion = \"0.0.0\"\n".to_vec(), + ); + } +} + +fn selected_paths(files: &BTreeMap>) -> Vec { + use socket_patch_cli::hosted_memory::{select_paths, SelectOptions, TreeEntryInput}; + let entries: Vec = files + .keys() + .map(|p| TreeEntryInput { + path: p.clone(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(1), + }) + .collect(); + let selection = select_paths(&entries, &SelectOptions::default()); + selection + .fetch_text + .into_iter() + .chain(selection.fetch_binary) + .chain(selection.present_only) + .collect() +} + +#[tokio::test] +async fn parity_cargo_vendor_tree_is_not_fetched() { + let dir = fixtures_root().join("redirect/cargo/cargo/workspace-member"); + let server = MockServer::start().await; + let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let mut files = fixture_files(&dir.join("input")); + with_cargo_vendor_tree(&mut files); + + let selected = selected_paths(&files); + assert!( + !selected.iter().any(|p| p.starts_with("vendor/")), + "{selected:?}" + ); + let disk = run_disk(&server, &files, false); + assert!( + !disk.changed.keys().any(|p| p.starts_with("vendor/")), + "{}", + describe(&disk.changed) + ); + let memory = run_engine(&server, selected_input(&files)).await; + let project = &memory.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + assert_eq!( + project.redirect, disk.envelope["redirect"], + "{}", + disk.stderr + ); + let memory_changed = engine_changed(&memory); + assert_eq!( + memory_changed, + disk.changed, + "memory:\n{}\ndisk:\n{}", + describe(&memory_changed), + describe(&disk.changed) + ); + assert!(!project.redirected.is_empty(), "{:#}", project.redirect); +} + +#[tokio::test] +async fn cargo_patch_path_into_vendor_tree_fails_closed() { + let dir = fixtures_root().join("redirect/cargo/cargo/workspace-member"); + let server = MockServer::start().await; + let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let mut files = fixture_files(&dir.join("input")); + let mut manifest = String::from_utf8(files["Cargo.toml"].clone()).unwrap(); + manifest.push_str("\n[patch.crates-io]\nfoo = { path = \"vendor/foo\" }\n"); + files.insert("Cargo.toml".into(), manifest.into_bytes()); + let mut lock = String::from_utf8(files["Cargo.lock"].clone()).unwrap(); + lock.push_str( + "\n[[package]]\nname = \"foo\"\nversion = \"0.1.0\"\ndependencies = [\n \"serde\",\n]\n", + ); + files.insert("Cargo.lock".into(), lock.into_bytes()); + files.insert( + "vendor/foo/Cargo.toml".into(), + b"[package]\nname = \"foo\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = \"1.0.190\"\n" + .to_vec(), + ); + files.insert( + "vendor/foo/.cargo-checksum.json".into(), + b"{\"files\":{},\"package\":\"00\"}".to_vec(), + ); + + let memory = run_engine(&server, selected_input(&files)).await; + let project = &memory.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + assert!(project.redirected.is_empty(), "{:#}", project.redirect); + assert!( + engine_changed(&memory).is_empty(), + "{}", + describe(&engine_changed(&memory)) + ); + assert!( + project + .redirect + .to_string() + .contains("redirect_cargo_transitive_dependents"), + "{:#}", + project.redirect + ); +} + +#[tokio::test] +async fn pipfile_advice_names_the_pipenv_major_option() { + let files = BTreeMap::from([ + ( + "Pipfile.lock".to_string(), + read_fixture("pipenv/2026.8.0/Pipfile.lock"), + ), + ( + "Pipfile".to_string(), + read_fixture("pipenv/2026.8.0/Pipfile"), + ), + ]); + let lock: Value = serde_json::from_slice(&files["Pipfile.lock"]).unwrap(); + let (name, entry) = lock["default"] + .as_object() + .and_then(|m| m.iter().next()) + .expect("a default package"); + let version = entry["version"].as_str().unwrap().trim_start_matches("=="); + let overrides = serde_json::json!([{ + "ecosystem": "pypi", "name": name, "version": version, + "token": "22222222-2222-4222-8222-222222222222", + "patchUuid": "e828efa5-5c6d-43f3-9909-03f5ac232b98", + "artifactUrl": format!("https://patch.socket.dev/patch/pypi/{name}/{version}/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/{name}-{version}-py3-none-any.whl"), + "integrity": {"sha256": "c".repeat(64)} + }]); + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("overrides.json"), overrides.to_string()).unwrap(); + let server = MockServer::start().await; + let patches = patches_from_overrides(&tmp.path().join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let memory = run_engine(&server, build_input(&files, &[], options(false))).await; + let detail = memory.projects[0].redirect["warnings"] + .as_array() + .and_then(|w| { + w.iter() + .find(|x| x["code"] == "redirect_pipenv_installer_unknown") + }) + .and_then(|w| w["detail"].as_str()) + .expect("pipenv advice warning") + .to_string(); + assert!(detail.contains("`pipenvMajor`"), "{detail}"); + assert!(!detail.contains("PATH"), "{detail}"); +} + +#[tokio::test] +async fn excluded_nested_cargo_project_is_its_own_root_through_selection() { + let ws = fixtures_root().join("redirect/cargo/cargo/workspace-member"); + let standalone = fixtures_root().join("redirect/cargo/cargo/basic"); + let server = MockServer::start().await; + let mut patches = patches_from_overrides(&ws.join("overrides.json"), Some(&server.uri())); + patches.extend(patches_from_overrides( + &standalone.join("overrides.json"), + Some(&server.uri()), + )); + mount_api(&server, &patches).await; + + let mut repo = fixture_files(&ws.join("input")); + let manifest = String::from_utf8(repo["Cargo.toml"].clone()) + .unwrap() + .replacen( + "[workspace]\n", + "[workspace]\nexclude = [\"tools/fuzz\"]\n", + 1, + ); + repo.insert("Cargo.toml".into(), manifest.into_bytes()); + let stale_member_lock = repo["Cargo.lock"].clone(); + repo.insert("a/Cargo.lock".into(), stale_member_lock); + let fuzz = fixture_files(&standalone.join("input")); + for (rel, bytes) in &fuzz { + repo.insert(format!("tools/fuzz/{rel}"), bytes.clone()); + } + + let memory = run_engine(&server, selected_input(&repo)).await; + let roots: Vec<&str> = memory.projects.iter().map(|p| p.root.as_str()).collect(); + assert!(roots.contains(&"tools/fuzz"), "{roots:?}"); + let changed = engine_changed(&memory); + + let disk = run_disk(&server, &fuzz, false); + let fuzz_project = memory + .projects + .iter() + .find(|p| p.root == "tools/fuzz") + .unwrap(); + assert!(fuzz_project.error.is_none(), "{:?}", fuzz_project.error); + assert_eq!(fuzz_project.redirect, disk.envelope["redirect"]); + let fuzz_changed: BTreeMap> = changed + .iter() + .filter_map(|(k, v)| { + k.strip_prefix("tools/fuzz/") + .map(|rel| (rel.to_string(), v.clone())) + }) + .collect(); + assert_eq!(fuzz_changed, disk.changed); + assert!( + !fuzz_project.redirected.is_empty(), + "{:#}", + fuzz_project.redirect + ); + + let workspace = memory.projects.iter().find(|p| p.root.is_empty()).unwrap(); + assert!(!workspace.redirected.is_empty(), "{:#}", workspace.redirect); + assert!( + !changed.contains_key("a/Cargo.lock"), + "{}", + describe(&changed) + ); + assert!( + memory.warnings.iter().any( + |w| w.code == "cargo_member_lock_ignored" && w.project_root.as_deref() == Some("a") + ), + "{:?}", + memory.warnings + ); +} diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index b0bc0424..e178d003 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -180,7 +180,7 @@ pub struct ApiClientOptions { /// Supports both the authenticated Socket API (`api.socket.dev`) and the /// public proxy (`patches-api.socket.dev`) which serves free patches /// without authentication. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ApiClient { client: reqwest::Client, /// Header-free twin of `client` (User-Agent only, never Authorization) @@ -2476,7 +2476,7 @@ fn convert_search_result_to_batch_info(patch: PatchSearchResult) -> BatchPatchIn /// reaches a caller. `scan` renders `packages[].patches` straight to the /// operator and treats the leading entry as the patch apply will install; /// both only hold because of this. -fn sort_batch_response(response: &mut BatchSearchResponse) { +pub fn sort_batch_response(response: &mut BatchSearchResponse) { for pkg in &mut response.packages { pkg.patches.sort_by(cmp_batch_infos); } @@ -2564,6 +2564,302 @@ pub enum ApiError { Other(String), } +// ── Patch API seam ──────────────────────────────────────────────────── + +/// A boxed, `Send` future returned by [`PatchApi`] methods. +pub type ApiFuture<'a, T> = + std::pin::Pin> + Send + 'a>>; + +/// The patch-data calls the hosted flow makes, as an object-safe trait so an +/// embedding host (the in-memory engine, a napi addon) can serve them +/// in-process. [`ApiClient`] implements it by forwarding to its inherent +/// methods, so existing callers are unaffected. +pub trait PatchApi: Send + Sync { + /// Whether calls go to the public patch proxy (free patches only). + fn uses_public_proxy(&self) -> bool; + + /// `POST …/patches/batch` for up to 500 purls, normalized through + /// [`sort_batch_response`]. + fn search_patches_batch<'a>( + &'a self, + purls: &'a [String], + ) -> ApiFuture<'a, BatchSearchResponse>; + + /// `GET …/patches/by-package/`, results best-first. + fn search_patches_by_package<'a>(&'a self, purl: &'a str) -> ApiFuture<'a, SearchResponse>; + + /// `POST …/patches/package` (hosted reference grants), keyed by uuid. + fn fetch_registry_references<'a>( + &'a self, + uuids: &'a [String], + ) -> ApiFuture<'a, std::collections::HashMap>; + + /// `GET …/patches/view/`; `None` when not found. + fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option>; + + /// Download a grant-tokenized artifact URL, refusing bodies over + /// `max_bytes`. + fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec>; +} + +impl PatchApi for ApiClient { + fn uses_public_proxy(&self) -> bool { + self.use_public_proxy + } + + fn search_patches_batch<'a>( + &'a self, + purls: &'a [String], + ) -> ApiFuture<'a, BatchSearchResponse> { + Box::pin(ApiClient::search_patches_batch(self, purls)) + } + + fn search_patches_by_package<'a>(&'a self, purl: &'a str) -> ApiFuture<'a, SearchResponse> { + Box::pin(ApiClient::search_patches_by_package(self, purl)) + } + + fn fetch_registry_references<'a>( + &'a self, + uuids: &'a [String], + ) -> ApiFuture<'a, std::collections::HashMap> { + Box::pin(ApiClient::fetch_registry_references(self, uuids)) + } + + fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option> { + Box::pin(ApiClient::fetch_patch(self, uuid)) + } + + fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec> { + Box::pin(self.download_artifact_capped(url, max_bytes)) + } +} + +/// Outcome of one capped artifact GET, and `Some(retry-after)` iff a +/// failure is retryable. +type CappedAttempt = (Result, ApiError>, Option>); + +impl ApiClient { + /// [`ApiClient::download_artifact`] with the caller's byte cap applied + /// while streaming (a declared `Content-Length` over it is refused + /// before the body is read), retrying transient failures on the vendor + /// retry policy. + async fn download_artifact_capped( + &self, + url: &str, + max_bytes: u64, + ) -> Result, ApiError> { + if !(url.starts_with("https://") || url.starts_with("http://")) { + return Err(ApiError::Other(format!( + "refusing non-http(s) artifact URL `{url}`" + ))); + } + let attempts = self.vendor_retry.attempts.max(1); + let mut attempt = 1; + loop { + match self.download_artifact_capped_once(url, max_bytes).await { + (Err(_), Some(retry_after)) if attempt < attempts => { + self.vendor_backoff(attempt, retry_after).await; + attempt += 1; + } + (outcome, _) => return outcome, + } + } + } + + async fn download_artifact_capped_once(&self, url: &str, max_bytes: u64) -> CappedAttempt { + use crate::utils::http::{read_capped_typed, ReadCappedError}; + let sent = tokio::time::timeout( + self.vendor_retry.attempt_timeout, + self.plain + .get(url) + .header(header::ACCEPT, "application/octet-stream") + .send(), + ) + .await; + let resp = match sent { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + return ( + Err(ApiError::Network(format!( + "Network error fetching artifact: {}", + network_error_detail(&e) + ))), + Some(None), + ) + } + Err(_) => { + return ( + Err(ApiError::Network(format!( + "Network error fetching artifact: no response within {:?}", + self.vendor_retry.attempt_timeout + ))), + Some(None), + ) + } + }; + let status = resp.status(); + match status { + StatusCode::OK => {} + StatusCode::NOT_FOUND | StatusCode::GONE => { + return ( + Err(ApiError::Other(format!("artifact not found: {url}"))), + None, + ) + } + StatusCode::REQUEST_TIMEOUT => { + return ( + Err(ApiError::Other(format!("artifact still building: {url}"))), + None, + ) + } + _ => { + let hint = + vendor_status_retryable(status).then(|| retry_after_secs(resp.headers())); + let err = classify_auth_error(status, true).unwrap_or_else(|| { + ApiError::Other(format!( + "artifact download failed with status {}", + status.as_u16() + )) + }); + return (Err(err), hint); + } + } + let body = tokio::time::timeout( + self.vendor_retry.body_timeout, + read_capped_typed(resp, max_bytes, "artifact"), + ) + .await + .unwrap_or_else(|_| { + Err(ReadCappedError::Truncated(format!( + "artifact body not received within {:?}", + self.vendor_retry.body_timeout + ))) + }); + match body { + Ok(bytes) => (Ok(bytes), None), + Err(ReadCappedError::Truncated(e)) => (Err(ApiError::Network(e)), Some(None)), + Err(ReadCappedError::CapExceeded(_)) => ( + Err(ApiError::Other(format!( + "artifact exceeds the {max_bytes}-byte limit" + ))), + None, + ), + } + } +} + +/// Never prints the bearer token: an `ApiClient` can end up in a panic +/// message or an error's `{:?}`. +impl std::fmt::Debug for ApiClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ApiClient") + .field("api_url", &self.api_url) + .field("api_token", &self.api_token.as_ref().map(|_| "")) + .field("use_public_proxy", &self.use_public_proxy) + .field("org_slug", &self.org_slug) + .field("vendor_retry", &self.vendor_retry) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod patch_api_seam_tests { + use super::*; + + #[test] + fn debug_redacts_the_token() { + let client = ApiClient::new(ApiClientOptions { + api_url: "https://api.example".into(), + api_token: Some("sktsec_secret_value_api".into()), + use_public_proxy: false, + org_slug: Some("org".into()), + }); + let rendered = format!("{client:?}"); + assert!(!rendered.contains("sktsec_secret_value_api"), "{rendered}"); + assert!(rendered.contains(""), "{rendered}"); + } + + #[test] + fn trait_reports_the_proxy_flag() { + let client = ApiClient::new(ApiClientOptions { + api_url: "https://api.example".into(), + api_token: None, + use_public_proxy: true, + org_slug: None, + }); + let api: &dyn PatchApi = &client; + assert!(api.uses_public_proxy()); + } + + #[tokio::test] + async fn trait_forwards_to_the_inherent_batch_search() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v0/orgs/org/patches/batch")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "packages": [ + {"purl": "pkg:npm/b@1", "patches": []}, + {"purl": "pkg:npm/a@1", "patches": []} + ], + "canAccessPaidPatches": true + }))) + .mount(&server) + .await; + let client = ApiClient::new(ApiClientOptions { + api_url: server.uri(), + api_token: Some("t".into()), + use_public_proxy: false, + org_slug: Some("org".into()), + }); + let api: &dyn PatchApi = &client; + let purls = vec!["pkg:npm/a@1".to_string(), "pkg:npm/b@1".to_string()]; + let response = api.search_patches_batch(&purls).await.unwrap(); + assert!(response.can_access_paid_patches); + let order: Vec<&str> = response.packages.iter().map(|p| p.purl.as_str()).collect(); + assert_eq!(order, vec!["pkg:npm/a@1", "pkg:npm/b@1"]); + } + + #[tokio::test] + async fn trait_download_enforces_the_callers_byte_cap() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/a.whl")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(vec![7u8; 64])) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/gone.whl")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + let client = ApiClient::new(ApiClientOptions { + api_url: server.uri(), + api_token: Some("t".into()), + use_public_proxy: false, + org_slug: Some("org".into()), + }); + let api: &dyn PatchApi = &client; + let url = format!("{}/a.whl", server.uri()); + assert_eq!( + api.download_artifact(&url, 64).await.unwrap(), + vec![7u8; 64] + ); + let err = api.download_artifact(&url, 63).await.unwrap_err(); + assert!(err.to_string().contains("63-byte limit"), "{err}"); + let gone = format!("{}/gone.whl", server.uri()); + let err = api.download_artifact(&gone, 64).await.unwrap_err(); + assert!(err.to_string().contains("not found"), "{err}"); + assert!(api + .download_artifact("file:///etc/passwd", 64) + .await + .is_err()); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/socket-patch-core/src/utils/cargo_workspace.rs b/crates/socket-patch-core/src/utils/cargo_workspace.rs index da7cb153..3721f0de 100644 --- a/crates/socket-patch-core/src/utils/cargo_workspace.rs +++ b/crates/socket-patch-core/src/utils/cargo_workspace.rs @@ -12,11 +12,12 @@ //! Cargo.lock dependents check refuses a crate one of them depends on. use std::collections::BTreeSet; -use std::path::{Component, Path, PathBuf}; +use std::path::{Component, Path}; use std::sync::Arc; use toml_edit::{DocumentMut, Item, Table}; +use crate::vendor::lock_inventory::view::{MemoryProject, ProjectView}; use crate::vendor::parse_memo::ParseMemo; /// Upper bound on discovered manifests — a runaway glob (or a hostile tree) @@ -27,7 +28,88 @@ const MAX_MANIFESTS: usize = 4096; /// in-root path dependencies of the project at `root`, sorted. Empty when /// `root/Cargo.toml` is absent or unparseable. pub fn member_manifests(root: &Path) -> Vec { - let Some(facts) = read_manifest(&root.join("Cargo.toml")) else { + member_manifests_with(&DiskTree(root)) +} + +/// [`member_manifests`] over a [`ProjectView`]: the in-memory variant +/// reads the manifests the host supplied and expands member globs against +/// the supplied paths (a directory exists when some supplied path lives +/// under it; symbolic links are never directories). +pub fn member_manifests_in(view: &ProjectView<'_>) -> Vec { + match view { + ProjectView::Disk(root) => member_manifests(root), + ProjectView::Memory(project) => member_manifests_with(&MemoryTree(project)), + } +} + +/// The three filesystem questions the member walk asks, keyed by +/// `/`-separated root-relative paths (`""` is the root). +trait Tree { + /// A regular (non-symlink) manifest file's walk facts. + fn read_manifest(&self, rel: &str) -> Option>; + /// A real directory (not a symbolic link). + fn is_real_dir(&self, rel: &str) -> bool; + /// The real sub-directory names of `rel`, or `None` when unreadable. + fn child_dirs(&self, rel: &str) -> Option>; +} + +struct DiskTree<'a>(&'a Path); + +impl Tree for DiskTree<'_> { + fn read_manifest(&self, rel: &str) -> Option> { + read_manifest(&self.0.join(rel)) + } + + fn is_real_dir(&self, rel: &str) -> bool { + is_real_dir(&self.0.join(rel)) + } + + fn child_dirs(&self, rel: &str) -> Option> { + let entries = std::fs::read_dir(self.0.join(rel)).ok()?; + Some( + entries + .filter_map(Result::ok) + .filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) + .filter_map(|e| e.file_name().to_str().map(str::to_string)) + .collect(), + ) + } +} + +struct MemoryTree<'a>(&'a MemoryProject); + +impl Tree for MemoryTree<'_> { + fn read_manifest(&self, rel: &str) -> Option> { + let doc: DocumentMut = self.0.text(rel)?.parse().ok()?; + Some(Arc::new(ManifestFacts::of(&doc))) + } + + fn is_real_dir(&self, rel: &str) -> bool { + self.0.is_dir(rel) + } + + fn child_dirs(&self, rel: &str) -> Option> { + self.0.is_dir(rel).then(|| { + self.0 + .children(rel) + .into_iter() + .filter(|(_, is_dir)| *is_dir) + .map(|(name, _)| name) + .collect() + }) + } +} + +fn join_rel(base: &str, seg: &str) -> String { + if base.is_empty() { + seg.to_string() + } else { + format!("{base}/{seg}") + } +} + +fn member_manifests_with(tree: &dyn Tree) -> Vec { + let Some(facts) = tree.read_manifest("Cargo.toml") else { return Vec::new(); }; let mut dirs: BTreeSet = BTreeSet::new(); @@ -35,24 +117,24 @@ pub fn member_manifests(root: &Path) -> Vec { if let Some((members, exclude)) = &facts.workspace { let excluded: BTreeSet = - exclude.iter().flat_map(|p| expand_glob(root, p)).collect(); + exclude.iter().flat_map(|p| expand_glob(tree, p)).collect(); for pattern in members { - for dir in expand_glob(root, pattern) { + for dir in expand_glob(tree, pattern) { if !excluded.contains(&dir) { - enqueue(root, dir, &mut dirs, &mut queue); + enqueue(tree, dir, &mut dirs, &mut queue); } } } } for dep_dir in &facts.path_deps { if let Some(dir) = normalize_rel("", dep_dir) { - enqueue(root, dir, &mut dirs, &mut queue); + enqueue(tree, dir, &mut dirs, &mut queue); } } while let Some((dir, facts)) = queue.pop() { for dep_dir in &facts.path_deps { if let Some(dep) = normalize_rel(&dir, dep_dir) { - enqueue(root, dep, &mut dirs, &mut queue); + enqueue(tree, dep, &mut dirs, &mut queue); } } } @@ -124,76 +206,28 @@ fn read_manifest(path: &Path) -> Option> { /// call), kept as the equivalence oracle. #[cfg(test)] fn member_manifests_unmemoized(root: &Path) -> Vec { - fn read_manifest(path: &Path) -> Option { - if !std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file()) { - return None; - } - std::fs::read_to_string(path).ok()?.parse().ok() - } - fn enqueue( - root: &Path, - dir: String, - dirs: &mut BTreeSet, - queue: &mut Vec<(String, DocumentMut)>, - ) { - if dir.is_empty() - || dirs.len() >= MAX_MANIFESTS - || dirs.contains(&dir) - || dir.split('/').any(|seg| seg == "target") - || !is_real_dir_path(root, &dir) - { - return; - } - let Some(doc) = read_manifest(&root.join(&dir).join("Cargo.toml")) else { - return; - }; - dirs.insert(dir.clone()); - queue.push((dir, doc)); - } - let Some(doc) = read_manifest(&root.join("Cargo.toml")) else { - return Vec::new(); - }; - let mut dirs: BTreeSet = BTreeSet::new(); - let mut queue: Vec<(String, DocumentMut)> = Vec::new(); - - if let Some(ws) = doc.get("workspace").and_then(Item::as_table_like) { - let patterns = |key: &str| -> Vec { - ws.get(key) - .and_then(Item::as_array) - .map(|a| { - a.iter() - .filter_map(|v| v.as_str().map(str::to_string)) - .collect() - }) - .unwrap_or_default() - }; - let excluded: BTreeSet = patterns("exclude") - .iter() - .flat_map(|p| expand_glob(root, p)) - .collect(); - for pattern in patterns("members") { - for dir in expand_glob(root, &pattern) { - if !excluded.contains(&dir) { - enqueue(root, dir, &mut dirs, &mut queue); - } + struct UnmemoizedDiskTree<'a>(&'a Path); + + impl Tree for UnmemoizedDiskTree<'_> { + fn read_manifest(&self, rel: &str) -> Option> { + let path = self.0.join(rel); + if !std::fs::symlink_metadata(&path).is_ok_and(|m| m.is_file()) { + return None; } + let doc: DocumentMut = std::fs::read_to_string(path).ok()?.parse().ok()?; + Some(Arc::new(ManifestFacts::of(&doc))) } - } - for dep_dir in path_dependencies(&doc) { - if let Some(dir) = normalize_rel("", &dep_dir) { - enqueue(root, dir, &mut dirs, &mut queue); + + fn is_real_dir(&self, rel: &str) -> bool { + DiskTree(self.0).is_real_dir(rel) } - } - while let Some((dir, doc)) = queue.pop() { - for dep_dir in path_dependencies(&doc) { - if let Some(dep) = normalize_rel(&dir, &dep_dir) { - enqueue(root, dep, &mut dirs, &mut queue); - } + + fn child_dirs(&self, rel: &str) -> Option> { + DiskTree(self.0).child_dirs(rel) } } - dirs.into_iter() - .map(|dir| format!("{dir}/Cargo.toml")) - .collect() + + member_manifests_with(&UnmemoizedDiskTree(root)) } /// A directory that is not itself a symbolic link. @@ -202,17 +236,17 @@ fn is_real_dir(path: &Path) -> bool { } /// Every component of repo-relative `dir` is a real directory under -/// `root` — none is a symbolic link (which may lead outside the root). -fn is_real_dir_path(root: &Path, dir: &str) -> bool { - let mut at = root.to_path_buf(); +/// the root — none is a symbolic link (which may lead outside the root). +fn is_real_dir_path(tree: &dyn Tree, dir: &str) -> bool { + let mut at = String::new(); dir.split('/').all(|seg| { - at.push(seg); - is_real_dir(&at) + at = join_rel(&at, seg); + tree.is_real_dir(&at) }) } fn enqueue( - root: &Path, + tree: &dyn Tree, dir: String, dirs: &mut BTreeSet, queue: &mut Vec<(String, Arc)>, @@ -226,11 +260,11 @@ fn enqueue( || dirs.len() >= MAX_MANIFESTS || dirs.contains(&dir) || dir.split('/').any(|seg| seg == "target") - || !is_real_dir_path(root, &dir) + || !is_real_dir_path(tree, &dir) { return; } - let Some(facts) = read_manifest(&root.join(&dir).join("Cargo.toml")) else { + let Some(facts) = tree.read_manifest(&format!("{dir}/Cargo.toml")) else { return; }; dirs.insert(dir.clone()); @@ -310,53 +344,51 @@ pub(crate) fn normalize_rel(base: &str, rel: &str) -> Option { /// Expand a cargo `members` / `exclude` glob (`*`, `?`, `**`) to the /// repo-relative directories it names. -fn expand_glob(root: &Path, pattern: &str) -> Vec { +fn expand_glob(tree: &dyn Tree, pattern: &str) -> Vec { let Some(normalized) = normalize_rel("", pattern.trim_end_matches('/')) else { return Vec::new(); }; let segments: Vec<&str> = normalized.split('/').filter(|s| !s.is_empty()).collect(); let mut out = Vec::new(); - expand_from(root, PathBuf::new(), &segments, &mut out); + expand_from(tree, String::new(), &segments, &mut out); out.sort(); out.dedup(); out } -fn expand_from(root: &Path, at: PathBuf, rest: &[&str], out: &mut Vec) { +fn expand_from(tree: &dyn Tree, at: String, rest: &[&str], out: &mut Vec) { if out.len() >= MAX_MANIFESTS { return; } let Some((seg, tail)) = rest.split_first() else { - out.push(at.to_string_lossy().replace('\\', "/")); + out.push(at); return; }; if !seg.contains(['*', '?']) { - let next = at.join(seg); - if is_real_dir(&root.join(&next)) { - expand_from(root, next, tail, out); + let next = join_rel(&at, seg); + if tree.is_real_dir(&next) { + expand_from(tree, next, tail, out); } return; } - let Ok(entries) = std::fs::read_dir(root.join(&at)) else { + let Some(entries) = tree.child_dirs(&at) else { return; }; let mut children: Vec = entries - .filter_map(Result::ok) - .filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) - .filter_map(|e| e.file_name().to_str().map(str::to_string)) + .into_iter() .filter(|name| !name.starts_with('.') && name != "target") .collect(); children.sort(); if *seg == "**" { - expand_from(root, at.clone(), tail, out); + expand_from(tree, at.clone(), tail, out); for child in children { - expand_from(root, at.join(child), rest, out); + expand_from(tree, join_rel(&at, &child), rest, out); } return; } for child in children { if wildcard_match(seg.as_bytes(), child.as_bytes()) { - expand_from(root, at.join(child), tail, out); + expand_from(tree, join_rel(&at, &child), tail, out); } } } diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/bun.rs b/crates/socket-patch-core/src/vendor/lock_inventory/bun.rs index 85153459..bbb20df1 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/bun.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/bun.rs @@ -3,10 +3,10 @@ use std::path::Path; use crate::constants::npm_family::{BUN_LOCK, BUN_LOCKB}; -use crate::utils::fs::{read_regular_to_bytes, read_regular_to_string}; use crate::vendor::bun_lock_text::{self, BunEntry}; use crate::vendor::bun_lockb::BunLockb; +use super::view::ProjectView; use super::{http_url, LockIntegrity, LockfileEntry, UnsupportedNpmLayout}; /// Every `packages` entry of a text `bun.lock`, read with the ONE @@ -30,21 +30,25 @@ pub(crate) fn bun_text_entries(text: &str) -> Result, String> { /// `bun.lockb` beside it is not the live lock. Lockfile discovery answers /// the same question with `DiscoverCtx::exists` (the same lstat). pub(crate) async fn bun_text_lock_present(root: &Path) -> bool { - tokio::fs::symlink_metadata(root.join(BUN_LOCK)) - .await - .is_ok() + bun_text_lock_present_in(&ProjectView::Disk(root)).await +} + +/// [`bun_text_lock_present`] over a [`ProjectView`]. +pub(crate) async fn bun_text_lock_present_in(view: &ProjectView<'_>) -> bool { + view.exists_no_follow(BUN_LOCK).await } // ── registry view ── -pub(super) async fn inventory_bun_binary( - root: &Path, +pub(super) async fn inventory_bun_binary_in( + view: &ProjectView<'_>, ) -> Result, UnsupportedNpmLayout> { let invalid = |detail: String| UnsupportedNpmLayout { code: "bun_lockb_invalid", detail: format!("cannot inventory bun.lockb: {detail}"), }; - let bytes = read_regular_to_bytes(&root.join(BUN_LOCKB)) + let bytes = view + .read_bytes(BUN_LOCKB) .await .map_err(|error| invalid(error.to_string()))?; let packages = BunLockb::parse_packages(&bytes).map_err(invalid)?; @@ -71,8 +75,13 @@ pub(super) async fn inventory_bun_binary( .collect()) } +#[cfg(test)] pub(super) async fn inventory_bun(root: &Path) -> Option> { - let text = read_regular_to_string(&root.join(BUN_LOCK)).await.ok()?; + inventory_bun_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_bun_in(view: &ProjectView<'_>) -> Option> { + let text = view.read_text(BUN_LOCK).await.ok()?; let entries = bun_text_entries(&text).ok()?; let mut out = Vec::new(); diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs b/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs index 8cc89d11..f334b46e 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs @@ -1,10 +1,12 @@ //! `Cargo.lock`: the registry view. +#[cfg(test)] use std::path::Path; use crate::utils::digest::is_hex; use crate::utils::purl::simple_purl; +use super::view::ProjectView; use super::{dedup_prefer_integrity, LockIntegrity, LockfileEntry, SourceKind}; // ── registry view ── @@ -22,18 +24,34 @@ use super::{dedup_prefer_integrity, LockIntegrity, LockfileEntry, SourceKind}; /// verifier (its checksum pins a tagged version no registry serves under /// the purl's version). A lock that is not TOML yields nothing — cargo /// itself refuses to build from it. +#[cfg(test)] pub(super) async fn inventory_cargo_lock(project_root: &Path) -> Option> { - inventory_cargo_lock_raw(project_root) + inventory_cargo_lock_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_cargo_lock`] over a [`ProjectView`]. +pub(super) async fn inventory_cargo_lock_in(view: &ProjectView<'_>) -> Option> { + inventory_cargo_lock_raw_in(view) .await .map(dedup_prefer_integrity) } /// [`inventory_cargo_lock`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_cargo_lock_raw(project_root: &Path) -> Option> { - let (_, doc, _) = crate::vendor::cargo_lock::read_lock(project_root) - .await - .ok()?; +pub(super) async fn inventory_cargo_lock_raw_in( + view: &ProjectView<'_>, +) -> Option> { + let doc: std::sync::Arc = match view { + ProjectView::Disk(project_root) => { + crate::vendor::cargo_lock::read_lock(project_root) + .await + .ok()? + .1 + } + ProjectView::Memory(_) => { + std::sync::Arc::new(view.read_text("Cargo.lock").await.ok()?.parse().ok()?) + } + }; let mut out = Vec::new(); for pkg in crate::vendor::cargo_lock::locked_packages(&doc) { let Some(source) = pkg.source else { diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/composer.rs b/crates/socket-patch-core/src/vendor/lock_inventory/composer.rs index 5223de1e..25227762 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/composer.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/composer.rs @@ -1,6 +1,7 @@ //! `composer.lock`: the shared entry walk ([`composer_lock_packages`]) and //! its registry view. +#[cfg(test)] use std::path::Path; use serde_json::Value; @@ -8,9 +9,9 @@ use serde_json::Value; use crate::crawlers::composer_crawler::normalize_version; use crate::patch::path_safety; use crate::utils::digest::sha1_hex; -use crate::utils::fs::read_regular_to_bytes; use crate::vendor::path::{parse_vendor_path, VendorPathParts}; +use super::view::ProjectView; use super::{dedup_prefer_integrity, http_url, LockIntegrity, LockfileEntry, SourceKind}; // ── entry model ── @@ -87,18 +88,26 @@ pub(crate) fn composer_lock_packages(doc: &Value) -> Vec /// discovery-only. Names lowercase to the canonical packagist form; /// versions drop the pretty leading `v`/`V` through the crawler's /// [`normalize_version`], so installed and lockfile rows agree. +#[cfg(test)] pub(super) async fn inventory_composer_lock(project_root: &Path) -> Option> { - inventory_composer_lock_raw(project_root) + inventory_composer_lock_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_composer_lock`] over a [`ProjectView`]. +pub(super) async fn inventory_composer_lock_in( + view: &ProjectView<'_>, +) -> Option> { + inventory_composer_lock_raw_in(view) .await .map(dedup_prefer_integrity) } /// [`inventory_composer_lock`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_composer_lock_raw(project_root: &Path) -> Option> { - let bytes = read_regular_to_bytes(&project_root.join("composer.lock")) - .await - .ok()?; +pub(super) async fn inventory_composer_lock_raw_in( + view: &ProjectView<'_>, +) -> Option> { + let bytes = view.read_bytes("composer.lock").await.ok()?; let doc: Value = serde_json::from_slice(&bytes).ok()?; let mut out = Vec::new(); for pkg in composer_lock_packages(&doc) { diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/gem.rs b/crates/socket-patch-core/src/vendor/lock_inventory/gem.rs index 885a0c69..ee6fb95c 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/gem.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/gem.rs @@ -7,6 +7,7 @@ use crate::utils::fs::read_regular_to_string; use crate::utils::purl::simple_purl; use crate::vendor::gemfile_lock::{self, Section}; +use super::view::ProjectView; use super::{dedup_prefer_integrity, http_url, LockIntegrity, LockfileEntry, SourceKind}; // ── registry view ── @@ -26,18 +27,26 @@ use super::{dedup_prefer_integrity, http_url, LockIntegrity, LockfileEntry, Sour /// distinct `remote:` lines is a legacy bundler 1.x multisource lock whose /// per-spec origin is genuinely ambiguous: its specs stay discovery-only /// (no resolved URL — the fetch layer then refuses), fail-closed. +#[cfg(test)] pub(super) async fn inventory_gemfile_lock(project_root: &Path) -> Option> { - inventory_gemfile_lock_raw(project_root) + inventory_gemfile_lock_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_gemfile_lock`] over a [`ProjectView`]. +pub(super) async fn inventory_gemfile_lock_in( + view: &ProjectView<'_>, +) -> Option> { + inventory_gemfile_lock_raw_in(view) .await .map(dedup_prefer_integrity) } /// [`inventory_gemfile_lock`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_gemfile_lock_raw(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("Gemfile.lock")) - .await - .ok()?; +pub(super) async fn inventory_gemfile_lock_raw_in( + view: &ProjectView<'_>, +) -> Option> { + let text = view.read_text("Gemfile.lock").await.ok()?; // The shared lock model (lockfile discovery reads it too); what bundler // would refuse (`problems`) still inventories whatever parsed — this is // read-only discovery. diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/golang.rs b/crates/socket-patch-core/src/vendor/lock_inventory/golang.rs index 3ef93542..25636f85 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/golang.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/golang.rs @@ -1,11 +1,12 @@ //! `go.sum`: the registry view. +#[cfg(test)] use std::path::Path; -use crate::utils::fs::read_regular_to_string; use crate::utils::purl::golang_purl; use crate::vendor::go_sum_edit::go_sum_lines; +use super::view::ProjectView; use super::{dedup_prefer_integrity, LockIntegrity, LockfileEntry, SourceKind}; // ── registry view ── @@ -14,18 +15,22 @@ use super::{dedup_prefer_integrity, LockIntegrity, LockfileEntry, SourceKind}; /// `/go.mod`-suffixed lines hash only the manifest and are skipped. go.sum /// may list more modules than the final build graph — acceptable for /// discovery, and the manifest decides what actually gets vendored. +#[cfg(test)] pub(super) async fn inventory_go_sum(project_root: &Path) -> Option> { - inventory_go_sum_raw(project_root) + inventory_go_sum_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_go_sum`] over a [`ProjectView`]. +pub(super) async fn inventory_go_sum_in(view: &ProjectView<'_>) -> Option> { + inventory_go_sum_raw_in(view) .await .map(dedup_prefer_integrity) } /// [`inventory_go_sum`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_go_sum_raw(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("go.sum")) - .await - .ok()?; +pub(super) async fn inventory_go_sum_raw_in(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("go.sum").await.ok()?; let mut out = Vec::new(); for line in go_sum_lines(&text) { if line.go_mod || !line.hash.starts_with("h1:") { diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs b/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs index 774ad377..11279bf8 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs @@ -65,23 +65,31 @@ pub(crate) mod npm_family; pub(crate) mod pnpm; pub(crate) mod pypi; pub(crate) mod recover; +pub mod view; pub(crate) mod wired; pub(crate) mod yarn; pub(crate) use self::composer::{composer_lock_packages, ComposerLockPackage}; pub(crate) use self::npm::{npm_lock_nodes, NpmLockNode}; +#[cfg(test)] pub(crate) use self::npm_family::inventory_npm_lock; pub(crate) use self::pnpm::pnpm_registry_key; pub(crate) use self::pypi::pipfile_lock_entries; pub use self::recover::recover_lock_entry; +pub use self::view::{MemoryEntry, MemoryProject, ProjectView}; pub use self::wired::wired_vendor_integrity; // The per-format views `inventory_project_diagnosed` unions (and the test // modules reach through `super::*`). +#[cfg(test)] use self::cargo::inventory_cargo_lock; +#[cfg(test)] use self::composer::inventory_composer_lock; +#[cfg(test)] use self::gem::inventory_gemfile_lock; +#[cfg(test)] use self::golang::inventory_go_sum; +#[cfg(test)] use self::pypi::inventory_pypi_locks; #[cfg(test)] use self::{ @@ -260,18 +268,33 @@ enum Instances { Every, } +/// [`inventory_project_diagnosed`] over a [`ProjectView`]: the same views, +/// precedence and guards, reading from disk or from an in-memory project. +pub async fn inventory_project_diagnosed_in( + view: &ProjectView<'_>, +) -> (Vec, Vec) { + union_views_in(view, Instances::Collapsed).await +} + /// The union of the per-format views, in the one precedence order. async fn union_views( project_root: &Path, instances: Instances, +) -> (Vec, Vec) { + union_views_in(&ProjectView::Disk(project_root), instances).await +} + +async fn union_views_in( + view: &ProjectView<'_>, + instances: Instances, ) -> (Vec, Vec) { let every = instances == Instances::Every; let mut out: Vec = Vec::new(); let mut unsupported: Vec = Vec::new(); let npm = if every { - npm_family::inventory_npm_lock_raw(project_root).await + npm_family::inventory_npm_lock_raw_in(view).await } else { - inventory_npm_lock(project_root).await + npm_family::inventory_npm_lock_in(view).await }; match npm { Ok(Some((_, entries))) => out.extend(entries), @@ -280,29 +303,29 @@ async fn union_views( } let views = [ if every { - cargo::inventory_cargo_lock_raw(project_root).await + cargo::inventory_cargo_lock_raw_in(view).await } else { - inventory_cargo_lock(project_root).await + cargo::inventory_cargo_lock_in(view).await }, if every { - golang::inventory_go_sum_raw(project_root).await + golang::inventory_go_sum_raw_in(view).await } else { - inventory_go_sum(project_root).await + golang::inventory_go_sum_in(view).await }, if every { - composer::inventory_composer_lock_raw(project_root).await + composer::inventory_composer_lock_raw_in(view).await } else { - inventory_composer_lock(project_root).await + composer::inventory_composer_lock_in(view).await }, if every { - gem::inventory_gemfile_lock_raw(project_root).await + gem::inventory_gemfile_lock_raw_in(view).await } else { - inventory_gemfile_lock(project_root).await + gem::inventory_gemfile_lock_in(view).await }, if every { - pypi::inventory_pypi_locks_raw(project_root).await + pypi::inventory_pypi_locks_raw_in(view).await } else { - inventory_pypi_locks(project_root).await + pypi::inventory_pypi_locks_in(view).await }, ]; out.extend(views.into_iter().flatten().flatten()); diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/npm.rs b/crates/socket-patch-core/src/vendor/lock_inventory/npm.rs index 2d892c9c..0353ec3c 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/npm.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/npm.rs @@ -1,15 +1,16 @@ //! `package-lock.json` / `npm-shrinkwrap.json`: the shared entry walk //! ([`npm_lock_nodes`]) and its registry view. +#[cfg(test)] use std::path::Path; use serde_json::Value; use crate::constants::npm_family::NPM_LOCKS; use crate::utils::digest::is_sri_pin; -use crate::utils::fs::read_regular_to_bytes; use crate::vendor::path::parse_vendor_path; +use super::view::ProjectView; use super::{http_url, LockIntegrity, LockfileEntry}; // ── entry model ── @@ -105,11 +106,18 @@ fn walk_npm_legacy_dependencies<'a>( // ── registry view ── +#[cfg(test)] pub(super) async fn inventory_package_lock(root: &Path) -> Option> { + inventory_package_lock_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_package_lock_in( + view: &ProjectView<'_>, +) -> Option> { // Shrinkwrap wins, mirroring `npm_lock::select_lockfile`. let mut bytes = None; for lock in NPM_LOCKS { - if let Ok(b) = read_regular_to_bytes(&root.join(lock)).await { + if let Ok(b) = view.read_bytes(lock).await { bytes = Some(b); break; } diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/npm_family.rs b/crates/socket-patch-core/src/vendor/lock_inventory/npm_family.rs index a82a4dda..d111a6ba 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/npm_family.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/npm_family.rs @@ -2,16 +2,20 @@ //! installs from ([`inventory_npm_lock`]), the migration-leftover sibling //! probe, and the shared name/version guard + dedup of npm entries. +#[cfg(test)] use std::path::Path; use crate::constants::npm_family::{BUN_LOCK, BUN_LOCKB, NPM_LOCKS, PNPM_SHRINKWRAP_LEGACY}; use crate::utils::purl::npm_purl; -use crate::vendor::npm_flavor::{detect_npm_lock_flavor, NpmLockFlavor}; +use crate::vendor::npm_flavor::NpmLockFlavor; -use super::bun::{bun_text_lock_present, inventory_bun, inventory_bun_binary}; -use super::npm::inventory_package_lock; -use super::pnpm::{inventory_pnpm_lock, inventory_pnpm_lock_at, inventory_rush_pnpm_locks}; -use super::yarn::{inventory_yarn_berry, inventory_yarn_classic}; +use super::bun::{bun_text_lock_present_in, inventory_bun_binary_in, inventory_bun_in}; +use super::npm::inventory_package_lock_in; +use super::pnpm::{ + inventory_pnpm_lock_in, inventory_pnpm_lock_rel_in, inventory_rush_pnpm_locks_in, +}; +use super::view::{detect_npm_lock_flavor_in, ProjectView}; +use super::yarn::{inventory_yarn_berry_in, inventory_yarn_classic_in}; use super::{dedup_prefer_integrity, LockfileEntry, UnsupportedNpmLayout}; // ── registry view ── @@ -27,24 +31,32 @@ use super::{dedup_prefer_integrity, LockfileEntry, UnsupportedNpmLayout}; /// `pnpm-lock.yaml` directly — unless a live sibling lock the router would /// otherwise have chosen sits beside it (a pnpm→yarn/npm migration /// leftover), in which case the SIBLING is inventoried instead -/// ([`inventory_live_sibling_lock`]) — and `vendor_lockfile_missing` reads +/// ([`inventory_live_sibling_lock_in`]) — and `vendor_lockfile_missing` reads /// the pnpm <=2-era `shrinkwrap.yaml` (same v5 grammar, older filename). /// Any remaining probe failure falls back to Rush's common lock when /// `rush.json` is present. +#[cfg(test)] pub(crate) async fn inventory_npm_lock( project_root: &Path, ) -> Result)>, UnsupportedNpmLayout> { - inventory_npm_lock_raw(project_root) + inventory_npm_lock_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_npm_lock`] over a [`ProjectView`]. +pub(super) async fn inventory_npm_lock_in( + view: &ProjectView<'_>, +) -> Result)>, UnsupportedNpmLayout> { + inventory_npm_lock_raw_in(view) .await .map(|found| found.map(|(flavor, raw)| (flavor, dedup_prefer_integrity(raw)))) } /// [`inventory_npm_lock`] before its collapse: every /// guarded instance ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_npm_lock_raw( - project_root: &Path, +pub(super) async fn inventory_npm_lock_raw_in( + view: &ProjectView<'_>, ) -> Result)>, UnsupportedNpmLayout> { - let (flavor, _warnings) = match detect_npm_lock_flavor(project_root).await { + let (flavor, _warnings) = match detect_npm_lock_flavor_in(view).await { Ok(found) => found, Err((code, detail)) => { // The PnP loaders are a refusal, not an absence: propagate the @@ -81,7 +93,7 @@ pub(super) async fn inventory_npm_lock_raw( // have chosen had the pnpm lock not shadowed it; only a // sibling-less project is a genuine old-pnpm project whose // lock the fallback may surface. - match inventory_live_sibling_lock(project_root).await { + match inventory_live_sibling_lock_in(view).await { Some((flavor, entries)) if !entries.is_empty() => { return Ok(Some((flavor, guard_npm(entries)))); } @@ -91,7 +103,7 @@ pub(super) async fn inventory_npm_lock_raw( // blind beats presenting dead resolutions as live. Some(_) => {} None => { - let pnpm = inventory_pnpm_lock(project_root).await.unwrap_or_default(); + let pnpm = inventory_pnpm_lock_in(view).await.unwrap_or_default(); if !pnpm.is_empty() { return Ok(Some((NpmLockFlavor::Pnpm, guard_npm(pnpm)))); } @@ -111,7 +123,7 @@ pub(super) async fn inventory_npm_lock_raw( // long-ago migration whose dead resolutions must not pose as // the live dependency set. if code == "vendor_lockfile_missing" { - let legacy = inventory_pnpm_lock_at(&project_root.join(PNPM_SHRINKWRAP_LEGACY)) + let legacy = inventory_pnpm_lock_rel_in(view, PNPM_SHRINKWRAP_LEGACY) .await .unwrap_or_default(); if !legacy.is_empty() { @@ -123,23 +135,23 @@ pub(super) async fn inventory_npm_lock_raw( // The flavor probe (root-relative) can't see it, so fall back // explicitly when the root lock is absent but rush.json is // present. - let rush = inventory_rush_pnpm_locks(project_root).await; + let rush = inventory_rush_pnpm_locks_in(view).await; return Ok((!rush.is_empty()).then(|| (NpmLockFlavor::Pnpm, guard_npm(rush)))); } }; let raw = match flavor { - NpmLockFlavor::PackageLock => inventory_package_lock(project_root).await, + NpmLockFlavor::PackageLock => inventory_package_lock_in(view).await, // The pnpm reader is grammar-agnostic (it already served legacy // 5.4/6.0 locks through the refusal fallback below before those // grammars had a wiring backend), so both pnpm flavors share it. - NpmLockFlavor::Pnpm | NpmLockFlavor::PnpmLegacy => inventory_pnpm_lock(project_root).await, - NpmLockFlavor::YarnClassic => inventory_yarn_classic(project_root).await, - NpmLockFlavor::YarnBerry => inventory_yarn_berry(project_root).await, + NpmLockFlavor::Pnpm | NpmLockFlavor::PnpmLegacy => inventory_pnpm_lock_in(view).await, + NpmLockFlavor::YarnClassic => inventory_yarn_classic_in(view).await, + NpmLockFlavor::YarnBerry => inventory_yarn_berry_in(view).await, NpmLockFlavor::Bun => { - if bun_text_lock_present(project_root).await { - inventory_bun(project_root).await + if bun_text_lock_present_in(view).await { + inventory_bun_in(view).await } else { - Some(inventory_bun_binary(project_root).await?) + Some(inventory_bun_binary_in(view).await?) } } }; @@ -155,27 +167,23 @@ pub(super) async fn inventory_npm_lock_raw( /// sibling's inventory (possibly empty: presence alone proves the pnpm lock /// is migration debris, so the caller must not fall back to it). Raw /// entries — the caller guards and collapses them. -pub(super) async fn inventory_live_sibling_lock( - root: &Path, +pub(super) async fn inventory_live_sibling_lock_in( + view: &ProjectView<'_>, ) -> Option<(NpmLockFlavor, Vec)> { - let exists = |name: &str| { - let p = root.join(name); - async move { tokio::fs::metadata(&p).await.is_ok() } - }; // bun.lock — router step 2. That step runs BEFORE the pnpm sniff, so // when the version refusal fired no bun.lock can actually be present; // probed anyway to keep this a literal transcription of the router's // order. The binary lock shares the same routing precedence. - if exists(BUN_LOCK).await { + if view.exists(BUN_LOCK).await { return Some(( NpmLockFlavor::Bun, - inventory_bun(root).await.unwrap_or_default(), + inventory_bun_in(view).await.unwrap_or_default(), )); } - if exists(BUN_LOCKB).await { + if view.exists(BUN_LOCKB).await { return Some(( NpmLockFlavor::Bun, - inventory_bun_binary(root).await.unwrap_or_default(), + inventory_bun_binary_in(view).await.unwrap_or_default(), )); } // yarn.lock — router step 4, where classic vs berry is a content @@ -184,22 +192,22 @@ pub(super) async fn inventory_live_sibling_lock( // fields vs berry's `resolution:` lines), so a non-empty result is the // sniff's answer. Berry PnP needs no carve-out: a PnP marker would have // refused at the router's step 1 with a code this fallback ignores. - if exists("yarn.lock").await { - let classic = inventory_yarn_classic(root).await.unwrap_or_default(); + if view.exists("yarn.lock").await { + let classic = inventory_yarn_classic_in(view).await.unwrap_or_default(); if !classic.is_empty() { return Some((NpmLockFlavor::YarnClassic, classic)); } return Some(( NpmLockFlavor::YarnBerry, - inventory_yarn_berry(root).await.unwrap_or_default(), + inventory_yarn_berry_in(view).await.unwrap_or_default(), )); } // npm — router step 5 (`inventory_package_lock` itself prefers the // shrinkwrap when both exist, mirroring npm). - if exists(NPM_LOCKS[0]).await || exists(NPM_LOCKS[1]).await { + if view.exists(NPM_LOCKS[0]).await || view.exists(NPM_LOCKS[1]).await { return Some(( NpmLockFlavor::PackageLock, - inventory_package_lock(root).await.unwrap_or_default(), + inventory_package_lock_in(view).await.unwrap_or_default(), )); } None diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs b/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs index 383adb5d..9d78c288 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs @@ -11,6 +11,7 @@ use crate::patch::redirect::pnpm; use crate::utils::fs::read_regular_to_string; use crate::vendor::path::parse_vendor_path; +use super::view::ProjectView; use super::{http_url, LockIntegrity, LockfileEntry}; // ── entry model ── @@ -190,8 +191,22 @@ pub(crate) async fn rush_lock_rels(root: &Path) -> Vec { // ── registry view ── +#[cfg(test)] pub(super) async fn inventory_pnpm_lock(root: &Path) -> Option> { - inventory_pnpm_lock_at(&root.join(PNPM_LOCK)).await + inventory_pnpm_lock_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_pnpm_lock_in(view: &ProjectView<'_>) -> Option> { + inventory_pnpm_lock_rel_in(view, PNPM_LOCK).await +} + +/// [`inventory_pnpm_lock_at`] for a project-relative lock path. +pub(super) async fn inventory_pnpm_lock_rel_in( + view: &ProjectView<'_>, + rel: &str, +) -> Option> { + let text = view.read_text(rel).await.ok()?; + pnpm_lock_text_inventory(&text) } /// Inventory a specific `pnpm-lock.yaml` (path given explicitly so the Rush @@ -203,6 +218,10 @@ pub(super) async fn inventory_pnpm_lock(root: &Path) -> Option Option> { let text = read_regular_to_string(lock_path).await.ok()?; + pnpm_lock_text_inventory(&text) +} + +fn pnpm_lock_text_inventory(text: &str) -> Option> { if !text .lines() .any(|l| l.trim_end_matches('\r') == "packages:") @@ -210,7 +229,7 @@ pub(super) async fn inventory_pnpm_lock_at(lock_path: &Path) -> Option Option Vec { +pub(super) async fn inventory_rush_pnpm_locks_in(view: &ProjectView<'_>) -> Vec { + let project = match view { + ProjectView::Disk(project_root) => return inventory_rush_pnpm_locks(project_root).await, + ProjectView::Memory(project) => *project, + }; + if !project.contains("rush.json") { + return Vec::new(); + } + let mut out = Vec::new(); + if let Some(entries) = inventory_pnpm_lock_rel_in(view, RUSH_COMMON_LOCK_REL).await { + out.extend(entries); + } + for (name, is_dir) in project.children(RUSH_SUBSPACES_DIR) { + if !is_dir { + continue; + } + let rel = format!("{RUSH_SUBSPACES_DIR}/{name}/{PNPM_LOCK}"); + if let Some(entries) = inventory_pnpm_lock_rel_in(view, &rel).await { + out.extend(entries); + } + } + out +} + +async fn inventory_rush_pnpm_locks(project_root: &Path) -> Vec { if tokio::fs::metadata(project_root.join("rush.json")) .await .is_err() diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs b/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs index 8bbd452a..b30c4484 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs @@ -2,19 +2,20 @@ //! requirements): the registry views, with the Pipfile.lock entry walk //! lockfile discovery shares ([`pipfile_lock_entries`]). +#[cfg(test)] use std::path::Path; use serde_json::Value; use toml_edit::{DocumentMut, Item, TableLike}; use crate::crawlers::python_crawler::canonicalize_pypi_name; -use crate::utils::fs::read_regular_to_string; use crate::utils::purl::{percent_decode_purl_component, pypi_purl}; use crate::utils::python_lock::{lock_package_collection, package_artifacts, UvSource}; use crate::utils::requirements::archive_filename_coords; use crate::utils::digest::{sha256_hex, sha256_prefixed}; +use super::view::ProjectView; use super::{dedup_prefer_integrity, http_url, LockIntegrity, LockfileEntry, SourceKind}; // pypi purls and lock entries compare in PEP 503 normalized form @@ -171,21 +172,45 @@ pub(crate) fn hosted_artifact_url(url: &str) -> Result Option> { - inventory_pypi_locks_raw(project_root) + inventory_pypi_locks_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_pypi_locks`] over a [`ProjectView`]. +pub(super) async fn inventory_pypi_locks_in(view: &ProjectView<'_>) -> Option> { + inventory_pypi_locks_raw_in(view) .await .map(dedup_prefer_integrity) } +/// The project-root Python lock names ([`crate::utils::python_lock::python_lock_paths`] +/// on disk; the in-memory project's root-level names otherwise), sorted. +pub(crate) fn python_lock_paths_in(view: &ProjectView<'_>) -> std::io::Result> { + match view { + ProjectView::Disk(root) => crate::utils::python_lock::python_lock_paths(root), + ProjectView::Memory(project) => Ok(project + .children("") + .into_iter() + .filter(|(name, is_dir)| { + !is_dir && crate::utils::python_lock::is_python_lock_name(name) + }) + .map(|(name, _)| name) + .collect()), + } +} + /// [`inventory_pypi_locks`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_pypi_locks_raw(project_root: &Path) -> Option> { +pub(super) async fn inventory_pypi_locks_raw_in( + view: &ProjectView<'_>, +) -> Option> { let mut out = Vec::new(); let mut found = false; let mut uv_lock = false; - if let Ok(paths) = crate::utils::python_lock::python_lock_paths(project_root) { + if let Ok(paths) = python_lock_paths_in(view) { for path in paths { - let Ok(text) = read_regular_to_string(&project_root.join(&path)).await else { + let Ok(text) = view.read_text(&path).await else { continue; }; if let Some(entries) = python_lock_inventory(&text) { @@ -209,10 +234,10 @@ pub(super) async fn inventory_pypi_locks_raw(project_root: &Path) -> Option Option Option> { /// API; it stays discovery-only otherwise (Poetry 0.12's /// `[metadata.hashes]` lists bare digests without filenames, so no wheel /// can be chosen there). A lock that is not TOML contributes nothing. -async fn inventory_poetry_lock(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("poetry.lock")) - .await - .ok()?; +async fn inventory_poetry_lock(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("poetry.lock").await.ok()?; let document: DocumentMut = text.parse().ok()?; let pure_wheel_sha = |files: Vec<&dyn TableLike>| { files.into_iter().find_map(|entry| { @@ -428,10 +451,8 @@ pub(super) fn socket_reference_coords(reference: &str) -> Option<(String, String /// our own already-wired file references. An unparseable lock contributes /// nothing, so the caller falls through to requirements.txt like an absent /// lock would. -async fn inventory_pipfile_lock(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("Pipfile.lock")) - .await - .ok()?; +async fn inventory_pipfile_lock(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("Pipfile.lock").await.ok()?; let value = parse_pipfile_lock(&text).ok()?; let root = value.as_object()?; // Digests are only fetchable through PyPI's JSON API when the lock @@ -515,10 +536,8 @@ async fn inventory_pipfile_lock(project_root: &Path) -> Option Option> { - let text = read_regular_to_string(&project_root.join("pdm.lock")) - .await - .ok()?; +async fn inventory_pdm_lock(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("pdm.lock").await.ok()?; let document: DocumentMut = text.parse().ok()?; let out: Vec = toml_package_coords(&document) .into_iter() @@ -542,10 +561,8 @@ async fn inventory_pdm_lock(project_root: &Path) -> Option> { /// logical lines with the shared requirements lexer /// ([`crate::utils::requirements`]: continuations joined, comments cut, one /// leading BOM dropped), the same one the planner and discovery use. -async fn inventory_requirements_txt(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("requirements.txt")) - .await - .ok()?; +async fn inventory_requirements_txt(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("requirements.txt").await.ok()?; let mut out = Vec::new(); for line in crate::utils::requirements::logical_lines(&text) { let t = crate::utils::requirements::strip_comment(&line.text).trim(); diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs new file mode 100644 index 00000000..72e4bf31 --- /dev/null +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -0,0 +1,498 @@ +//! Where the registry views read a project from: the filesystem +//! ([`ProjectView::Disk`]) or an in-memory file map +//! ([`ProjectView::Memory`]) handed in by a host that never materializes +//! the repository (the hosted in-memory engine). The disk variant calls the +//! exact readers the views always used, so on-disk behavior is unchanged. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io; +use std::path::Path; +use std::sync::Arc; + +use crate::constants::npm_family::{BUN_LOCK, BUN_LOCKB, NPM_LOCKS, PNPM_LOCK, PNP_MARKERS}; +use crate::utils::fs::{ + read_regular_to_bytes, read_regular_to_string, read_regular_to_string_sync, +}; +use crate::vendor::npm_flavor::NpmLockFlavor; +use crate::vendor::pnpm_lock_legacy::{sniff_lock_grammar, PnpmLockGrammar}; +use crate::vendor::VendorWarning; + +/// One in-memory file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MemoryEntry { + /// UTF-8 content. + Text(Arc), + /// Raw bytes (e.g. `bun.lockb`). + Binary(Arc<[u8]>), + /// Known to exist, content not provided (presence-only markers, files + /// the host skipped as oversize, binary, or LFS pointers). + Present, + /// A symbolic link: exists, never readable, never writable. + Symlink, +} + +/// A project's files, keyed by `/`-separated project-relative path. +/// Directories are implied by the keys. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MemoryProject { + entries: BTreeMap, +} + +impl MemoryProject { + pub fn new() -> Self { + Self::default() + } + + pub fn insert(&mut self, rel: impl Into, entry: MemoryEntry) { + self.entries.insert(rel.into(), entry); + } + + pub fn insert_text(&mut self, rel: impl Into, text: impl Into>) { + self.insert(rel, MemoryEntry::Text(text.into())); + } + + pub fn insert_binary(&mut self, rel: impl Into, bytes: impl Into>) { + self.insert(rel, MemoryEntry::Binary(bytes.into())); + } + + pub fn insert_present(&mut self, rel: impl Into) { + self.insert(rel, MemoryEntry::Present); + } + + pub fn insert_symlink(&mut self, rel: impl Into) { + self.insert(rel, MemoryEntry::Symlink); + } + + pub fn remove(&mut self, rel: &str) -> Option { + self.entries.remove(rel) + } + + pub fn get(&self, rel: &str) -> Option<&MemoryEntry> { + self.entries.get(rel) + } + + pub fn contains(&self, rel: &str) -> bool { + self.entries.contains_key(rel) + } + + pub fn text(&self, rel: &str) -> Option<&str> { + match self.entries.get(rel)? { + MemoryEntry::Text(text) => Some(text), + _ => None, + } + } + + pub fn is_symlink(&self, rel: &str) -> bool { + matches!(self.entries.get(rel), Some(MemoryEntry::Symlink)) + } + + pub fn paths(&self) -> impl Iterator { + self.entries.keys().map(String::as_str) + } + + pub fn entries(&self) -> impl Iterator { + self.entries.iter().map(|(k, v)| (k.as_str(), v)) + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Whether `rel` is an implied directory (some key lives under it). + /// The project root (`""`) is always a directory. + pub fn is_dir(&self, rel: &str) -> bool { + if rel.is_empty() { + return true; + } + let prefix = format!("{rel}/"); + self.entries + .range(prefix.clone()..) + .next() + .is_some_and(|(k, _)| k.starts_with(&prefix)) + } + + /// The direct children of directory `rel`, sorted: `(name, is_dir)`. + pub fn children(&self, rel: &str) -> Vec<(String, bool)> { + let prefix = if rel.is_empty() { + String::new() + } else { + format!("{rel}/") + }; + let mut files: BTreeSet = BTreeSet::new(); + let mut dirs: BTreeSet = BTreeSet::new(); + for key in self + .entries + .range(prefix.clone()..) + .map(|(k, _)| k) + .take_while(|k| k.starts_with(&prefix)) + { + let rest = &key[prefix.len()..]; + match rest.split_once('/') { + Some((dir, _)) => { + dirs.insert(dir.to_string()); + } + None => { + files.insert(rest.to_string()); + } + } + } + let mut out: Vec<(String, bool)> = files + .into_iter() + .filter(|f| !dirs.contains(f)) + .map(|f| (f, false)) + .collect(); + out.extend(dirs.into_iter().map(|d| (d, true))); + out.sort(); + out + } + + fn read_bytes(&self, rel: &str) -> io::Result> { + match self.entries.get(rel) { + Some(MemoryEntry::Text(text)) => Ok(text.as_bytes().to_vec()), + Some(MemoryEntry::Binary(bytes)) => Ok(bytes.to_vec()), + Some(MemoryEntry::Present) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "file content was not provided", + )), + Some(MemoryEntry::Symlink) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "is a symbolic link", + )), + None if self.is_dir(rel) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "not a regular file", + )), + None => Err(io::Error::new(io::ErrorKind::NotFound, "not found")), + } + } + + fn read_text(&self, rel: &str) -> io::Result { + match self.entries.get(rel) { + Some(MemoryEntry::Text(text)) => Ok(text.to_string()), + Some(MemoryEntry::Binary(bytes)) => String::from_utf8(bytes.to_vec()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ) + }), + _ => self.read_bytes(rel).map(|_| String::new()), + } + } +} + +/// A directory entry as the registry views need it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirEntryInfo { + pub name: String, + /// A real directory (a symbolic link to one is not). + pub is_dir: bool, +} + +/// Where the registry views read the project from. +#[derive(Debug, Clone, Copy)] +pub enum ProjectView<'a> { + Disk(&'a Path), + Memory(&'a MemoryProject), +} + +impl ProjectView<'_> { + /// FIFO-safe regular-file text read. + pub async fn read_text(&self, rel: &str) -> io::Result { + match self { + ProjectView::Disk(root) => read_regular_to_string(&root.join(rel)).await, + ProjectView::Memory(project) => project.read_text(rel), + } + } + + /// FIFO-safe regular-file byte read. + pub async fn read_bytes(&self, rel: &str) -> io::Result> { + match self { + ProjectView::Disk(root) => read_regular_to_bytes(&root.join(rel)).await, + ProjectView::Memory(project) => project.read_bytes(rel), + } + } + + /// Synchronous twin of [`Self::read_text`]. + pub fn read_text_sync(&self, rel: &str) -> io::Result { + match self { + ProjectView::Disk(root) => read_regular_to_string_sync(&root.join(rel)), + ProjectView::Memory(project) => project.read_text(rel), + } + } + + /// `metadata` (follows links) succeeds. + pub async fn exists(&self, rel: &str) -> bool { + match self { + ProjectView::Disk(root) => tokio::fs::metadata(root.join(rel)).await.is_ok(), + ProjectView::Memory(project) => project.contains(rel) || project.is_dir(rel), + } + } + + /// `symlink_metadata` (does not follow links) succeeds. + pub async fn exists_no_follow(&self, rel: &str) -> bool { + match self { + ProjectView::Disk(root) => tokio::fs::symlink_metadata(root.join(rel)).await.is_ok(), + ProjectView::Memory(project) => project.contains(rel) || project.is_dir(rel), + } + } + + /// A regular file (following links on disk). + pub fn is_file(&self, rel: &str) -> bool { + match self { + ProjectView::Disk(root) => root.join(rel).is_file(), + ProjectView::Memory(project) => matches!( + project.get(rel), + Some(MemoryEntry::Text(_) | MemoryEntry::Binary(_) | MemoryEntry::Present) + ), + } + } + + /// The path itself is a symbolic link. + pub fn is_symlink(&self, rel: &str) -> bool { + match self { + ProjectView::Disk(root) => { + std::fs::symlink_metadata(root.join(rel)).is_ok_and(|m| m.file_type().is_symlink()) + } + ProjectView::Memory(project) => project.is_symlink(rel), + } + } + + /// The UTF-8-named entries of directory `rel`, sorted by name. + pub async fn list_dir(&self, rel: &str) -> io::Result> { + match self { + ProjectView::Disk(root) => { + let mut dir = tokio::fs::read_dir(root.join(rel)).await?; + let mut out = Vec::new(); + while let Ok(Some(entry)) = dir.next_entry().await { + let Some(name) = entry.file_name().to_str().map(str::to_string) else { + continue; + }; + let is_dir = entry.file_type().await.is_ok_and(|t| t.is_dir()); + out.push(DirEntryInfo { name, is_dir }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) + } + ProjectView::Memory(project) => { + if !project.is_dir(rel) { + return Err(io::Error::new(io::ErrorKind::NotFound, "not found")); + } + Ok(project + .children(rel) + .into_iter() + .map(|(name, is_dir)| DirEntryInfo { name, is_dir }) + .collect()) + } + } + } +} + +/// How many head lines the yarn content sniff reads (mirrors the disk +/// probe). +const YARN_SNIFF_HEAD_LINES: usize = 30; + +/// [`crate::vendor::npm_flavor::detect_npm_lock_flavor`] over a +/// [`ProjectView`]. The disk variant IS the disk probe; the memory variant +/// follows the same decision table, with pnpm's own Plug'n'Play layout +/// never detected (there is no installed store in memory). +pub(crate) async fn detect_npm_lock_flavor_in( + view: &ProjectView<'_>, +) -> Result<(NpmLockFlavor, Vec), (&'static str, String)> { + let project = match view { + ProjectView::Disk(root) => { + return crate::vendor::npm_flavor::detect_npm_lock_flavor(root).await + } + ProjectView::Memory(project) => *project, + }; + let exists = |name: &str| project.contains(name); + let read_lock = |name: &str| -> Result { + project.read_text(name).map_err(|e| { + ( + "vendor_lockfile_missing", + format!("cannot read {name}: {e}"), + ) + }) + }; + + if let Some(marker) = PNP_MARKERS.iter().find(|m| exists(m)) { + return Err(( + "vendor_yarn_berry_unsupported", + format!( + "found `{marker}`: this is a yarn berry Plug'n'Play project — packages \ + live inside .yarn/cache/ zips, not node_modules/, so there is nothing \ + vendor could stage or rewire; use `yarn patch ` instead" + ), + )); + } + + let detected = 'flavor: { + if exists(BUN_LOCK) || exists(BUN_LOCKB) { + break 'flavor NpmLockFlavor::Bun; + } + if exists(PNPM_LOCK) { + let text = read_lock(PNPM_LOCK)?; + match sniff_lock_grammar(&text) { + Ok(PnpmLockGrammar::V9) => break 'flavor NpmLockFlavor::Pnpm, + Ok(PnpmLockGrammar::V54 | PnpmLockGrammar::V60) => { + break 'flavor NpmLockFlavor::PnpmLegacy + } + Err(detail) => return Err(("vendor_lockfile_version_unsupported", detail)), + } + } + if exists("yarn.lock") { + let text = read_lock("yarn.lock")?; + let head: Vec<&str> = text + .strip_prefix('\u{feff}') + .unwrap_or(&text) + .lines() + .take(YARN_SNIFF_HEAD_LINES) + .collect(); + if head.iter().any(|l| l.starts_with("__metadata:")) { + break 'flavor NpmLockFlavor::YarnBerry; + } + if head.iter().any(|l| l.trim() == "# yarn lockfile v1") { + break 'flavor NpmLockFlavor::YarnClassic; + } + return Err(( + "vendor_lockfile_version_unsupported", + "yarn.lock carries neither the `# yarn lockfile v1` header nor a berry \ + `__metadata:` key; cannot identify the lockfile version" + .to_string(), + )); + } + if exists(NPM_LOCKS[0]) || exists(NPM_LOCKS[1]) { + break 'flavor NpmLockFlavor::PackageLock; + } + if exists("rush.json") { + return Err(( + "vendor_rush_unsupported", + format!( + "found rush.json: this is a Rush monorepo — its single pnpm lockfile \ + lives at {}; use `socket-patch scan --mode hosted`, which edits it in \ + place", + crate::constants::npm_family::RUSH_COMMON_LOCK_REL + ), + )); + } + return Err(( + "vendor_lockfile_missing", + "no package-lock.json, npm-shrinkwrap.json, yarn.lock, pnpm-lock.yaml, bun.lock, \ + or bun.lockb in the project root" + .to_string(), + )); + }; + Ok((detected, Vec::new())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn project(files: &[(&str, MemoryEntry)]) -> MemoryProject { + let mut p = MemoryProject::new(); + for (k, v) in files { + p.insert(*k, v.clone()); + } + p + } + + fn text(s: &str) -> MemoryEntry { + MemoryEntry::Text(Arc::from(s)) + } + + #[test] + fn children_lists_files_and_implied_dirs_sorted() { + let p = project(&[ + ("b.txt", text("")), + ("a/x.toml", text("")), + ("a/y/z.toml", text("")), + ("c", MemoryEntry::Symlink), + ]); + assert_eq!( + p.children(""), + vec![ + ("a".to_string(), true), + ("b.txt".to_string(), false), + ("c".to_string(), false) + ] + ); + assert_eq!( + p.children("a"), + vec![("x.toml".to_string(), false), ("y".to_string(), true)] + ); + assert!(p.is_dir("a/y")); + assert!(!p.is_dir("a/x.toml")); + assert!(!p.is_dir("ab")); + } + + #[tokio::test] + async fn memory_reads_classify_like_the_disk_reader() { + let p = project(&[ + ("t", text("hello")), + ("b", MemoryEntry::Binary(Arc::from(vec![0xffu8, 0xfe]))), + ("p", MemoryEntry::Present), + ("s", MemoryEntry::Symlink), + ("d/f", text("")), + ]); + let view = ProjectView::Memory(&p); + assert_eq!(view.read_text("t").await.unwrap(), "hello"); + assert_eq!( + view.read_text("b").await.unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert_eq!(view.read_bytes("b").await.unwrap(), vec![0xff, 0xfe]); + assert!(view.read_text("p").await.is_err()); + assert!(view.read_text("s").await.is_err()); + assert_eq!( + view.read_text("missing").await.unwrap_err().kind(), + io::ErrorKind::NotFound + ); + assert_eq!( + view.read_text("d").await.unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert!(view.exists("s").await); + assert!(view.is_symlink("s")); + assert!(!view.is_file("s")); + assert!(view.is_file("p")); + } + + #[tokio::test] + async fn memory_flavor_probe_follows_the_disk_decision_table() { + let berry = project(&[("yarn.lock", text("__metadata:\n version: 8\n"))]); + assert_eq!( + detect_npm_lock_flavor_in(&ProjectView::Memory(&berry)) + .await + .unwrap() + .0, + NpmLockFlavor::YarnBerry + ); + let bun_over_npm = project(&[ + ("package-lock.json", text("{}")), + ("bun.lockb", MemoryEntry::Present), + ]); + assert_eq!( + detect_npm_lock_flavor_in(&ProjectView::Memory(&bun_over_npm)) + .await + .unwrap() + .0, + NpmLockFlavor::Bun + ); + let pnp = project(&[(".pnp.cjs", MemoryEntry::Present), ("yarn.lock", text(""))]); + assert_eq!( + detect_npm_lock_flavor_in(&ProjectView::Memory(&pnp)) + .await + .unwrap_err() + .0, + "vendor_yarn_berry_unsupported" + ); + let empty = MemoryProject::new(); + assert_eq!( + detect_npm_lock_flavor_in(&ProjectView::Memory(&empty)) + .await + .unwrap_err() + .0, + "vendor_lockfile_missing" + ); + } +} diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/yarn.rs b/crates/socket-patch-core/src/vendor/lock_inventory/yarn.rs index d7c2413c..91b48921 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/yarn.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/yarn.rs @@ -1,10 +1,10 @@ //! `yarn.lock`, classic and berry: the entry models lockfile discovery //! shares ([`classic_entries`], [`berry_entries`]) and the registry views. +#[cfg(test)] use std::path::Path; use crate::utils::digest::is_hex; -use crate::utils::fs::read_regular_to_string; use crate::vendor::yarn_berry_lock::{ berry_field, berry_metadata, parse_berry_locator, BerryLocator, }; @@ -13,6 +13,7 @@ use crate::vendor::yarn_classic_lock::{ split_resolved_sha1, LockBlock, }; +use super::view::ProjectView; use super::{http_url, LockIntegrity, LockfileEntry}; // ── entry model ── @@ -105,8 +106,15 @@ pub(crate) fn berry_checksum_pin(value: &str, cache_key: Option<&str>) -> Option // ── registry view ── +#[cfg(test)] pub(super) async fn inventory_yarn_classic(root: &Path) -> Option> { - let text = read_regular_to_string(&root.join("yarn.lock")).await.ok()?; + inventory_yarn_classic_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_yarn_classic_in( + view: &ProjectView<'_>, +) -> Option> { + let text = view.read_text("yarn.lock").await.ok()?; Some(classic_registry_view(&text)) } @@ -147,8 +155,13 @@ fn classic_registry_view(text: &str) -> Vec { out } +#[cfg(test)] pub(super) async fn inventory_yarn_berry(root: &Path) -> Option> { - let text = read_regular_to_string(&root.join("yarn.lock")).await.ok()?; + inventory_yarn_berry_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_yarn_berry_in(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("yarn.lock").await.ok()?; Some(berry_registry_view(&text)) } diff --git a/crates/socket-patch-core/src/vendor/pypi.rs b/crates/socket-patch-core/src/vendor/pypi.rs index 182b24de..2c2f6c3a 100644 --- a/crates/socket-patch-core/src/vendor/pypi.rs +++ b/crates/socket-patch-core/src/vendor/pypi.rs @@ -86,7 +86,7 @@ fn validate_hosted_wheel_sha256(sha256: &str) -> Result<(), String> { Ok(()) } -fn decode_hosted_wheel_metadata(bytes: &[u8], sha256: &str) -> Result, String> { +pub fn decode_hosted_wheel_metadata(bytes: &[u8], sha256: &str) -> Result, String> { validate_hosted_wheel_sha256(sha256)?; if !hex::encode(Sha256::digest(bytes)).eq_ignore_ascii_case(sha256) { return Err("hosted wheel sha256 does not match the published artifact".to_string()); diff --git a/crates/socket-patch-node/Cargo.toml b/crates/socket-patch-node/Cargo.toml new file mode 100644 index 00000000..d95cb71a --- /dev/null +++ b/crates/socket-patch-node/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "socket-patch-node" +description = "Node addon that runs the socket-patch hosted redirect engine in-process" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[lib] +name = "socket_patch_node" +path = "src/lib.rs" +crate-type = ["cdylib"] +# A cdylib that links against the host's `napi_*` symbols has nothing to +# test outside Node; the engine's tests live in socket-patch-cli and the +# addon's in npm/test/smoke.mjs. +test = false +doctest = false + +[dependencies] +socket-patch-cli = { path = "../socket-patch-cli" } +socket-patch-core = { workspace = true } +napi = { workspace = true } +napi-derive = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } + +[build-dependencies] +napi-build = { workspace = true } diff --git a/crates/socket-patch-node/build.rs b/crates/socket-patch-node/build.rs new file mode 100644 index 00000000..0f1b0100 --- /dev/null +++ b/crates/socket-patch-node/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/crates/socket-patch-node/npm/.gitignore b/crates/socket-patch-node/npm/.gitignore new file mode 100644 index 00000000..5797b59e --- /dev/null +++ b/crates/socket-patch-node/npm/.gitignore @@ -0,0 +1,2 @@ +socket_patch_node.node +node_modules/ diff --git a/crates/socket-patch-node/npm/index.d.ts b/crates/socket-patch-node/npm/index.d.ts new file mode 100644 index 00000000..e5fc61dc --- /dev/null +++ b/crates/socket-patch-node/npm/index.d.ts @@ -0,0 +1,82 @@ +export type Ecosystem = 'npm' | 'pypi' | 'cargo' | 'golang' | 'gem' | 'composer' | 'maven' | 'nuget' + +export interface TreeEntryInput { path: string; mode: string; type: 'blob' | 'tree' | 'commit'; size?: number } +export interface PathSelection { + roots: string[] // detected project roots, repo-relative ('' = repo root), sorted + fetchText: string[] // stream these as UTF-8 text files + fetchBinary: string[] // stream these as raw bytes (e.g. bun.lockb) + presentOnly: string[] // engine only needs to know they exist (e.g. .pnp.cjs, rush repo-state.json) + symlinks: string[] // candidate paths that are symlinks (mode 120000) — refuse-to-write + ignoredCount: number + ignoredSample: { path: string; reason: string }[] // ≤100 +} +export function selectHostedScanPaths(entries: TreeEntryInput[], options?: { projectRoots?: string[]; ecosystems?: Ecosystem[] }): PathSelection +export function hostedScanCandidateFiles(): string[] // debug listing only +export function engineVersion(): string // "+" + +export type ProviderErrorKind = 'unauthorized' | 'forbidden' | 'rate_limited' | 'network' | 'parse' | 'not_found' | 'other' +export type ProviderResult = { ok: true; value: T } | { ok: false; error: { kind: ProviderErrorKind; message: string } } + +// Request/response bodies are EXACTLY the api-v0 HTTP JSON bodies (camelCase), so the Rust serde types are reused unchanged. +export interface BatchPatchInfo { uuid: string; purl: string; tier: string; cveIds: string[]; ghsaIds: string[]; severity: string | null; title: string; publishedAt?: string } +export interface BatchSearchResponse { packages: { purl: string; patches: BatchPatchInfo[] }[]; canAccessPaidPatches: boolean } +export interface PatchSearchResult { uuid: string; purl: string; publishedAt: string; description: string; license: string; tier: string; vulnerabilities: Record } +export interface SearchResponse { patches: PatchSearchResult[]; canAccessPaidPatches: boolean } +export interface PackageVendorResult { status: string; url: string | null; purl: string | null; artifacts: unknown[] | null; registryOverride: unknown | null } // exact api-v0 package.ts serialization (null-filled) +export interface PatchResponse { uuid: string; purl: string; publishedAt: string; files: Record; vulnerabilities: Record; description: string; license: string; tier: string } + +export interface PatchProvider { + searchPatchesBatch(request: { components: { purl: string }[] }): Promise> + searchPatchesByPackage(request: { purl: string }): Promise> + fetchRegistryReferences(request: { uuids: string[] }): Promise }>> + fetchPatch(request: { uuid: string }): Promise> + downloadArtifact(request: { url: string; maxBytes: number }): Promise> +} +// Providers MUST resolve (never reject); the JS loader wraps provider methods so a thrown/rejected call becomes {ok:false, kind:'other'}. +// A 'not_found' failure mirrors the HTTP API's 404 per method: searchPatchesByPackage => no patches, fetchRegistryReferences => no references, fetchPatch => null, searchPatchesBatch and downloadArtifact => error. + +export interface HostedScanLimits { maxFileBytes?: number /*20 MiB*/; maxTotalBytes?: number /*64 MiB*/; maxFiles?: number /*2000*/; maxPurls?: number /*20000*/; maxProjects?: number /*200*/; maxArtifactBytes?: number /*32 MiB*/ } +export interface HostedScanSessionOptions { + orgSlug: string + ecosystems?: Ecosystem[] + batchSize?: number // 1..500, default 100 + dryRun?: boolean + pipenvMajor?: number // never spawns pipenv; absent => same default as CLI when pipenv unavailable + trustLockfileConfig?: boolean // default true + npmAllowRemoteConfig?: boolean // default true + projectRoots?: string[] // must match selectHostedScanPaths input + providerConcurrency?: number // default 8 + requestTimeoutMs?: number // per provider call, default 60000 + limits?: HostedScanLimits +} +export class HostedScanSession { + constructor(options: HostedScanSessionOptions, provider: PatchProvider) + pushChunk(path: string, chunk: Buffer): void // throws on limit breach, unknown state, or after finish + endFile(path: string): void + markPresent(path: string, kind: 'present' | 'symlink' | 'binary_skipped' | 'oversize' | 'lfs_pointer'): void + finish(): Promise // runs off the JS thread; rejects only on engine bug/limit/cancel + cancel(): void // cooperative; finish() rejects with code 'cancelled' +} +export interface EngineWarning { code: string; detail: string; projectRoot?: string } +export interface ProjectResult { + root: string + redirect: Record // same shape as CLI `--json` `redirect` block + summary: { scannedPackages: number; packagesWithPatches: number; totalPatches: number; freePatches: number; paidPatches: number; canAccessPaidPatches: boolean } + redirected: { purl: string; uuid: string }[] + skipped: { purl: string; uuid: string; reason: string; detail?: string }[] + error?: { code: string; message: string } // project-level failure (e.g. corrupt_ledger, patch_lookup_failed) +} +export interface HostedScanResult { + projects: ProjectResult[] + changedFiles: { path: string; content: string }[] // repo-relative, sorted, only byte-changed, includes ledgers (wet runs only) + changedBinaryFiles: { path: string; content: Buffer }[] + deletedFiles: string[] + warnings: EngineWarning[] + stats: { projects: number; filesInput: number; bytesInput: number; packagesScanned: number; packagesWithPatches: number; patchesSelected: number; patchesRedirected: number; filesChanged: number; providerCalls: Record; phaseMs: Record } + engineVersion: string +} + +export class SocketPatchAddonUnavailableError extends Error { + readonly code: 'addon_unavailable' + readonly attempted: string[] +} diff --git a/crates/socket-patch-node/npm/index.js b/crates/socket-patch-node/npm/index.js new file mode 100644 index 00000000..65d695e9 --- /dev/null +++ b/crates/socket-patch-node/npm/index.js @@ -0,0 +1,362 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') + +const ADDON_ENV = 'SOCKET_PATCH_NODE_ADDON_PATH' +const PROVIDER_ERROR_KINDS = new Set([ + 'unauthorized', + 'forbidden', + 'rate_limited', + 'network', + 'parse', + 'not_found', + 'other', +]) +const JSON_PROVIDER_METHODS = [ + 'searchPatchesBatch', + 'searchPatchesByPackage', + 'fetchRegistryReferences', + 'fetchPatch', +] +const MAX_PROVIDER_MESSAGE = 2000 +const REQUIRED_EXPORTS = [ + 'selectHostedScanPathsJson', + 'hostedScanCandidateFiles', + 'engineVersion', + 'createHostedScanSession', + 'hostedScanSessionPushChunk', + 'hostedScanSessionEndFile', + 'hostedScanSessionMarkPresent', + 'hostedScanSessionFinish', + 'hostedScanSessionCancel', +] + +class SocketPatchAddonUnavailableError extends Error { + constructor(message, attempted, options) { + super(message, options) + this.name = 'SocketPatchAddonUnavailableError' + this.code = 'addon_unavailable' + this.attempted = attempted + } +} + +function libraryNames() { + if (process.platform === 'win32') { + return ['socket_patch_node.dll'] + } + if (process.platform === 'darwin') { + return ['libsocket_patch_node.dylib'] + } + return ['libsocket_patch_node.so'] +} + +function candidatePaths() { + const explicit = process.env[ADDON_ENV] + if (explicit) { + return [path.resolve(explicit)] + } + const candidates = [path.join(__dirname, 'socket_patch_node.node')] + if (process.env.NODE_ENV !== 'production') { + const targetDir = path.resolve(__dirname, '..', '..', '..', 'target') + for (const profile of ['release', 'debug']) { + for (const name of libraryNames()) { + candidates.push(path.join(targetDir, profile, name)) + } + } + } + return candidates +} + +function loadFile(file) { + if (path.extname(file) === '.node') { + return require(file) + } + const addonModule = { exports: {} } + process.dlopen(addonModule, file) + return addonModule.exports +} + +let cachedBinding = null + +function binding() { + if (cachedBinding) { + return cachedBinding + } + const attempted = candidatePaths() + const found = attempted.find((file) => fs.existsSync(file)) + if (!found) { + const hint = process.env[ADDON_ENV] + ? `${ADDON_ENV} points at a missing file` + : `build it with \`pnpm run build:addon\` or set ${ADDON_ENV}` + throw new SocketPatchAddonUnavailableError( + `socket-patch-node addon not found (${hint}); tried: ${attempted.join(', ')}`, + attempted, + ) + } + let loaded + try { + loaded = loadFile(found) + } catch (error) { + throw new SocketPatchAddonUnavailableError( + `socket-patch-node addon at ${found} failed to load: ${describe(error)}`, + attempted, + { cause: error }, + ) + } + const missing = REQUIRED_EXPORTS.filter((name) => loaded[name] === undefined) + if (missing.length > 0) { + throw new SocketPatchAddonUnavailableError( + `socket-patch-node addon at ${found} is missing exports: ${missing.join(', ')}`, + attempted, + ) + } + cachedBinding = loaded + return loaded +} + +function describe(error) { + let text + if (error instanceof Error) { + text = error.message + } else { + try { + text = String(error) + } catch { + text = 'unprintable error' + } + } + return text.length > MAX_PROVIDER_MESSAGE + ? `${text.slice(0, MAX_PROVIDER_MESSAGE)}…` + : text +} + +function engineError(code, kind, message) { + const error = new Error(message) + error.name = 'HostedScanError' + error.code = code + error.kind = kind + return error +} + +function fromNativeError(error) { + const reason = error instanceof Error ? error.message : undefined + if (typeof reason === 'string') { + try { + const parsed = JSON.parse(reason) + if ( + parsed && + typeof parsed.code === 'string' && + typeof parsed.kind === 'string' && + typeof parsed.message === 'string' + ) { + return engineError(parsed.code, parsed.kind, parsed.message) + } + } catch {} + } + return engineError('addon_internal', 'internal', describe(error)) +} + +function callNative(call) { + try { + return call() + } catch (error) { + throw fromNativeError(error) + } +} + +function providerFailure(kind, message) { + return { + ok: false, + error: { kind: PROVIDER_ERROR_KINDS.has(kind) ? kind : 'other', message }, + } +} + +function normalizeJsonResult(result) { + if (result === null || typeof result !== 'object' || typeof result.ok !== 'boolean') { + return providerFailure('other', 'provider returned a malformed result') + } + if (result.ok) { + return { ok: true, value: result.value === undefined ? null : result.value } + } + const error = result.error !== null && typeof result.error === 'object' ? result.error : {} + return providerFailure( + typeof error.kind === 'string' ? error.kind : 'other', + typeof error.message === 'string' ? describe(error.message) : '', + ) +} + +function serializeResult(result) { + try { + return JSON.stringify(result) + } catch (error) { + return JSON.stringify( + providerFailure('parse', `provider result is not serializable: ${describe(error)}`), + ) + } +} + +function invokeProvider(provider, method, requestJson) { + return Promise.resolve().then(() => { + const request = JSON.parse(requestJson) + const fn = provider[method] + if (typeof fn !== 'function') { + throw new TypeError(`provider.${method} is not a function`) + } + return fn.call(provider, request) + }) +} + +function wrapJsonMethod(provider, method) { + return (requestJson) => + invokeProvider(provider, method, requestJson) + .then(normalizeJsonResult, (error) => providerFailure('other', describe(error))) + .then(serializeResult) + .catch(() => serializeResult(providerFailure('other', `provider.${method} failed`))) +} + +function toBuffer(value) { + if (Buffer.isBuffer(value)) { + return value + } + if (value instanceof Uint8Array) { + return Buffer.from(value.buffer, value.byteOffset, value.byteLength) + } + return null +} + +function flatDownloadFailure(kind, message) { + return { + ok: false, + value: undefined, + kind: PROVIDER_ERROR_KINDS.has(kind) ? kind : 'other', + message, + } +} + +function normalizeDownloadResult(result) { + if (result === null || typeof result !== 'object' || typeof result.ok !== 'boolean') { + return flatDownloadFailure('other', 'provider returned a malformed result') + } + if (result.ok) { + const bytes = toBuffer(result.value) + if (!bytes) { + return flatDownloadFailure('parse', 'downloadArtifact must resolve a Buffer') + } + return { ok: true, value: bytes, kind: undefined, message: undefined } + } + const error = result.error !== null && typeof result.error === 'object' ? result.error : {} + return flatDownloadFailure( + typeof error.kind === 'string' ? error.kind : 'other', + typeof error.message === 'string' ? describe(error.message) : '', + ) +} + +function wrapDownloadMethod(provider) { + return (requestJson) => + invokeProvider(provider, 'downloadArtifact', requestJson) + .then(normalizeDownloadResult, (error) => flatDownloadFailure('other', describe(error))) + .catch(() => flatDownloadFailure('other', 'provider.downloadArtifact failed')) +} + +function wrapProvider(provider) { + if (provider === null || (typeof provider !== 'object' && typeof provider !== 'function')) { + throw new TypeError('provider must be an object implementing PatchProvider') + } + const wrapped = {} + for (const method of JSON_PROVIDER_METHODS) { + wrapped[method] = wrapJsonMethod(provider, method) + } + wrapped.downloadArtifact = wrapDownloadMethod(provider) + return wrapped +} + +function requirePath(value) { + if (typeof value !== 'string') { + throw new TypeError('path must be a string') + } + return value +} + +function selectHostedScanPaths(entries, options) { + if (!Array.isArray(entries)) { + throw new TypeError('entries must be an array of tree entries') + } + const native = binding() + const optionsJson = options === undefined || options === null ? null : JSON.stringify(options) + return JSON.parse( + callNative(() => native.selectHostedScanPathsJson(JSON.stringify(entries), optionsJson)), + ) +} + +function hostedScanCandidateFiles() { + return binding().hostedScanCandidateFiles() +} + +function engineVersion() { + return binding().engineVersion() +} + +class HostedScanSession { + #binding + #native + + constructor(options, provider) { + if (options === null || typeof options !== 'object') { + throw new TypeError('options must be a HostedScanSessionOptions object') + } + const native = binding() + const wrapped = wrapProvider(provider) + const optionsJson = JSON.stringify(options) + this.#binding = native + this.#native = callNative(() => native.createHostedScanSession(optionsJson, wrapped)) + } + + pushChunk(path, chunk) { + const bytes = toBuffer(chunk) + if (!bytes) { + throw new TypeError('chunk must be a Buffer or Uint8Array') + } + callNative(() => this.#binding.hostedScanSessionPushChunk(this.#native, requirePath(path), bytes)) + } + + endFile(path) { + callNative(() => this.#binding.hostedScanSessionEndFile(this.#native, requirePath(path))) + } + + markPresent(path, kind) { + if (typeof kind !== 'string') { + throw new TypeError('kind must be a string') + } + callNative(() => this.#binding.hostedScanSessionMarkPresent(this.#native, requirePath(path), kind)) + } + + async finish() { + const outcome = await callNative(() => this.#binding.hostedScanSessionFinish(this.#native)) + if (!outcome.ok) { + throw engineError( + outcome.errorCode ?? 'engine_internal', + outcome.errorKind ?? 'internal', + outcome.errorMessage ?? 'the hosted scan failed', + ) + } + const result = JSON.parse(outcome.resultJson) + result.changedBinaryFiles = (outcome.binaryFiles ?? []).map((file) => ({ + path: file.path, + content: file.content, + })) + return result + } + + cancel() { + callNative(() => this.#binding.hostedScanSessionCancel(this.#native)) + } +} + +module.exports = { + HostedScanSession, + SocketPatchAddonUnavailableError, + engineVersion, + hostedScanCandidateFiles, + selectHostedScanPaths, +} diff --git a/crates/socket-patch-node/npm/package.json b/crates/socket-patch-node/npm/package.json new file mode 100644 index 00000000..1bd82062 --- /dev/null +++ b/crates/socket-patch-node/npm/package.json @@ -0,0 +1,19 @@ +{ + "name": "@socketsecurity/socket-patch-node", + "version": "4.0.0", + "private": true, + "description": "In-process Node addon for the socket-patch hosted redirect engine", + "license": "MIT", + "main": "index.js", + "types": "index.d.ts", + "files": [ + "index.js", + "index.d.ts" + ], + "scripts": { + "build:addon": "node scripts/build-addon.mjs" + }, + "engines": { + "node": ">=18" + } +} diff --git a/crates/socket-patch-node/npm/scripts/build-addon.mjs b/crates/socket-patch-node/npm/scripts/build-addon.mjs new file mode 100644 index 00000000..dc8d74e7 --- /dev/null +++ b/crates/socket-patch-node/npm/scripts/build-addon.mjs @@ -0,0 +1,39 @@ +import { spawnSync } from 'node:child_process' +import { copyFileSync, existsSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const packageDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const repoRoot = path.resolve(packageDir, '..', '..', '..') +const profile = process.env.SOCKET_PATCH_NODE_CARGO_PROFILE || 'release' +const targetDir = process.env.CARGO_TARGET_DIR + ? path.resolve(process.env.CARGO_TARGET_DIR) + : path.join(repoRoot, 'target') + +const cargo = spawnSync( + 'cargo', + ['build', '--locked', '--profile', profile, '-p', 'socket-patch-node'], + { cwd: repoRoot, stdio: 'inherit' }, +) +if (cargo.error) { + throw cargo.error +} +if (cargo.status !== 0) { + process.exit(cargo.status ?? 1) +} + +const libraryName = + process.platform === 'win32' + ? 'socket_patch_node.dll' + : process.platform === 'darwin' + ? 'libsocket_patch_node.dylib' + : 'libsocket_patch_node.so' +const profileDir = profile === 'dev' ? 'debug' : profile +const built = path.join(targetDir, profileDir, libraryName) +if (!existsSync(built)) { + console.error(`build:addon: ${built} was not produced`) + process.exit(1) +} +const destination = path.join(packageDir, 'socket_patch_node.node') +copyFileSync(built, destination) +console.log(`build:addon: copied ${built} -> ${destination}`) diff --git a/crates/socket-patch-node/npm/test/smoke.mjs b/crates/socket-patch-node/npm/test/smoke.mjs new file mode 100644 index 00000000..a333ade1 --- /dev/null +++ b/crates/socket-patch-node/npm/test/smoke.mjs @@ -0,0 +1,461 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import path from 'node:path' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' + +const require = createRequire(import.meta.url) +const addon = require('../index.js') + +const here = path.dirname(fileURLToPath(import.meta.url)) +const fixtureDir = path.resolve( + here, + '../../../socket-patch-core/tests/fixtures/redirect/npm/package-lock-v3/basic', +) +const inputLock = readFileSync(path.join(fixtureDir, 'input/package-lock.json')) +const expectedLock = readFileSync(path.join(fixtureDir, 'expected/package-lock.json'), 'utf8') +const overrides = JSON.parse(readFileSync(path.join(fixtureDir, 'overrides.json'), 'utf8')) + +const patches = overrides.map((o) => ({ + purl: `pkg:${o.ecosystem}/${o.namespace ? `${o.namespace}/` : ''}${o.name}@${o.version}`, + uuid: o.patchUuid, + reference: { + status: 'granted', + url: o.artifactUrl, + purl: null, + artifacts: [{ kind: 'tarball', url: o.artifactUrl, integrity: o.integrity }], + registryOverride: o.registryOverride ?? null, + }, +})) + +const vulnerabilities = { + 'GHSA-test-aaaa-bbbb': { + cves: ['CVE-2024-0001'], + summary: 's', + severity: 'high', + description: 'd', + }, +} + +function fakeProvider(overridesByMethod = {}) { + const calls = { + searchPatchesBatch: 0, + searchPatchesByPackage: 0, + fetchRegistryReferences: 0, + fetchPatch: 0, + downloadArtifact: 0, + } + const base = { + async searchPatchesBatch({ components }) { + const packages = [] + for (const { purl } of components) { + const matches = patches.filter((p) => p.purl === purl) + if (matches.length > 0) { + packages.push({ + purl, + patches: matches.map((p) => ({ + uuid: p.uuid, + purl, + tier: 'free', + cveIds: [], + ghsaIds: ['GHSA-test-aaaa-bbbb'], + severity: 'high', + title: 'fixture', + })), + }) + } + } + return { ok: true, value: { packages, canAccessPaidPatches: false } } + }, + async searchPatchesByPackage({ purl }) { + return { + ok: true, + value: { + patches: patches + .filter((p) => p.purl === purl) + .map((p) => ({ + uuid: p.uuid, + purl, + publishedAt: '2024-01-01T00:00:00Z', + description: 'fixture', + license: 'MIT', + tier: 'free', + vulnerabilities, + })), + canAccessPaidPatches: false, + }, + } + }, + async fetchRegistryReferences({ uuids }) { + const results = {} + for (const uuid of uuids) { + const patch = patches.find((p) => p.uuid === uuid) + if (patch) { + results[uuid] = patch.reference + } + } + return { ok: true, value: { results } } + }, + async fetchPatch({ uuid }) { + const patch = patches.find((p) => p.uuid === uuid) + if (!patch) { + return { ok: true, value: null } + } + return { + ok: true, + value: { + uuid: patch.uuid, + purl: patch.purl, + publishedAt: '2024-01-01T00:00:00Z', + files: { + 'package/index.js': { beforeHash: 'a'.repeat(64), afterHash: 'b'.repeat(64) }, + }, + vulnerabilities, + description: 'fixture', + license: 'MIT', + tier: 'free', + }, + } + }, + async downloadArtifact() { + return { ok: false, error: { kind: 'not_found', message: 'no artifacts in this fixture' } } + }, + } + const provider = {} + for (const method of Object.keys(calls)) { + const impl = overridesByMethod[method] ?? base[method] + provider[method] = (request) => { + calls[method] += 1 + return impl(request) + } + } + return { provider, calls } +} + +const tree = [ + { path: 'package.json', mode: '100644', type: 'blob', size: 40 }, + { path: 'package-lock.json', mode: '100644', type: 'blob', size: inputLock.length }, + { path: 'src', mode: '040000', type: 'tree' }, + { path: 'src/index.js', mode: '100644', type: 'blob', size: 10 }, + { path: 'node_modules/left-pad/package-lock.json', mode: '100644', type: 'blob', size: 10 }, + { path: 'test/fixtures/app/package-lock.json', mode: '100644', type: 'blob', size: 10 }, +] + +const files = { + 'package-lock.json': inputLock, + 'package.json': Buffer.from('{"name":"consumer","version":"1.0.0"}\n'), +} + +function streamSelection(session, selection, chunkSize = 7) { + for (const file of [...selection.fetchText, ...selection.fetchBinary]) { + const bytes = files[file] + if (bytes === undefined) { + session.markPresent(file, 'present') + continue + } + for (let offset = 0; offset < bytes.length; offset += chunkSize) { + session.pushChunk(file, bytes.subarray(offset, offset + chunkSize)) + } + session.endFile(file) + } + for (const file of selection.presentOnly) { + session.markPresent(file, 'present') + } + for (const file of selection.symlinks) { + session.markPresent(file, 'symlink') + } +} + +test('engineVersion and hostedScanCandidateFiles', () => { + assert.match(addon.engineVersion(), /^\d+\.\d+\.\d+\+.+$/) + const candidates = addon.hostedScanCandidateFiles() + assert.ok(Array.isArray(candidates)) + assert.ok(candidates.includes('package-lock.json')) +}) + +test('selectHostedScanPaths picks the root lockfile and ignores vendored trees', () => { + const selection = addon.selectHostedScanPaths(tree) + assert.deepEqual(selection.roots, ['']) + assert.ok(selection.fetchText.includes('package-lock.json')) + assert.ok(!selection.fetchText.includes('node_modules/left-pad/package-lock.json')) + assert.ok(!selection.fetchText.includes('test/fixtures/app/package-lock.json')) + assert.deepEqual(selection.fetchBinary, []) + assert.equal(typeof selection.ignoredCount, 'number') + assert.ok(selection.ignoredSample.length <= 100) +}) + +test('streamed session redirects the package-lock fixture', async () => { + const { provider, calls } = fakeProvider() + const selection = addon.selectHostedScanPaths(tree, { ecosystems: ['npm'] }) + const session = new addon.HostedScanSession( + { orgSlug: 'test-org', ecosystems: ['npm'] }, + provider, + ) + streamSelection(session, selection) + const result = await session.finish() + + assert.equal(result.projects.length, 1) + const [project] = result.projects + assert.equal(project.root, '') + assert.equal(project.error, undefined) + assert.deepEqual(project.redirected, [ + { purl: 'pkg:npm/left-pad@1.3.0', uuid: '22222222-2222-2222-2222-222222222222' }, + ]) + assert.equal(typeof project.redirect, 'object') + + const lock = result.changedFiles.find((f) => f.path === 'package-lock.json') + assert.ok(lock, 'package-lock.json changed') + assert.equal(lock.content, expectedLock) + const paths = result.changedFiles.map((f) => f.path) + assert.deepEqual(paths, [...paths].sort()) + assert.ok( + paths.some((p) => p.startsWith('.socket/')), + `a wet run writes the ledger (changed: ${paths.join(', ')})`, + ) + assert.deepEqual(result.changedBinaryFiles, []) + assert.deepEqual(result.deletedFiles, []) + assert.equal(result.engineVersion, addon.engineVersion()) + assert.equal(result.stats.patchesRedirected, 1) + assert.equal(calls.searchPatchesBatch, 1) + assert.equal(calls.fetchRegistryReferences, 1) + assert.ok(calls.fetchPatch >= 1) +}) + +test('dry run previews the lockfile without a ledger or patch fetches', async () => { + const { provider, calls } = fakeProvider() + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession({ orgSlug: 'test-org', dryRun: true }, provider) + streamSelection(session, selection, 4096) + const result = await session.finish() + const paths = result.changedFiles.map((f) => f.path) + assert.ok(paths.includes('package-lock.json')) + assert.ok(!paths.some((p) => p.startsWith('.socket/'))) + assert.equal(calls.fetchPatch, 0) +}) + +test('provider failures become project errors, never rejections', async () => { + for (const searchPatchesBatch of [ + async () => ({ ok: false, error: { kind: 'unauthorized', message: 'token revoked' } }), + async () => { + throw new Error('database unavailable') + }, + () => { + throw new Error('synchronous provider bug') + }, + async () => ({ nonsense: true }), + ]) { + const { provider } = fakeProvider({ searchPatchesBatch }) + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + streamSelection(session, selection) + const result = await session.finish() + assert.equal(result.projects.length, 1) + assert.equal(result.projects[0].error?.code, 'patch_lookup_failed') + assert.deepEqual(result.changedFiles, []) + } +}) + +test('cancel rejects a running finish with code cancelled', async () => { + let entered + const reached = new Promise((resolve) => { + entered = resolve + }) + const { provider } = fakeProvider({ + searchPatchesBatch: () => { + entered() + return new Promise(() => {}) + }, + }) + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + streamSelection(session, selection) + const pending = session.finish() + await reached + session.cancel() + await assert.rejects(pending, (error) => { + assert.equal(error.code, 'cancelled') + assert.equal(error.kind, 'cancelled') + return true + }) + await assert.rejects(session.finish(), (error) => error.code === 'cancelled') +}) + +test('a provider call that never settles times out into a project error', async () => { + const { provider } = fakeProvider({ searchPatchesBatch: () => new Promise(() => {}) }) + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession( + { orgSlug: 'test-org', requestTimeoutMs: 50 }, + provider, + ) + streamSelection(session, selection) + const result = await session.finish() + assert.equal(result.projects[0].error?.code, 'patch_lookup_failed') +}) + +test('cancel before finish rejects and blocks further input', async () => { + const { provider, calls } = fakeProvider() + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + session.cancel() + assert.throws( + () => session.pushChunk('package-lock.json', inputLock), + (error) => error.code === 'cancelled', + ) + await assert.rejects(session.finish(), (error) => error.code === 'cancelled') + assert.equal(calls.searchPatchesBatch, 0) +}) + +test('cancel before finish frees the buffered input right away', async () => { + const { provider } = fakeProvider() + // One whole-file chunk each: one allocation per file, which every + // allocator returns to the OS on free, so RSS reflects what is retained. + const fileMiB = 16 + const chunk = Buffer.alloc(fileMiB * 1024 * 1024, 0x61) + const roots = ['a', 'b', 'c'] + const sessions = [] + const before = process.memoryUsage().rss + for (let i = 0; i < 10; i += 1) { + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + for (const root of roots) { + session.pushChunk(`${root}/package-lock.json`, chunk) + } + session.cancel() + sessions.push(session) + } + const grownMiB = (process.memoryUsage().rss - before) / (1024 * 1024) + const bufferedMiB = sessions.length * roots.length * fileMiB + assert.ok( + grownMiB < bufferedMiB / 3, + `RSS grew ${grownMiB.toFixed(0)} MiB while ${bufferedMiB} MiB was buffered and cancelled`, + ) + for (const session of sessions) { + await assert.rejects(session.finish(), (error) => error.code === 'cancelled') + } +}) + +function bigLock(count) { + const packages = { '': { name: 'big', version: '1.0.0', dependencies: {} } } + for (let i = 0; i < count; i += 1) { + const name = `pkg-${i}` + packages[''].dependencies[name] = '1.0.0' + packages[`node_modules/${name}`] = { + version: '1.0.0', + resolved: `https://registry.npmjs.org/${name}/-/${name}-1.0.0.tgz`, + integrity: `sha512-${'A'.repeat(86)}==`, + } + } + return Buffer.from( + JSON.stringify({ name: 'big', version: '1.0.0', lockfileVersion: 3, requires: true, packages }), + ) +} + +test('cancel settles finish without waiting out a synchronous engine phase', async () => { + const lock = bigLock(60000) + const roots = ['a', 'b', 'c'] + const { provider } = fakeProvider({ + searchPatchesBatch: async () => ({ + ok: true, + value: { packages: [], canAccessPaidPatches: false }, + }), + }) + const start = () => { + const session = new addon.HostedScanSession( + { orgSlug: 'test-org', dryRun: true, projectRoots: roots, limits: { maxPurls: 200000 } }, + provider, + ) + for (const root of roots) { + session.pushChunk(`${root}/package-lock.json`, lock) + session.endFile(`${root}/package-lock.json`) + session.markPresent(`${root}/package.json`, 'present') + } + return session + } + + const baselineStart = performance.now() + const baseline = await start().finish() + const baselineMs = performance.now() - baselineStart + assert.equal(baseline.projects.length, roots.length) + const inventoryMs = baseline.stats.phaseMs.inventory + if (inventoryMs < 150) { + return + } + + const session = start() + const pending = session.finish() + await new Promise((resolve) => setTimeout(resolve, 20)) + const cancelledAt = performance.now() + session.cancel() + await assert.rejects(pending, (error) => error.code === 'cancelled') + const latencyMs = performance.now() - cancelledAt + assert.ok( + latencyMs < inventoryMs / 3, + `finish() settled ${latencyMs.toFixed(0)} ms after cancel (inventory ${inventoryMs} ms, run ${baselineMs.toFixed(0)} ms)`, + ) +}) + +test('not_found failures map per method like the HTTP API 404', async () => { + const notFound = async () => ({ ok: false, error: { kind: 'not_found', message: 'missing' } }) + const run = async (overrides) => { + const { provider, calls } = fakeProvider(overrides) + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + streamSelection(session, selection) + const result = await session.finish() + assert.equal(result.projects.length, 1) + return { project: result.projects[0], result, calls } + } + + const byPackage = await run({ searchPatchesByPackage: notFound }) + assert.equal(byPackage.calls.searchPatchesByPackage, 1) + assert.equal(byPackage.project.error, undefined) + assert.deepEqual(byPackage.project.redirected, []) + + const references = await run({ fetchRegistryReferences: notFound }) + assert.equal(references.calls.fetchRegistryReferences, 1) + assert.equal(references.project.error, undefined) + assert.deepEqual(references.project.redirected, []) + + const patch = await run({ fetchPatch: notFound }) + assert.ok(patch.calls.fetchPatch >= 1) + assert.equal(patch.project.error, undefined) + + const batch = await run({ searchPatchesBatch: notFound }) + assert.equal(batch.project.error?.code, 'patch_lookup_failed') + assert.match(batch.project.error.message, /not_found/) + assert.deepEqual(batch.result.changedFiles, []) +}) + +test('limits are enforced while streaming and poison the session', async () => { + const { provider } = fakeProvider() + const session = new addon.HostedScanSession( + { orgSlug: 'test-org', limits: { maxFileBytes: 16 } }, + provider, + ) + session.pushChunk('package-lock.json', inputLock.subarray(0, 10)) + assert.throws( + () => session.pushChunk('package-lock.json', inputLock.subarray(10, 30)), + (error) => error.code === 'max_file_bytes' && error.kind === 'limit', + ) + assert.throws( + () => session.endFile('package-lock.json'), + (error) => error.code === 'max_file_bytes', + ) + await assert.rejects(session.finish(), (error) => error.code === 'max_file_bytes') +}) + +test('invalid options and inputs throw typed errors', () => { + const { provider } = fakeProvider() + assert.throws( + () => new addon.HostedScanSession({ orgSlug: 'test-org', batchSize: 0 }, provider), + (error) => error.code === 'invalid_batch_size' && error.kind === 'invalid_input', + ) + assert.throws( + () => new addon.HostedScanSession({ orgSlug: 'test-org' }, null), + TypeError, + ) + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + assert.throws( + () => session.markPresent('.pnp.cjs', 'bogus'), + (error) => error.code === 'invalid_mark_kind', + ) +}) diff --git a/crates/socket-patch-node/src/lib.rs b/crates/socket-patch-node/src/lib.rs new file mode 100644 index 00000000..5f1c8ed7 --- /dev/null +++ b/crates/socket-patch-node/src/lib.rs @@ -0,0 +1,369 @@ +//! Node addon for the in-memory hosted redirect engine +//! (`socket_patch_cli::hosted_memory`). +//! +//! This is the private native half of `@socketsecurity/socket-patch-node`; +//! npm/index.js is the public surface (npm/index.d.ts). Options, tree +//! entries and results cross as JSON strings so the engine's serde types +//! are the single definition of every shape; bytes cross as `Buffer`s. + +mod provider; + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use napi::bindgen_prelude::{Buffer, External, Function, JsObjectValue, Object, PromiseRaw}; +use napi::{Env, Status}; +use napi_derive::napi; +use socket_patch_cli::hosted_memory::{ + self, EngineError, HostedScanOptions, HostedScanOutput, PresentKind, SelectOptions, + SessionBuilder, TreeEntryInput, +}; +use socket_patch_core::api::client::PatchApi; +use tokio_util::sync::CancellationToken; + +use provider::{JsPatchApi, ProviderRefs}; + +/// Every error the addon throws carries `{code, kind, message}` as a JSON +/// reason; the loader rethrows it as an `Error` with those properties. +fn js_error(code: &str, kind: &str, message: impl Into) -> napi::Error { + let reason = serde_json::json!({ "code": code, "kind": kind, "message": message.into() }); + napi::Error::new(Status::GenericFailure, reason.to_string()) +} + +fn engine_js_error(error: &EngineError) -> napi::Error { + js_error(error.code(), error.kind(), error.to_string()) +} + +fn invalid_input(code: &str, message: impl Into) -> napi::Error { + js_error(code, "invalid_input", message) +} + +fn napi_js_error(error: napi::Error) -> napi::Error { + js_error("addon_internal", "internal", error.reason.clone()) +} + +#[napi(js_name = "selectHostedScanPathsJson")] +pub fn select_hosted_scan_paths_json( + entries_json: String, + options_json: Option, +) -> napi::Result { + let entries: Vec = serde_json::from_str(&entries_json) + .map_err(|e| invalid_input("invalid_entries", format!("tree entries: {e}")))?; + let options: SelectOptions = match options_json.as_deref() { + None => SelectOptions::default(), + Some(text) => serde_json::from_str(text) + .map_err(|e| invalid_input("invalid_options", format!("selection options: {e}")))?, + }; + let selection = hosted_memory::select_paths(&entries, &options); + serde_json::to_string(&selection) + .map_err(|e| js_error("addon_internal", "internal", e.to_string())) +} + +#[napi(js_name = "hostedScanCandidateFiles")] +pub fn hosted_scan_candidate_files() -> Vec { + hosted_memory::candidate_files() +} + +#[napi(js_name = "engineVersion")] +pub fn engine_version() -> String { + hosted_memory::engine_version() +} + +#[napi(object)] +pub struct NativeBinaryFile { + pub path: String, + pub content: Buffer, +} + +/// `finish()`'s settlement. The loader turns a failure into a rejected +/// promise whose error carries `code` and `kind`, so the native promise +/// itself always resolves. +#[napi(object)] +pub struct NativeFinishOutcome { + pub ok: bool, + /// `HostedScanResult` with `changedBinaryFiles` left empty. + pub result_json: Option, + pub binary_files: Option>, + pub error_code: Option, + pub error_kind: Option, + pub error_message: Option, +} + +impl NativeFinishOutcome { + fn failure(code: &str, kind: &str, message: impl Into) -> Self { + Self { + ok: false, + result_json: None, + binary_files: None, + error_code: Some(code.to_string()), + error_kind: Some(kind.to_string()), + error_message: Some(message.into()), + } + } + + fn engine_failure(error: &EngineError) -> Self { + Self::failure(error.code(), error.kind(), error.to_string()) + } + + fn success(mut output: HostedScanOutput) -> Self { + let binaries = std::mem::take(&mut output.changed_binary_files); + match serde_json::to_string(&output) { + Ok(json) => Self { + ok: true, + result_json: Some(json), + binary_files: Some( + binaries + .into_iter() + .map(|file| NativeBinaryFile { + path: file.path, + content: Buffer::from(file.content), + }) + .collect(), + ), + error_code: None, + error_kind: None, + error_message: None, + }, + Err(e) => Self::failure( + "engine_internal", + "internal", + format!("serializing the result failed: {e}"), + ), + } + } +} + +enum SessionState { + Open(Box), + Failed(EngineError), + Finished, +} + +type OutcomeFuture = Pin + Send>>; + +async fn run_engine( + input: hosted_memory::HostedScanInput, + provider: Arc, + cancel: CancellationToken, +) -> NativeFinishOutcome { + let engine_cancel = cancel.child_token(); + let _stop_engine_if_dropped = engine_cancel.clone().drop_guard(); + let mut task = tokio::spawn(hosted_memory::run_in_memory(input, provider, engine_cancel)); + // The engine only observes cancellation between synchronous phases, so + // racing the join settles finish() without waiting out a long parse. + let joined = tokio::select! { + biased; + _ = cancel.cancelled() => { + return NativeFinishOutcome::engine_failure(&EngineError::Cancelled); + } + joined = &mut task => joined, + }; + match joined { + Ok(Ok(output)) => NativeFinishOutcome::success(output), + Ok(Err(error)) => NativeFinishOutcome::engine_failure(&error), + Err(join) if join.is_panic() => NativeFinishOutcome::failure( + "engine_panic", + "internal", + "the hosted scan engine panicked", + ), + Err(_) => NativeFinishOutcome::failure( + "engine_internal", + "internal", + "the hosted scan engine task was aborted", + ), + } +} + +fn ready(outcome: NativeFinishOutcome) -> OutcomeFuture { + Box::pin(async move { outcome }) +} + +pub struct NativeHostedScanSession { + state: SessionState, + provider: Option, + cancel: CancellationToken, +} + +fn provider_function<'a, Return>( + provider: &Object<'a>, + name: &str, +) -> napi::Result> +where + Return: napi::bindgen_prelude::FromNapiValue, +{ + provider + .get_named_property::>(name) + .map_err(|_| { + invalid_input( + "invalid_provider", + format!("provider.{name} must be a function"), + ) + }) +} + +impl NativeHostedScanSession { + pub fn new(options_json: String, provider: Object<'_>) -> napi::Result { + let options: HostedScanOptions = serde_json::from_str(&options_json) + .map_err(|e| invalid_input("invalid_options", format!("session options: {e}")))?; + let builder = SessionBuilder::new(options).map_err(|e| engine_js_error(&e))?; + let refs = ProviderRefs { + search_patches_batch: provider_function(&provider, "searchPatchesBatch")? + .create_ref() + .map_err(napi_js_error)?, + search_patches_by_package: provider_function(&provider, "searchPatchesByPackage")? + .create_ref() + .map_err(napi_js_error)?, + fetch_registry_references: provider_function(&provider, "fetchRegistryReferences")? + .create_ref() + .map_err(napi_js_error)?, + fetch_patch: provider_function(&provider, "fetchPatch")? + .create_ref() + .map_err(napi_js_error)?, + download_artifact: provider_function(&provider, "downloadArtifact")? + .create_ref() + .map_err(napi_js_error)?, + }; + Ok(Self { + state: SessionState::Open(Box::new(builder)), + provider: Some(refs), + cancel: CancellationToken::new(), + }) + } + + fn with_builder( + &mut self, + step: impl FnOnce(&mut SessionBuilder) -> Result<(), EngineError>, + ) -> napi::Result<()> { + if self.cancel.is_cancelled() { + return Err(engine_js_error(&EngineError::Cancelled)); + } + let builder = match &mut self.state { + SessionState::Open(builder) => builder, + SessionState::Failed(error) => return Err(engine_js_error(error)), + SessionState::Finished => { + return Err(invalid_input( + "session_finished", + "the session has already finished", + )) + } + }; + match step(builder) { + Ok(()) => Ok(()), + Err(error) => { + let thrown = engine_js_error(&error); + self.state = SessionState::Failed(error); + self.provider = None; + Err(thrown) + } + } + } + + pub fn push_chunk(&mut self, path: String, chunk: Buffer) -> napi::Result<()> { + self.with_builder(|builder| builder.push_chunk(&path, &chunk)) + } + + pub fn end_file(&mut self, path: String) -> napi::Result<()> { + self.with_builder(|builder| builder.end_file(&path)) + } + + pub fn mark_present(&mut self, path: String, kind: String) -> napi::Result<()> { + let Some(mark) = PresentKind::parse(&kind) else { + return Err(invalid_input( + "invalid_mark_kind", + format!("`{kind}` is not a markPresent kind"), + )); + }; + self.with_builder(|builder| builder.mark_present(&path, mark)) + } + + pub fn finish<'env>( + &mut self, + env: &'env Env, + ) -> napi::Result> { + let state = std::mem::replace(&mut self.state, SessionState::Finished); + let refs = self.provider.take(); + let outcome: OutcomeFuture = match state { + _ if self.cancel.is_cancelled() => { + ready(NativeFinishOutcome::engine_failure(&EngineError::Cancelled)) + } + SessionState::Finished => ready(NativeFinishOutcome::failure( + "session_finished", + "invalid_input", + "the session has already finished", + )), + SessionState::Failed(error) => ready(NativeFinishOutcome::engine_failure(&error)), + SessionState::Open(builder) => match (builder.finish(), refs) { + (Err(error), _) => ready(NativeFinishOutcome::engine_failure(&error)), + (Ok(_), None) => ready(NativeFinishOutcome::failure( + "engine_internal", + "internal", + "the session has no provider", + )), + (Ok(input), Some(refs)) => { + let api = JsPatchApi::new(env, &refs).map_err(napi_js_error)?; + let provider: Arc = Arc::new(api); + Box::pin(run_engine(input, provider, self.cancel.clone())) + } + }, + }; + env.spawn_future(async move { Ok(outcome.await) }) + } + + pub fn cancel(&mut self) { + self.cancel.cancel(); + // Buffered chunks are native memory V8 does not see, so waiting for + // the wrapper's finalizer could hold up to maxTotalBytes per session. + if matches!(self.state, SessionState::Open(_)) { + self.state = SessionState::Failed(EngineError::Cancelled); + } + self.provider = None; + } +} + +#[napi(js_name = "createHostedScanSession")] +pub fn create_hosted_scan_session( + options_json: String, + provider: Object<'_>, +) -> napi::Result> { + NativeHostedScanSession::new(options_json, provider).map(External::new) +} + +#[napi(js_name = "hostedScanSessionPushChunk")] +pub fn hosted_scan_session_push_chunk( + session: &mut External, + path: String, + chunk: Buffer, +) -> napi::Result<()> { + session.push_chunk(path, chunk) +} + +#[napi(js_name = "hostedScanSessionEndFile")] +pub fn hosted_scan_session_end_file( + session: &mut External, + path: String, +) -> napi::Result<()> { + session.end_file(path) +} + +#[napi(js_name = "hostedScanSessionMarkPresent")] +pub fn hosted_scan_session_mark_present( + session: &mut External, + path: String, + kind: String, +) -> napi::Result<()> { + session.mark_present(path, kind) +} + +#[napi(js_name = "hostedScanSessionFinish")] +pub fn hosted_scan_session_finish<'env>( + env: &'env Env, + session: &mut External, +) -> napi::Result> { + session.finish(env) +} + +#[napi(js_name = "hostedScanSessionCancel")] +pub fn hosted_scan_session_cancel(session: &mut External) { + session.cancel(); +} diff --git a/crates/socket-patch-node/src/provider.rs b/crates/socket-patch-node/src/provider.rs new file mode 100644 index 00000000..13b132ce --- /dev/null +++ b/crates/socket-patch-node/src/provider.rs @@ -0,0 +1,280 @@ +//! [`PatchApi`] served by the host's JavaScript `PatchProvider`. +//! +//! Each provider method crosses as a threadsafe function. The JSON methods +//! take the api-v0 request body as a JSON string and resolve with a +//! `ProviderResult` envelope as a JSON string; `downloadArtifact` resolves +//! with a [`NativeDownload`] so the bytes stay a `Buffer`. The loader +//! (npm/index.js) wraps every provider method so these promises always +//! resolve: a rejected promise would carry a reference to a JS value into +//! Rust, which is exactly what the envelopes avoid. + +use std::collections::HashMap; + +use napi::bindgen_prelude::{Buffer, FunctionRef, Promise}; +use napi::threadsafe_function::ThreadsafeFunction; +use napi::{Env, Status}; +use napi_derive::napi; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use socket_patch_core::api::client::{sort_batch_response, ApiError, ApiFuture, PatchApi}; +use socket_patch_core::api::types::{ + BatchSearchResponse, PackageVendorResult, PatchResponse, SearchResponse, +}; + +type JsonCall = ThreadsafeFunction, String, Status, false>; +type DownloadCall = ThreadsafeFunction, String, Status, false>; + +pub(crate) type JsonRef = FunctionRef>; +pub(crate) type DownloadRef = FunctionRef>; + +/// `downloadArtifact`'s reply as the loader flattens it. +#[napi(object)] +pub struct NativeDownload { + pub ok: bool, + pub value: Option, + pub kind: Option, + pub message: Option, +} + +/// The provider's functions, held on the JS thread until `finish()` turns +/// them into threadsafe functions. +pub(crate) struct ProviderRefs { + pub(crate) search_patches_batch: JsonRef, + pub(crate) search_patches_by_package: JsonRef, + pub(crate) fetch_registry_references: JsonRef, + pub(crate) fetch_patch: JsonRef, + pub(crate) download_artifact: DownloadRef, +} + +pub(crate) struct JsPatchApi { + search_patches_batch: JsonCall, + search_patches_by_package: JsonCall, + fetch_registry_references: JsonCall, + fetch_patch: JsonCall, + download_artifact: DownloadCall, +} + +fn json_call(env: &Env, function: &JsonRef) -> napi::Result { + function + .borrow_back(env)? + .build_threadsafe_function::() + .callee_handled::() + .build() +} + +impl JsPatchApi { + /// Must run on the JS thread. The threadsafe functions keep the event + /// loop alive until this value is dropped, which happens when the + /// engine future completes or is cancelled. + pub(crate) fn new(env: &Env, refs: &ProviderRefs) -> napi::Result { + Ok(Self { + search_patches_batch: json_call(env, &refs.search_patches_batch)?, + search_patches_by_package: json_call(env, &refs.search_patches_by_package)?, + fetch_registry_references: json_call(env, &refs.fetch_registry_references)?, + fetch_patch: json_call(env, &refs.fetch_patch)?, + download_artifact: refs + .download_artifact + .borrow_back(env)? + .build_threadsafe_function::() + .callee_handled::() + .build()?, + }) + } +} + +#[derive(Deserialize)] +struct ProviderFailure { + #[serde(default)] + kind: String, + #[serde(default)] + message: String, +} + +#[derive(Deserialize)] +struct Envelope { + ok: bool, + #[serde(default = "none")] + value: Option, + #[serde(default)] + error: Option, +} + +fn none() -> Option { + None +} + +#[derive(Deserialize)] +struct ReferencesBody { + #[serde(default)] + results: HashMap, +} + +fn provider_error(method: &str, kind: &str, message: &str) -> ApiError { + let detail = if message.is_empty() { + format!("patch provider {method} failed ({kind})") + } else { + format!("patch provider {method} failed ({kind}): {message}") + }; + match kind { + "unauthorized" => ApiError::Unauthorized(detail), + "forbidden" => ApiError::Forbidden(detail), + "rate_limited" => ApiError::RateLimited(detail), + "network" => ApiError::Network(detail), + "parse" => ApiError::Parse(detail), + _ => ApiError::Other(detail), + } +} + +fn transport_error(method: &str, error: &napi::Error) -> ApiError { + ApiError::Network(format!( + "patch provider {method} unavailable: {}", + error.reason + )) +} + +enum Reply { + /// A successful reply; `None` when its value is `null` (a missing patch). + Value(Option), + /// A `not_found` failure, which each method maps the way the HTTP client + /// maps that route's 404. + NotFound(String), +} + +async fn call_json( + method: &str, + function: &JsonCall, + request: String, +) -> Result, ApiError> { + let promise = function + .call_async_catch(request) + .await + .map_err(|e| transport_error(method, &e))?; + let reply = promise.await.map_err(|e| transport_error(method, &e))?; + let envelope: Envelope = serde_json::from_str(&reply).map_err(|e| { + ApiError::Parse(format!( + "patch provider {method} returned an unusable result: {e}" + )) + })?; + if envelope.ok { + return Ok(Reply::Value(envelope.value)); + } + let failure = envelope.error.unwrap_or(ProviderFailure { + kind: "other".to_string(), + message: String::new(), + }); + if failure.kind == "not_found" { + return Ok(Reply::NotFound(failure.message)); + } + Err(provider_error(method, &failure.kind, &failure.message)) +} + +fn missing_value(method: &str) -> ApiError { + ApiError::Parse(format!( + "patch provider {method} resolved ok without a value" + )) +} + +fn request_json(value: serde_json::Value) -> String { + value.to_string() +} + +impl PatchApi for JsPatchApi { + fn uses_public_proxy(&self) -> bool { + false + } + + fn search_patches_batch<'a>( + &'a self, + purls: &'a [String], + ) -> ApiFuture<'a, BatchSearchResponse> { + Box::pin(async move { + let components: Vec = purls + .iter() + .map(|purl| serde_json::json!({ "purl": purl })) + .collect(); + let request = request_json(serde_json::json!({ "components": components })); + let method = "searchPatchesBatch"; + // Like the HTTP collection-route 404: "no patches" is an empty + // success, so a miss is a misconfiguration, not zero patches. + let mut response: BatchSearchResponse = + match call_json(method, &self.search_patches_batch, request).await? { + Reply::Value(value) => value.ok_or_else(|| missing_value(method))?, + Reply::NotFound(message) => { + return Err(provider_error(method, "not_found", &message)) + } + }; + sort_batch_response(&mut response); + Ok(response) + }) + } + + fn search_patches_by_package<'a>(&'a self, purl: &'a str) -> ApiFuture<'a, SearchResponse> { + Box::pin(async move { + let method = "searchPatchesByPackage"; + let request = request_json(serde_json::json!({ "purl": purl })); + match call_json(method, &self.search_patches_by_package, request).await? { + Reply::Value(value) => value.ok_or_else(|| missing_value(method)), + Reply::NotFound(_) => Ok(SearchResponse { + patches: Vec::new(), + can_access_paid_patches: false, + }), + } + }) + } + + fn fetch_registry_references<'a>( + &'a self, + uuids: &'a [String], + ) -> ApiFuture<'a, HashMap> { + Box::pin(async move { + if uuids.is_empty() { + return Ok(HashMap::new()); + } + let method = "fetchRegistryReferences"; + let request = request_json(serde_json::json!({ "uuids": uuids })); + match call_json::(method, &self.fetch_registry_references, request) + .await? + { + Reply::Value(value) => Ok(value.ok_or_else(|| missing_value(method))?.results), + Reply::NotFound(_) => Ok(HashMap::new()), + } + }) + } + + fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option> { + Box::pin(async move { + let request = request_json(serde_json::json!({ "uuid": uuid })); + match call_json("fetchPatch", &self.fetch_patch, request).await? { + Reply::Value(value) => Ok(value), + Reply::NotFound(_) => Ok(None), + } + }) + } + + fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec> { + Box::pin(async move { + let method = "downloadArtifact"; + let request = request_json(serde_json::json!({ "url": url, "maxBytes": max_bytes })); + let promise = self + .download_artifact + .call_async_catch(request) + .await + .map_err(|e| transport_error(method, &e))?; + let reply = promise.await.map_err(|e| transport_error(method, &e))?; + if !reply.ok { + return Err(provider_error( + method, + reply.kind.as_deref().unwrap_or("other"), + reply.message.as_deref().unwrap_or(""), + )); + } + let buffer = reply.value.ok_or_else(|| missing_value(method))?; + if buffer.len() as u64 > max_bytes { + return Err(ApiError::Other(format!( + "patch provider {method} returned more than {max_bytes} bytes" + ))); + } + Ok(buffer.to_vec()) + }) + } +}