From f2f0677a37fa04c0ef69ee3b57bd25b33fb53078 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Sun, 11 Oct 2026 18:20:38 +0100 Subject: [PATCH 01/17] fix(tunnel): keep tunnel captures out of git Opening the capture database first adds `inspector.db*` to `.lpm/.gitignore`, so captures, which hold request headers and bodies, stay out of git wherever the project sits in a repository, without editing the project's own `.gitignore`. A linked `.lpm/.gitignore` is left alone. The capture warning names files git already tracks, which an ignore rule can't stop from being committed, and prints once: as a human warning, or on the log when that's hidden. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/dev.rs | 2 +- .../lpm-cli/src/commands/install/gitignore.rs | 8 +- crates/lpm-cli/src/commands/npmrc.rs | 2 +- crates/lpm-cli/src/commands/tunnel.rs | 108 +++++++++++++++-- crates/lpm-inspect/src/db.rs | 109 ++++++++++++++++++ 5 files changed, 210 insertions(+), 19 deletions(-) diff --git a/crates/lpm-cli/src/commands/dev.rs b/crates/lpm-cli/src/commands/dev.rs index 7ee054f47..ea5c415f8 100644 --- a/crates/lpm-cli/src/commands/dev.rs +++ b/crates/lpm-cli/src/commands/dev.rs @@ -1573,7 +1573,7 @@ pub async fn run( let (webhook_tx, mut webhook_rx) = tokio::sync::mpsc::channel::(64); - crate::commands::tunnel::warn_capture_persistence(!quiet); + crate::commands::tunnel::warn_capture_persistence(project_dir, !quiet); // Dashboard webhook channel: when --dashboard is active, webhooks are // forwarded to the dashboard TUI via a std::sync channel. diff --git a/crates/lpm-cli/src/commands/install/gitignore.rs b/crates/lpm-cli/src/commands/install/gitignore.rs index 0b77cf964..6d1c270ab 100644 --- a/crates/lpm-cli/src/commands/install/gitignore.rs +++ b/crates/lpm-cli/src/commands/install/gitignore.rs @@ -107,11 +107,9 @@ pub(super) fn maintain_project_linker_layout(project_dir: &Path, json_output: bo /// — ensure `.gitignore` contains an entry for /// `.lpm/wrappers/`. /// -/// Mirrors [`ensure_skills_gitignore`] (and its siblings in -/// `lpm-vault::ensure_lpm_dir_gitignore` and -/// `commands::npmrc::ensure_lpm_dir_gitignore`) — runtime "ensure -/// once" pattern, idempotent, OpenOptions-append to minimize the -/// read-then-write TOCTOU window. +/// Mirrors [`ensure_skills_gitignore`] — runtime "ensure once" pattern, +/// idempotent, OpenOptions-append to minimize the read-then-write TOCTOU +/// window. /// /// Why runtime instead of template emission: existing projects also /// need the entry, and the repo has a precedent for runtime diff --git a/crates/lpm-cli/src/commands/npmrc.rs b/crates/lpm-cli/src/commands/npmrc.rs index d77229baf..5b8de6613 100644 --- a/crates/lpm-cli/src/commands/npmrc.rs +++ b/crates/lpm-cli/src/commands/npmrc.rs @@ -535,7 +535,7 @@ pub(super) fn reject_tracked_npmrc(project_dir: &Path) -> Result<(), LpmError> { ))) } -fn hardened_git_command(project_dir: &Path, args: &[&str]) -> std::process::Command { +pub(crate) fn hardened_git_command(project_dir: &Path, args: &[&str]) -> std::process::Command { let mut command = std::process::Command::new("git"); command .args(args) diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index e0fff6142..ca7977b9e 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -476,7 +476,7 @@ pub(crate) async fn run_start( let inspector_db = open_inspector_db(project_dir)?; let inspector_state = lpm_inspect::state::InspectorState::with_db_for_target(local_target.clone(), inspector_db); - warn_capture_persistence(show_capture_warning); + warn_capture_persistence(project_dir, show_capture_warning); let inspector_handle = if no_inspect { None } else { @@ -747,24 +747,66 @@ pub(crate) async fn run_start( } } -pub(crate) fn warn_capture_persistence(show_human_warning: bool) { - tracing::warn!( - target: "lpm_cli::tunnel", +/// Warns that captures persist, and that git tracks them where it does: an +/// ignore rule doesn't stop a tracked file from being committed. +pub(crate) fn warn_capture_persistence(project_dir: &Path, show_human_warning: bool) { + capture_warning( "tunnel webhook capture persists full request/response bodies and headers \ - (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db — the \ - database uses mode 0600 on Unix and inherits project ACLs on Windows; \ - it survives commits / backups / IDE indexing. \ - Add `.lpm/inspector.db*` to .gitignore if you haven't already." + (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db (mode 0600 \ + on Unix; project ACLs on Windows); .lpm/.gitignore keeps it out of git", + show_human_warning, ); + let tracked = tracked_capture_files(project_dir); + if !tracked.is_empty() { + capture_warning( + &format!( + "git tracks {}, so captures can still be committed; run \ + `git rm --cached .lpm/inspector.db*` and commit", + tracked.join(", ") + ), + show_human_warning, + ); + } +} + +/// The human warning, or the same text on the log when it isn't shown, so +/// it isn't printed twice. +fn capture_warning(message: &str, show_human_warning: bool) { if show_human_warning { - install_ui::warn( - "tunnel webhook capture persists full request/response bodies and headers \ - in .lpm/inspector.db (mode 0600 on Unix; project ACLs on Windows). \ - Add `.lpm/inspector.db*` to .gitignore.", + install_ui::warn_untrusted(message); + } else { + tracing::warn!( + target: "lpm_cli::tunnel", + "{}", + lpm_common::sanitize_terminal_inline(message) ); } } +/// The capture files git tracks in `project_dir`. None when it isn't a +/// repository or git can't be run. +fn tracked_capture_files(project_dir: &Path) -> Vec { + let output = crate::commands::npmrc::hardened_git_command( + project_dir, + &["ls-files", "-z", "--", ".lpm/inspector.db*"], + ) + .stdin(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .output(); + let Ok(output) = output else { + return Vec::new(); + }; + if !output.status.success() { + return Vec::new(); + } + output + .stdout + .split(|byte| *byte == 0) + .filter(|name| !name.is_empty()) + .map(|name| String::from_utf8_lossy(name).into_owned()) + .collect() +} + fn compose_tunnel_control_and_connect( control_result: Result<(), LpmError>, connect_result: Result<(), LpmError>, @@ -1883,6 +1925,48 @@ fn is_valid_tunnel_domain(domain: &str) -> bool { mod tests { use super::*; + fn git(directory: &Path, args: &[&str]) { + let output = crate::commands::npmrc::hardened_git_command(directory, args) + .output() + .unwrap(); + assert!(output.status.success(), "git {args:?}: {output:?}"); + } + + #[test] + fn names_the_capture_files_git_tracks() { + let project = tempfile::tempdir().unwrap(); + git(project.path(), &["init", "--quiet"]); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + for name in ["inspector.db", "inspector.db-wal", "added-sources.json"] { + std::fs::write(project.path().join(".lpm").join(name), b"x").unwrap(); + } + git( + project.path(), + &[ + "add", + ".lpm/inspector.db", + ".lpm/inspector.db-wal", + ".lpm/added-sources.json", + ], + ); + + assert_eq!( + tracked_capture_files(project.path()), + [".lpm/inspector.db", ".lpm/inspector.db-wal"] + ); + } + + #[test] + fn names_no_capture_files_outside_a_repository_or_when_untracked() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + std::fs::write(project.path().join(".lpm/inspector.db"), b"x").unwrap(); + + assert!(tracked_capture_files(project.path()).is_empty()); + git(project.path(), &["init", "--quiet"]); + assert!(tracked_capture_files(project.path()).is_empty()); + } + #[test] fn non_interactive_tunnel_auth_output_never_contains_the_secret() { let output = tunnel_auth_header_summary("super-secret", false); diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index 59b21867f..af44c6771 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -64,6 +64,12 @@ impl InspectorDb { let lpm_dir = project_dir.join(".lpm"); std::fs::create_dir_all(&lpm_dir) .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?; + if let Err(error) = ensure_captures_ignored(&lpm_dir) { + tracing::warn!( + "could not add `{CAPTURES_IGNORE_LINE}` to {}: {error}", + lpm_dir.join(".gitignore").display() + ); + } let db_path = lpm_dir.join("inspector.db"); @@ -754,6 +760,41 @@ impl InspectorDb { } } +/// The `.lpm/.gitignore` line that keeps the capture database and its +/// SQLite sidecars out of git, since captures hold request headers and +/// bodies. +pub const CAPTURES_IGNORE_LINE: &str = "inspector.db*"; + +const GITIGNORE_SIZE_CAP_BYTES: u64 = 64 * 1024; + +/// Adds [`CAPTURES_IGNORE_LINE`] to `.lpm/.gitignore` unless it's there. +/// An ignore file inside `.lpm/` holds wherever the project sits in a +/// repository, and leaves the project's own `.gitignore` alone. A linked +/// `.lpm/.gitignore` is left as it is. +fn ensure_captures_ignored(lpm_dir: &Path) -> std::io::Result<()> { + let path = lpm_dir.join(".gitignore"); + let existing = match lpm_common::read_text_file_capped_nofollow(&path, GITIGNORE_SIZE_CAP_BYTES) + { + Ok(text) => text, + Err(lpm_common::BoundedReadError::NotFound { .. }) => String::new(), + Err(error) => return Err(std::io::Error::other(error)), + }; + if existing + .lines() + .any(|line| line.trim() == CAPTURES_IGNORE_LINE) + { + return Ok(()); + } + let mut updated = existing; + if !updated.is_empty() && !updated.ends_with('\n') { + updated.push('\n'); + } + updated.push_str("# LPM tunnel captures hold request headers and bodies.\n"); + updated.push_str(CAPTURES_IGNORE_LINE); + updated.push('\n'); + lpm_common::write_file_atomic(&path, updated) +} + // ── Schema initialization ───────────────────────────────────────────── #[cfg(unix)] @@ -1253,6 +1294,74 @@ mod tests { use lpm_tunnel::webhook::WebhookProvider; use std::collections::HashMap; + const CAPTURES_IGNORED: &str = + "# LPM tunnel captures hold request headers and bodies.\ninspector.db*\n"; + + fn lpm_gitignore(project: &Path) -> String { + std::fs::read_to_string(project.join(".lpm/.gitignore")).unwrap() + } + + #[tokio::test] + async fn opening_keeps_captures_out_of_git_once() { + let project = tempfile::tempdir().unwrap(); + + drop(InspectorDb::open(project.path()).unwrap()); + drop(InspectorDb::open(project.path()).unwrap()); + + assert_eq!(lpm_gitignore(project.path()), CAPTURES_IGNORED); + } + + #[tokio::test] + async fn opening_adds_the_capture_rule_to_an_lpm_gitignore_of_the_project() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + std::fs::write(project.path().join(".lpm/.gitignore"), "skills/").unwrap(); + + drop(InspectorDb::open(project.path()).unwrap()); + + assert_eq!( + lpm_gitignore(project.path()), + format!("skills/\n{CAPTURES_IGNORED}") + ); + } + + #[tokio::test] + async fn opening_leaves_an_lpm_gitignore_that_already_ignores_captures() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + std::fs::write( + project.path().join(".lpm/.gitignore"), + " inspector.db* \n", + ) + .unwrap(); + + drop(InspectorDb::open(project.path()).unwrap()); + + assert_eq!(lpm_gitignore(project.path()), " inspector.db* \n"); + } + + #[cfg(unix)] + #[tokio::test] + async fn opening_leaves_a_linked_lpm_gitignore_and_its_target_alone() { + let project = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let target = outside.path().join("profile"); + std::fs::write(&target, "export PATH\n").unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let link = project.path().join(".lpm/.gitignore"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + drop(InspectorDb::open(project.path()).unwrap()); + + assert_eq!(std::fs::read_to_string(&target).unwrap(), "export PATH\n"); + assert!( + std::fs::symlink_metadata(&link) + .unwrap() + .file_type() + .is_symlink() + ); + } + #[test] fn capture_schema_upgrade_serializes_concurrent_openers() { std::thread_local! { From c650ea4410e42db97b0187a5316312e0b0521560 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Sun, 11 Oct 2026 20:26:46 +0100 Subject: [PATCH 02/17] fix(tunnel): run no repository code, and say only what git confirms about captures The git call uses `-c core.fsmonitor=false -c safe.bareRepository=explicit`, so neither an archived project's fsmonitor nor a bare repository a cloned project carries can run code, and runs async with a 2 s timeout. The capture warning asks git which capture files it tracks or doesn't ignore, case-insensitively, says captures are kept out of git only when git ignores them all, and names tracked files quoted, with a `git rm --cached` command that works and the step to rotate what they held. `.lpm/.gitignore` lines are read as git reads them, a later negation needs the rule again, CRLF files keep their line endings, and a read-only file is left alone. A linked `.lpm` or capture file is refused, the database is created owner-only before SQLite opens it with no-follow, and existing capture files are made owner-only. Workflow tests check the warning appears once and names a tracked capture. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/dev.rs | 4 +- crates/lpm-cli/src/commands/npmrc.rs | 10 + crates/lpm-cli/src/commands/tunnel.rs | 291 ++++++++++++++++++----- crates/lpm-inspect/src/db.rs | 318 ++++++++++++++++++++++---- tests/workflows/tests/dev_tunnel.rs | 69 +++++- 5 files changed, 586 insertions(+), 106 deletions(-) diff --git a/crates/lpm-cli/src/commands/dev.rs b/crates/lpm-cli/src/commands/dev.rs index ea5c415f8..f1a49011e 100644 --- a/crates/lpm-cli/src/commands/dev.rs +++ b/crates/lpm-cli/src/commands/dev.rs @@ -1542,7 +1542,7 @@ pub async fn run( let inspector_db = lpm_inspect::db::InspectorDb::open(project_dir).map_err(|error| { LpmError::Tunnel(format!( "failed to open this project's .lpm/inspector.db: {error}. \ - Repair or move the corrupt database and retry" + Move or repair it and retry" )) })?; let inspector_state = lpm_inspect::state::InspectorState::with_db_pending(inspector_db); @@ -1573,7 +1573,7 @@ pub async fn run( let (webhook_tx, mut webhook_rx) = tokio::sync::mpsc::channel::(64); - crate::commands::tunnel::warn_capture_persistence(project_dir, !quiet); + crate::commands::tunnel::warn_capture_persistence(project_dir, !quiet).await; // Dashboard webhook channel: when --dashboard is active, webhooks are // forwarded to the dashboard TUI via a std::sync channel. diff --git a/crates/lpm-cli/src/commands/npmrc.rs b/crates/lpm-cli/src/commands/npmrc.rs index 5b8de6613..5308d6e5f 100644 --- a/crates/lpm-cli/src/commands/npmrc.rs +++ b/crates/lpm-cli/src/commands/npmrc.rs @@ -535,9 +535,19 @@ pub(super) fn reject_tracked_npmrc(project_dir: &Path) -> Result<(), LpmError> { ))) } +/// A git command for a project that may be untrusted: no `GIT_*` variable +/// from the environment, no `core.fsmonitor` program, which reading the +/// index would run, and no repository found as a bare one, which a cloned +/// project can contain with its own config. pub(crate) fn hardened_git_command(project_dir: &Path, args: &[&str]) -> std::process::Command { let mut command = std::process::Command::new("git"); command + .args([ + "-c", + "core.fsmonitor=false", + "-c", + "safe.bareRepository=explicit", + ]) .args(args) .current_dir(project_dir) .env("LC_ALL", "C"); diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index ca7977b9e..c9000473d 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -476,7 +476,7 @@ pub(crate) async fn run_start( let inspector_db = open_inspector_db(project_dir)?; let inspector_state = lpm_inspect::state::InspectorState::with_db_for_target(local_target.clone(), inspector_db); - warn_capture_persistence(project_dir, show_capture_warning); + warn_capture_persistence(project_dir, show_capture_warning).await; let inspector_handle = if no_inspect { None } else { @@ -747,22 +747,52 @@ pub(crate) async fn run_start( } } -/// Warns that captures persist, and that git tracks them where it does: an -/// ignore rule doesn't stop a tracked file from being committed. -pub(crate) fn warn_capture_persistence(project_dir: &Path, show_human_warning: bool) { +/// Warns that captures persist, and says how git treats them: kept out of +/// git only where git confirms it ignores them; files git tracks, which an +/// ignore rule can't keep from being committed; and files it neither tracks +/// nor ignores. +pub(crate) async fn warn_capture_persistence(project_dir: &Path, show_human_warning: bool) { + let state = capture_git_state(project_dir).await; + let kept_out = + matches!(state, Some(ref files) if files.tracked.is_empty() && files.unignored.is_empty()); capture_warning( - "tunnel webhook capture persists full request/response bodies and headers \ - (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db (mode 0600 \ - on Unix; project ACLs on Windows); .lpm/.gitignore keeps it out of git", + &format!( + "tunnel webhook capture persists full request/response bodies and headers \ + (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db (owner-only \ + on Unix; project ACLs on Windows){}", + if kept_out { + "; .lpm/.gitignore keeps it out of git" + } else { + "" + } + ), show_human_warning, ); - let tracked = tracked_capture_files(project_dir); - if !tracked.is_empty() { + let Some(files) = state else { + return; + }; + if !files.tracked.is_empty() { + let names = files + .tracked + .iter() + .map(|name| shell_quote(name)) + .collect::>() + .join(" "); capture_warning( &format!( - "git tracks {}, so captures can still be committed; run \ - `git rm --cached .lpm/inspector.db*` and commit", - tracked.join(", ") + "git tracks {}, so captures there can be committed and stay in its \ + history; run `git rm --cached -- {names}`, commit, and rotate any \ + credentials they held", + listed(&files.tracked) + ), + show_human_warning, + ); + } + if !files.unignored.is_empty() { + capture_warning( + &format!( + "git doesn't ignore {}; add `inspector.db*` to .lpm/.gitignore", + listed(&files.unignored) ), show_human_warning, ); @@ -783,28 +813,84 @@ fn capture_warning(message: &str, show_human_warning: bool) { } } -/// The capture files git tracks in `project_dir`. None when it isn't a -/// repository or git can't be run. -fn tracked_capture_files(project_dir: &Path) -> Vec { - let output = crate::commands::npmrc::hardened_git_command( +/// Capture files git tracks, and those it neither tracks nor ignores. +#[derive(Debug, Default, PartialEq, Eq)] +struct CaptureGitFiles { + tracked: Vec, + unignored: Vec, +} + +const CAPTURE_GIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + +/// How git treats the capture files in `project_dir`, matching their names +/// without regard to case, as a case-insensitive disk does; None when it +/// isn't a repository, git can't be run, or it doesn't answer in time. +async fn capture_git_state(project_dir: &Path) -> Option { + let mut command = tokio::process::Command::from(crate::commands::npmrc::hardened_git_command( project_dir, - &["ls-files", "-z", "--", ".lpm/inspector.db*"], - ) - .stdin(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .output(); - let Ok(output) = output else { - return Vec::new(); - }; + &[ + "ls-files", + "-z", + "-t", + "--cached", + "--others", + "--exclude-standard", + "--", + ":(glob,icase).lpm/inspector.db*", + ], + )); + command + .stdin(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true); + let output = tokio::time::timeout(CAPTURE_GIT_TIMEOUT, command.output()) + .await + .ok()? + .ok()?; if !output.status.success() { - return Vec::new(); - } - output - .stdout - .split(|byte| *byte == 0) - .filter(|name| !name.is_empty()) - .map(|name| String::from_utf8_lossy(name).into_owned()) - .collect() + return None; + } + Some(parse_capture_git_files(&output.stdout)) +} + +fn parse_capture_git_files(stdout: &[u8]) -> CaptureGitFiles { + let mut files = CaptureGitFiles::default(); + for record in stdout.split(|byte| *byte == 0) { + // `-t` tags each name: `?` for one git neither tracks nor ignores, + // a letter for one it tracks. + let (Some(&tag), Some(name)) = (record.first(), record.get(2..)) else { + continue; + }; + let name = String::from_utf8_lossy(name).into_owned(); + if tag == b'?' { + files.unignored.push(name); + } else { + files.tracked.push(name); + } + } + files +} + +const LISTED_FILES: usize = 5; + +/// Names quoted, at most `LISTED_FILES` of them, so text from the +/// repository reads as names in the warning. +fn listed(names: &[String]) -> String { + let mut listed = names + .iter() + .take(LISTED_FILES) + .map(|name| format!("'{name}'")) + .collect::>() + .join(", "); + if names.len() > LISTED_FILES { + listed.push_str(&format!(" and {} more", names.len() - LISTED_FILES)); + } + listed +} + +/// A POSIX shell word for `value`. +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) } fn compose_tunnel_control_and_connect( @@ -1318,7 +1404,7 @@ fn open_inspector_db(project_dir: &Path) -> Result tempfile::TempDir { let project = tempfile::tempdir().unwrap(); git(project.path(), &["init", "--quiet"]); - std::fs::create_dir(project.path().join(".lpm")).unwrap(); - for name in ["inspector.db", "inspector.db-wal", "added-sources.json"] { - std::fs::write(project.path().join(".lpm").join(name), b"x").unwrap(); + for name in files { + let path = project.path().join(name); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, b"x").unwrap(); } + project + } + + fn names(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_string()).collect() + } + + #[tokio::test] + async fn tells_capture_files_git_tracks_from_those_it_neither_tracks_nor_ignores() { + let project = capture_project(&[ + ".lpm/inspector.db", + ".lpm/inspector.db-wal", + ".lpm/inspector.db-shm", + ".lpm/added-sources.json", + ".lpm/inspector.db.d/notes.md", + ]); git( project.path(), - &[ - "add", - ".lpm/inspector.db", - ".lpm/inspector.db-wal", - ".lpm/added-sources.json", - ], + &["add", ".lpm/inspector.db", ".lpm/added-sources.json"], ); + let files = capture_git_state(project.path()).await.unwrap(); + + assert_eq!(files.tracked, names(&[".lpm/inspector.db"])); + let mut unignored = files.unignored; + unignored.sort(); assert_eq!( - tracked_capture_files(project.path()), - [".lpm/inspector.db", ".lpm/inspector.db-wal"] + unignored, + names(&[".lpm/inspector.db-shm", ".lpm/inspector.db-wal"]) ); } - #[test] - fn names_no_capture_files_outside_a_repository_or_when_untracked() { + #[tokio::test] + async fn finds_capture_files_ignored_by_lpm_gitignore_kept_out() { + let project = capture_project(&[".lpm/inspector.db", ".lpm/inspector.db-wal"]); + std::fs::write(project.path().join(".lpm/.gitignore"), "inspector.db*\n").unwrap(); + + assert_eq!( + capture_git_state(project.path()).await, + Some(CaptureGitFiles::default()) + ); + } + + #[tokio::test] + async fn names_tracked_capture_files_whatever_their_case() { + let project = capture_project(&[".LPM/Inspector.DB"]); + git(project.path(), &["add", ".LPM/Inspector.DB"]); + + let files = capture_git_state(project.path()).await.unwrap(); + + assert_eq!(files.tracked, names(&[".LPM/Inspector.DB"])); + } + + #[tokio::test] + async fn says_nothing_of_git_outside_a_repository() { let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); std::fs::write(project.path().join(".lpm/inspector.db"), b"x").unwrap(); - assert!(tracked_capture_files(project.path()).is_empty()); - git(project.path(), &["init", "--quiet"]); - assert!(tracked_capture_files(project.path()).is_empty()); + assert_eq!(capture_git_state(project.path()).await, None); + } + + #[cfg(unix)] + #[tokio::test] + async fn runs_no_fsmonitor_program_from_the_repository() { + let project = capture_project(&[".lpm/inspector.db"]); + let marker = project.path().join("fsmonitor-ran"); + git( + project.path(), + &[ + "config", + "core.fsmonitor", + &format!("touch '{}'; false", marker.display()), + ], + ); + + capture_git_state(project.path()).await; + + assert!(!marker.exists()); + } + + #[cfg(unix)] + #[tokio::test] + async fn refuses_a_bare_repository_a_project_carries() { + let project = tempfile::tempdir().unwrap(); + let web = project.path().join("web"); + for directory in ["objects", "refs", ".lpm"] { + std::fs::create_dir_all(web.join(directory)).unwrap(); + } + std::fs::write(web.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + let marker = project.path().join("fsmonitor-ran"); + std::fs::write( + web.join("config"), + format!( + "[core]\n\tbare = false\n\tworktree = .\n\tfsmonitor = touch '{}'; false\n", + marker.display() + ), + ) + .unwrap(); + std::fs::write(web.join(".lpm/inspector.db"), b"x").unwrap(); + + assert_eq!(capture_git_state(&web).await, None); + assert!(!marker.exists()); + } + + #[test] + fn lists_at_most_five_names_quoted() { + let many: Vec = (0..7) + .map(|index| format!(".lpm/inspector.db{index}")) + .collect(); + + assert_eq!( + listed(&many), + "'.lpm/inspector.db0', '.lpm/inspector.db1', '.lpm/inspector.db2', \ + '.lpm/inspector.db3', '.lpm/inspector.db4' and 2 more" + ); + } + + #[test] + fn quotes_names_for_a_posix_shell() { + assert_eq!(shell_quote(".lpm/inspector.db"), "'.lpm/inspector.db'"); + assert_eq!(shell_quote(".lpm/it's.db"), "'.lpm/it'\\''s.db'"); } #[test] diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index af44c6771..d3714cffa 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -62,26 +62,21 @@ impl InspectorDb { /// write-batching task. pub fn open(project_dir: &Path) -> Result { let lpm_dir = project_dir.join(".lpm"); - std::fs::create_dir_all(&lpm_dir) + let db_path = lpm_dir.join("inspector.db"); + prepare_capture_files(&lpm_dir, &db_path) .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?; if let Err(error) = ensure_captures_ignored(&lpm_dir) { - tracing::warn!( - "could not add `{CAPTURES_IGNORE_LINE}` to {}: {error}", - lpm_dir.join(".gitignore").display() - ); + tracing::warn!("could not keep tunnel captures out of git: {error}"); } - let db_path = lpm_dir.join("inspector.db"); - // Write connection (owned by the background flush task) - let write_conn = Connection::open(&db_path)?; - restrict_database_permissions(&db_path)?; + let write_conn = open_capture_connection(&db_path)?; init_schema(&write_conn)?; // Read connection (shared across API handlers). // busy_timeout ensures low-frequency mutations (tag updates, session renames) // wait gracefully if the write connection holds a lock during batch flushes. - let read_conn = Connection::open(&db_path)?; + let read_conn = open_capture_connection(&db_path)?; read_conn.execute_batch( "PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; @@ -763,55 +758,192 @@ impl InspectorDb { /// The `.lpm/.gitignore` line that keeps the capture database and its /// SQLite sidecars out of git, since captures hold request headers and /// bodies. -pub const CAPTURES_IGNORE_LINE: &str = "inspector.db*"; +const CAPTURES_IGNORE_LINE: &str = "inspector.db*"; + +/// The capture database and the sidecar files SQLite keeps beside it. +fn capture_files(db_path: &Path) -> [std::path::PathBuf; 4] { + let named = |suffix: &str| { + let mut name = db_path.as_os_str().to_os_string(); + name.push(suffix); + std::path::PathBuf::from(name) + }; + [ + db_path.to_path_buf(), + named("-wal"), + named("-shm"), + named("-journal"), + ] +} + +fn linked_or_not(path: &Path, expected: &str) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "{} is a link or not a {expected}; tunnel captures, which hold request \ + headers and bodies, are kept only in the project's own files", + path.display() + ), + ) +} + +/// Makes `.lpm/` and the capture files safe to write: neither `.lpm` nor +/// any capture file may be a link, which a cloned repository could point +/// anywhere, such as into a directory it serves; existing capture files are +/// made owner-only, as a checkout or an older release may have left them +/// readable; and a new database is created owner-only before SQLite opens +/// it, so it's never readable by others. +fn prepare_capture_files(lpm_dir: &Path, db_path: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(lpm_dir) { + Ok(metadata) if lpm_common::is_symlink_or_junction(&metadata) || !metadata.is_dir() => { + return Err(linked_or_not(lpm_dir, "directory")); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir_all(lpm_dir)?; + } + Err(error) => return Err(error), + } + for path in capture_files(db_path) { + match std::fs::symlink_metadata(&path) { + Ok(metadata) + if lpm_common::is_symlink_or_junction(&metadata) || !metadata.is_file() => + { + return Err(linked_or_not(&path, "file")); + } + Ok(_) => make_owner_only(&path)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + create_owner_only(db_path) +} -const GITIGNORE_SIZE_CAP_BYTES: u64 = 64 * 1024; +#[cfg(unix)] +fn make_owner_only(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) +} -/// Adds [`CAPTURES_IGNORE_LINE`] to `.lpm/.gitignore` unless it's there. -/// An ignore file inside `.lpm/` holds wherever the project sits in a -/// repository, and leaves the project's own `.gitignore` alone. A linked -/// `.lpm/.gitignore` is left as it is. +#[cfg(not(unix))] +fn make_owner_only(_path: &Path) -> std::io::Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn create_owner_only(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::OpenOptionsExt; + + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + { + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), + Err(error) => Err(error), + } +} + +#[cfg(not(unix))] +fn create_owner_only(_path: &Path) -> std::io::Result<()> { + Ok(()) +} + +/// Opens the capture database without following a link at its path. +/// SQLite's no-follow open refuses a link anywhere in the path, as macOS's +/// `/var` and `/tmp` are, so it gets the directory's real path, which leaves +/// only the database itself to refuse as a link. +fn open_capture_connection(path: &Path) -> Result { + let directory = match path.parent() { + Some(directory) if !directory.as_os_str().is_empty() => directory, + _ => Path::new("."), + }; + let real_path = std::fs::canonicalize(directory) + .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))? + .join(path.file_name().unwrap_or(path.as_os_str())); + Connection::open_with_flags( + real_path, + rusqlite::OpenFlags::default() | rusqlite::OpenFlags::SQLITE_OPEN_NOFOLLOW, + ) +} + +/// A `.gitignore` line as git reads it: without a trailing carriage return +/// or trailing spaces that aren't escaped. Leading whitespace stays part of +/// the pattern, as it does for git. +fn git_pattern(line: &str) -> &str { + let line = line.strip_suffix('\r').unwrap_or(line); + let bytes = line.as_bytes(); + let mut end = bytes.len(); + while end > 0 && bytes[end - 1] == b' ' && !(end >= 2 && bytes[end - 2] == b'\\') { + end -= 1; + } + &line[..end] +} + +/// Whether these `.gitignore` lines leave the capture files ignored: the +/// rule, unanchored or anchored to `.lpm/`, after the last line that could +/// negate it. +fn ignores_captures(lines: &str) -> bool { + let mut ignored = false; + for line in lines.lines() { + let pattern = git_pattern(line); + if pattern == CAPTURES_IGNORE_LINE + || pattern.strip_prefix('/') == Some(CAPTURES_IGNORE_LINE) + { + ignored = true; + } else if pattern.starts_with('!') && pattern.contains("inspector.db") { + ignored = false; + } + } + ignored +} + +/// Adds [`CAPTURES_IGNORE_LINE`] to `.lpm/.gitignore` unless the file +/// already ignores the capture files. An ignore file inside `.lpm/` holds +/// wherever the project sits in a repository, and leaves the project's own +/// `.gitignore` alone. A linked or read-only `.lpm/.gitignore` is left as it +/// is; the CLI's capture warning then says git doesn't ignore the captures. fn ensure_captures_ignored(lpm_dir: &Path) -> std::io::Result<()> { let path = lpm_dir.join(".gitignore"); - let existing = match lpm_common::read_text_file_capped_nofollow(&path, GITIGNORE_SIZE_CAP_BYTES) - { + let existing = match lpm_common::read_text_file_capped_nofollow( + &path, + lpm_common::CONFIG_FILE_SIZE_CAP_BYTES, + ) { Ok(text) => text, Err(lpm_common::BoundedReadError::NotFound { .. }) => String::new(), Err(error) => return Err(std::io::Error::other(error)), }; - if existing - .lines() - .any(|line| line.trim() == CAPTURES_IGNORE_LINE) - { + if ignores_captures(&existing) { return Ok(()); } + if !existing.is_empty() && std::fs::metadata(&path)?.permissions().readonly() { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!("{} is read-only", path.display()), + )); + } + let newline = if existing.contains("\r\n") { + "\r\n" + } else { + "\n" + }; let mut updated = existing; if !updated.is_empty() && !updated.ends_with('\n') { - updated.push('\n'); + updated.push_str(newline); } - updated.push_str("# LPM tunnel captures hold request headers and bodies.\n"); + updated.push_str("# LPM tunnel captures hold request headers and bodies."); + updated.push_str(newline); updated.push_str(CAPTURES_IGNORE_LINE); - updated.push('\n'); + updated.push_str(newline); lpm_common::write_file_atomic(&path, updated) } // ── Schema initialization ───────────────────────────────────────────── -#[cfg(unix)] -fn restrict_database_permissions(path: &Path) -> Result<(), rusqlite::Error> { - use std::os::unix::fs::PermissionsExt; - - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) - .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error))) -} - -#[cfg(not(unix))] -fn restrict_database_permissions(_path: &Path) -> Result<(), rusqlite::Error> { - Ok(()) -} - fn open_control_connection(path: &Path) -> Result { - let connection = Connection::open(path)?; + let connection = open_capture_connection(path)?; connection.execute_batch( "PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; @@ -1325,19 +1457,113 @@ mod tests { ); } + fn opened_with_gitignore(content: &str) -> (tempfile::TempDir, String) { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + std::fs::write(project.path().join(".lpm/.gitignore"), content).unwrap(); + drop(InspectorDb::open(project.path()).unwrap()); + let written = lpm_gitignore(project.path()); + (project, written) + } + + #[tokio::test] + async fn opening_leaves_an_lpm_gitignore_git_reads_as_ignoring_captures() { + for content in [ + "inspector.db*\n", + "inspector.db* \n", + "inspector.db*\r\n", + "/inspector.db*\n", + "!inspector.db\ninspector.db*\n", + ] { + let (_project, written) = opened_with_gitignore(content); + assert_eq!(written, content, "{content:?}"); + } + } + + #[tokio::test] + async fn opening_adds_the_rule_where_git_wouldnt_read_one() { + for content in [ + " inspector.db*\n", + "\tinspector.db*\n", + "inspector.db*\t\n", + "inspector.db*\\ \n", + "inspector.db*\n!inspector.db\n", + ] { + let (_project, written) = opened_with_gitignore(content); + assert_eq!( + written, + format!("{content}{CAPTURES_IGNORED}"), + "{content:?}" + ); + } + } + + #[tokio::test] + async fn opening_adds_the_rule_with_the_file_s_line_endings() { + let (_project, written) = opened_with_gitignore("skills/\r\n"); + + assert_eq!( + written, + "skills/\r\n# LPM tunnel captures hold request headers and bodies.\r\ninspector.db*\r\n" + ); + } + + #[cfg(unix)] #[tokio::test] - async fn opening_leaves_an_lpm_gitignore_that_already_ignores_captures() { + async fn opening_leaves_a_read_only_lpm_gitignore_alone() { + use std::os::unix::fs::PermissionsExt; + let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); - std::fs::write( - project.path().join(".lpm/.gitignore"), - " inspector.db* \n", - ) - .unwrap(); + let path = project.path().join(".lpm/.gitignore"); + std::fs::write(&path, "skills/\n").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o444)).unwrap(); drop(InspectorDb::open(project.path()).unwrap()); - assert_eq!(lpm_gitignore(project.path()), " inspector.db* \n"); + assert_eq!(lpm_gitignore(project.path()), "skills/\n"); + } + + #[cfg(unix)] + #[tokio::test] + async fn opening_refuses_a_linked_lpm_directory_or_capture_file() { + let outside = tempfile::tempdir().unwrap(); + let linked_dir = tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink(outside.path(), linked_dir.path().join(".lpm")).unwrap(); + assert!(InspectorDb::open(linked_dir.path()).is_err()); + assert!(!outside.path().join("inspector.db").exists()); + + for name in ["inspector.db", "inspector.db-wal"] { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let target = outside.path().join(format!("{name}-target")); + std::os::unix::fs::symlink(&target, project.path().join(".lpm").join(name)).unwrap(); + + assert!(InspectorDb::open(project.path()).is_err(), "{name}"); + assert!(!target.exists(), "{name}"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn opening_keeps_capture_files_owner_only() { + use std::os::unix::fs::PermissionsExt; + + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let wal = project.path().join(".lpm/inspector.db-wal"); + std::fs::write(&wal, b"").unwrap(); + std::fs::set_permissions(&wal, std::fs::Permissions::from_mode(0o644)).unwrap(); + + drop(InspectorDb::open(project.path()).unwrap()); + + for name in ["inspector.db", "inspector.db-wal"] { + let mode = std::fs::metadata(project.path().join(".lpm").join(name)) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "{name}"); + } } #[cfg(unix)] diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index 1e803e4ff..e67dd388f 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1265,8 +1265,9 @@ async fn tunnel_json_reports_a_retry_before_a_successful_reconnection() { "#); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn tunnel_start_warns_that_capture_history_persists_sensitive_request_data() { +/// Starts `lpm tunnel 5173` against a relay that greets it and closes, in a +/// project `prepare` sets up, and returns its stderr. +async fn tunnel_start_stderr(prepare: impl FnOnce(&Path) + Send + 'static) -> String { let relay = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let relay_url = format!("ws://{}/connect", relay.local_addr().unwrap()); let relay_task = tokio::spawn(async move { @@ -1296,6 +1297,7 @@ async fn tunnel_start_warns_that_capture_history_persists_sensitive_request_data let output_task = tokio::task::spawn_blocking(move || { let project = TempProject::empty(r#"{"name":"tunnel","version":"1.0.0"}"#); + prepare(project.path()); let mut command = lpm_spawnable(&project); command.env("LPM_TUNNEL_RELAY", relay_url).args([ "--token", @@ -1313,16 +1315,71 @@ async fn tunnel_start_warns_that_capture_history_persists_sensitive_request_data "tunnel command failed: {}", String::from_utf8_lossy(&output.stderr) ); - let stderr = String::from_utf8_lossy(&output.stderr); + String::from_utf8_lossy(&output.stderr).into_owned() +} + +fn git(directory: &Path, args: &[&str]) { + let output = std::process::Command::new("git") + .args(args) + .current_dir(directory) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_NOSYSTEM", "1") + .output() + .unwrap(); + assert!(output.status.success(), "git {args:?}: {output:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tunnel_start_warns_that_capture_history_persists_sensitive_request_data() { + let stderr = tunnel_start_stderr(|_| {}).await; + + assert_eq!( + stderr + .matches("tunnel webhook capture persists full request/response bodies and headers") + .count(), + 1, + "human tunnel startup must disclose capture persistence once:\n{stderr}" + ); assert!( - stderr.contains("tunnel webhook capture persists full request/response bodies and headers") - && stderr.contains(".lpm/inspector.db") - && stderr.contains("mode 0600 on Unix") + stderr.contains(".lpm/inspector.db") + && stderr.contains("owner-only on Unix") && stderr.contains("project ACLs on Windows"), "human tunnel startup must disclose capture persistence:\n{stderr}" ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tunnel_start_names_capture_files_git_tracks_and_how_to_stop_it() { + let stderr = tunnel_start_stderr(|project| { + git(project, &["init", "--quiet"]); + std::fs::create_dir(project.join(".lpm")).unwrap(); + std::fs::write(project.join(".lpm/inspector.db"), b"").unwrap(); + git(project, &["add", ".lpm/inspector.db"]); + }) + .await; + + assert!( + stderr.contains("git tracks '.lpm/inspector.db'") + && stderr.contains("git rm --cached -- '.lpm/inspector.db'") + && stderr.contains("rotate any credentials"), + "tunnel startup must name tracked capture files:\n{stderr}" + ); + assert!( + !stderr.contains("keeps it out of git"), + "a tracked capture isn't kept out of git:\n{stderr}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tunnel_start_says_captures_are_kept_out_of_git_once_git_ignores_them() { + let stderr = tunnel_start_stderr(|project| git(project, &["init", "--quiet"])).await; + + assert!( + stderr.contains(".lpm/.gitignore keeps it out of git") && !stderr.contains("git tracks"), + "{stderr}" + ); +} + #[test] fn tunnel_claim_without_auth_under_json_emits_error_envelope_on_stdout() { let project = TempProject::empty(r#"{"name":"tunnel","version":"1.0.0"}"#); From 37556b84b7167e8da07f5a5b4cf001cf6eb4f408 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Sun, 11 Oct 2026 21:02:39 +0100 Subject: [PATCH 03/17] test(tunnel): check linked sidecars and leftover capture file modes directly A linked sidecar now points at an existing outside file, so following it would change that file's mode or content. Leftover sidecars are made owner-only by the preparation step itself, before SQLite opens them. The corrupt-database workflow test matches the reworded advice. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-inspect/src/db.rs | 46 ++++++++++++++++++++++++++++- tests/workflows/tests/dev_tunnel.rs | 2 +- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index d3714cffa..b39601dc8 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -1534,13 +1534,57 @@ mod tests { assert!(!outside.path().join("inspector.db").exists()); for name in ["inspector.db", "inspector.db-wal"] { + use std::os::unix::fs::PermissionsExt; + let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let target = outside.path().join(format!("{name}-target")); + std::fs::write(&target, "outside\n").unwrap(); + std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o644)).unwrap(); std::os::unix::fs::symlink(&target, project.path().join(".lpm").join(name)).unwrap(); assert!(InspectorDb::open(project.path()).is_err(), "{name}"); - assert!(!target.exists(), "{name}"); + assert_eq!( + std::fs::read_to_string(&target).unwrap(), + "outside\n", + "{name}" + ); + let mode = std::fs::metadata(&target).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o644, "{name}"); + } + } + + #[cfg(unix)] + #[test] + fn preparing_makes_leftover_capture_files_owner_only_before_sqlite_opens_them() { + use std::os::unix::fs::PermissionsExt; + + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + for name in [ + "inspector.db-wal", + "inspector.db-shm", + "inspector.db-journal", + ] { + let path = lpm_dir.join(name); + std::fs::write(&path, b"").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + } + + prepare_capture_files(&lpm_dir, &lpm_dir.join("inspector.db")).unwrap(); + + for name in [ + "inspector.db", + "inspector.db-wal", + "inspector.db-shm", + "inspector.db-journal", + ] { + let mode = std::fs::metadata(lpm_dir.join(name)) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "{name}"); } } diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index e67dd388f..9415cf6ef 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1615,7 +1615,7 @@ fn tunnel_inspect_reports_an_actionable_corrupt_database_error() { String::from_utf8_lossy(&output.stderr) ); assert!(rendered.contains(".lpm/inspector.db")); - assert!(rendered.contains("Repair or move")); + assert!(rendered.contains("Move or repair")); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From 2edf4d13a364f5c722056755b64ef8363c58af82 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Sun, 11 Oct 2026 21:10:08 +0100 Subject: [PATCH 04/17] fix(tunnel): keep the untrack advice as short as the list it follows A repository tracking more capture files than the warning lists gets the capture pathspec in its `git rm --cached` command instead of every name. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/tunnel.rs | 50 +++++++++++++++++++++------ 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index c9000473d..ff755d0e9 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -772,18 +772,12 @@ pub(crate) async fn warn_capture_persistence(project_dir: &Path, show_human_warn return; }; if !files.tracked.is_empty() { - let names = files - .tracked - .iter() - .map(|name| shell_quote(name)) - .collect::>() - .join(" "); capture_warning( &format!( "git tracks {}, so captures there can be committed and stay in its \ - history; run `git rm --cached -- {names}`, commit, and rotate any \ - credentials they held", - listed(&files.tracked) + history; run `{}`, commit, and rotate any credentials they held", + listed(&files.tracked), + untrack_command(&files.tracked) ), show_human_warning, ); @@ -820,6 +814,8 @@ struct CaptureGitFiles { unignored: Vec, } +const CAPTURES_PATHSPEC: &str = ":(glob,icase).lpm/inspector.db*"; + const CAPTURE_GIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); /// How git treats the capture files in `project_dir`, matching their names @@ -836,7 +832,7 @@ async fn capture_git_state(project_dir: &Path) -> Option { "--others", "--exclude-standard", "--", - ":(glob,icase).lpm/inspector.db*", + CAPTURES_PATHSPEC, ], )); command @@ -888,6 +884,21 @@ fn listed(names: &[String]) -> String { listed } +/// The command that stops git tracking `tracked`: their names, or the +/// capture pathspec when there are more than the warning lists. +fn untrack_command(tracked: &[String]) -> String { + let paths = if tracked.len() > LISTED_FILES { + shell_quote(CAPTURES_PATHSPEC) + } else { + tracked + .iter() + .map(|name| shell_quote(name)) + .collect::>() + .join(" ") + }; + format!("git rm --cached -- {paths}") +} + /// A POSIX shell word for `value`. fn shell_quote(value: &str) -> String { format!("'{}'", value.replace('\'', "'\\''")) @@ -2148,6 +2159,25 @@ mod tests { ); } + #[test] + fn untracks_listed_names_or_every_capture_file_past_the_list() { + let names = |count: usize| -> Vec { + (0..count) + .map(|index| format!(".lpm/inspector.db{index}")) + .collect() + }; + + assert_eq!( + untrack_command(&names(5)), + "git rm --cached -- '.lpm/inspector.db0' '.lpm/inspector.db1' \ + '.lpm/inspector.db2' '.lpm/inspector.db3' '.lpm/inspector.db4'" + ); + assert_eq!( + untrack_command(&names(6)), + "git rm --cached -- ':(glob,icase).lpm/inspector.db*'" + ); + } + #[test] fn quotes_names_for_a_posix_shell() { assert_eq!(shell_quote(".lpm/inspector.db"), "'.lpm/inspector.db'"); From d7e983a8a16d83ed53dabede4c8321a42d5c5182 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Sun, 11 Oct 2026 23:35:51 +0100 Subject: [PATCH 05/17] fix(tunnel): restrict capture files through their handle, and read .lpm/.gitignore as git does - A capture file is made owner-only through a handle opened without following a link, so a link swapped in after the check isn't followed; a file SQLite removed meanwhile needs nothing (a concurrent `lpm` could fail the open), and one with other hard links is refused. - `.lpm/.gitignore` is written before the database is created. A negation that could match a capture file (`!*`, `!*.db`, `!inspector*`) needs the rule again, a bare `*` already covers the captures, a line ending in two carriage returns isn't the rule, and an empty read-only file is left alone like any other. - On Windows a link is a symlink or a junction, not any reparse point, so a synced folder's placeholders aren't refused, and the database path is no longer canonicalized there. - The warning says git ignores the captures without crediting a file; for a linked `.lpm/.gitignore`, which git doesn't read, it says to replace it with a file. Names in the untrack command are bare when plain, and double-quoted where every shell reads them alike, so the command works in cmd.exe and PowerShell too. A conflicted file is listed once. - A local capture database error no longer comes with the tunnel's network hint. - The capture check's tests no longer depend on the developer's global ignore file, and its timeout is tested. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/dev.rs | 7 +- crates/lpm-cli/src/commands/tunnel.rs | 120 ++++++++++++---- crates/lpm-common/src/bounded_read.rs | 14 +- crates/lpm-common/src/lib.rs | 6 +- crates/lpm-inspect/src/db.rs | 198 +++++++++++++++++++------- tests/workflows/tests/dev_tunnel.rs | 37 ++++- 6 files changed, 286 insertions(+), 96 deletions(-) diff --git a/crates/lpm-cli/src/commands/dev.rs b/crates/lpm-cli/src/commands/dev.rs index f1a49011e..c494ac41b 100644 --- a/crates/lpm-cli/src/commands/dev.rs +++ b/crates/lpm-cli/src/commands/dev.rs @@ -1539,12 +1539,7 @@ pub async fn run( // It runs alongside the dashboard's webhook tab (which is // text-only) so the user can pick whichever fits the moment. // Quiet by default — we never auto-open a browser tab here. - let inspector_db = lpm_inspect::db::InspectorDb::open(project_dir).map_err(|error| { - LpmError::Tunnel(format!( - "failed to open this project's .lpm/inspector.db: {error}. \ - Move or repair it and retry" - )) - })?; + let inspector_db = crate::commands::tunnel::open_inspector_db(project_dir)?; let inspector_state = lpm_inspect::state::InspectorState::with_db_pending(inspector_db); multi_inspector_state = Some(inspector_state.clone()); if !no_inspect { diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index ff755d0e9..ebb60fa0c 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -760,11 +760,7 @@ pub(crate) async fn warn_capture_persistence(project_dir: &Path, show_human_warn "tunnel webhook capture persists full request/response bodies and headers \ (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db (owner-only \ on Unix; project ACLs on Windows){}", - if kept_out { - "; .lpm/.gitignore keeps it out of git" - } else { - "" - } + if kept_out { "; git ignores it" } else { "" } ), show_human_warning, ); @@ -783,10 +779,18 @@ pub(crate) async fn warn_capture_persistence(project_dir: &Path, show_human_warn ); } if !files.unignored.is_empty() { + // Git doesn't read a linked ignore file, so the rule can't go there. + let linked_ignore_file = std::fs::symlink_metadata(project_dir.join(".lpm/.gitignore")) + .is_ok_and(|metadata| metadata.file_type().is_symlink()); capture_warning( &format!( - "git doesn't ignore {}; add `inspector.db*` to .lpm/.gitignore", - listed(&files.unignored) + "git doesn't ignore {}; {}", + listed(&files.unignored), + if linked_ignore_file { + "replace the linked .lpm/.gitignore with a file that holds `inspector.db*`" + } else { + "add `inspector.db*` to .lpm/.gitignore" + } ), show_human_warning, ); @@ -822,7 +826,7 @@ const CAPTURE_GIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs( /// without regard to case, as a case-insensitive disk does; None when it /// isn't a repository, git can't be run, or it doesn't answer in time. async fn capture_git_state(project_dir: &Path) -> Option { - let mut command = tokio::process::Command::from(crate::commands::npmrc::hardened_git_command( + let command = tokio::process::Command::from(crate::commands::npmrc::hardened_git_command( project_dir, &[ "ls-files", @@ -835,11 +839,21 @@ async fn capture_git_state(project_dir: &Path) -> Option { CAPTURES_PATHSPEC, ], )); + capture_git_output(command, CAPTURE_GIT_TIMEOUT).await +} + +/// Runs the capture check's git command, giving up after `timeout` and +/// killing it, so a slow repository delays the tunnel's start by that much +/// at most. +async fn capture_git_output( + mut command: tokio::process::Command, + timeout: std::time::Duration, +) -> Option { command .stdin(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .kill_on_drop(true); - let output = tokio::time::timeout(CAPTURE_GIT_TIMEOUT, command.output()) + let output = tokio::time::timeout(timeout, command.output()) .await .ok()? .ok()?; @@ -858,10 +872,14 @@ fn parse_capture_git_files(stdout: &[u8]) -> CaptureGitFiles { continue; }; let name = String::from_utf8_lossy(name).into_owned(); - if tag == b'?' { - files.unignored.push(name); + // A conflicted file is listed once for each stage, one after another. + let list = if tag == b'?' { + &mut files.unignored } else { - files.tracked.push(name); + &mut files.tracked + }; + if list.last() != Some(&name) { + list.push(name); } } files @@ -888,19 +906,29 @@ fn listed(names: &[String]) -> String { /// capture pathspec when there are more than the warning lists. fn untrack_command(tracked: &[String]) -> String { let paths = if tracked.len() > LISTED_FILES { - shell_quote(CAPTURES_PATHSPEC) + shell_word(CAPTURES_PATHSPEC) } else { tracked .iter() - .map(|name| shell_quote(name)) + .map(|name| shell_word(name)) .collect::>() .join(" ") }; format!("git rm --cached -- {paths}") } -/// A POSIX shell word for `value`. -fn shell_quote(value: &str) -> String { +/// `value` as one word for a shell: bare when it holds only characters no +/// shell treats specially, in double quotes when nothing in it expands +/// inside them, which POSIX shells, PowerShell and cmd.exe all read alike, +/// and otherwise single-quoted for a POSIX shell. +fn shell_word(value: &str) -> String { + let plain = |c: char| c.is_ascii_alphanumeric() || matches!(c, '.' | '/' | '_' | '-'); + if !value.is_empty() && value.chars().all(plain) { + return value.to_string(); + } + if !value.contains(['"', '$', '`', '\\', '!', '%']) { + return format!("\"{value}\""); + } format!("'{}'", value.replace('\'', "'\\''")) } @@ -1411,17 +1439,23 @@ async fn run_log(project_dir: &Path, args: &[String], json_output: bool) -> Resu // ── Helper functions ──────────────────────────────────────────────── -fn open_inspector_db(project_dir: &Path) -> Result { +/// Opens the project's capture database. Its failures are local, so they're +/// reported as such, not as a tunnel's. +pub(crate) fn open_inspector_db( + project_dir: &Path, +) -> Result { lpm_inspect::db::InspectorDb::open(project_dir).map_err(|error| { - LpmError::Tunnel(format!( + LpmError::Io(std::io::Error::other(format!( "failed to open this project's .lpm/inspector.db: {error}. \ Move or repair it and retry" - )) + ))) }) } fn inspector_query_error(error: impl Display) -> LpmError { - LpmError::Tunnel(format!("failed to read .lpm/inspector.db: {error}")) + LpmError::Io(std::io::Error::other(format!( + "failed to read .lpm/inspector.db: {error}" + ))) } async fn read_capture_entries( @@ -2036,6 +2070,14 @@ mod tests { fn capture_project(files: &[&str]) -> tempfile::TempDir { let project = tempfile::tempdir().unwrap(); git(project.path(), &["init", "--quiet"]); + // The check runs git with the user's own configuration, so a global + // ignore file mustn't hide the files these tests make. + let no_excludes = project.path().join(".git/no-excludes"); + std::fs::write(&no_excludes, b"").unwrap(); + git( + project.path(), + &["config", "core.excludesFile", no_excludes.to_str().unwrap()], + ); for name in files { let path = project.path().join(name); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); @@ -2169,19 +2211,45 @@ mod tests { assert_eq!( untrack_command(&names(5)), - "git rm --cached -- '.lpm/inspector.db0' '.lpm/inspector.db1' \ - '.lpm/inspector.db2' '.lpm/inspector.db3' '.lpm/inspector.db4'" + "git rm --cached -- .lpm/inspector.db0 .lpm/inspector.db1 \ + .lpm/inspector.db2 .lpm/inspector.db3 .lpm/inspector.db4" ); assert_eq!( untrack_command(&names(6)), - "git rm --cached -- ':(glob,icase).lpm/inspector.db*'" + "git rm --cached -- \":(glob,icase).lpm/inspector.db*\"" ); } #[test] - fn quotes_names_for_a_posix_shell() { - assert_eq!(shell_quote(".lpm/inspector.db"), "'.lpm/inspector.db'"); - assert_eq!(shell_quote(".lpm/it's.db"), "'.lpm/it'\\''s.db'"); + fn writes_names_as_words_every_shell_reads_alike() { + assert_eq!(shell_word(".lpm/inspector.db-wal"), ".lpm/inspector.db-wal"); + assert_eq!(shell_word(".lpm/inspector db"), "\".lpm/inspector db\""); + assert_eq!(shell_word(".lpm/it's.db"), "\".lpm/it's.db\""); + assert_eq!(shell_word(".lpm/$HOME.db"), "'.lpm/$HOME.db'"); + assert_eq!(shell_word(".lpm/it's $x"), "'.lpm/it'\\''s $x'"); + } + + #[test] + fn lists_a_conflicted_capture_file_once() { + let files = parse_capture_git_files( + b"M .lpm/inspector.db\0M .lpm/inspector.db\0M .lpm/inspector.db\0? .lpm/inspector.db-wal\0", + ); + + assert_eq!(files.tracked, names(&[".lpm/inspector.db"])); + assert_eq!(files.unignored, names(&[".lpm/inspector.db-wal"])); + } + + #[cfg(unix)] + #[tokio::test] + async fn gives_up_on_a_slow_repository_and_stops_its_git() { + let started = std::time::Instant::now(); + let mut command = tokio::process::Command::new("sleep"); + command.arg("5"); + + let state = capture_git_output(command, std::time::Duration::from_millis(100)).await; + + assert_eq!(state, None); + assert!(started.elapsed() < std::time::Duration::from_secs(2)); } #[test] diff --git a/crates/lpm-common/src/bounded_read.rs b/crates/lpm-common/src/bounded_read.rs index f420cafcd..bec17876b 100644 --- a/crates/lpm-common/src/bounded_read.rs +++ b/crates/lpm-common/src/bounded_read.rs @@ -241,8 +241,11 @@ pub fn read_text_file_capped(path: &Path, limit: u64) -> Result Result { +/// Open a file for reading without following a linked leaf or blocking on a +/// special file: the handle is the entry at `path` itself, so checks made on +/// it hold for what the handle reads or changes, even if the path is +/// replaced concurrently. +pub fn open_nofollow(path: &Path) -> io::Result { let mut options = std::fs::OpenOptions::new(); options.read(true); #[cfg(unix)] @@ -258,7 +261,12 @@ pub fn read_text_file_capped_nofollow(path: &Path, limit: u64) -> Result Result { + let file = open_nofollow(path).map_err(|source| { if source.kind() == io::ErrorKind::NotFound { BoundedReadError::NotFound { path: path.to_path_buf(), diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index c4795d61b..04146f8cb 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -39,9 +39,9 @@ pub use atomic_write::{ pub use bounded_read::permissions_are_owner_only; pub use bounded_read::{ BoundedReadError, CONFIG_FILE_SIZE_CAP_BYTES, NPMRC_FILE_SIZE_CAP_BYTES, - TLS_MATERIAL_AGGREGATE_CAP_BYTES, TLS_MATERIAL_FILE_SIZE_CAP_BYTES, read_file_capped, - read_file_capped_from_open_file, read_regular_file_capped_with_metadata, read_stream_capped, - read_text_file_capped, read_text_file_capped_from_open_file, + TLS_MATERIAL_AGGREGATE_CAP_BYTES, TLS_MATERIAL_FILE_SIZE_CAP_BYTES, open_nofollow, + read_file_capped, read_file_capped_from_open_file, read_regular_file_capped_with_metadata, + read_stream_capped, read_text_file_capped, read_text_file_capped_from_open_file, read_text_file_capped_from_open_file_with_known_size, read_text_file_capped_nofollow, read_text_file_capped_with_metadata, read_text_regular_file_capped_with_metadata, }; diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index b39601dc8..549502f8b 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -63,11 +63,12 @@ impl InspectorDb { pub fn open(project_dir: &Path) -> Result { let lpm_dir = project_dir.join(".lpm"); let db_path = lpm_dir.join("inspector.db"); - prepare_capture_files(&lpm_dir, &db_path) - .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?; + let io_error = |error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)); + prepare_lpm_dir(&lpm_dir).map_err(io_error)?; if let Err(error) = ensure_captures_ignored(&lpm_dir) { tracing::warn!("could not keep tunnel captures out of git: {error}"); } + prepare_capture_files(&db_path).map_err(io_error)?; // Write connection (owned by the background flush task) let write_conn = open_capture_connection(&db_path)?; @@ -786,28 +787,37 @@ fn linked_or_not(path: &Path, expected: &str) -> std::io::Error { ) } -/// Makes `.lpm/` and the capture files safe to write: neither `.lpm` nor -/// any capture file may be a link, which a cloned repository could point -/// anywhere, such as into a directory it serves; existing capture files are -/// made owner-only, as a checkout or an older release may have left them -/// readable; and a new database is created owner-only before SQLite opens -/// it, so it's never readable by others. -fn prepare_capture_files(lpm_dir: &Path, db_path: &Path) -> std::io::Result<()> { +/// Whether an entry is a link: a symbolic link, or on Windows a junction. +/// Other reparse points, such as a synced folder's cloud placeholders, are +/// the project's own files. +fn is_link(metadata: &std::fs::Metadata) -> bool { + metadata.file_type().is_symlink() +} + +/// Makes sure `.lpm` is a directory of the project's own, creating it if +/// needed: a link, which a cloned repository could point anywhere, such as +/// into a directory it serves, is refused. +fn prepare_lpm_dir(lpm_dir: &Path) -> std::io::Result<()> { match std::fs::symlink_metadata(lpm_dir) { - Ok(metadata) if lpm_common::is_symlink_or_junction(&metadata) || !metadata.is_dir() => { - return Err(linked_or_not(lpm_dir, "directory")); + Ok(metadata) if is_link(&metadata) || !metadata.is_dir() => { + Err(linked_or_not(lpm_dir, "directory")) } - Ok(_) => {} + Ok(_) => Ok(()), Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - std::fs::create_dir_all(lpm_dir)?; + std::fs::create_dir_all(lpm_dir) } - Err(error) => return Err(error), + Err(error) => Err(error), } +} + +/// Makes the capture files safe to write: no capture file may be a link; +/// existing ones are made owner-only, as a checkout or an older release may +/// have left them readable; and a new database is created owner-only before +/// SQLite opens it, so it's never readable by others. +fn prepare_capture_files(db_path: &Path) -> std::io::Result<()> { for path in capture_files(db_path) { match std::fs::symlink_metadata(&path) { - Ok(metadata) - if lpm_common::is_symlink_or_junction(&metadata) || !metadata.is_file() => - { + Ok(metadata) if is_link(&metadata) || !metadata.is_file() => { return Err(linked_or_not(&path, "file")); } Ok(_) => make_owner_only(&path)?, @@ -818,11 +828,26 @@ fn prepare_capture_files(lpm_dir: &Path, db_path: &Path) -> std::io::Result<()> create_owner_only(db_path) } +/// Makes an existing capture file owner-only through a handle that doesn't +/// follow a link, so a link swapped in after the check isn't followed +/// either. A file that's gone needs nothing: SQLite removes `-wal` and +/// `-shm` when another process closes its last connection. A file with +/// other hard links, which a project could make point at one of its own +/// files, is refused. #[cfg(unix)] fn make_owner_only(path: &Path) -> std::io::Result<()> { - use std::os::unix::fs::PermissionsExt; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + let file = match lpm_common::open_nofollow(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.nlink() > 1 { + return Err(linked_or_not(path, "file")); + } + file.set_permissions(std::fs::Permissions::from_mode(0o600)) } #[cfg(not(unix))] @@ -851,27 +876,31 @@ fn create_owner_only(_path: &Path) -> std::io::Result<()> { Ok(()) } -/// Opens the capture database without following a link at its path. -/// SQLite's no-follow open refuses a link anywhere in the path, as macOS's -/// `/var` and `/tmp` are, so it gets the directory's real path, which leaves -/// only the database itself to refuse as a link. +/// Opens the capture database without following a link at its path. On +/// Unix, SQLite's no-follow open refuses a link anywhere in the path, as +/// macOS's `/var` and `/tmp` are, so it gets the directory's real path, +/// which leaves only the database itself to refuse as a link. On Windows +/// the flag does nothing, and the path is opened as it is. fn open_capture_connection(path: &Path) -> Result { - let directory = match path.parent() { - Some(directory) if !directory.as_os_str().is_empty() => directory, - _ => Path::new("."), + #[cfg(unix)] + let path = { + let directory = match path.parent() { + Some(directory) if !directory.as_os_str().is_empty() => directory, + _ => Path::new("."), + }; + std::fs::canonicalize(directory) + .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))? + .join(path.file_name().unwrap_or(path.as_os_str())) }; - let real_path = std::fs::canonicalize(directory) - .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))? - .join(path.file_name().unwrap_or(path.as_os_str())); Connection::open_with_flags( - real_path, + path, rusqlite::OpenFlags::default() | rusqlite::OpenFlags::SQLITE_OPEN_NOFOLLOW, ) } -/// A `.gitignore` line as git reads it: without a trailing carriage return -/// or trailing spaces that aren't escaped. Leading whitespace stays part of -/// the pattern, as it does for git. +/// A `.gitignore` line as git reads it: without one trailing carriage +/// return or trailing spaces that aren't escaped. Leading whitespace stays +/// part of the pattern, as it does for git. fn git_pattern(line: &str) -> &str { let line = line.strip_suffix('\r').unwrap_or(line); let bytes = line.as_bytes(); @@ -882,18 +911,22 @@ fn git_pattern(line: &str) -> &str { &line[..end] } -/// Whether these `.gitignore` lines leave the capture files ignored: the -/// rule, unanchored or anchored to `.lpm/`, after the last line that could -/// negate it. +/// Whether these `.gitignore` lines leave the capture files ignored: a rule +/// covering them, unanchored or anchored to `.lpm/` (`inspector.db*`, or +/// `*` as a tool directory's ignore file has), after the last negation that +/// could match one. A negation naming another file outright, such as +/// `!.gitignore`, can't. fn ignores_captures(lines: &str) -> bool { + const COVERING: [&str; 3] = [CAPTURES_IGNORE_LINE, "*", "**"]; let mut ignored = false; - for line in lines.lines() { + for line in lines.split('\n') { let pattern = git_pattern(line); - if pattern == CAPTURES_IGNORE_LINE - || pattern.strip_prefix('/') == Some(CAPTURES_IGNORE_LINE) - { + let unanchored = pattern.strip_prefix('/').unwrap_or(pattern); + if COVERING.contains(&unanchored) { ignored = true; - } else if pattern.starts_with('!') && pattern.contains("inspector.db") { + } else if let Some(negated) = pattern.strip_prefix('!') + && (negated.contains(['*', '?', '[']) || negated.contains("inspector.db")) + { ignored = false; } } @@ -907,18 +940,18 @@ fn ignores_captures(lines: &str) -> bool { /// is; the CLI's capture warning then says git doesn't ignore the captures. fn ensure_captures_ignored(lpm_dir: &Path) -> std::io::Result<()> { let path = lpm_dir.join(".gitignore"); - let existing = match lpm_common::read_text_file_capped_nofollow( + let (existing, exists) = match lpm_common::read_text_file_capped_nofollow( &path, lpm_common::CONFIG_FILE_SIZE_CAP_BYTES, ) { - Ok(text) => text, - Err(lpm_common::BoundedReadError::NotFound { .. }) => String::new(), + Ok(text) => (text, true), + Err(lpm_common::BoundedReadError::NotFound { .. }) => (String::new(), false), Err(error) => return Err(std::io::Error::other(error)), }; if ignores_captures(&existing) { return Ok(()); } - if !existing.is_empty() && std::fs::metadata(&path)?.permissions().readonly() { + if exists && std::fs::metadata(&path)?.permissions().readonly() { return Err(std::io::Error::new( std::io::ErrorKind::PermissionDenied, format!("{} is read-only", path.display()), @@ -1474,6 +1507,9 @@ mod tests { "inspector.db*\r\n", "/inspector.db*\n", "!inspector.db\ninspector.db*\n", + "*\n!.gitignore\n", + "/*\n", + "**\n", ] { let (_project, written) = opened_with_gitignore(content); assert_eq!(written, content, "{content:?}"); @@ -1488,6 +1524,11 @@ mod tests { "inspector.db*\t\n", "inspector.db*\\ \n", "inspector.db*\n!inspector.db\n", + "inspector.db*\n!*\n", + "inspector.db*\n!*.db\n", + "inspector.db*\n!inspector*\n", + "inspector.db*\n!**\n", + "inspector.db\n", ] { let (_project, written) = opened_with_gitignore(content); assert_eq!( @@ -1498,6 +1539,17 @@ mod tests { } } + #[tokio::test] + async fn opening_adds_the_rule_after_one_with_two_carriage_returns_which_git_reads_as_another() + { + let (_project, written) = opened_with_gitignore("inspector.db*\r\r\n"); + + assert_eq!( + written, + "inspector.db*\r\r\n# LPM tunnel captures hold request headers and bodies.\r\ninspector.db*\r\n" + ); + } + #[tokio::test] async fn opening_adds_the_rule_with_the_file_s_line_endings() { let (_project, written) = opened_with_gitignore("skills/\r\n"); @@ -1510,18 +1562,58 @@ mod tests { #[cfg(unix)] #[tokio::test] - async fn opening_leaves_a_read_only_lpm_gitignore_alone() { + async fn opening_leaves_a_read_only_lpm_gitignore_alone_even_empty() { use std::os::unix::fs::PermissionsExt; + for content in ["skills/\n", ""] { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let path = project.path().join(".lpm/.gitignore"); + std::fs::write(&path, content).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o444)).unwrap(); + + drop(InspectorDb::open(project.path()).unwrap()); + + assert_eq!(lpm_gitignore(project.path()), content, "{content:?}"); + } + } + + #[tokio::test] + async fn opening_writes_the_rule_before_it_creates_the_database() { let project = tempfile::tempdir().unwrap(); - std::fs::create_dir(project.path().join(".lpm")).unwrap(); - let path = project.path().join(".lpm/.gitignore"); - std::fs::write(&path, "skills/\n").unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o444)).unwrap(); + // A database path that can't be a file makes the open fail right + // after the rule is written, and before any database exists. + std::fs::create_dir_all(project.path().join(".lpm/inspector.db")).unwrap(); - drop(InspectorDb::open(project.path()).unwrap()); + assert!(InspectorDb::open(project.path()).is_err()); + + assert_eq!(lpm_gitignore(project.path()), CAPTURES_IGNORED); + } + + #[cfg(unix)] + #[test] + fn restricting_a_capture_file_that_vanished_needs_nothing() { + let project = tempfile::tempdir().unwrap(); + + make_owner_only(&project.path().join("inspector.db-wal")).unwrap(); + } + + #[cfg(unix)] + #[test] + fn preparing_refuses_a_capture_file_with_other_hard_links() { + use std::os::unix::fs::PermissionsExt; + + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + let served = project.path().join("public.txt"); + std::fs::write(&served, b"").unwrap(); + std::fs::set_permissions(&served, std::fs::Permissions::from_mode(0o644)).unwrap(); + std::fs::hard_link(&served, lpm_dir.join("inspector.db")).unwrap(); - assert_eq!(lpm_gitignore(project.path()), "skills/\n"); + assert!(prepare_capture_files(&lpm_dir.join("inspector.db")).is_err()); + let mode = std::fs::metadata(&served).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o644); } #[cfg(unix)] @@ -1572,7 +1664,7 @@ mod tests { std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); } - prepare_capture_files(&lpm_dir, &lpm_dir.join("inspector.db")).unwrap(); + prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap(); for name in [ "inspector.db", diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index 9415cf6ef..b11d09ca6 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1360,22 +1360,45 @@ async fn tunnel_start_names_capture_files_git_tracks_and_how_to_stop_it() { assert!( stderr.contains("git tracks '.lpm/inspector.db'") - && stderr.contains("git rm --cached -- '.lpm/inspector.db'") + && stderr.contains("git rm --cached -- .lpm/inspector.db") && stderr.contains("rotate any credentials"), "tunnel startup must name tracked capture files:\n{stderr}" ); assert!( - !stderr.contains("keeps it out of git"), - "a tracked capture isn't kept out of git:\n{stderr}" + !stderr.contains("git ignores it"), + "a tracked capture isn't ignored by git:\n{stderr}" ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn tunnel_start_says_captures_are_kept_out_of_git_once_git_ignores_them() { +async fn tunnel_start_says_git_ignores_captures_once_it_does() { let stderr = tunnel_start_stderr(|project| git(project, &["init", "--quiet"])).await; assert!( - stderr.contains(".lpm/.gitignore keeps it out of git") && !stderr.contains("git tracks"), + stderr.contains("; git ignores it") && !stderr.contains("git tracks"), + "{stderr}" + ); +} + +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tunnel_start_says_to_replace_a_linked_lpm_gitignore_git_doesnt_read() { + let outside = tempfile::tempdir().unwrap(); + let rules = outside.path().join("rules"); + std::fs::write(&rules, "inspector.db*\n").unwrap(); + let stderr = tunnel_start_stderr(move |project| { + git(project, &["init", "--quiet"]); + std::fs::create_dir(project.join(".lpm")).unwrap(); + std::os::unix::fs::symlink(&rules, project.join(".lpm/.gitignore")).unwrap(); + }) + .await; + + assert!( + stderr.contains("git doesn't ignore") + && stderr.contains( + "replace the linked .lpm/.gitignore with a file that holds `inspector.db*`" + ) + && !stderr.contains("git ignores it"), "{stderr}" ); } @@ -1616,6 +1639,10 @@ fn tunnel_inspect_reports_an_actionable_corrupt_database_error() { ); assert!(rendered.contains(".lpm/inspector.db")); assert!(rendered.contains("Move or repair")); + assert!( + !rendered.contains("network connection"), + "a local database error isn't a network one:\n{rendered}" + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From 92efd3a415255e9580a86512ea442831af3efed5 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Mon, 12 Oct 2026 07:55:34 +0100 Subject: [PATCH 06/17] fix(tunnel): keep captures from other users and planted files, and ask git inside .lpm .lpm must be the user's and writable only by them, and LPM's opens take turns under a lock. A capture database others could open is replaced with an owner-only copy once no other process has it open, since making it owner-only doesn't revoke an earlier handle. Capture files that are links, hard-linked or another user's are refused, and so is a database with triggers or views; connections are defensive and trust no schema code. Switching to WAL is retried on SQLITE_BUSY, and clearing history takes its write lock up front. .lpm/.gitignore is read as bytes, and any negation that could spell a capture file's name undoes the rule. One that's read-only to its owner or another user's is left alone, and a cloud placeholder is read on Windows. The capture check asks git inside .lpm, so a repository there answers, and falls back to the project only when git refuses. Only plain names are shown or put in the untrack command, which runs from the current directory. Database write errors are IO errors. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/dev.rs | 22 +- crates/lpm-cli/src/commands/tunnel.rs | 703 ++++++++--- crates/lpm-common/src/bounded_read.rs | 161 +++ crates/lpm-common/src/lib.rs | 21 +- crates/lpm-common/src/paths.rs | 69 ++ crates/lpm-inspect/Cargo.toml | 2 +- crates/lpm-inspect/src/db.rs | 1037 ++++++++++++++--- .../tests/atomic_write_inventory.rs | 4 + tests/workflows/tests/dev_tunnel.rs | 60 +- 9 files changed, 1757 insertions(+), 322 deletions(-) diff --git a/crates/lpm-cli/src/commands/dev.rs b/crates/lpm-cli/src/commands/dev.rs index c494ac41b..6023424e5 100644 --- a/crates/lpm-cli/src/commands/dev.rs +++ b/crates/lpm-cli/src/commands/dev.rs @@ -707,9 +707,10 @@ fn publish_initial_runtime( && let Err(finalize_error) = publication.finalize() { let error = rollback_inspector_session_after( - LpmError::Tunnel(format!( - "failed to publish the inspector session: {finalize_error}" - )), + crate::commands::tunnel::inspector_write_error( + "publish the inspector session", + finalize_error, + ), published_inspector_session, ); return Err(rollback_published_proxy_after( @@ -940,7 +941,7 @@ fn publish_tunnel_connection( prepare_tunnel_connection(inspector_state, publication)? .finalize() .map_err(|error| { - LpmError::Tunnel(format!("failed to publish the inspector session: {error}")) + crate::commands::tunnel::inspector_write_error("publish the inspector session", error) })?; render_tunnel_connection(publication); Ok(()) @@ -959,9 +960,7 @@ fn prepare_tunnel_connection( publication.session.tunnel_url.clone(), ) .map_err(|error| { - LpmError::Tunnel(format!( - "failed to persist inspector session start: {error}" - )) + crate::commands::tunnel::inspector_write_error("persist inspector session start", error) }) } @@ -3247,12 +3246,13 @@ async fn finalize_multi_service_tunnel_after( }; let persistence_result = if let Some(state) = inspector_state { let end_result = state.end_session().await.map_err(|error| { - LpmError::Tunnel(format!("failed to persist inspector session end: {error}")) + crate::commands::tunnel::inspector_write_error("persist inspector session end", error) }); let flush_result = state.flush().await.map_err(|error| { - LpmError::Tunnel(format!( - "failed to commit captures to .lpm/inspector.db: {error}" - )) + crate::commands::tunnel::inspector_write_error( + "commit captures to .lpm/inspector.db", + error, + ) }); combine_tunnel_cleanup_results(end_result, flush_result) } else { diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index ebb60fa0c..d7a5d102a 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -3,7 +3,7 @@ use lpm_common::{LpmError, format_bytes}; use lpm_registry::RegistryClient; use std::fmt::Display; use std::io::IsTerminal; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -575,11 +575,7 @@ pub(crate) async fn run_start( let name = session_name_owned.clone(); state .start_session_immediate(session_id, domain, local, name) - .map_err(|error| { - LpmError::Tunnel(format!( - "failed to persist inspector session start: {error}" - )) - })?; + .map_err(|error| inspector_write_error("persist inspector session start", error))?; tokio::spawn(async move { state.set_tunnel_url(url).await; }); @@ -718,14 +714,14 @@ pub(crate) async fn run_start( let websocket_result = ws_consumer .await .map_err(|error| LpmError::Tunnel(format!("WebSocket capture task failed: {error}"))); - let session_result = inspector_state.end_session().await.map_err(|error| { - LpmError::Tunnel(format!("failed to persist inspector session end: {error}")) - }); - let flush_result = inspector_state.flush().await.map_err(|error| { - LpmError::Tunnel(format!( - "failed to commit captures to .lpm/inspector.db: {error}" - )) - }); + let session_result = inspector_state + .end_session() + .await + .map_err(|error| inspector_write_error("persist inspector session end", error)); + let flush_result = inspector_state + .flush() + .await + .map_err(|error| inspector_write_error("commit captures to .lpm/inspector.db", error)); let inspector_result = match inspector_handle { Some(handle) => handle.shutdown().await, None => Ok(()), @@ -747,54 +743,75 @@ pub(crate) async fn run_start( } } -/// Warns that captures persist, and says how git treats them: kept out of -/// git only where git confirms it ignores them; files git tracks, which an -/// ignore rule can't keep from being committed; and files it neither tracks -/// nor ignores. +/// Warns that captures persist, and says how git treats them. pub(crate) async fn warn_capture_persistence(project_dir: &Path, show_human_warning: bool) { let state = capture_git_state(project_dir).await; - let kept_out = - matches!(state, Some(ref files) if files.tracked.is_empty() && files.unignored.is_empty()); - capture_warning( - &format!( - "tunnel webhook capture persists full request/response bodies and headers \ - (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db (owner-only \ - on Unix; project ACLs on Windows){}", - if kept_out { "; git ignores it" } else { "" } - ), - show_human_warning, - ); - let Some(files) = state else { - return; + for warning in capture_warnings( + state.as_ref(), + &git_base(project_dir), + ignore_file_git_skips(project_dir), + ) { + capture_warning(&warning, show_human_warning); + } +} + +/// The capture warnings: that captures persist, kept out of git only where +/// git confirms it ignores them; files git tracks, which an ignore rule +/// can't keep from being committed; and files it neither tracks nor +/// ignores. +fn capture_warnings( + state: Option<&CaptureGitState>, + base: &GitBase, + ignore_file_skipped: bool, +) -> Vec { + let kept_out = state.is_some_and(|state| { + state.asked_in_lpm && state.files.tracked.is_empty() && state.files.unignored.is_empty() + }); + let mut warnings = vec![format!( + "tunnel webhook capture persists full request/response bodies and headers \ + (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db (owner-only \ + on Unix; project ACLs on Windows){}", + if kept_out { "; git ignores it" } else { "" } + )]; + let Some(state) = state else { + return warnings; }; - if !files.tracked.is_empty() { - capture_warning( - &format!( - "git tracks {}, so captures there can be committed and stay in its \ - history; run `{}`, commit, and rotate any credentials they held", - listed(&files.tracked), - untrack_command(&files.tracked) - ), - show_human_warning, + if !state.asked_in_lpm { + warnings.push( + "git couldn't be asked inside .lpm, so if .lpm holds a repository of its own, \ + check that it doesn't track inspector.db*" + .to_string(), ); } + let files = &state.files; + if !files.tracked.is_empty() { + warnings.push(format!( + "git tracks {}, so captures there can be committed and stay in its history; \ + run `{}`, commit, and rotate any credentials they held", + listed(&state.shown(&files.tracked)), + untrack_command(&files.tracked, state.asked_in_lpm, base) + )); + } if !files.unignored.is_empty() { - // Git doesn't read a linked ignore file, so the rule can't go there. - let linked_ignore_file = std::fs::symlink_metadata(project_dir.join(".lpm/.gitignore")) - .is_ok_and(|metadata| metadata.file_type().is_symlink()); - capture_warning( - &format!( - "git doesn't ignore {}; {}", - listed(&files.unignored), - if linked_ignore_file { - "replace the linked .lpm/.gitignore with a file that holds `inspector.db*`" - } else { - "add `inspector.db*` to .lpm/.gitignore" - } - ), - show_human_warning, - ); + warnings.push(format!( + "git doesn't ignore {}; {}", + listed(&state.shown(&files.unignored)), + if ignore_file_skipped { + "replace .lpm/.gitignore, which git doesn't read as a file of the \ + project's, with a file that holds `inspector.db*`" + } else { + "add `inspector.db*` to .lpm/.gitignore" + } + )); } + warnings +} + +/// Whether git skips `.lpm/.gitignore`: a link, which git doesn't follow, +/// or an entry that isn't a regular file, such as a directory. +fn ignore_file_git_skips(project_dir: &Path) -> bool { + std::fs::symlink_metadata(project_dir.join(".lpm/.gitignore")) + .is_ok_and(|metadata| metadata.file_type().is_symlink() || !metadata.is_file()) } /// The human warning, or the same text on the log when it isn't shown, so @@ -811,23 +828,96 @@ fn capture_warning(message: &str, show_human_warning: bool) { } } -/// Capture files git tracks, and those it neither tracks nor ignores. -#[derive(Debug, Default, PartialEq, Eq)] +/// Capture files git tracks, and those it neither tracks nor ignores, by +/// name relative to the directory git was asked in. +#[derive(Debug, Default, Clone, PartialEq, Eq)] struct CaptureGitFiles { tracked: Vec, unignored: Vec, } -const CAPTURES_PATHSPEC: &str = ":(glob,icase).lpm/inspector.db*"; +#[derive(Debug, Clone, PartialEq, Eq)] +struct CaptureGitState { + files: CaptureGitFiles, + /// Whether git answered inside `.lpm`, where a repository of its own, + /// such as a submodule, keeps the files from the project's repository. + /// Otherwise the project's repository answered. + asked_in_lpm: bool, +} + +impl CaptureGitState { + /// Names as the project sees them. + fn shown(&self, names: &[String]) -> Vec { + if self.asked_in_lpm { + names.iter().map(|name| format!(".lpm/{name}")).collect() + } else { + names.to_vec() + } + } +} + +const CAPTURES_PATHSPEC: &str = ":(glob,icase)inspector.db*"; + +const PROJECT_CAPTURES_PATHSPEC: &str = ":(glob,icase).lpm/inspector.db*"; const CAPTURE_GIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); +/// Why git didn't say how it treats the capture files. +#[derive(Debug, PartialEq, Eq)] +enum CaptureGitFailure { + /// Git ran and refused, as outside a repository, or for one it won't use. + Refused, + /// Git couldn't be run, or didn't answer in time. + Unavailable, +} + /// How git treats the capture files in `project_dir`, matching their names -/// without regard to case, as a case-insensitive disk does; None when it +/// without regard to case, as a case-insensitive disk does. Git is asked +/// inside `.lpm`, and only if it refuses there, in the project. None when it /// isn't a repository, git can't be run, or it doesn't answer in time. -async fn capture_git_state(project_dir: &Path) -> Option { - let command = tokio::process::Command::from(crate::commands::npmrc::hardened_git_command( - project_dir, +async fn capture_git_state(project_dir: &Path) -> Option { + capture_git_state_by(project_dir, |directory, pathspec| { + capture_git_output( + capture_git_command(&directory, pathspec), + CAPTURE_GIT_TIMEOUT, + ) + }) + .await +} + +async fn capture_git_state_by(project_dir: &Path, ask: Ask) -> Option +where + Ask: Fn(PathBuf, &'static str) -> Answer, + Answer: std::future::Future>, +{ + let lpm_dir = project_dir.join(".lpm"); + let in_lpm = if lpm_dir.is_dir() { + ask(lpm_dir, CAPTURES_PATHSPEC).await + } else { + Err(CaptureGitFailure::Refused) + }; + match in_lpm { + Ok(files) => Some(CaptureGitState { + files, + asked_in_lpm: true, + }), + // A git too slow to answer once would be as slow again. + Err(CaptureGitFailure::Unavailable) => None, + Err(CaptureGitFailure::Refused) => { + ask(project_dir.to_path_buf(), PROJECT_CAPTURES_PATHSPEC) + .await + .ok() + .map(|files| CaptureGitState { + files, + asked_in_lpm: false, + }) + } + } +} + +fn capture_git_command(directory: &Path, pathspec: &str) -> tokio::process::Command { + tokio::process::Command::from(crate::commands::npmrc::hardened_git_command( + directory, &[ "ls-files", "-z", @@ -836,10 +926,9 @@ async fn capture_git_state(project_dir: &Path) -> Option { "--others", "--exclude-standard", "--", - CAPTURES_PATHSPEC, + pathspec, ], - )); - capture_git_output(command, CAPTURE_GIT_TIMEOUT).await + )) } /// Runs the capture check's git command, giving up after `timeout` and @@ -848,19 +937,19 @@ async fn capture_git_state(project_dir: &Path) -> Option { async fn capture_git_output( mut command: tokio::process::Command, timeout: std::time::Duration, -) -> Option { +) -> Result { command .stdin(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .kill_on_drop(true); let output = tokio::time::timeout(timeout, command.output()) .await - .ok()? - .ok()?; + .map_err(|_| CaptureGitFailure::Unavailable)? + .map_err(|_| CaptureGitFailure::Unavailable)?; if !output.status.success() { - return None; + return Err(CaptureGitFailure::Refused); } - Some(parse_capture_git_files(&output.stdout)) + Ok(parse_capture_git_files(&output.stdout)) } fn parse_capture_git_files(stdout: &[u8]) -> CaptureGitFiles { @@ -887,49 +976,90 @@ fn parse_capture_git_files(stdout: &[u8]) -> CaptureGitFiles { const LISTED_FILES: usize = 5; -/// Names quoted, at most `LISTED_FILES` of them, so text from the -/// repository reads as names in the warning. +/// Whether a name holds only characters no shell treats specially and no +/// terminal reads as anything but text, so it can go into the warning and +/// its command as it is. +fn is_plain(name: &str) -> bool { + !name.is_empty() + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '/' | '_' | '-')) +} + +/// At most `LISTED_FILES` plain names, quoted; other names are only +/// counted, so text from the repository can't read as part of the warning. fn listed(names: &[String]) -> String { - let mut listed = names + let shown: Vec = names .iter() + .filter(|name| is_plain(name)) .take(LISTED_FILES) .map(|name| format!("'{name}'")) - .collect::>() - .join(", "); - if names.len() > LISTED_FILES { - listed.push_str(&format!(" and {} more", names.len() - LISTED_FILES)); + .collect(); + let rest = names.len() - shown.len(); + match (shown.is_empty(), rest) { + (true, 1) => "1 capture file".to_string(), + (true, count) => format!("{count} capture files"), + (false, 0) => shown.join(", "), + (false, count) => format!("{} and {count} more", shown.join(", ")), + } +} + +/// The command that stops git tracking `tracked`, run from the current +/// directory: their names when the warning lists them all, and otherwise +/// the capture pathspec, which every shell reads alike in double quotes. +fn untrack_command(tracked: &[String], asked_in_lpm: bool, base: &GitBase) -> String { + let paths = if tracked.len() <= LISTED_FILES && tracked.iter().all(|name| is_plain(name)) { + tracked.join(" ") + } else if asked_in_lpm { + format!("\"{CAPTURES_PATHSPEC}\"") + } else { + format!("\"{PROJECT_CAPTURES_PATHSPEC}\"") + }; + let directory = if asked_in_lpm { + Some(base.join(".lpm")) + } else { + base.project() + }; + match directory { + Some(directory) => format!("git -C {directory} rm --cached -- {paths}"), + None => format!("git rm --cached -- {paths}"), } - listed } -/// The command that stops git tracking `tracked`: their names, or the -/// capture pathspec when there are more than the warning lists. -fn untrack_command(tracked: &[String]) -> String { - let paths = if tracked.len() > LISTED_FILES { - shell_word(CAPTURES_PATHSPEC) - } else { - tracked - .iter() - .map(|name| shell_word(name)) - .collect::>() - .join(" ") - }; - format!("git rm --cached -- {paths}") +/// The project directory as the current directory reaches it: `..` once for +/// each level the current directory is below it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct GitBase { + levels_up: usize, } -/// `value` as one word for a shell: bare when it holds only characters no -/// shell treats specially, in double quotes when nothing in it expands -/// inside them, which POSIX shells, PowerShell and cmd.exe all read alike, -/// and otherwise single-quoted for a POSIX shell. -fn shell_word(value: &str) -> String { - let plain = |c: char| c.is_ascii_alphanumeric() || matches!(c, '.' | '/' | '_' | '-'); - if !value.is_empty() && value.chars().all(plain) { - return value.to_string(); +impl GitBase { + /// The project's own directory, or None when it's the current one. + fn project(&self) -> Option { + (self.levels_up > 0).then(|| vec![".."; self.levels_up].join("/")) } - if !value.contains(['"', '$', '`', '\\', '!', '%']) { - return format!("\"{value}\""); + + fn join(&self, name: &str) -> String { + match self.project() { + Some(project) => format!("{project}/{name}"), + None => name.to_string(), + } } - format!("'{}'", value.replace('\'', "'\\''")) +} + +/// How many levels the current directory is below `project_dir`, or none +/// when it isn't below it. +fn git_base(project_dir: &Path) -> GitBase { + let levels_up = std::env::current_dir() + .ok() + .and_then(|current| { + current + .strip_prefix(project_dir) + .ok() + .map(|below| below.components().count()) + }) + .unwrap_or(0); + GitBase { levels_up } } fn compose_tunnel_control_and_connect( @@ -1452,6 +1582,12 @@ pub(crate) fn open_inspector_db( }) } +/// A failure writing `.lpm/inspector.db`, which is the disk's, not the +/// tunnel connection's. +pub(crate) fn inspector_write_error(what: &str, error: impl Display) -> LpmError { + LpmError::Io(std::io::Error::other(format!("failed to {what}: {error}"))) +} + fn inspector_query_error(error: impl Display) -> LpmError { LpmError::Io(std::io::Error::other(format!( "failed to read .lpm/inspector.db: {error}" @@ -2090,6 +2226,8 @@ mod tests { names.iter().map(|name| (*name).to_string()).collect() } + const HERE: GitBase = GitBase { levels_up: 0 }; + #[tokio::test] async fn tells_capture_files_git_tracks_from_those_it_neither_tracks_nor_ignores() { let project = capture_project(&[ @@ -2104,15 +2242,13 @@ mod tests { &["add", ".lpm/inspector.db", ".lpm/added-sources.json"], ); - let files = capture_git_state(project.path()).await.unwrap(); + let state = capture_git_state(project.path()).await.unwrap(); - assert_eq!(files.tracked, names(&[".lpm/inspector.db"])); - let mut unignored = files.unignored; + assert!(state.asked_in_lpm); + assert_eq!(state.files.tracked, names(&["inspector.db"])); + let mut unignored = state.files.unignored; unignored.sort(); - assert_eq!( - unignored, - names(&[".lpm/inspector.db-shm", ".lpm/inspector.db-wal"]) - ); + assert_eq!(unignored, names(&["inspector.db-shm", "inspector.db-wal"])); } #[tokio::test] @@ -2122,18 +2258,65 @@ mod tests { assert_eq!( capture_git_state(project.path()).await, - Some(CaptureGitFiles::default()) + Some(CaptureGitState { + files: CaptureGitFiles::default(), + asked_in_lpm: true, + }) ); } #[tokio::test] async fn names_tracked_capture_files_whatever_their_case() { - let project = capture_project(&[".LPM/Inspector.DB"]); - git(project.path(), &["add", ".LPM/Inspector.DB"]); + let project = capture_project(&[".lpm/Inspector.DB"]); + git(project.path(), &["add", ".lpm/Inspector.DB"]); + + let state = capture_git_state(project.path()).await.unwrap(); + + assert_eq!(state.files.tracked, names(&["Inspector.DB"])); + } - let files = capture_git_state(project.path()).await.unwrap(); + #[tokio::test] + async fn asks_a_repository_inside_lpm_about_the_files_it_keeps() { + let project = capture_project(&[".lpm/inspector.db"]); + let lpm_dir = project.path().join(".lpm"); + git(&lpm_dir, &["init", "--quiet"]); + git(&lpm_dir, &["add", "inspector.db"]); + + let state = capture_git_state(project.path()).await.unwrap(); - assert_eq!(files.tracked, names(&[".LPM/Inspector.DB"])); + assert!(state.asked_in_lpm); + assert_eq!(state.files.tracked, names(&["inspector.db"])); + assert_eq!( + untrack_command(&state.files.tracked, state.asked_in_lpm, &HERE), + "git -C .lpm rm --cached -- inspector.db" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn asks_the_project_when_git_refuses_inside_lpm_and_says_so() { + let project = capture_project(&[".lpm/inspector.db"]); + git(project.path(), &["add", ".lpm/inspector.db"]); + // An `.lpm` that looks like a bare repository, which the check's + // git refuses to use. + let lpm_dir = project.path().join(".lpm"); + for directory in ["objects", "refs"] { + std::fs::create_dir(lpm_dir.join(directory)).unwrap(); + } + std::fs::write(lpm_dir.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + + let state = capture_git_state(project.path()).await.unwrap(); + + assert!(!state.asked_in_lpm); + assert_eq!(state.files.tracked, names(&[".lpm/inspector.db"])); + assert_eq!( + state.shown(&state.files.tracked), + names(&[".lpm/inspector.db"]) + ); + assert_eq!( + untrack_command(&state.files.tracked, state.asked_in_lpm, &HERE), + "git rm --cached -- .lpm/inspector.db" + ); } #[tokio::test] @@ -2188,6 +2371,163 @@ mod tests { assert!(!marker.exists()); } + /// Asks with canned answers, one per call, recording where each was asked. + async fn state_from_answers( + project_dir: &Path, + answers: Vec>, + ) -> (Option, Vec<(PathBuf, &'static str)>) { + let answers = std::sync::Mutex::new(answers.into_iter()); + let asked = std::sync::Mutex::new(Vec::new()); + let state = capture_git_state_by(project_dir, |directory, pathspec| { + asked.lock().unwrap().push((directory, pathspec)); + let answer = answers.lock().unwrap().next().unwrap(); + async move { answer } + }) + .await; + (state, asked.into_inner().unwrap()) + } + + #[tokio::test] + async fn asks_the_project_only_when_git_refuses_inside_lpm() { + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + let tracked = CaptureGitFiles { + tracked: names(&[".lpm/inspector.db"]), + unignored: Vec::new(), + }; + + let (state, asked) = state_from_answers( + project.path(), + vec![Err(CaptureGitFailure::Refused), Ok(tracked)], + ) + .await; + assert_eq!(state.unwrap().files.tracked, names(&[".lpm/inspector.db"])); + assert_eq!( + asked, + vec![ + (lpm_dir.clone(), CAPTURES_PATHSPEC), + (project.path().to_path_buf(), PROJECT_CAPTURES_PATHSPEC), + ] + ); + + let (state, asked) = + state_from_answers(project.path(), vec![Err(CaptureGitFailure::Unavailable)]).await; + assert_eq!(state, None); + assert_eq!(asked.len(), 1); + + let (state, asked) = + state_from_answers(project.path(), vec![Ok(CaptureGitFiles::default())]).await; + assert!(state.unwrap().asked_in_lpm); + assert_eq!(asked.len(), 1); + } + + fn warnings_for(state: Option, ignore_file_skipped: bool) -> Vec { + capture_warnings(state.as_ref(), &HERE, ignore_file_skipped) + } + + fn state(asked_in_lpm: bool, tracked: &[&str], unignored: &[&str]) -> CaptureGitState { + CaptureGitState { + files: CaptureGitFiles { + tracked: names(tracked), + unignored: names(unignored), + }, + asked_in_lpm, + } + } + + #[test] + fn says_git_ignores_captures_only_when_git_inside_lpm_says_so() { + assert!(warnings_for(Some(state(true, &[], &[])), false)[0].ends_with("; git ignores it")); + for unconfirmed in [ + None, + Some(state(false, &[], &[])), + Some(state(true, &["inspector.db"], &[])), + Some(state(true, &[], &["inspector.db-wal"])), + ] { + assert!( + !warnings_for(unconfirmed.clone(), false)[0].contains("git ignores it"), + "{unconfirmed:?}" + ); + } + } + + #[test] + fn says_when_git_couldnt_be_asked_inside_lpm() { + let warnings = warnings_for(Some(state(false, &[], &[])), false); + + assert_eq!(warnings.len(), 2); + assert!(warnings[1].starts_with("git couldn't be asked inside .lpm")); + assert_eq!(warnings_for(Some(state(true, &[], &[])), false).len(), 1); + assert_eq!(warnings_for(None, false).len(), 1); + } + + #[test] + fn names_tracked_captures_and_the_command_that_stops_tracking_them() { + let warnings = warnings_for(Some(state(true, &["inspector.db"], &[])), false); + + assert_eq!(warnings.len(), 2); + assert_eq!( + warnings[1], + "git tracks '.lpm/inspector.db', so captures there can be committed and stay in \ + its history; run `git -C .lpm rm --cached -- inspector.db`, commit, and rotate \ + any credentials they held" + ); + } + + #[test] + fn advises_on_captures_git_neither_tracks_nor_ignores() { + let added = warnings_for(Some(state(true, &[], &["inspector.db-wal"])), false); + let replaced = warnings_for(Some(state(true, &[], &["inspector.db-wal"])), true); + + assert_eq!( + added[1], + "git doesn't ignore '.lpm/inspector.db-wal'; add `inspector.db*` to .lpm/.gitignore" + ); + assert_eq!( + replaced[1], + "git doesn't ignore '.lpm/inspector.db-wal'; replace .lpm/.gitignore, which git \ + doesn't read as a file of the project's, with a file that holds `inspector.db*`" + ); + } + + #[test] + fn reports_inspector_database_reads_as_disk_errors() { + let error = inspector_query_error("database disk image is malformed"); + + assert_eq!(error.error_code(), "io"); + assert!( + error + .to_string() + .contains("failed to read .lpm/inspector.db: database disk image is malformed") + ); + } + + #[test] + fn reports_inspector_database_writes_as_disk_errors() { + let error = inspector_write_error("commit captures to .lpm/inspector.db", "disk I/O error"); + + assert_eq!(error.error_code(), "io"); + assert!( + error + .to_string() + .contains("failed to commit captures to .lpm/inspector.db: disk I/O error") + ); + } + + #[test] + fn shows_names_git_gave_inside_lpm_as_the_project_sees_them() { + let state = CaptureGitState { + files: CaptureGitFiles::default(), + asked_in_lpm: true, + }; + + assert_eq!( + state.shown(&names(&["inspector.db"])), + names(&[".lpm/inspector.db"]) + ); + } + #[test] fn lists_at_most_five_names_quoted() { let many: Vec = (0..7) @@ -2201,55 +2541,154 @@ mod tests { ); } + #[test] + fn lists_names_with_other_characters_only_by_count() { + assert_eq!( + listed(&names(&[ + ".lpm/inspector.db", + ".lpm/inspector.db\u{202e}bd.x", + ".lpm/it\u{2019}s.db", + ])), + "'.lpm/inspector.db' and 2 more" + ); + assert_eq!(listed(&names(&[".lpm/inspector db"])), "1 capture file"); + assert_eq!( + listed(&names(&[".lpm/inspector db", ".lpm/inspector\u{200b}.db"])), + "2 capture files" + ); + } + #[test] fn untracks_listed_names_or_every_capture_file_past_the_list() { let names = |count: usize| -> Vec { (0..count) - .map(|index| format!(".lpm/inspector.db{index}")) + .map(|index| format!("inspector.db{index}")) .collect() }; assert_eq!( - untrack_command(&names(5)), - "git rm --cached -- .lpm/inspector.db0 .lpm/inspector.db1 \ - .lpm/inspector.db2 .lpm/inspector.db3 .lpm/inspector.db4" + untrack_command(&names(5), true, &HERE), + "git -C .lpm rm --cached -- inspector.db0 inspector.db1 \ + inspector.db2 inspector.db3 inspector.db4" + ); + assert_eq!( + untrack_command(&names(6), true, &HERE), + "git -C .lpm rm --cached -- \":(glob,icase)inspector.db*\"" ); assert_eq!( - untrack_command(&names(6)), + untrack_command(&names(6), false, &HERE), "git rm --cached -- \":(glob,icase).lpm/inspector.db*\"" ); } #[test] - fn writes_names_as_words_every_shell_reads_alike() { - assert_eq!(shell_word(".lpm/inspector.db-wal"), ".lpm/inspector.db-wal"); - assert_eq!(shell_word(".lpm/inspector db"), "\".lpm/inspector db\""); - assert_eq!(shell_word(".lpm/it's.db"), "\".lpm/it's.db\""); - assert_eq!(shell_word(".lpm/$HOME.db"), "'.lpm/$HOME.db'"); - assert_eq!(shell_word(".lpm/it's $x"), "'.lpm/it'\\''s $x'"); + fn untracks_every_capture_file_when_a_name_has_other_characters() { + for name in [ + "inspector db", + "inspector\u{201d}.db", + "inspector\u{2019}.db", + "!'$(touch x)'.db", + "inspector\".db", + "%PATH%.db", + "", + ] { + assert_eq!( + untrack_command(&names(&["inspector.db", name]), true, &HERE), + "git -C .lpm rm --cached -- \":(glob,icase)inspector.db*\"", + "{name:?}" + ); + } + } + + #[test] + fn untracks_from_a_directory_below_the_project() { + let below = GitBase { levels_up: 2 }; + + assert_eq!( + untrack_command(&names(&["inspector.db"]), true, &below), + "git -C ../../.lpm rm --cached -- inspector.db" + ); + assert_eq!( + untrack_command(&names(&[".lpm/inspector.db"]), false, &below), + "git -C ../.. rm --cached -- .lpm/inspector.db" + ); + } + + #[test] + fn counts_the_levels_from_the_project_to_the_current_directory() { + let current = std::env::current_dir().unwrap(); + + assert_eq!(git_base(¤t), HERE); + assert_eq!( + git_base(current.parent().unwrap()), + GitBase { levels_up: 1 } + ); + assert_eq!(git_base(Path::new("/nowhere-near")), HERE); + } + + #[cfg(unix)] + #[test] + fn tells_an_lpm_gitignore_git_skips_from_one_it_reads() { + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + assert!(!ignore_file_git_skips(project.path())); + + std::fs::write(lpm_dir.join(".gitignore"), "skills/\n").unwrap(); + assert!(!ignore_file_git_skips(project.path())); + + std::fs::remove_file(lpm_dir.join(".gitignore")).unwrap(); + std::fs::create_dir(lpm_dir.join(".gitignore")).unwrap(); + assert!(ignore_file_git_skips(project.path())); + + std::fs::remove_dir(lpm_dir.join(".gitignore")).unwrap(); + std::os::unix::fs::symlink(project.path().join("elsewhere"), lpm_dir.join(".gitignore")) + .unwrap(); + assert!(ignore_file_git_skips(project.path())); } #[test] fn lists_a_conflicted_capture_file_once() { let files = parse_capture_git_files( - b"M .lpm/inspector.db\0M .lpm/inspector.db\0M .lpm/inspector.db\0? .lpm/inspector.db-wal\0", + b"M inspector.db\0M inspector.db\0M inspector.db\0? inspector.db-wal\0", ); - assert_eq!(files.tracked, names(&[".lpm/inspector.db"])); - assert_eq!(files.unignored, names(&[".lpm/inspector.db-wal"])); + assert_eq!(files.tracked, names(&["inspector.db"])); + assert_eq!(files.unignored, names(&["inspector.db-wal"])); } #[cfg(unix)] #[tokio::test] async fn gives_up_on_a_slow_repository_and_stops_its_git() { + let directory = tempfile::tempdir().unwrap(); + let pid_file = directory.path().join("pid"); let started = std::time::Instant::now(); - let mut command = tokio::process::Command::new("sleep"); - command.arg("5"); + let mut command = tokio::process::Command::new("sh"); + command + .arg("-c") + .arg("echo $$ > \"$1\"; exec sleep 5") + .arg("sh") + .arg(&pid_file); - let state = capture_git_output(command, std::time::Duration::from_millis(100)).await; + let state = capture_git_output(command, std::time::Duration::from_millis(300)).await; - assert_eq!(state, None); + assert_eq!(state, Err(CaptureGitFailure::Unavailable)); assert!(started.elapsed() < std::time::Duration::from_secs(2)); + let pid = std::fs::read_to_string(&pid_file).unwrap(); + let stopped = |pid: &str| { + let output = std::process::Command::new("ps") + .args(["-o", "stat=", "-p", pid.trim()]) + .output() + .unwrap(); + let status = String::from_utf8_lossy(&output.stdout); + // Gone, or killed and not yet reaped. + status.trim().is_empty() || status.trim_start().starts_with('Z') + }; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while !stopped(&pid) && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(stopped(&pid), "the slow command is still running"); } #[test] diff --git a/crates/lpm-common/src/bounded_read.rs b/crates/lpm-common/src/bounded_read.rs index bec17876b..2a10c41f4 100644 --- a/crates/lpm-common/src/bounded_read.rs +++ b/crates/lpm-common/src/bounded_read.rs @@ -264,6 +264,114 @@ pub fn open_nofollow(path: &Path) -> io::Result { options.open(path) } +/// The effective user id of this process. +#[cfg(unix)] +pub fn current_uid() -> u32 { + // SAFETY: geteuid takes no arguments, can't fail, and touches no memory + // of the caller's. + unsafe { libc::geteuid() } +} + +/// How many names the open file has, as hard links, on Unix and on Windows, +/// where the standard library doesn't say. +pub fn link_count(file: &std::fs::File) -> io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + + Ok(file.metadata()?.nlink()) + } + #[cfg(windows)] + { + Ok(u64::from(handle_information(file)?.nNumberOfLinks)) + } + #[cfg(not(any(unix, windows)))] + { + let _ = file; + Ok(1) + } +} + +#[cfg(windows)] +fn handle_information( + file: &std::fs::File, +) -> io::Result { + use std::os::windows::io::AsRawHandle as _; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + + // SAFETY: an all-zero BY_HANDLE_FILE_INFORMATION is a valid value of its + // plain integer fields, and the call only writes into it. + let mut information: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + // SAFETY: the handle is the open file's, valid while `file` lives. + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut information) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(information) +} + +/// Read a bounded regular file unless the entry at `path` is a link: a +/// symbolic link, or on Windows a junction or another name-surrogate reparse +/// point. Unlike [`read_text_file_capped_nofollow`], other reparse points, +/// such as a synced folder's cloud placeholders, are read through, from the +/// same file the check opened. The metadata is the read file's. +pub fn read_file_capped_unlinked( + path: &Path, + limit: u64, +) -> Result<(Vec, Metadata), BoundedReadError> { + let io_error = |source| BoundedReadError::Io { + path: path.to_path_buf(), + source, + }; + let file = open_nofollow(path).map_err(|source| { + if source.kind() == io::ErrorKind::NotFound { + BoundedReadError::NotFound { + path: path.to_path_buf(), + } + } else { + io_error(source) + } + })?; + let metadata = file.metadata().map_err(io_error)?; + if !metadata.is_file() { + return Err(io_error(io::Error::new( + io::ErrorKind::InvalidData, + "a link or not a regular file", + ))); + } + #[cfg(windows)] + let file = if metadata_is_link_or_reparse(&metadata) { + open_reparse_point_target(&file, path).map_err(io_error)? + } else { + file + }; + read_file_capped_from_open_file(file, path, limit) +} + +/// Opens a reparse point that isn't a link through its filter, as a cloud +/// placeholder must be read, and checks the file opened is the one `checked` +/// holds, so an entry swapped in between isn't read. +#[cfg(windows)] +fn open_reparse_point_target(checked: &std::fs::File, path: &Path) -> io::Result { + let target = std::fs::File::open(path)?; + let identity = + |information: windows_sys::Win32::Storage::FileSystem::BY_HANDLE_FILE_INFORMATION| { + ( + information.dwVolumeSerialNumber, + information.nFileIndexHigh, + information.nFileIndexLow, + ) + }; + if identity(handle_information(checked)?) != identity(handle_information(&target)?) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the file changed while it was opened", + )); + } + Ok(target) +} + /// Read bounded UTF-8 text without following a linked leaf or blocking on a special file. pub fn read_text_file_capped_nofollow(path: &Path, limit: u64) -> Result { let file = open_nofollow(path).map_err(|source| { @@ -575,4 +683,57 @@ mod tests { Err(BoundedReadError::TooLarge { limit: 5, .. }) )); } + + #[test] + fn unlinked_read_returns_a_regular_file_s_bytes_and_metadata() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(".gitignore"); + std::fs::write(&path, b"skills/\n\xff").unwrap(); + + let (bytes, metadata) = read_file_capped_unlinked(&path, 32).unwrap(); + + assert_eq!(bytes, b"skills/\n\xff"); + assert_eq!(metadata.len(), 9); + assert!(matches!( + read_file_capped_unlinked(&dir.path().join("missing"), 32), + Err(BoundedReadError::NotFound { .. }) + )); + } + + #[cfg(unix)] + #[test] + fn unlinked_read_refuses_a_link_and_a_fifo_without_blocking() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("target"); + std::fs::write(&target, b"outside").unwrap(); + let link = dir.path().join("link"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + let fifo = dir.path().join("fifo"); + let fifo_name = std::ffi::CString::new(fifo.as_os_str().as_encoded_bytes()).unwrap(); + // SAFETY: the name is a valid NUL-terminated path for the call. + assert_eq!(unsafe { libc::mkfifo(fifo_name.as_ptr(), 0o600) }, 0); + + for path in [&link, &fifo] { + assert!( + matches!( + read_file_capped_unlinked(path, 32), + Err(BoundedReadError::Io { .. }) + ), + "{}", + path.display() + ); + } + } + + #[test] + fn link_count_counts_a_file_s_hard_links() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("file"); + std::fs::write(&path, b"").unwrap(); + assert_eq!(link_count(&File::open(&path).unwrap()).unwrap(), 1); + + std::fs::hard_link(&path, dir.path().join("other-name")).unwrap(); + + assert_eq!(link_count(&File::open(&path).unwrap()).unwrap(), 2); + } } diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index 04146f8cb..97d19925e 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -36,14 +36,17 @@ pub use atomic_write::{ write_file_atomic_in_dir_with_options, write_file_atomic_with, write_file_atomic_with_options, }; #[cfg(unix)] +pub use bounded_read::current_uid; +#[cfg(unix)] pub use bounded_read::permissions_are_owner_only; pub use bounded_read::{ BoundedReadError, CONFIG_FILE_SIZE_CAP_BYTES, NPMRC_FILE_SIZE_CAP_BYTES, - TLS_MATERIAL_AGGREGATE_CAP_BYTES, TLS_MATERIAL_FILE_SIZE_CAP_BYTES, open_nofollow, - read_file_capped, read_file_capped_from_open_file, read_regular_file_capped_with_metadata, - read_stream_capped, read_text_file_capped, read_text_file_capped_from_open_file, - read_text_file_capped_from_open_file_with_known_size, read_text_file_capped_nofollow, - read_text_file_capped_with_metadata, read_text_regular_file_capped_with_metadata, + TLS_MATERIAL_AGGREGATE_CAP_BYTES, TLS_MATERIAL_FILE_SIZE_CAP_BYTES, link_count, open_nofollow, + read_file_capped, read_file_capped_from_open_file, read_file_capped_unlinked, + read_regular_file_capped_with_metadata, read_stream_capped, read_text_file_capped, + read_text_file_capped_from_open_file, read_text_file_capped_from_open_file_with_known_size, + read_text_file_capped_nofollow, read_text_file_capped_with_metadata, + read_text_regular_file_capped_with_metadata, }; pub use command_name::{PortableCommandNameError, validate_portable_command_name}; pub use error::{ @@ -69,10 +72,10 @@ pub use paths::{ ExclusiveLockHandle, FsKind, GLOBAL_INSTALL_PATH_BUDGET, INSTALL_READY_MARKER, LpmRoot, ProjectLockDirectory, ProjectLockKind, SharedLockHandle, SingleFileExclusiveLockHandle, SingleFileSharedLockHandle, acquire_exclusive_lock, acquire_single_file_exclusive_lock, - acquire_single_file_exclusive_lock_from_file, acquire_single_file_shared_lock, - acquire_single_file_shared_lock_from_file, as_extended_path, check_install_path_budget, - is_local_fs, parent_install_lock_environment, project_install_lock, project_publish_lock, - try_acquire_capability_exclusive_lock, try_acquire_exclusive_lock, + acquire_single_file_exclusive_lock_from_file, acquire_single_file_exclusive_lock_nofollow, + acquire_single_file_shared_lock, acquire_single_file_shared_lock_from_file, as_extended_path, + check_install_path_budget, is_local_fs, parent_install_lock_environment, project_install_lock, + project_publish_lock, try_acquire_capability_exclusive_lock, try_acquire_exclusive_lock, try_acquire_single_file_exclusive_lock_from_file, try_with_exclusive_lock, with_capability_exclusive_lock, with_exclusive_lock, with_exclusive_lock_async, with_project_exclusive_lock, with_project_exclusive_lock_async, with_project_shared_lock, diff --git a/crates/lpm-common/src/paths.rs b/crates/lpm-common/src/paths.rs index 904d8c8d4..40d244ccf 100644 --- a/crates/lpm-common/src/paths.rs +++ b/crates/lpm-common/src/paths.rs @@ -718,6 +718,34 @@ fn open_lock_file(lock_path: &Path) -> std::io::Result { options.open(lock_path) } +fn open_lock_file_nofollow(lock_path: &Path) -> std::io::Result { + let mut options = std::fs::OpenOptions::new(); + options.create(true).read(true).write(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + + options + .mode(0o600) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt as _; + use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OPEN_REPARSE_POINT; + + options.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT); + } + let file = options.open(lock_path)?; + if !file.metadata()?.is_file() { + return Err(std::io::Error::other(format!( + "refusing lock path that is a link or not a regular file: {}", + lock_path.display() + ))); + } + Ok(file) +} + fn open_capability_lock_file( directory: &cap_std::fs::Dir, name: &OsStr, @@ -1291,6 +1319,16 @@ pub fn acquire_single_file_exclusive_lock( acquire_single_file_exclusive_lock_from_file(file) } +/// Acquire a single-file exclusive advisory lock whose file is opened without +/// following a link at `lock_path`, for a lock in a directory whose entries a +/// repository can supply, such as a project's `.lpm`. +pub fn acquire_single_file_exclusive_lock_nofollow( + lock_path: impl AsRef, +) -> Result { + let file = open_lock_file_nofollow(lock_path.as_ref())?; + acquire_single_file_exclusive_lock_from_file(file) +} + pub fn acquire_single_file_exclusive_lock_from_file( file: std::fs::File, ) -> Result { @@ -2223,6 +2261,37 @@ mod tests { ); } + #[test] + fn nofollow_single_file_lock_shares_the_lock_domain_of_its_path() { + let tmp = TempDir::new().unwrap(); + let lock_path = tmp.path().join("inspector.db-lock"); + let held = acquire_single_file_exclusive_lock_nofollow(&lock_path).unwrap(); + + assert!( + try_acquire_single_file_exclusive_lock_from_file(open_lock_file(&lock_path).unwrap()) + .unwrap() + .is_none() + ); + drop(held); + assert!( + try_acquire_single_file_exclusive_lock_from_file(open_lock_file(&lock_path).unwrap()) + .unwrap() + .is_some() + ); + } + + #[cfg(unix)] + #[test] + fn nofollow_single_file_lock_refuses_a_linked_path_and_creates_nothing() { + let tmp = TempDir::new().unwrap(); + let target = tmp.path().join("elsewhere"); + let lock_path = tmp.path().join("inspector.db-lock"); + std::os::unix::fs::symlink(&target, &lock_path).unwrap(); + + assert!(acquire_single_file_exclusive_lock_nofollow(&lock_path).is_err()); + assert!(!target.exists()); + } + /// Shared lock blocks an exclusive acquire until released. #[test] fn exclusive_blocks_while_shared_held() { diff --git a/crates/lpm-inspect/Cargo.toml b/crates/lpm-inspect/Cargo.toml index 443744a19..a742cde31 100644 --- a/crates/lpm-inspect/Cargo.toml +++ b/crates/lpm-inspect/Cargo.toml @@ -23,7 +23,7 @@ async-stream = "0.3" futures-util = "0.3" base64 = { workspace = true } reqwest = { workspace = true } -rusqlite = { version = "0.34", features = ["bundled"] } +rusqlite = { version = "0.34", features = ["backup", "bundled"] } chrono = { workspace = true } rand = { workspace = true } subtle = "2" diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index 549502f8b..a2102f05b 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -63,27 +63,24 @@ impl InspectorDb { pub fn open(project_dir: &Path) -> Result { let lpm_dir = project_dir.join(".lpm"); let db_path = lpm_dir.join("inspector.db"); - let io_error = |error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)); - prepare_lpm_dir(&lpm_dir).map_err(io_error)?; + prepare_lpm_dir(&lpm_dir).map_err(sqlite_io_error)?; + let _opening = lock_capture_files(&db_path)?; if let Err(error) = ensure_captures_ignored(&lpm_dir) { tracing::warn!("could not keep tunnel captures out of git: {error}"); } - prepare_capture_files(&db_path).map_err(io_error)?; + if prepare_capture_files(&db_path).map_err(sqlite_io_error)? { + replace_with_owner_only_copy(&db_path)?; + } // Write connection (owned by the background flush task) - let write_conn = open_capture_connection(&db_path)?; + let write_conn = open_capture_connection(&db_path, "NORMAL")?; + refuse_foreign_schema(&write_conn, &db_path)?; init_schema(&write_conn)?; - // Read connection (shared across API handlers). - // busy_timeout ensures low-frequency mutations (tag updates, session renames) - // wait gracefully if the write connection holds a lock during batch flushes. - let read_conn = open_capture_connection(&db_path)?; - read_conn.execute_batch( - "PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA foreign_keys=ON; - PRAGMA busy_timeout=5000;", - )?; + // Read connection (shared across API handlers). The busy timeout + // lets low-frequency mutations (tag updates, session renames) wait + // while the write connection flushes a batch. + let read_conn = open_capture_connection(&db_path, "NORMAL")?; let control_conn = open_control_connection(&db_path)?; let (write_tx, write_rx) = mpsc::channel(WRITE_QUEUE_CAPACITY); @@ -124,15 +121,11 @@ impl InspectorDb { )); let write_conn = Connection::open(&tmp)?; + configure_connection(&write_conn, "NORMAL")?; init_schema(&write_conn)?; let read_conn = Connection::open(&tmp)?; - read_conn.execute_batch( - "PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA foreign_keys=ON; - PRAGMA busy_timeout=5000;", - )?; + configure_connection(&read_conn, "NORMAL")?; let control_conn = open_control_connection(&tmp)?; let (write_tx, write_rx) = mpsc::channel(WRITE_QUEUE_CAPACITY); @@ -738,7 +731,11 @@ impl InspectorDb { /// captures without violating the request-to-session foreign key. pub async fn clear_history(&self) -> Result<(), rusqlite::Error> { let conn = self.read_conn.lock().await; - let tx = conn.unchecked_transaction()?; + // Immediate: a deferred transaction that reads before it writes gets + // SQLITE_BUSY at once, without waiting, if another connection + // commits in between. + let tx = + rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; tx.execute("DELETE FROM requests_fts", [])?; tx.execute("DELETE FROM requests", [])?; tx.execute("DELETE FROM sessions WHERE ended_at IS NOT NULL", [])?; @@ -761,21 +758,39 @@ impl InspectorDb { /// bodies. const CAPTURES_IGNORE_LINE: &str = "inspector.db*"; +/// How long a connection waits for another's lock, and how long opening +/// keeps retrying what SQLite refuses at once while another connection is +/// busy. +const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// The suffixes of the files SQLite keeps beside the capture database. +const SQLITE_SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; + +/// The file beside the capture database whose name ends in `suffix`. +fn capture_file(db_path: &Path, suffix: &str) -> PathBuf { + let mut name = db_path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + /// The capture database and the sidecar files SQLite keeps beside it. -fn capture_files(db_path: &Path) -> [std::path::PathBuf; 4] { - let named = |suffix: &str| { - let mut name = db_path.as_os_str().to_os_string(); - name.push(suffix); - std::path::PathBuf::from(name) - }; +fn capture_files(db_path: &Path) -> [PathBuf; 4] { [ db_path.to_path_buf(), - named("-wal"), - named("-shm"), - named("-journal"), + capture_file(db_path, SQLITE_SIDECARS[0]), + capture_file(db_path, SQLITE_SIDECARS[1]), + capture_file(db_path, SQLITE_SIDECARS[2]), ] } +fn sqlite_io_error(error: std::io::Error) -> rusqlite::Error { + rusqlite::Error::ToSqlConversionFailure(Box::new(error)) +} + +fn with_path(path: &Path, error: std::io::Error) -> std::io::Error { + std::io::Error::new(error.kind(), format!("{}: {error}", path.display())) +} + fn linked_or_not(path: &Path, expected: &str) -> std::io::Error { std::io::Error::new( std::io::ErrorKind::InvalidInput, @@ -787,6 +802,30 @@ fn linked_or_not(path: &Path, expected: &str) -> std::io::Error { ) } +#[cfg(unix)] +fn another_users(path: &Path) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!( + "{} belongs to another user; tunnel captures, which hold request headers \ + and bodies, are kept only in your own files. Run the tunnel as its owner, \ + or make it yours", + path.display() + ), + ) +} + +fn hard_linked(path: &Path) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "{} has other hard links, so captures written to it would change those \ + files too; move it aside and retry", + path.display() + ), + ) +} + /// Whether an entry is a link: a symbolic link, or on Windows a junction. /// Other reparse points, such as a synced folder's cloud placeholders, are /// the project's own files. @@ -796,63 +835,153 @@ fn is_link(metadata: &std::fs::Metadata) -> bool { /// Makes sure `.lpm` is a directory of the project's own, creating it if /// needed: a link, which a cloned repository could point anywhere, such as -/// into a directory it serves, is refused. +/// into a directory it serves, is refused. On Unix it must also be the +/// user's own, and is made writable by no one else, so no other user can +/// swap the capture files' names between the checks below and SQLite's +/// opens. fn prepare_lpm_dir(lpm_dir: &Path) -> std::io::Result<()> { match std::fs::symlink_metadata(lpm_dir) { Ok(metadata) if is_link(&metadata) || !metadata.is_dir() => { - Err(linked_or_not(lpm_dir, "directory")) + return Err(linked_or_not(lpm_dir, "directory")); } - Ok(_) => Ok(()), + Ok(_) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - std::fs::create_dir_all(lpm_dir) + std::fs::create_dir_all(lpm_dir).map_err(|error| with_path(lpm_dir, error))?; } - Err(error) => Err(error), + Err(error) => return Err(with_path(lpm_dir, error)), + } + restrict_lpm_dir(lpm_dir) +} + +#[cfg(unix)] +fn restrict_lpm_dir(lpm_dir: &Path) -> std::io::Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let directory = + lpm_common::open_nofollow(lpm_dir).map_err(|error| with_path(lpm_dir, error))?; + let metadata = directory.metadata()?; + if !metadata.is_dir() { + return Err(linked_or_not(lpm_dir, "directory")); + } + if metadata.uid() != lpm_common::current_uid() { + return Err(another_users(lpm_dir)); } + let mode = metadata.mode() & 0o7777; + if mode & 0o022 != 0 { + directory + .set_permissions(std::fs::Permissions::from_mode(mode & !0o022)) + .map_err(|error| with_path(lpm_dir, error))?; + } + Ok(()) +} + +#[cfg(not(unix))] +fn restrict_lpm_dir(_lpm_dir: &Path) -> std::io::Result<()> { + Ok(()) +} + +/// Holds other LPM processes off the capture files while this one checks, +/// replaces and opens them. A process that opened the database while it was +/// being replaced would go on writing to the old file. +fn lock_capture_files( + db_path: &Path, +) -> Result { + let lock_path = capture_file(db_path, "-lock"); + lpm_common::acquire_single_file_exclusive_lock_nofollow(&lock_path).map_err(|error| { + sqlite_io_error(std::io::Error::other(format!( + "{}: {error}", + lock_path.display() + ))) + }) } -/// Makes the capture files safe to write: no capture file may be a link; -/// existing ones are made owner-only, as a checkout or an older release may -/// have left them readable; and a new database is created owner-only before -/// SQLite opens it, so it's never readable by others. -fn prepare_capture_files(db_path: &Path) -> std::io::Result<()> { +/// Makes the capture files safe to write before SQLite opens them, and says +/// whether the database must be replaced, as others could open it or a +/// sidecar. No capture file may be a link, another user's, or have other +/// hard links. Existing ones are made owner-only, as a checkout or an older +/// release may have left them readable (see [`restrict_capture_file`]). +/// Sidecars without a database, which SQLite could read into a new one, are +/// removed, and a new database is created owner-only, so it's never +/// readable by others. +fn prepare_capture_files(db_path: &Path) -> std::io::Result { + let mut open_to_others = false; + let mut database_exists = false; for path in capture_files(db_path) { match std::fs::symlink_metadata(&path) { Ok(metadata) if is_link(&metadata) || !metadata.is_file() => { return Err(linked_or_not(&path, "file")); } - Ok(_) => make_owner_only(&path)?, + Ok(metadata) => { + open_to_others |= restrict_capture_file(&path, &metadata)?; + database_exists |= path == db_path; + } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error), + Err(error) => return Err(with_path(&path, error)), } } - create_owner_only(db_path) + if database_exists { + return Ok(open_to_others); + } + for suffix in SQLITE_SIDECARS { + remove_if_present(&capture_file(db_path, suffix))?; + } + create_owner_only(db_path)?; + Ok(false) } -/// Makes an existing capture file owner-only through a handle that doesn't -/// follow a link, so a link swapped in after the check isn't followed -/// either. A file that's gone needs nothing: SQLite removes `-wal` and -/// `-shm` when another process closes its last connection. A file with -/// other hard links, which a project could make point at one of its own -/// files, is refused. +fn remove_if_present(path: &Path) -> std::io::Result<()> { + match std::fs::remove_file(path) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(with_path(path, error)), + _ => Ok(()), + } +} + +/// Checks an existing capture file and says whether others could open it. +/// One they couldn't is made owner-only. One they could keeps its mode, +/// made only the owner's to read and write as well, until the database is +/// replaced, so a replacement that can't run yet runs on a later open. +/// Modes are changed by name, which no other user can swap in a directory +/// only its owner can write, rather than through a handle, whose closing +/// would drop the locks SQLite holds on the file in this process, and which +/// a file without read permission can't have. A file that's gone needs +/// nothing: SQLite removes `-wal` and `-shm` when another process closes +/// its last connection. #[cfg(unix)] -fn make_owner_only(path: &Path) -> std::io::Result<()> { +fn restrict_capture_file(path: &Path, metadata: &std::fs::Metadata) -> std::io::Result { use std::os::unix::fs::{MetadataExt, PermissionsExt}; - let file = match lpm_common::open_nofollow(path) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(error), - }; - let metadata = file.metadata()?; - if !metadata.is_file() || metadata.nlink() > 1 { - return Err(linked_or_not(path, "file")); + if metadata.uid() != lpm_common::current_uid() { + return Err(another_users(path)); } - file.set_permissions(std::fs::Permissions::from_mode(0o600)) + if metadata.nlink() > 1 { + return Err(hard_linked(path)); + } + let mode = metadata.mode() & 0o7777; + let open_to_others = mode & 0o077 != 0; + let wanted = if open_to_others { mode | 0o600 } else { 0o600 }; + if mode != wanted { + match std::fs::set_permissions(path, std::fs::Permissions::from_mode(wanted)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(with_path(path, error)), + } + } + Ok(open_to_others) } +/// Refuses an existing capture file with other hard links, read through a +/// handle that doesn't follow a link. Access to it is the project's ACLs. #[cfg(not(unix))] -fn make_owner_only(_path: &Path) -> std::io::Result<()> { - Ok(()) +fn restrict_capture_file(path: &Path, _metadata: &std::fs::Metadata) -> std::io::Result { + let file = match lpm_common::open_nofollow(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(with_path(path, error)), + }; + if lpm_common::link_count(&file).map_err(|error| with_path(path, error))? > 1 { + return Err(hard_linked(path)); + } + Ok(false) } #[cfg(unix)] @@ -867,7 +996,7 @@ fn create_owner_only(path: &Path) -> std::io::Result<()> { { Ok(_) => Ok(()), Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), - Err(error) => Err(error), + Err(error) => Err(with_path(path, error)), } } @@ -876,12 +1005,75 @@ fn create_owner_only(_path: &Path) -> std::io::Result<()> { Ok(()) } -/// Opens the capture database without following a link at its path. On -/// Unix, SQLite's no-follow open refuses a link anywhere in the path, as +/// Replaces a capture database others could open with an owner-only copy. +/// Making the file owner-only stops new opens, but not a handle another user +/// opened while it was readable, which would go on reading every capture +/// written after; the copy is a new file such a handle doesn't reach. A +/// database another process has open isn't replaced, since that process +/// would go on writing to the old file. +fn replace_with_owner_only_copy(db_path: &Path) -> Result<(), rusqlite::Error> { + let in_use = || { + sqlite_io_error(std::io::Error::new( + std::io::ErrorKind::ResourceBusy, + format!( + "{} is open to other users, and another process has it open, so it \ + can't be replaced with an owner-only copy yet; stop other `lpm tunnel` \ + and `lpm dev` runs in this project and retry", + db_path.display() + ), + )) + }; + let database = open_nofollow_connection(db_path)?; + // Leaving WAL needs the only connection, so it also proves no other + // process has the database open, and folds the WAL into the file. + match database.query_row("PRAGMA journal_mode=DELETE", [], |row| { + row.get::<_, String>(0) + }) { + Ok(mode) if mode.eq_ignore_ascii_case("delete") => {} + Ok(_) => return Err(in_use()), + Err(error) if is_busy(&error) => return Err(in_use()), + Err(error) => return Err(error), + } + + let copy_path = capture_file(db_path, "-copy"); + for path in [©_path, &capture_file(©_path, "-journal")] { + remove_if_present(path).map_err(sqlite_io_error)?; + } + create_owner_only(©_path).map_err(sqlite_io_error)?; + { + let mut copy = open_nofollow_connection(©_path)?; + let backup = rusqlite::backup::Backup::new(&database, &mut copy)?; + let deadline = std::time::Instant::now() + BUSY_TIMEOUT; + loop { + match backup.step(-1)? { + rusqlite::backup::StepResult::Done => break, + _ if std::time::Instant::now() < deadline => { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + _ => return Err(in_use()), + } + } + } + drop(database); + for suffix in SQLITE_SIDECARS { + remove_if_present(&capture_file(db_path, suffix)).map_err(sqlite_io_error)?; + } + std::fs::rename(©_path, db_path).map_err(|error| sqlite_io_error(with_path(db_path, error))) +} + +/// Opens a capture database without following a link at its path, with the +/// settings every capture connection has (see [`harden_connection`]). +fn open_nofollow_connection(path: &Path) -> Result { + let connection = connect_nofollow(path)?; + harden_connection(&connection)?; + Ok(connection) +} + +/// On Unix, SQLite's no-follow open refuses a link anywhere in the path, as /// macOS's `/var` and `/tmp` are, so it gets the directory's real path, -/// which leaves only the database itself to refuse as a link. On Windows -/// the flag does nothing, and the path is opened as it is. -fn open_capture_connection(path: &Path) -> Result { +/// which leaves only the database itself to refuse as a link. On Windows the +/// flag does nothing, and the path is opened as it is. +fn connect_nofollow(path: &Path) -> Result { #[cfg(unix)] let path = { let directory = match path.parent() { @@ -889,7 +1081,7 @@ fn open_capture_connection(path: &Path) -> Result { _ => Path::new("."), }; std::fs::canonicalize(directory) - .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))? + .map_err(|error| sqlite_io_error(with_path(directory, error)))? .join(path.file_name().unwrap_or(path.as_os_str())) }; Connection::open_with_flags( @@ -898,34 +1090,105 @@ fn open_capture_connection(path: &Path) -> Result { ) } +/// What every capture connection has: a database from a repository or +/// another tool can't run schema code with side effects or write SQLite's +/// internal tables, and a busy database is waited for. +fn harden_connection(connection: &Connection) -> Result<(), rusqlite::Error> { + use rusqlite::config::DbConfig; + + connection.set_db_config(DbConfig::SQLITE_DBCONFIG_DEFENSIVE, true)?; + connection.set_db_config(DbConfig::SQLITE_DBCONFIG_TRUSTED_SCHEMA, false)?; + connection.busy_timeout(BUSY_TIMEOUT) +} + +/// Opens a capture connection in WAL, with `synchronous` as its sync level. +fn open_capture_connection(path: &Path, synchronous: &str) -> Result { + let connection = connect_nofollow(path)?; + configure_connection(&connection, synchronous)?; + Ok(connection) +} + +fn configure_connection(connection: &Connection, synchronous: &str) -> Result<(), rusqlite::Error> { + harden_connection(connection)?; + use_wal(connection)?; + connection.execute_batch(&format!( + "PRAGMA synchronous={synchronous}; PRAGMA foreign_keys=ON;" + )) +} + +/// Switches the database to WAL. Connections switching a new database at +/// once can get SQLITE_BUSY without the busy timeout being waited, so the +/// switch is retried, a jittered few milliseconds apart, until it passes. +fn use_wal(connection: &Connection) -> Result<(), rusqlite::Error> { + use rand::Rng as _; + + let deadline = std::time::Instant::now() + BUSY_TIMEOUT; + loop { + match connection.query_row("PRAGMA journal_mode=WAL", [], |_| Ok(())) { + Err(error) if is_busy(&error) && std::time::Instant::now() < deadline => { + let pause = rand::thread_rng().gen_range(5..=25); + std::thread::sleep(std::time::Duration::from_millis(pause)); + } + result => return result, + } + } +} + +fn is_busy(error: &rusqlite::Error) -> bool { + matches!( + error.sqlite_error_code(), + Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked) + ) +} + +/// Refuses a database whose schema holds triggers or views, which LPM never +/// creates: one from a repository or another tool could run them on every +/// capture written. +fn refuse_foreign_schema(connection: &Connection, db_path: &Path) -> Result<(), rusqlite::Error> { + let foreign: bool = connection.query_row( + "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type IN ('trigger', 'view'))", + [], + |row| row.get(0), + )?; + if foreign { + return Err(sqlite_io_error(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "{} holds triggers or views, which LPM never creates; move it aside \ + and retry", + db_path.display() + ), + ))); + } + Ok(()) +} + /// A `.gitignore` line as git reads it: without one trailing carriage /// return or trailing spaces that aren't escaped. Leading whitespace stays /// part of the pattern, as it does for git. -fn git_pattern(line: &str) -> &str { - let line = line.strip_suffix('\r').unwrap_or(line); - let bytes = line.as_bytes(); - let mut end = bytes.len(); - while end > 0 && bytes[end - 1] == b' ' && !(end >= 2 && bytes[end - 2] == b'\\') { +fn git_pattern(line: &[u8]) -> &[u8] { + let line = line.strip_suffix(b"\r").unwrap_or(line); + let mut end = line.len(); + while end > 0 && line[end - 1] == b' ' && !(end >= 2 && line[end - 2] == b'\\') { end -= 1; } &line[..end] } /// Whether these `.gitignore` lines leave the capture files ignored: a rule -/// covering them, unanchored or anchored to `.lpm/` (`inspector.db*`, or -/// `*` as a tool directory's ignore file has), after the last negation that -/// could match one. A negation naming another file outright, such as -/// `!.gitignore`, can't. -fn ignores_captures(lines: &str) -> bool { - const COVERING: [&str; 3] = [CAPTURES_IGNORE_LINE, "*", "**"]; +/// covering them, unanchored or anchored to `.lpm/` (`inspector.db*`, or `*` +/// as a tool directory's ignore file has), after the last negation that +/// could match one. The lines are bytes, as git reads them. +fn ignores_captures(lines: &[u8]) -> bool { + const COVERING: [&[u8]; 3] = [CAPTURES_IGNORE_LINE.as_bytes(), b"*", b"**"]; let mut ignored = false; - for line in lines.split('\n') { + for line in lines.split(|byte| *byte == b'\n') { let pattern = git_pattern(line); - let unanchored = pattern.strip_prefix('/').unwrap_or(pattern); + let unanchored = pattern.strip_prefix(b"/").unwrap_or(pattern); if COVERING.contains(&unanchored) { ignored = true; - } else if let Some(negated) = pattern.strip_prefix('!') - && (negated.contains(['*', '?', '[']) || negated.contains("inspector.db")) + } else if let Some(negated) = pattern.strip_prefix(b"!") + && could_name_a_capture_file(negated) { ignored = false; } @@ -933,67 +1196,94 @@ fn ignores_captures(lines: &str) -> bool { ignored } +/// Whether a negated pattern could match a capture file. Only one of plain +/// characters that doesn't hold `inspector.db` in any case can't: escapes, +/// wildcards and ranges can all spell a capture file's name, and a +/// repository on a case-insensitive disk matches it in any case. +fn could_name_a_capture_file(pattern: &[u8]) -> bool { + const NAME: &[u8] = b"inspector.db"; + let plain = pattern + .iter() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'/' | b'_' | b'-')); + !plain + || pattern + .windows(NAME.len()) + .any(|window| window.eq_ignore_ascii_case(NAME)) +} + /// Adds [`CAPTURES_IGNORE_LINE`] to `.lpm/.gitignore` unless the file /// already ignores the capture files. An ignore file inside `.lpm/` holds /// wherever the project sits in a repository, and leaves the project's own -/// `.gitignore` alone. A linked or read-only `.lpm/.gitignore` is left as it -/// is; the CLI's capture warning then says git doesn't ignore the captures. +/// `.gitignore` alone. One that git wouldn't read as a file of the +/// project's, being a link or not a regular file, or that its owner keeps +/// from being written, being read-only or on Unix another user's, is left +/// as it is; the CLI's capture warning then says git doesn't ignore the +/// captures. fn ensure_captures_ignored(lpm_dir: &Path) -> std::io::Result<()> { let path = lpm_dir.join(".gitignore"); - let (existing, exists) = match lpm_common::read_text_file_capped_nofollow( + let (existing, metadata) = match lpm_common::read_file_capped_unlinked( &path, lpm_common::CONFIG_FILE_SIZE_CAP_BYTES, ) { - Ok(text) => (text, true), - Err(lpm_common::BoundedReadError::NotFound { .. }) => (String::new(), false), + Ok((bytes, metadata)) => (bytes, Some(metadata)), + Err(lpm_common::BoundedReadError::NotFound { .. }) => (Vec::new(), None), Err(error) => return Err(std::io::Error::other(error)), }; if ignores_captures(&existing) { return Ok(()); } - if exists && std::fs::metadata(&path)?.permissions().readonly() { - return Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - format!("{} is read-only", path.display()), - )); + if let Some(metadata) = &metadata { + refuse_kept_from_writing(&path, metadata)?; } - let newline = if existing.contains("\r\n") { - "\r\n" + let newline: &[u8] = if existing.windows(2).any(|pair| pair == b"\r\n") { + b"\r\n" } else { - "\n" + b"\n" }; let mut updated = existing; - if !updated.is_empty() && !updated.ends_with('\n') { - updated.push_str(newline); + if !updated.is_empty() && !updated.ends_with(b"\n") { + updated.extend_from_slice(newline); } - updated.push_str("# LPM tunnel captures hold request headers and bodies."); - updated.push_str(newline); - updated.push_str(CAPTURES_IGNORE_LINE); - updated.push_str(newline); + updated.extend_from_slice(b"# LPM tunnel captures hold request headers and bodies."); + updated.extend_from_slice(newline); + updated.extend_from_slice(CAPTURES_IGNORE_LINE.as_bytes()); + updated.extend_from_slice(newline); lpm_common::write_file_atomic(&path, updated) } +fn refuse_kept_from_writing(path: &Path, metadata: &std::fs::Metadata) -> std::io::Result<()> { + #[cfg(unix)] + let read_only = { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + if metadata.uid() != lpm_common::current_uid() { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!("{} belongs to another user", path.display()), + )); + } + metadata.permissions().mode() & 0o200 == 0 + }; + #[cfg(not(unix))] + let read_only = metadata.permissions().readonly(); + if read_only { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!("{} is read-only", path.display()), + )); + } + Ok(()) +} + // ── Schema initialization ───────────────────────────────────────────── fn open_control_connection(path: &Path) -> Result { - let connection = open_capture_connection(path)?; - connection.execute_batch( - "PRAGMA journal_mode=WAL; - PRAGMA synchronous=FULL; - PRAGMA foreign_keys=ON; - PRAGMA busy_timeout=5000;", - )?; - Ok(connection) + open_capture_connection(path, "FULL") } fn init_schema(conn: &Connection) -> Result<(), rusqlite::Error> { conn.execute_batch( - "PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA foreign_keys=ON; - PRAGMA busy_timeout=5000; - - CREATE TABLE IF NOT EXISTS sessions ( + "CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, name TEXT, domain TEXT, @@ -1466,6 +1756,36 @@ mod tests { std::fs::read_to_string(project.join(".lpm/.gitignore")).unwrap() } + #[cfg(unix)] + fn mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + + std::fs::symlink_metadata(path) + .unwrap() + .permissions() + .mode() + & 0o7777 + } + + #[cfg(unix)] + fn set_mode(path: &Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + /// A file of root's, for checks of files that belong to another user; + /// None when the tests run as root. + #[cfg(unix)] + fn another_users_file() -> Option<(PathBuf, std::fs::Metadata)> { + if lpm_common::current_uid() == 0 { + return None; + } + let path = PathBuf::from("/etc/hosts"); + let metadata = std::fs::symlink_metadata(&path).unwrap(); + Some((path, metadata)) + } + #[tokio::test] async fn opening_keeps_captures_out_of_git_once() { let project = tempfile::tempdir().unwrap(); @@ -1490,12 +1810,12 @@ mod tests { ); } - fn opened_with_gitignore(content: &str) -> (tempfile::TempDir, String) { + fn opened_with_gitignore(content: &[u8]) -> (tempfile::TempDir, Vec) { let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); std::fs::write(project.path().join(".lpm/.gitignore"), content).unwrap(); drop(InspectorDb::open(project.path()).unwrap()); - let written = lpm_gitignore(project.path()); + let written = std::fs::read(project.path().join(".lpm/.gitignore")).unwrap(); (project, written) } @@ -1508,11 +1828,12 @@ mod tests { "/inspector.db*\n", "!inspector.db\ninspector.db*\n", "*\n!.gitignore\n", + "*\n!skills/\n!/added-sources.json\n!Skills_2-x\n", "/*\n", "**\n", ] { - let (_project, written) = opened_with_gitignore(content); - assert_eq!(written, content, "{content:?}"); + let (_project, written) = opened_with_gitignore(content.as_bytes()); + assert_eq!(written, content.as_bytes(), "{content:?}"); } } @@ -1528,49 +1849,87 @@ mod tests { "inspector.db*\n!*.db\n", "inspector.db*\n!inspector*\n", "inspector.db*\n!**\n", + "inspector.db*\n!inspector.d?\n", + "inspector.db*\n!inspector.d[b]\n", + "inspector.db*\n!inspector.d\\b\n", + "inspector.db*\n!inspector\\.db\n", + "inspector.db*\n!/inspector.d\\b\n", + "inspector.db*\n!inspecto\\r.db-wal\n", + "inspector.db*\n!Inspector.db\n", + "inspector.db*\n!INSPECTOR.DB-WAL\n", + "**\n!INSPECTOR.DB\n", + "*\n!.lpm/inspector.db\n", "inspector.db\n", ] { - let (_project, written) = opened_with_gitignore(content); + let (_project, written) = opened_with_gitignore(content.as_bytes()); assert_eq!( written, - format!("{content}{CAPTURES_IGNORED}"), + format!("{content}{CAPTURES_IGNORED}").as_bytes(), "{content:?}" ); } } + #[tokio::test] + async fn opening_adds_the_rule_to_an_lpm_gitignore_that_isnt_utf8() { + let content = b"skills/\n\xff\xfe-notes\n"; + + let (_project, written) = opened_with_gitignore(content); + + let mut expected = content.to_vec(); + expected.extend_from_slice(CAPTURES_IGNORED.as_bytes()); + assert_eq!(written, expected); + } + + #[tokio::test] + async fn opening_reads_a_rule_beside_lines_that_arent_utf8() { + let content = b"\xff\xfe\ninspector.db*\n"; + + let (_project, written) = opened_with_gitignore(content); + + assert_eq!(written, content); + } + #[tokio::test] async fn opening_adds_the_rule_after_one_with_two_carriage_returns_which_git_reads_as_another() { - let (_project, written) = opened_with_gitignore("inspector.db*\r\r\n"); + let (_project, written) = opened_with_gitignore(b"inspector.db*\r\r\n"); + + assert_eq!( + written, + b"inspector.db*\r\r\n# LPM tunnel captures hold request headers and bodies.\r\ninspector.db*\r\n" + ); + } + + #[tokio::test] + async fn opening_reads_a_negation_ending_in_a_carriage_return_as_git_does() { + let (_project, written) = opened_with_gitignore(b"inspector.db*\r\n!Inspector.DB\r\n"); assert_eq!( written, - "inspector.db*\r\r\n# LPM tunnel captures hold request headers and bodies.\r\ninspector.db*\r\n" + b"inspector.db*\r\n!Inspector.DB\r\n# LPM tunnel captures hold request headers and bodies.\r\ninspector.db*\r\n" ); } #[tokio::test] async fn opening_adds_the_rule_with_the_file_s_line_endings() { - let (_project, written) = opened_with_gitignore("skills/\r\n"); + let (_project, written) = opened_with_gitignore(b"skills/\r\n"); assert_eq!( written, - "skills/\r\n# LPM tunnel captures hold request headers and bodies.\r\ninspector.db*\r\n" + b"skills/\r\n# LPM tunnel captures hold request headers and bodies.\r\ninspector.db*\r\n" ); } #[cfg(unix)] #[tokio::test] async fn opening_leaves_a_read_only_lpm_gitignore_alone_even_empty() { - use std::os::unix::fs::PermissionsExt; - for content in ["skills/\n", ""] { let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let path = project.path().join(".lpm/.gitignore"); std::fs::write(&path, content).unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o444)).unwrap(); + set_mode(&path, 0o444); drop(InspectorDb::open(project.path()).unwrap()); @@ -1578,6 +1937,46 @@ mod tests { } } + #[cfg(unix)] + #[tokio::test] + async fn opening_leaves_an_lpm_gitignore_its_owner_cant_write_alone() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let path = project.path().join(".lpm/.gitignore"); + std::fs::write(&path, "skills/\n").unwrap(); + set_mode(&path, 0o464); + + drop(InspectorDb::open(project.path()).unwrap()); + + assert_eq!(lpm_gitignore(project.path()), "skills/\n"); + } + + #[cfg(unix)] + #[test] + fn leaves_another_user_s_lpm_gitignore_alone() { + let Some((path, metadata)) = another_users_file() else { + return; + }; + + let error = refuse_kept_from_writing(&path, &metadata).unwrap_err(); + + assert!( + error.to_string().contains("belongs to another user"), + "{error}" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn opening_leaves_an_lpm_gitignore_that_isnt_a_regular_file_alone() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(project.path().join(".lpm/.gitignore")).unwrap(); + + drop(InspectorDb::open(project.path()).unwrap()); + + assert!(project.path().join(".lpm/.gitignore").is_dir()); + } + #[tokio::test] async fn opening_writes_the_rule_before_it_creates_the_database() { let project = tempfile::tempdir().unwrap(); @@ -1590,30 +1989,77 @@ mod tests { assert_eq!(lpm_gitignore(project.path()), CAPTURES_IGNORED); } + #[cfg(unix)] + #[tokio::test] + async fn opening_makes_an_lpm_directory_others_could_write_theirs_alone() { + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + set_mode(&lpm_dir, 0o777); + + drop(InspectorDb::open(project.path()).unwrap()); + + assert_eq!(mode(&lpm_dir), 0o755); + } + + #[cfg(unix)] + #[test] + fn refuses_another_user_s_lpm_directory() { + if lpm_common::current_uid() == 0 { + return; + } + + let error = restrict_lpm_dir(Path::new("/usr")).unwrap_err(); + + assert!( + error.to_string().contains("belongs to another user"), + "{error}" + ); + } + + #[cfg(unix)] + #[test] + fn refuses_another_user_s_capture_file() { + let Some((path, metadata)) = another_users_file() else { + return; + }; + + let error = restrict_capture_file(&path, &metadata).unwrap_err(); + + assert!( + error.to_string().contains("belongs to another user"), + "{error}" + ); + } + #[cfg(unix)] #[test] fn restricting_a_capture_file_that_vanished_needs_nothing() { let project = tempfile::tempdir().unwrap(); + let path = project.path().join("inspector.db-wal"); + std::fs::write(&path, b"").unwrap(); + set_mode(&path, 0o400); + let metadata = std::fs::symlink_metadata(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); - make_owner_only(&project.path().join("inspector.db-wal")).unwrap(); + assert!(!restrict_capture_file(&path, &metadata).unwrap()); } #[cfg(unix)] #[test] fn preparing_refuses_a_capture_file_with_other_hard_links() { - use std::os::unix::fs::PermissionsExt; - let project = tempfile::tempdir().unwrap(); let lpm_dir = project.path().join(".lpm"); std::fs::create_dir(&lpm_dir).unwrap(); let served = project.path().join("public.txt"); std::fs::write(&served, b"").unwrap(); - std::fs::set_permissions(&served, std::fs::Permissions::from_mode(0o644)).unwrap(); + set_mode(&served, 0o644); std::fs::hard_link(&served, lpm_dir.join("inspector.db")).unwrap(); - assert!(prepare_capture_files(&lpm_dir.join("inspector.db")).is_err()); - let mode = std::fs::metadata(&served).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o644); + let error = prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap_err(); + + assert!(error.to_string().contains("other hard links"), "{error}"); + assert_eq!(mode(&served), 0o644); } #[cfg(unix)] @@ -1626,13 +2072,11 @@ mod tests { assert!(!outside.path().join("inspector.db").exists()); for name in ["inspector.db", "inspector.db-wal"] { - use std::os::unix::fs::PermissionsExt; - let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let target = outside.path().join(format!("{name}-target")); std::fs::write(&target, "outside\n").unwrap(); - std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o644)).unwrap(); + set_mode(&target, 0o644); std::os::unix::fs::symlink(&target, project.path().join(".lpm").join(name)).unwrap(); assert!(InspectorDb::open(project.path()).is_err(), "{name}"); @@ -1641,64 +2085,329 @@ mod tests { "outside\n", "{name}" ); - let mode = std::fs::metadata(&target).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o644, "{name}"); + assert_eq!(mode(&target), 0o644, "{name}"); } } #[cfg(unix)] #[test] - fn preparing_makes_leftover_capture_files_owner_only_before_sqlite_opens_them() { - use std::os::unix::fs::PermissionsExt; - + fn preparing_makes_leftover_capture_files_others_couldnt_open_owner_only() { let project = tempfile::tempdir().unwrap(); let lpm_dir = project.path().join(".lpm"); std::fs::create_dir(&lpm_dir).unwrap(); + for (name, leftover_mode) in [ + ("inspector.db", 0o000), + ("inspector.db-wal", 0o400), + ("inspector.db-shm", 0o700), + ("inspector.db-journal", 0o600), + ] { + let path = lpm_dir.join(name); + std::fs::write(&path, b"").unwrap(); + set_mode(&path, leftover_mode); + } + + assert!(!prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap()); + for name in [ + "inspector.db", "inspector.db-wal", "inspector.db-shm", "inspector.db-journal", ] { + assert_eq!(mode(&lpm_dir.join(name)), 0o600, "{name}"); + } + } + + #[cfg(unix)] + #[test] + fn preparing_keeps_a_leftover_others_could_open_for_the_database_s_replacement() { + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + for (name, leftover_mode) in [("inspector.db", 0o600), ("inspector.db-wal", 0o204)] { let path = lpm_dir.join(name); std::fs::write(&path, b"").unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + set_mode(&path, leftover_mode); + } + + assert!(prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap()); + + assert_eq!(mode(&lpm_dir.join("inspector.db-wal")), 0o604); + } + + #[cfg(unix)] + #[test] + fn preparing_removes_sidecars_left_without_a_database() { + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + for name in [ + "inspector.db-wal", + "inspector.db-shm", + "inspector.db-journal", + ] { + let path = lpm_dir.join(name); + std::fs::write(&path, b"leftover").unwrap(); + set_mode(&path, 0o644); } - prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap(); + assert!(!prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap()); for name in [ - "inspector.db", "inspector.db-wal", "inspector.db-shm", "inspector.db-journal", ] { - let mode = std::fs::metadata(lpm_dir.join(name)) - .unwrap() - .permissions() - .mode(); - assert_eq!(mode & 0o777, 0o600, "{name}"); + assert!(!lpm_dir.join(name).exists(), "{name}"); } + assert_eq!(mode(&lpm_dir.join("inspector.db")), 0o600); } #[cfg(unix)] #[tokio::test] async fn opening_keeps_capture_files_owner_only() { - use std::os::unix::fs::PermissionsExt; - let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let wal = project.path().join(".lpm/inspector.db-wal"); std::fs::write(&wal, b"").unwrap(); - std::fs::set_permissions(&wal, std::fs::Permissions::from_mode(0o644)).unwrap(); + set_mode(&wal, 0o644); drop(InspectorDb::open(project.path()).unwrap()); for name in ["inspector.db", "inspector.db-wal"] { - let mode = std::fs::metadata(project.path().join(".lpm").join(name)) + assert_eq!( + mode(&project.path().join(".lpm").join(name)), + 0o600, + "{name}" + ); + } + } + + /// A project whose capture database holds one capture, with no + /// connection left open. + #[cfg(unix)] + fn project_with_a_capture() -> tempfile::TempDir { + let project = tempfile::tempdir().unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let db = InspectorDb::open(project.path()).unwrap(); + db.insert_durable_request(Arc::new(make_webhook("kept", 200)), None) + .await + .unwrap(); + }); + // Dropping the runtime drops the flush task and its connection. + drop(runtime); + project + } + + #[cfg(unix)] + #[test] + fn opening_replaces_a_database_others_could_open_with_an_owner_only_copy() { + use std::os::unix::fs::MetadataExt; + + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + set_mode(&db_path, 0o644); + // A handle another user could have opened while the file was + // readable. + let earlier_handle = std::fs::File::open(&db_path).unwrap(); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let kept = runtime.block_on(async { + let db = InspectorDb::open(project.path()).unwrap(); + db.get_webhook("kept").await.unwrap() + }); + + assert!(kept.is_some()); + assert_eq!(mode(&db_path), 0o600); + let replaced = earlier_handle.metadata().unwrap(); + assert_eq!(replaced.nlink(), 0); + assert_ne!(replaced.ino(), std::fs::metadata(&db_path).unwrap().ino()); + assert!(!project.path().join(".lpm/inspector.db-copy").exists()); + } + + #[cfg(unix)] + #[test] + fn opening_waits_to_replace_a_database_another_process_has_open() { + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + set_mode(&db_path, 0o644); + let other = Connection::open(&db_path).unwrap(); + other + .query_row("SELECT count(*) FROM requests", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let started = std::time::Instant::now(); + + let error = runtime + .block_on(async { InspectorDb::open(project.path()).map(drop) }) + .unwrap_err(); + + assert!( + error.to_string().contains("another process has it open"), + "{error}" + ); + // Leaving WAL tells at once; the copy's own wait is the busy timeout. + assert!(started.elapsed() < std::time::Duration::from_secs(4)); + assert_eq!(mode(&db_path), 0o644); + drop(other); + runtime.block_on(async { drop(InspectorDb::open(project.path()).unwrap()) }); + assert_eq!(mode(&db_path), 0o600); + } + + #[tokio::test] + async fn opening_refuses_a_database_with_triggers_or_views() { + for statement in [ + "CREATE TRIGGER copy_requests AFTER INSERT ON requests BEGIN SELECT 1; END;", + "CREATE VIEW recent AS SELECT * FROM requests;", + ] { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let db_path = project.path().join(".lpm/inspector.db"); + Connection::open(&db_path) + .unwrap() + .execute_batch(&format!("CREATE TABLE requests (id TEXT); {statement}")) + .unwrap(); + #[cfg(unix)] + set_mode(&db_path, 0o600); + + let error = InspectorDb::open(project.path()).map(drop).unwrap_err(); + + assert!(error.to_string().contains("triggers or views"), "{error}"); + } + } + + #[cfg(unix)] + #[test] + fn capture_connections_refuse_a_linked_database() { + let directory = tempfile::tempdir().unwrap(); + let target = directory.path().join("elsewhere.db"); + Connection::open(&target) + .unwrap() + .execute_batch("CREATE TABLE t (x);") + .unwrap(); + let link = directory.path().join("inspector.db"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + assert!(open_nofollow_connection(&link).is_err()); + } + + #[test] + fn capture_connections_trust_no_schema_code() { + let directory = tempfile::tempdir().unwrap(); + + let connection = + open_capture_connection(&directory.path().join("inspector.db"), "NORMAL").unwrap(); + + let trusted: i64 = connection + .query_row("PRAGMA trusted_schema", [], |row| row.get(0)) + .unwrap(); + assert_eq!(trusted, 0); + assert!( + connection + .db_config(rusqlite::config::DbConfig::SQLITE_DBCONFIG_DEFENSIVE) .unwrap() - .permissions() - .mode(); - assert_eq!(mode & 0o777, 0o600, "{name}"); + ); + } + + #[test] + fn switching_to_wal_waits_out_a_connection_that_refuses_it_at_once() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("inspector.db"); + let holder = Connection::open(&path).unwrap(); + holder + .execute_batch("CREATE TABLE t (x); BEGIN; SELECT * FROM t;") + .unwrap(); + let switcher = Connection::open(&path).unwrap(); + switcher.busy_timeout(std::time::Duration::ZERO).unwrap(); + let released = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(100)); + holder.execute_batch("COMMIT").unwrap(); + holder + }); + + use_wal(&switcher).unwrap(); + + let mode: String = switcher + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .unwrap(); + assert_eq!(mode, "wal"); + drop(released.join().unwrap()); + } + + #[test] + fn switching_to_wal_gives_up_at_once_on_anything_but_a_busy_database() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("inspector.db"); + std::fs::write(&path, vec![b'x'; 4096]).unwrap(); + let connection = Connection::open(&path).unwrap(); + let started = std::time::Instant::now(); + + assert!(use_wal(&connection).is_err()); + + assert!(started.elapsed() < std::time::Duration::from_secs(1)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn opening_a_new_database_from_several_processes_at_once_succeeds() { + for _ in 0..10 { + let project = tempfile::tempdir().unwrap(); + let openers: Vec<_> = (0..8) + .map(|_| { + let path = project.path().to_path_buf(); + tokio::task::spawn_blocking(move || InspectorDb::open(&path).map(drop)) + }) + .collect(); + for opener in openers { + opener.await.unwrap().unwrap(); + } + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn opening_waits_for_another_lpm_process_opening_the_captures() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let lock = lpm_common::acquire_single_file_exclusive_lock_nofollow( + project.path().join(".lpm/inspector.db-lock"), + ) + .unwrap(); + let path = project.path().to_path_buf(); + let opener = tokio::task::spawn_blocking(move || { + InspectorDb::open(&path).map(drop)?; + Ok::<_, rusqlite::Error>(std::time::Instant::now()) + }); + + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let released = std::time::Instant::now(); + drop(lock); + let opened = opener.await.unwrap().unwrap(); + + assert!(opened >= released, "opened before the lock was released"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn clearing_history_from_two_processes_at_once_succeeds() { + let project = tempfile::tempdir().unwrap(); + let first = InspectorDb::open(project.path()).unwrap(); + let second = InspectorDb::open(project.path()).unwrap(); + for round in 0..200 { + first + .insert_durable_request(Arc::new(make_webhook(&format!("r{round}"), 200)), None) + .await + .unwrap(); + let (first_clear, second_clear) = (first.clone(), second.clone()); + let (one, two) = tokio::join!( + tokio::spawn(async move { first_clear.clear_history().await }), + tokio::spawn(async move { second_clear.clear_history().await }), + ); + one.unwrap().unwrap(); + two.unwrap().unwrap(); } } @@ -1937,16 +2646,10 @@ mod tests { let project = tempfile::tempdir().unwrap(); let db_path = project.path().join("inspector.db"); let write_conn = Connection::open(&db_path).unwrap(); + configure_connection(&write_conn, "NORMAL").unwrap(); init_schema(&write_conn).unwrap(); let read_conn = Connection::open(&db_path).unwrap(); - read_conn - .execute_batch( - "PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA foreign_keys=ON; - PRAGMA busy_timeout=5000;", - ) - .unwrap(); + configure_connection(&read_conn, "NORMAL").unwrap(); let control_conn = open_control_connection(&db_path).unwrap(); let (write_tx, write_rx) = mpsc::channel(1); let queued_capture_bytes = Arc::new(AtomicUsize::new(0)); diff --git a/tests/integration/tests/atomic_write_inventory.rs b/tests/integration/tests/atomic_write_inventory.rs index d48d73048..8d733bb47 100644 --- a/tests/integration/tests/atomic_write_inventory.rs +++ b/tests/integration/tests/atomic_write_inventory.rs @@ -158,6 +158,10 @@ fn reviewed_raw_atomic_writer(key: &str) -> bool { } const REVIEWED_RAW_ATOMIC_WRITERS: &[(&str, &str)] = &[ + ( + "crates/lpm-inspect/src/db.rs::replace_with_owner_only_copy", + "renames SQLite's finished backup, written into an exclusively created owner-only file beside the capture database, over a database others could open, under the capture files' lock", + ), ( "crates/lpm-cli/src/cli/install_timeline.rs::export_with", "serializes and flushes an exclusively created same-directory NamedTempFile, then publishes with persist_noclobber without replacing an existing destination", diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index b11d09ca6..280630859 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1360,7 +1360,7 @@ async fn tunnel_start_names_capture_files_git_tracks_and_how_to_stop_it() { assert!( stderr.contains("git tracks '.lpm/inspector.db'") - && stderr.contains("git rm --cached -- .lpm/inspector.db") + && stderr.contains("git -C .lpm rm --cached -- inspector.db") && stderr.contains("rotate any credentials"), "tunnel startup must name tracked capture files:\n{stderr}" ); @@ -1370,6 +1370,26 @@ async fn tunnel_start_names_capture_files_git_tracks_and_how_to_stop_it() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tunnel_start_asks_a_repository_inside_lpm_about_the_captures_it_tracks() { + let stderr = tunnel_start_stderr(|project| { + git(project, &["init", "--quiet"]); + let lpm_dir = project.join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + git(&lpm_dir, &["init", "--quiet"]); + std::fs::write(lpm_dir.join("inspector.db"), b"").unwrap(); + git(&lpm_dir, &["add", "inspector.db"]); + }) + .await; + + assert!( + stderr.contains("git tracks '.lpm/inspector.db'") + && stderr.contains("git -C .lpm rm --cached -- inspector.db") + && !stderr.contains("git ignores it"), + "a repository inside .lpm must answer for the captures it tracks:\n{stderr}" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tunnel_start_says_git_ignores_captures_once_it_does() { let stderr = tunnel_start_stderr(|project| git(project, &["init", "--quiet"])).await; @@ -1396,7 +1416,8 @@ async fn tunnel_start_says_to_replace_a_linked_lpm_gitignore_git_doesnt_read() { assert!( stderr.contains("git doesn't ignore") && stderr.contains( - "replace the linked .lpm/.gitignore with a file that holds `inspector.db*`" + "replace .lpm/.gitignore, which git doesn't read as a file of the project's, \ + with a file that holds `inspector.db*`" ) && !stderr.contains("git ignores it"), "{stderr}" @@ -1452,6 +1473,41 @@ fn tunnel_list_without_auth_under_json_emits_error_envelope_on_stdout() { assert_eq!(envelope["error_code"], serde_json::json!("tunnel")); } +#[test] +fn tunnel_inspect_reports_a_capture_database_it_cant_open_as_a_disk_error_under_json() { + let project = TempProject::empty(r#"{"name":"tunnel","version":"1.0.0"}"#); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + std::fs::write(project.path().join(".lpm/inspector.db"), b"not a database").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions( + project.path().join(".lpm/inspector.db"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + } + + let output = lpm(&project) + .args(["--json", "tunnel", "inspect"]) + .output() + .expect("failed to run lpm --json tunnel inspect"); + + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let envelope: serde_json::Value = serde_json::from_str(stdout.trim()).unwrap_or_else(|e| { + panic!("--json tunnel inspect error path must emit JSON: {e}\n---\n{stdout}") + }); + assert_eq!(envelope["success"], serde_json::json!(false)); + assert_eq!(envelope["error_code"], serde_json::json!("io")); + assert!( + envelope["error"] + .as_str() + .is_some_and(|message| message.contains("Move or repair it")), + "{envelope}" + ); +} + #[test] fn tunnel_inspect_without_auth_reads_project_inspector_database_under_json() { let project = TempProject::empty(r#"{"name":"tunnel","version":"1.0.0"}"#); From fd2f611dd79dc9941165ce1d84a2e9004acfd398 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Mon, 12 Oct 2026 12:02:09 +0100 Subject: [PATCH 07/17] fix(tunnel): review captures without replacing, and refuse planted schema, ACLs and damage - inspect, log, replay and inspect --ui open the capture database for review: they change no file's access, so they work while a tunnel records to it, and warn when others could read it. Only lpm tunnel and lpm dev --tunnel replace a database others could read. - Others could read a capture file when its group or anyone may read it, or on macOS when it has an extended ACL; a writer clears .lpm's ACL and new files' inherited entries. The ACL helpers move to lpm-common. - Root accepts the project directory owner's files and its own, and gives what it creates to that owner. - A filesystem without Unix modes keeps its database, and the tunnel's warning says it's open to other users. - A database holding schema LPM never creates is refused, compared with LPM's own schema built in memory. The replacement runs quick_check first. Connections check cell sizes, overwrite deleted content and cap values at 128 MiB. rusqlite 0.40.2 bundles SQLite 3.53.2. - log --clear rebuilds the full-text index and truncates the WAL, so no cleared capture stays on disk. - Every open removes a leftover replacement copy, and the open lock gives up after 10 s. - The capture warning asks the project's repository and git inside .lpm at once, with the untrack command for each. - lpm dev keeps disk failures IO errors through cleanup failures and restores, and a failed clear is reported as a write. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 39 +- crates/lpm-cli/Cargo.toml | 2 + crates/lpm-cli/src/commands/dev.rs | 188 ++- crates/lpm-cli/src/commands/tunnel.rs | 769 +++++---- crates/lpm-common/src/lib.rs | 9 +- crates/lpm-common/src/macos_acl.rs | 155 ++ crates/lpm-common/src/paths.rs | 88 +- crates/lpm-inspect/Cargo.toml | 3 +- crates/lpm-inspect/src/db.rs | 1548 ++++++++++++++++--- crates/lpm-vault/src/storage_transaction.rs | 92 +- tests/workflows/tests/dev_tunnel.rs | 56 +- 11 files changed, 2314 insertions(+), 635 deletions(-) create mode 100644 crates/lpm-common/src/macos_acl.rs diff --git a/Cargo.lock b/Cargo.lock index 54c03e0b4..e1b5acff3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2411,11 +2411,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.10.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.17.1", ] [[package]] @@ -3207,9 +3207,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.32.0" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" dependencies = [ "cc", "pkg-config", @@ -3452,6 +3452,7 @@ dependencies = [ "rcgen", "reqwest", "roxmltree", + "rusqlite", "rustix", "same-file", "secrecy", @@ -3678,6 +3679,7 @@ dependencies = [ "serde_json", "subtle", "tempfile", + "thiserror 2.0.18", "tokio", "tower", "tower-http", @@ -6205,6 +6207,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.18", +] + [[package]] name = "rtoolbox" version = "0.0.5" @@ -6217,9 +6229,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.34.0" +version = "0.40.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" dependencies = [ "bitflags 2.13.0", "fallible-iterator", @@ -6227,6 +6239,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -6849,6 +6862,18 @@ dependencies = [ "der", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/crates/lpm-cli/Cargo.toml b/crates/lpm-cli/Cargo.toml index bf484dff3..2a5a738f3 100644 --- a/crates/lpm-cli/Cargo.toml +++ b/crates/lpm-cli/Cargo.toml @@ -245,6 +245,8 @@ wiremock = "0.6" assert_cmd = "2" insta = { version = "1", features = ["json"] } lpm-vault = { workspace = true, features = ["acceptance-test-hooks"] } +# Damages a test project's capture database to exercise disk failures. +rusqlite = { version = "0.40", features = ["bundled"] } # Phase 46 P4 Chunk 2: generate synthetic x509 certs with URI SANs in # unit tests so the SAN-extractor is exercised against deterministic diff --git a/crates/lpm-cli/src/commands/dev.rs b/crates/lpm-cli/src/commands/dev.rs index 6023424e5..a89bbda9f 100644 --- a/crates/lpm-cli/src/commands/dev.rs +++ b/crates/lpm-cli/src/commands/dev.rs @@ -749,9 +749,33 @@ fn rollback_inspector_session_after( }; match publication.rollback() { Ok(()) => error, - Err(rollback_error) => LpmError::Tunnel(format!( - "{error}; restoring the inspector session also failed: {rollback_error}" + Err(rollback_error) => error_also( + error, + &format!("restoring the inspector session also failed: {rollback_error}"), + ), + } +} + +/// An error's message, without the prefix its kind adds. +fn error_detail(error: &LpmError) -> String { + match error { + LpmError::Io(error) => error.to_string(), + LpmError::Tunnel(message) => message.clone(), + other => other.to_string(), + } +} + +/// `first` with `also` added to its message. A disk error stays one when a +/// tunnel cleanup fails after it, and a tunnel error stays a tunnel's; any +/// other is reported as the tunnel's, as it was raised running one. +fn error_also(first: LpmError, also: &str) -> LpmError { + match first { + LpmError::Io(error) => LpmError::Io(std::io::Error::new( + error.kind(), + format!("{error}; {also}"), )), + LpmError::Tunnel(message) => LpmError::Tunnel(format!("{message}; {also}")), + other => LpmError::Tunnel(format!("{other}; {also}")), } } @@ -1539,6 +1563,7 @@ pub async fn run( // text-only) so the user can pick whichever fits the moment. // Quiet by default — we never auto-open a browser tab here. let inspector_db = crate::commands::tunnel::open_inspector_db(project_dir)?; + let capture_store = crate::commands::tunnel::CaptureStore::of(&inspector_db); let inspector_state = lpm_inspect::state::InspectorState::with_db_pending(inspector_db); multi_inspector_state = Some(inspector_state.clone()); if !no_inspect { @@ -1567,7 +1592,7 @@ pub async fn run( let (webhook_tx, mut webhook_rx) = tokio::sync::mpsc::channel::(64); - crate::commands::tunnel::warn_capture_persistence(project_dir, !quiet).await; + crate::commands::tunnel::warn_capture_persistence(project_dir, capture_store, !quiet).await; // Dashboard webhook channel: when --dashboard is active, webhooks are // forwarded to the dashboard TUI via a std::sync channel. @@ -3245,15 +3270,8 @@ async fn finalize_multi_service_tunnel_after( None => Ok(()), }; let persistence_result = if let Some(state) = inspector_state { - let end_result = state.end_session().await.map_err(|error| { - crate::commands::tunnel::inspector_write_error("persist inspector session end", error) - }); - let flush_result = state.flush().await.map_err(|error| { - crate::commands::tunnel::inspector_write_error( - "commit captures to .lpm/inspector.db", - error, - ) - }); + let (end_result, flush_result) = + crate::commands::tunnel::finish_capture_session(&state).await; combine_tunnel_cleanup_results(end_result, flush_result) } else { Ok(()) @@ -3275,7 +3293,7 @@ fn combine_tunnel_cleanup_results( match (first, second) { (Ok(()), Ok(())) => Ok(()), (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Err(first), Err(second)) => Err(LpmError::Tunnel(format!("{first}; {second}"))), + (Err(first), Err(second)) => Err(error_also(first, &error_detail(&second))), } } @@ -3289,9 +3307,13 @@ fn compose_tunnel_runtime_and_cleanup( (Err(primary), Ok(())) => Err(primary), (Err(primary), Err(cleanup_error)) => { dev_ui::warn(&format!("tunnel capture cleanup failed: {cleanup_error}")); - Err(LpmError::Tunnel(format!( - "{primary}; tunnel capture cleanup also failed: {cleanup_error}" - ))) + Err(error_also( + primary, + &format!( + "tunnel capture cleanup also failed: {}", + error_detail(&cleanup_error) + ), + )) } } } @@ -4823,6 +4845,140 @@ mod tests { assert!(error.contains("inspector session end failed"), "{error}"); } + /// An inspector whose capture database lost its tables after it was + /// opened, so its every write fails as the disk's would. + fn inspector_whose_writes_fail( + project: &Path, + ) -> ( + lpm_inspect::state::InspectorState, + lpm_inspect::state::PublishedInspectorSession, + ) { + let db = lpm_inspect::db::InspectorDb::open(project).unwrap(); + let state = lpm_inspect::state::InspectorState::with_db(3000, db); + state + .start_session_immediate("running".to_string(), None, 3000, None) + .unwrap(); + let publication = state + .start_session_reversible( + "published".to_string(), + None, + 3000, + None, + "https://example.lpm.fyi".to_string(), + ) + .unwrap(); + rusqlite::Connection::open(project.join(".lpm/inspector.db")) + .unwrap() + .execute_batch( + "PRAGMA foreign_keys=OFF; DROP TABLE requests_fts; DROP TABLE requests; \ + DROP TABLE pending_sessions; DROP TABLE sessions;", + ) + .unwrap(); + (state, publication) + } + + fn ready_publication() -> TunnelReadyPublication { + TunnelReadyPublication { + session: lpm_tunnel::TunnelSession { + tunnel_url: "https://example.lpm.fyi".to_string(), + domain: "example.lpm.fyi".to_string(), + session_id: "session".to_string(), + local_port: 3000, + plan: None, + base_domain: None, + domain_kind: None, + session_expires_at: None, + session_max_ms: None, + limits: None, + }, + local_target_url: "http://127.0.0.1:3000".to_string(), + tunnel_auth: None, + usage: None, + } + } + + #[tokio::test] + async fn capture_store_failures_stay_disk_errors_through_dev_s_tunnel_lifecycle() { + let project = TempDir::new().unwrap(); + let (state, publication) = inspector_whose_writes_fail(project.path()); + + let started = prepare_tunnel_connection(&state, &ready_publication()) + .map(drop) + .unwrap_err(); + let rolled_back = rollback_inspector_session_after( + LpmError::Io(std::io::Error::other("disk full")), + Some(publication), + ); + state + .push(lpm_tunnel::webhook::CapturedWebhook { + id: "queued".to_string(), + timestamp: "2026-08-14T12:00:00Z".to_string(), + method: "POST".to_string(), + path: "/hook".to_string(), + request_headers: HashMap::new(), + request_body: b"request".to_vec(), + response_status: 202, + response_headers: HashMap::new(), + response_body: b"accepted".to_vec(), + duration_ms: 1, + provider: None, + summary: "queued capture".to_string(), + signature_diagnostic: None, + auto_acked: false, + response_body_incomplete: false, + }) + .await; + let finished = finalize_multi_service_tunnel_after(Ok(()), None, Some(state), None) + .await + .unwrap_err(); + + assert!( + matches!(&started, LpmError::Io(error) if error.to_string().contains("persist inspector session start")), + "{started:?}" + ); + assert!( + matches!(&rolled_back, LpmError::Io(error) if error.to_string().contains("disk full") + && error.to_string().contains("restoring the inspector session also failed")), + "{rolled_back:?}" + ); + assert!( + matches!(&finished, LpmError::Io(error) if error.to_string().contains("persist inspector session end") + && error.to_string().contains("commit captures")), + "{finished:?}" + ); + } + + #[test] + fn a_cleanup_failure_after_another_keeps_the_first_one_s_kind() { + let io = || { + LpmError::Io(std::io::Error::new( + std::io::ErrorKind::StorageFull, + "disk full", + )) + }; + let tunnel = || LpmError::Tunnel("relay closed".to_string()); + + let disk_first = combine_tunnel_cleanup_results(Err(io()), Err(tunnel())).unwrap_err(); + let tunnel_first = combine_tunnel_cleanup_results(Err(tunnel()), Err(io())).unwrap_err(); + let primary_disk = + compose_tunnel_runtime_and_cleanup(Err(io()), Err(tunnel())).unwrap_err(); + + assert!( + matches!(&disk_first, LpmError::Io(error) if error.kind() == std::io::ErrorKind::StorageFull + && error.to_string() == "disk full; relay closed"), + "{disk_first:?}" + ); + assert!( + matches!(&tunnel_first, LpmError::Tunnel(message) if message == "relay closed; disk full"), + "{tunnel_first:?}" + ); + assert!( + matches!(&primary_disk, LpmError::Io(error) + if error.to_string() == "disk full; tunnel capture cleanup also failed: relay closed"), + "{primary_disk:?}" + ); + } + #[test] fn tunnel_shutdown_boundary_returns_only_after_task_acknowledgement() { let (boundary, shutdown_rx, completion) = TunnelShutdownBoundary::new(); diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index d7a5d102a..a5cdbdc3f 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -474,9 +474,10 @@ pub(crate) async fn run_start( }; let inspector_db = open_inspector_db(project_dir)?; + let store = CaptureStore::of(&inspector_db); let inspector_state = lpm_inspect::state::InspectorState::with_db_for_target(local_target.clone(), inspector_db); - warn_capture_persistence(project_dir, show_capture_warning).await; + warn_capture_persistence(project_dir, store, show_capture_warning).await; let inspector_handle = if no_inspect { None } else { @@ -714,14 +715,7 @@ pub(crate) async fn run_start( let websocket_result = ws_consumer .await .map_err(|error| LpmError::Tunnel(format!("WebSocket capture task failed: {error}"))); - let session_result = inspector_state - .end_session() - .await - .map_err(|error| inspector_write_error("persist inspector session end", error)); - let flush_result = inspector_state - .flush() - .await - .map_err(|error| inspector_write_error("commit captures to .lpm/inspector.db", error)); + let (session_result, flush_result) = finish_capture_session(&inspector_state).await; let inspector_result = match inspector_handle { Some(handle) => handle.shutdown().await, None => Ok(()), @@ -744,58 +738,79 @@ pub(crate) async fn run_start( } /// Warns that captures persist, and says how git treats them. -pub(crate) async fn warn_capture_persistence(project_dir: &Path, show_human_warning: bool) { +pub(crate) async fn warn_capture_persistence( + project_dir: &Path, + store: CaptureStore, + show_human_warning: bool, +) { let state = capture_git_state(project_dir).await; - for warning in capture_warnings( - state.as_ref(), - &git_base(project_dir), - ignore_file_git_skips(project_dir), - ) { + for warning in capture_warnings(state.as_ref(), ignore_file_git_skips(project_dir), store) { capture_warning(&warning, show_human_warning); } } /// The capture warnings: that captures persist, kept out of git only where -/// git confirms it ignores them; files git tracks, which an ignore rule -/// can't keep from being committed; and files it neither tracks nor -/// ignores. +/// git confirms it ignores them; files a repository tracks, which an ignore +/// rule can't keep from being committed, with the command for each +/// repository; and files git neither tracks nor ignores. fn capture_warnings( state: Option<&CaptureGitState>, - base: &GitBase, ignore_file_skipped: bool, + store: CaptureStore, ) -> Vec { - let kept_out = state.is_some_and(|state| { - state.asked_in_lpm && state.files.tracked.is_empty() && state.files.unignored.is_empty() - }); + let kept_out = state.is_some_and(CaptureGitState::kept_out); + let access = match store { + CaptureStore::OwnerOnly => "owner-only on Unix; project ACLs on Windows", + CaptureStore::OpenToOthers => { + "open to other users, as this filesystem doesn't keep Unix file modes" + } + }; let mut warnings = vec![format!( "tunnel webhook capture persists full request/response bodies and headers \ - (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db (owner-only \ - on Unix; project ACLs on Windows){}", + (incl. Authorization, Cookie, *-Signature) in .lpm/inspector.db ({access}){}", if kept_out { "; git ignores it" } else { "" } )]; let Some(state) = state else { return warnings; }; - if !state.asked_in_lpm { + if state.lpm_unasked { warnings.push( - "git couldn't be asked inside .lpm, so if .lpm holds a repository of its own, \ - check that it doesn't track inspector.db*" + "git couldn't be asked about .lpm, so check that git ignores and doesn't \ + track inspector.db*" .to_string(), ); } - let files = &state.files; - if !files.tracked.is_empty() { + if !state.project_tracked.is_empty() || !state.lpm_tracked.is_empty() { + let tracked: Vec = state + .project_tracked + .iter() + .cloned() + .chain(state.lpm_tracked.iter().map(|name| format!(".lpm/{name}"))) + .collect(); + let commands: Vec = [ + (&state.project_tracked, CaptureRepository::Project), + (&state.lpm_tracked, CaptureRepository::Lpm), + ] + .into_iter() + .filter(|(names, _)| !names.is_empty()) + .map(|(names, repository)| format!("`{}`", untrack_command(names, repository))) + .collect(); warnings.push(format!( "git tracks {}, so captures there can be committed and stay in its history; \ - run `{}`, commit, and rotate any credentials they held", - listed(&state.shown(&files.tracked)), - untrack_command(&files.tracked, state.asked_in_lpm, base) + run {}, commit, and rotate any credentials they held", + listed(&tracked), + commands.join(" and ") )); } - if !files.unignored.is_empty() { + if !state.unignored.is_empty() { + let unignored: Vec = state + .unignored + .iter() + .map(|name| format!(".lpm/{name}")) + .collect(); warnings.push(format!( "git doesn't ignore {}; {}", - listed(&state.shown(&files.unignored)), + listed(&unignored), if ignore_file_skipped { "replace .lpm/.gitignore, which git doesn't read as a file of the \ project's, with a file that holds `inspector.db*`" @@ -828,34 +843,63 @@ fn capture_warning(message: &str, show_human_warning: bool) { } } -/// Capture files git tracks, and those it neither tracks nor ignores, by -/// name relative to the directory git was asked in. +/// Who the capture database a writer opened lets read it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CaptureStore { + OwnerOnly, + /// On a filesystem that shows every file open to others, which no + /// replacement improves. + OpenToOthers, +} + +impl CaptureStore { + pub(crate) fn of(db: &lpm_inspect::db::InspectorDb) -> Self { + if db.open_to_others() { + Self::OpenToOthers + } else { + Self::OwnerOnly + } + } +} + +/// Capture files a git query listed as tracked, and as neither tracked nor +/// ignored, by name relative to the directory it was asked in. #[derive(Debug, Default, Clone, PartialEq, Eq)] struct CaptureGitFiles { tracked: Vec, unignored: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] +/// How git treats the capture files. +#[derive(Debug, Default, Clone, PartialEq, Eq)] struct CaptureGitState { - files: CaptureGitFiles, - /// Whether git answered inside `.lpm`, where a repository of its own, - /// such as a submodule, keeps the files from the project's repository. - /// Otherwise the project's repository answered. - asked_in_lpm: bool, + /// Those the project's repository tracks, by name in the project. + project_tracked: Vec, + /// Those a repository of `.lpm`'s own tracks, by name in `.lpm`. + lpm_tracked: Vec, + /// Those git neither tracks nor ignores, by name in `.lpm`. + unignored: Vec, + /// Whether git refused to answer for `.lpm`, as for one that looks like a + /// bare repository. + lpm_unasked: bool, } impl CaptureGitState { - /// Names as the project sees them. - fn shown(&self, names: &[String]) -> Vec { - if self.asked_in_lpm { - names.iter().map(|name| format!(".lpm/{name}")).collect() - } else { - names.to_vec() - } + fn kept_out(&self) -> bool { + !self.lpm_unasked + && self.project_tracked.is_empty() + && self.lpm_tracked.is_empty() + && self.unignored.is_empty() } } +/// The repository whose index a tracked capture file is in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CaptureRepository { + Project, + Lpm, +} + const CAPTURES_PATHSPEC: &str = ":(glob,icase)inspector.db*"; const PROJECT_CAPTURES_PATHSPEC: &str = ":(glob,icase).lpm/inspector.db*"; @@ -871,67 +915,101 @@ enum CaptureGitFailure { Unavailable, } +/// A git query of the capture files: the directory it runs in, its +/// pathspec, and whether it lists files neither tracked nor ignored too. +#[derive(Debug, Clone, PartialEq, Eq)] +struct CaptureGitQuery { + directory: PathBuf, + pathspec: &'static str, + unignored: bool, +} + /// How git treats the capture files in `project_dir`, matching their names -/// without regard to case, as a case-insensitive disk does. Git is asked -/// inside `.lpm`, and only if it refuses there, in the project. None when it -/// isn't a repository, git can't be run, or it doesn't answer in time. +/// without regard to case, as a case-insensitive disk does. Two queries run +/// at once: the project's repository, for the capture files its index holds +/// whatever the case of their directory; and git in `.lpm`, for those +/// neither tracked nor ignored there, and for those a repository of `.lpm`'s +/// own tracks, such as a submodule, whose files the project's index doesn't +/// list. None when neither is a repository, git can't be run, or it doesn't +/// answer in time. async fn capture_git_state(project_dir: &Path) -> Option { - capture_git_state_by(project_dir, |directory, pathspec| { - capture_git_output( - capture_git_command(&directory, pathspec), - CAPTURE_GIT_TIMEOUT, - ) + capture_git_state_by(project_dir, |query| { + capture_git_output(capture_git_command(&query), CAPTURE_GIT_TIMEOUT) }) .await } async fn capture_git_state_by(project_dir: &Path, ask: Ask) -> Option where - Ask: Fn(PathBuf, &'static str) -> Answer, + Ask: Fn(CaptureGitQuery) -> Answer, Answer: std::future::Future>, { let lpm_dir = project_dir.join(".lpm"); - let in_lpm = if lpm_dir.is_dir() { - ask(lpm_dir, CAPTURES_PATHSPEC).await - } else { - Err(CaptureGitFailure::Refused) - }; - match in_lpm { - Ok(files) => Some(CaptureGitState { - files, - asked_in_lpm: true, - }), - // A git too slow to answer once would be as slow again. - Err(CaptureGitFailure::Unavailable) => None, - Err(CaptureGitFailure::Refused) => { - ask(project_dir.to_path_buf(), PROJECT_CAPTURES_PATHSPEC) - .await - .ok() - .map(|files| CaptureGitState { - files, - asked_in_lpm: false, + let lpm_exists = lpm_dir.is_dir(); + let own_repository = std::fs::symlink_metadata(lpm_dir.join(".git")).is_ok(); + let project = ask(CaptureGitQuery { + directory: project_dir.to_path_buf(), + pathspec: PROJECT_CAPTURES_PATHSPEC, + unignored: false, + }); + let inside = async { + if lpm_exists { + Some( + ask(CaptureGitQuery { + directory: lpm_dir, + pathspec: CAPTURES_PATHSPEC, + unignored: true, }) + .await, + ) + } else { + None } - } + }; + let (project, inside) = tokio::join!(project, inside); + + let mut state = CaptureGitState::default(); + let project_answered = match project { + Ok(files) => { + state.project_tracked = files.tracked; + true + } + Err(CaptureGitFailure::Refused) => false, + Err(CaptureGitFailure::Unavailable) => return None, + }; + let inside_answered = match inside { + Some(Ok(files)) => { + // Without a repository of its own, git in `.lpm` lists the + // project's tracked files, which the project's query has. + if own_repository { + state.lpm_tracked = files.tracked; + } + state.unignored = files.unignored; + true + } + Some(Err(CaptureGitFailure::Refused)) => { + state.lpm_unasked = true; + false + } + Some(Err(CaptureGitFailure::Unavailable)) => return None, + None => false, + }; + (project_answered || inside_answered).then_some(state) } -fn capture_git_command(directory: &Path, pathspec: &str) -> tokio::process::Command { +fn capture_git_command(query: &CaptureGitQuery) -> tokio::process::Command { + let mut args = vec!["ls-files", "-z", "-t", "--cached"]; + if query.unignored { + args.extend(["--others", "--exclude-standard"]); + } + args.extend(["--", query.pathspec]); tokio::process::Command::from(crate::commands::npmrc::hardened_git_command( - directory, - &[ - "ls-files", - "-z", - "-t", - "--cached", - "--others", - "--exclude-standard", - "--", - pathspec, - ], + &query.directory, + &args, )) } -/// Runs the capture check's git command, giving up after `timeout` and +/// Runs a capture query's git command, giving up after `timeout` and /// killing it, so a slow repository delays the tunnel's start by that much /// at most. async fn capture_git_output( @@ -1004,64 +1082,23 @@ fn listed(names: &[String]) -> String { } } -/// The command that stops git tracking `tracked`, run from the current -/// directory: their names when the warning lists them all, and otherwise -/// the capture pathspec, which every shell reads alike in double quotes. -fn untrack_command(tracked: &[String], asked_in_lpm: bool, base: &GitBase) -> String { - let paths = if tracked.len() <= LISTED_FILES && tracked.iter().all(|name| is_plain(name)) { - tracked.join(" ") - } else if asked_in_lpm { - format!("\"{CAPTURES_PATHSPEC}\"") - } else { - format!("\"{PROJECT_CAPTURES_PATHSPEC}\"") +/// The command, run in the project directory, that stops `repository` +/// tracking `tracked`: their names when the warning can list them all, and +/// otherwise the repository's capture pathspec, which every shell reads +/// alike in double quotes. +fn untrack_command(tracked: &[String], repository: CaptureRepository) -> String { + let named = tracked.len() <= LISTED_FILES && tracked.iter().all(|name| is_plain(name)); + let (command, pathspec) = match repository { + CaptureRepository::Project => ("git rm --cached --", PROJECT_CAPTURES_PATHSPEC), + CaptureRepository::Lpm => ("git -C .lpm rm --cached --", CAPTURES_PATHSPEC), }; - let directory = if asked_in_lpm { - Some(base.join(".lpm")) + if named { + format!("{command} {}", tracked.join(" ")) } else { - base.project() - }; - match directory { - Some(directory) => format!("git -C {directory} rm --cached -- {paths}"), - None => format!("git rm --cached -- {paths}"), + format!("{command} \"{pathspec}\"") } } -/// The project directory as the current directory reaches it: `..` once for -/// each level the current directory is below it. -#[derive(Debug, Clone, PartialEq, Eq)] -struct GitBase { - levels_up: usize, -} - -impl GitBase { - /// The project's own directory, or None when it's the current one. - fn project(&self) -> Option { - (self.levels_up > 0).then(|| vec![".."; self.levels_up].join("/")) - } - - fn join(&self, name: &str) -> String { - match self.project() { - Some(project) => format!("{project}/{name}"), - None => name.to_string(), - } - } -} - -/// How many levels the current directory is below `project_dir`, or none -/// when it isn't below it. -fn git_base(project_dir: &Path) -> GitBase { - let levels_up = std::env::current_dir() - .ok() - .and_then(|current| { - current - .strip_prefix(project_dir) - .ok() - .map(|below| below.components().count()) - }) - .unwrap_or(0); - GitBase { levels_up } -} - fn compose_tunnel_control_and_connect( control_result: Result<(), LpmError>, connect_result: Result<(), LpmError>, @@ -1249,7 +1286,7 @@ async fn run_domains(client: &RegistryClient, json_output: bool) -> Result<(), L /// against any other local service); `Some(n)` binds that exact port and /// fails loudly on `AddrInUse`. async fn run_inspect_ui(project_dir: &Path, inspect_port: Option) -> Result<(), LpmError> { - let state = project_inspector_state(project_dir)?; + let state = project_inspector_state(project_dir, true)?; let handle = lpm_inspect::start(state, inspect_port.unwrap_or(0)).await?; if let Err(error) = open::that(&handle.url) { install_ui::warn_untrusted(&format!("failed to open inspector in browser: {error}")); @@ -1274,8 +1311,9 @@ async fn run_inspect_ui(project_dir: &Path, inspect_port: Option) -> Result fn project_inspector_state( project_dir: &Path, + show_human_warning: bool, ) -> Result { - let db = open_inspector_db(project_dir)?; + let db = open_inspector_db_for_review(project_dir, show_human_warning)?; let sessions = lpm_runner::dev_session::discover_active_sessions()?; let local_target = observer_replay_target(&sessions); Ok(lpm_inspect::state::InspectorState::with_db_observer_for_target(local_target, db)) @@ -1304,7 +1342,7 @@ async fn run_inspect( args: &[String], json_output: bool, ) -> Result<(), LpmError> { - let db = open_inspector_db(project_dir)?; + let db = open_inspector_db_for_review(project_dir, !json_output)?; let last = parse_flag_usize(args, "--last", "-n").unwrap_or(20); let filter_provider = parse_flag_str(args, "--filter"); @@ -1399,7 +1437,7 @@ async fn run_replay( default_port: Option, json_output: bool, ) -> Result<(), LpmError> { - let db = open_inspector_db(project_dir)?; + let db = open_inspector_db_for_review(project_dir, !json_output)?; let local_target = match parse_flag_u16(args, "--port", "-p")? { Some(port) => lpm_common::LocalTarget::loopback(lpm_common::LocalScheme::Http, port), None => resolve_local_target(default_port)?, @@ -1519,10 +1557,10 @@ async fn run_replay( /// --status — filter by status class /// --clear — delete all webhook logs async fn run_log(project_dir: &Path, args: &[String], json_output: bool) -> Result<(), LpmError> { - let db = open_inspector_db(project_dir)?; + let db = open_inspector_db_for_review(project_dir, !json_output)?; if args.contains(&"--clear".to_string()) { - db.clear_history().await.map_err(inspector_query_error)?; + clear_captures(&db).await?; if json_output { println!( "{}", @@ -1569,17 +1607,38 @@ async fn run_log(project_dir: &Path, args: &[String], json_output: bool) -> Resu // ── Helper functions ──────────────────────────────────────────────── -/// Opens the project's capture database. Its failures are local, so they're -/// reported as such, not as a tunnel's. +/// Opens the project's capture database to record captures. pub(crate) fn open_inspector_db( project_dir: &Path, ) -> Result { - lpm_inspect::db::InspectorDb::open(project_dir).map_err(|error| { - LpmError::Io(std::io::Error::other(format!( - "failed to open this project's .lpm/inspector.db: {error}. \ - Move or repair it and retry" - ))) - }) + lpm_inspect::db::InspectorDb::open(project_dir).map_err(inspector_open_error) +} + +/// Opens the project's capture database to read and manage the captures it +/// holds, warning when other users could open it. +fn open_inspector_db_for_review( + project_dir: &Path, + show_human_warning: bool, +) -> Result { + let db = + lpm_inspect::db::InspectorDb::open_for_review(project_dir).map_err(inspector_open_error)?; + if db.open_to_others() { + capture_warning(CAPTURES_OPEN_TO_OTHERS, show_human_warning); + } + Ok(db) +} + +const CAPTURES_OPEN_TO_OTHERS: &str = ".lpm/inspector.db is open to other users, so they may \ + read its captures; an `lpm tunnel` or `lpm dev --tunnel` run in this project makes it \ + owner-only where the filesystem keeps Unix file modes"; + +/// The capture database's open failures are local, so they're reported as +/// such, not as a tunnel's; each says what to do. +fn inspector_open_error(error: lpm_inspect::db::OpenError) -> LpmError { + LpmError::Io(std::io::Error::new( + error.kind(), + format!("failed to open this project's .lpm/inspector.db: {error}"), + )) } /// A failure writing `.lpm/inspector.db`, which is the disk's, not the @@ -1594,6 +1653,28 @@ fn inspector_query_error(error: impl Display) -> LpmError { ))) } +async fn clear_captures(db: &lpm_inspect::db::InspectorDb) -> Result<(), LpmError> { + db.clear_history() + .await + .map_err(|error| inspector_write_error("clear .lpm/inspector.db", error)) +} + +/// Ends the inspector session and commits the captures still queued: the +/// session end's result, then the commit's. +pub(crate) async fn finish_capture_session( + state: &lpm_inspect::state::InspectorState, +) -> (Result<(), LpmError>, Result<(), LpmError>) { + let ended = state + .end_session() + .await + .map_err(|error| inspector_write_error("persist inspector session end", error)); + let committed = state + .flush() + .await + .map_err(|error| inspector_write_error("commit captures to .lpm/inspector.db", error)); + (ended, committed) +} + async fn read_capture_entries( db: &lpm_inspect::db::InspectorDb, count: usize, @@ -2226,8 +2307,6 @@ mod tests { names.iter().map(|name| (*name).to_string()).collect() } - const HERE: GitBase = GitBase { levels_up: 0 }; - #[tokio::test] async fn tells_capture_files_git_tracks_from_those_it_neither_tracks_nor_ignores() { let project = capture_project(&[ @@ -2244,9 +2323,9 @@ mod tests { let state = capture_git_state(project.path()).await.unwrap(); - assert!(state.asked_in_lpm); - assert_eq!(state.files.tracked, names(&["inspector.db"])); - let mut unignored = state.files.unignored; + assert_eq!(state.project_tracked, names(&[".lpm/inspector.db"])); + assert!(state.lpm_tracked.is_empty()); + let mut unignored = state.unignored; unignored.sort(); assert_eq!(unignored, names(&["inspector.db-shm", "inspector.db-wal"])); } @@ -2256,13 +2335,10 @@ mod tests { let project = capture_project(&[".lpm/inspector.db", ".lpm/inspector.db-wal"]); std::fs::write(project.path().join(".lpm/.gitignore"), "inspector.db*\n").unwrap(); - assert_eq!( - capture_git_state(project.path()).await, - Some(CaptureGitState { - files: CaptureGitFiles::default(), - asked_in_lpm: true, - }) - ); + let state = capture_git_state(project.path()).await.unwrap(); + + assert_eq!(state, CaptureGitState::default()); + assert!(state.kept_out()); } #[tokio::test] @@ -2272,7 +2348,19 @@ mod tests { let state = capture_git_state(project.path()).await.unwrap(); - assert_eq!(state.files.tracked, names(&["Inspector.DB"])); + assert_eq!(state.project_tracked, names(&[".lpm/Inspector.DB"])); + } + + #[tokio::test] + async fn names_a_capture_the_project_tracks_under_another_case_of_lpm() { + // As a repository committed on a case-insensitive disk carries it. + let project = capture_project(&[".LPM/inspector.db"]); + git(project.path(), &["add", ".LPM/inspector.db"]); + + let state = capture_git_state(project.path()).await.unwrap(); + + assert_eq!(state.project_tracked, names(&[".LPM/inspector.db"])); + assert!(!state.kept_out()); } #[tokio::test] @@ -2284,17 +2372,53 @@ mod tests { let state = capture_git_state(project.path()).await.unwrap(); - assert!(state.asked_in_lpm); - assert_eq!(state.files.tracked, names(&["inspector.db"])); + assert!(state.project_tracked.is_empty()); + assert_eq!(state.lpm_tracked, names(&["inspector.db"])); assert_eq!( - untrack_command(&state.files.tracked, state.asked_in_lpm, &HERE), + untrack_command(&state.lpm_tracked, CaptureRepository::Lpm), "git -C .lpm rm --cached -- inspector.db" ); } + #[tokio::test] + async fn names_the_project_s_capture_and_a_repository_inside_lpm_s_alike() { + let project = capture_project(&[".lpm/inspector.db"]); + git(project.path(), &["add", ".lpm/inspector.db"]); + let lpm_dir = project.path().join(".lpm"); + git(&lpm_dir, &["init", "--quiet"]); + git(&lpm_dir, &["add", "inspector.db"]); + + let state = capture_git_state(project.path()).await.unwrap(); + + assert_eq!(state.project_tracked, names(&[".lpm/inspector.db"])); + assert_eq!(state.lpm_tracked, names(&["inspector.db"])); + let warnings = capture_warnings(Some(&state), false, CaptureStore::OwnerOnly); + assert!( + warnings[1].contains( + "run `git rm --cached -- .lpm/inspector.db` and \ + `git -C .lpm rm --cached -- inspector.db`" + ), + "{warnings:?}" + ); + } + + #[tokio::test] + async fn names_a_capture_the_project_tracks_when_lpm_points_git_elsewhere() { + let project = capture_project(&[".lpm/inspector.db"]); + git(project.path(), &["add", ".lpm/inspector.db"]); + // As a downloaded archive can carry: git inside `.lpm` uses the + // project's repository with `.lpm` as its working tree. + std::fs::write(project.path().join(".lpm/.git"), "gitdir: ../.git\n").unwrap(); + + let state = capture_git_state(project.path()).await.unwrap(); + + assert_eq!(state.project_tracked, names(&[".lpm/inspector.db"])); + assert!(!state.kept_out()); + } + #[cfg(unix)] #[tokio::test] - async fn asks_the_project_when_git_refuses_inside_lpm_and_says_so() { + async fn says_when_git_refuses_to_answer_for_lpm() { let project = capture_project(&[".lpm/inspector.db"]); git(project.path(), &["add", ".lpm/inspector.db"]); // An `.lpm` that looks like a bare repository, which the check's @@ -2307,16 +2431,9 @@ mod tests { let state = capture_git_state(project.path()).await.unwrap(); - assert!(!state.asked_in_lpm); - assert_eq!(state.files.tracked, names(&[".lpm/inspector.db"])); - assert_eq!( - state.shown(&state.files.tracked), - names(&[".lpm/inspector.db"]) - ); - assert_eq!( - untrack_command(&state.files.tracked, state.asked_in_lpm, &HERE), - "git rm --cached -- .lpm/inspector.db" - ); + assert!(state.lpm_unasked); + assert_eq!(state.project_tracked, names(&[".lpm/inspector.db"])); + assert!(!state.kept_out()); } #[tokio::test] @@ -2371,79 +2488,150 @@ mod tests { assert!(!marker.exists()); } - /// Asks with canned answers, one per call, recording where each was asked. + /// Asks with canned answers, by whether a query is the project's, + /// recording each query. async fn state_from_answers( project_dir: &Path, - answers: Vec>, - ) -> (Option, Vec<(PathBuf, &'static str)>) { - let answers = std::sync::Mutex::new(answers.into_iter()); + project: Result, + inside: Result, + ) -> (Option, Vec) { + let answers = std::sync::Mutex::new((Some(project), Some(inside))); let asked = std::sync::Mutex::new(Vec::new()); - let state = capture_git_state_by(project_dir, |directory, pathspec| { - asked.lock().unwrap().push((directory, pathspec)); - let answer = answers.lock().unwrap().next().unwrap(); + let state = capture_git_state_by(project_dir, |query| { + let answer = { + let mut answers = answers.lock().unwrap(); + if query.unignored { + answers.1.take() + } else { + answers.0.take() + } + } + .unwrap(); + asked.lock().unwrap().push(query); async move { answer } }) .await; (state, asked.into_inner().unwrap()) } + fn files(tracked: &[&str], unignored: &[&str]) -> CaptureGitFiles { + CaptureGitFiles { + tracked: names(tracked), + unignored: names(unignored), + } + } + #[tokio::test] - async fn asks_the_project_only_when_git_refuses_inside_lpm() { + async fn asks_the_project_s_index_and_git_in_lpm_together() { let project = tempfile::tempdir().unwrap(); let lpm_dir = project.path().join(".lpm"); std::fs::create_dir(&lpm_dir).unwrap(); - let tracked = CaptureGitFiles { - tracked: names(&[".lpm/inspector.db"]), - unignored: Vec::new(), - }; - let (state, asked) = state_from_answers( + let (state, mut asked) = state_from_answers( project.path(), - vec![Err(CaptureGitFailure::Refused), Ok(tracked)], + Ok(files(&[".lpm/inspector.db"], &[])), + Ok(files(&["inspector.db"], &["inspector.db-wal"])), ) .await; - assert_eq!(state.unwrap().files.tracked, names(&[".lpm/inspector.db"])); + + asked.sort_by_key(|query| query.unignored); assert_eq!( asked, vec![ - (lpm_dir.clone(), CAPTURES_PATHSPEC), - (project.path().to_path_buf(), PROJECT_CAPTURES_PATHSPEC), + CaptureGitQuery { + directory: project.path().to_path_buf(), + pathspec: PROJECT_CAPTURES_PATHSPEC, + unignored: false, + }, + CaptureGitQuery { + directory: lpm_dir, + pathspec: CAPTURES_PATHSPEC, + unignored: true, + }, ] ); + // Without a repository of `.lpm`'s own, git there lists the + // project's files, which the project's query has. + assert_eq!( + state, + Some(CaptureGitState { + project_tracked: names(&[".lpm/inspector.db"]), + lpm_tracked: Vec::new(), + unignored: names(&["inspector.db-wal"]), + lpm_unasked: false, + }) + ); + } + + #[tokio::test] + async fn counts_what_a_repository_of_lpm_s_own_tracks() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(project.path().join(".lpm/.git")).unwrap(); + + let (state, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Refused), + Ok(files(&["inspector.db"], &[])), + ) + .await; - let (state, asked) = - state_from_answers(project.path(), vec![Err(CaptureGitFailure::Unavailable)]).await; - assert_eq!(state, None); - assert_eq!(asked.len(), 1); + assert_eq!(state.unwrap().lpm_tracked, names(&["inspector.db"])); + } - let (state, asked) = - state_from_answers(project.path(), vec![Ok(CaptureGitFiles::default())]).await; - assert!(state.unwrap().asked_in_lpm); - assert_eq!(asked.len(), 1); + #[tokio::test] + async fn says_nothing_of_git_when_either_query_doesn_t_answer_in_time() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + + for (project_answer, inside_answer) in [ + ( + Err(CaptureGitFailure::Unavailable), + Ok(CaptureGitFiles::default()), + ), + ( + Ok(CaptureGitFiles::default()), + Err(CaptureGitFailure::Unavailable), + ), + ( + Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::Refused), + ), + ] { + let (state, _) = + state_from_answers(project.path(), project_answer, inside_answer).await; + assert_eq!(state, None); + } } fn warnings_for(state: Option, ignore_file_skipped: bool) -> Vec { - capture_warnings(state.as_ref(), &HERE, ignore_file_skipped) + capture_warnings(state.as_ref(), ignore_file_skipped, CaptureStore::OwnerOnly) } - fn state(asked_in_lpm: bool, tracked: &[&str], unignored: &[&str]) -> CaptureGitState { + fn state( + project_tracked: &[&str], + lpm_tracked: &[&str], + unignored: &[&str], + ) -> CaptureGitState { CaptureGitState { - files: CaptureGitFiles { - tracked: names(tracked), - unignored: names(unignored), - }, - asked_in_lpm, + project_tracked: names(project_tracked), + lpm_tracked: names(lpm_tracked), + unignored: names(unignored), + lpm_unasked: false, } } #[test] - fn says_git_ignores_captures_only_when_git_inside_lpm_says_so() { - assert!(warnings_for(Some(state(true, &[], &[])), false)[0].ends_with("; git ignores it")); + fn says_git_ignores_captures_only_when_every_query_says_so() { + assert!(warnings_for(Some(state(&[], &[], &[])), false)[0].ends_with("; git ignores it")); for unconfirmed in [ None, - Some(state(false, &[], &[])), - Some(state(true, &["inspector.db"], &[])), - Some(state(true, &[], &["inspector.db-wal"])), + Some(CaptureGitState { + lpm_unasked: true, + ..state(&[], &[], &[]) + }), + Some(state(&[".lpm/inspector.db"], &[], &[])), + Some(state(&[], &["inspector.db"], &[])), + Some(state(&[], &[], &["inspector.db-wal"])), ] { assert!( !warnings_for(unconfirmed.clone(), false)[0].contains("git ignores it"), @@ -2453,32 +2641,38 @@ mod tests { } #[test] - fn says_when_git_couldnt_be_asked_inside_lpm() { - let warnings = warnings_for(Some(state(false, &[], &[])), false); + fn says_when_git_couldnt_be_asked_about_lpm() { + let warnings = warnings_for( + Some(CaptureGitState { + lpm_unasked: true, + ..state(&[], &[], &[]) + }), + false, + ); assert_eq!(warnings.len(), 2); - assert!(warnings[1].starts_with("git couldn't be asked inside .lpm")); - assert_eq!(warnings_for(Some(state(true, &[], &[])), false).len(), 1); + assert!(warnings[1].starts_with("git couldn't be asked about .lpm")); + assert_eq!(warnings_for(Some(state(&[], &[], &[])), false).len(), 1); assert_eq!(warnings_for(None, false).len(), 1); } #[test] fn names_tracked_captures_and_the_command_that_stops_tracking_them() { - let warnings = warnings_for(Some(state(true, &["inspector.db"], &[])), false); + let warnings = warnings_for(Some(state(&[".lpm/inspector.db"], &[], &[])), false); assert_eq!(warnings.len(), 2); assert_eq!( warnings[1], "git tracks '.lpm/inspector.db', so captures there can be committed and stay in \ - its history; run `git -C .lpm rm --cached -- inspector.db`, commit, and rotate \ - any credentials they held" + its history; run `git rm --cached -- .lpm/inspector.db`, commit, and rotate any \ + credentials they held" ); } #[test] fn advises_on_captures_git_neither_tracks_nor_ignores() { - let added = warnings_for(Some(state(true, &[], &["inspector.db-wal"])), false); - let replaced = warnings_for(Some(state(true, &[], &["inspector.db-wal"])), true); + let added = warnings_for(Some(state(&[], &[], &["inspector.db-wal"])), false); + let replaced = warnings_for(Some(state(&[], &[], &["inspector.db-wal"])), true); assert_eq!( added[1], @@ -2491,6 +2685,58 @@ mod tests { ); } + #[test] + fn reports_a_capture_database_it_cant_open_yet_as_busy_with_its_own_advice() { + let error = inspector_open_error(lpm_inspect::db::OpenError::Files(std::io::Error::new( + std::io::ErrorKind::ResourceBusy, + "inspector.db is open to other users, and another process has it open; retry", + ))); + + let LpmError::Io(error) = error else { + panic!("{error:?}"); + }; + assert_eq!(error.kind(), std::io::ErrorKind::ResourceBusy); + assert_eq!( + error.to_string(), + "failed to open this project's .lpm/inspector.db: inspector.db is open to other \ + users, and another process has it open; retry" + ); + } + + #[test] + fn the_capture_warning_says_when_the_filesystem_leaves_captures_open_to_others() { + let open = capture_warnings(None, false, CaptureStore::OpenToOthers); + let owner_only = capture_warnings(None, false, CaptureStore::OwnerOnly); + + assert!( + open[0] + .contains("(open to other users, as this filesystem doesn't keep Unix file modes)"), + "{open:?}" + ); + assert!(!open[0].contains("owner-only"), "{open:?}"); + assert!(owner_only[0].contains("(owner-only on Unix; project ACLs on Windows)")); + } + + #[tokio::test] + async fn reports_a_failed_clear_as_a_write() { + let project = tempfile::tempdir().unwrap(); + let db = lpm_inspect::db::InspectorDb::open(project.path()).unwrap(); + rusqlite::Connection::open(project.path().join(".lpm/inspector.db")) + .unwrap() + .execute_batch("DROP TABLE requests_fts;") + .unwrap(); + + let error = clear_captures(&db).await.unwrap_err(); + + assert_eq!(error.error_code(), "io"); + assert!( + error + .to_string() + .contains("failed to clear .lpm/inspector.db: no such table: requests_fts"), + "{error}" + ); + } + #[test] fn reports_inspector_database_reads_as_disk_errors() { let error = inspector_query_error("database disk image is malformed"); @@ -2515,19 +2761,6 @@ mod tests { ); } - #[test] - fn shows_names_git_gave_inside_lpm_as_the_project_sees_them() { - let state = CaptureGitState { - files: CaptureGitFiles::default(), - asked_in_lpm: true, - }; - - assert_eq!( - state.shown(&names(&["inspector.db"])), - names(&[".lpm/inspector.db"]) - ); - } - #[test] fn lists_at_most_five_names_quoted() { let many: Vec = (0..7) @@ -2560,24 +2793,24 @@ mod tests { #[test] fn untracks_listed_names_or_every_capture_file_past_the_list() { - let names = |count: usize| -> Vec { + let names = |count: usize, prefix: &str| -> Vec { (0..count) - .map(|index| format!("inspector.db{index}")) + .map(|index| format!("{prefix}inspector.db{index}")) .collect() }; assert_eq!( - untrack_command(&names(5), true, &HERE), - "git -C .lpm rm --cached -- inspector.db0 inspector.db1 \ - inspector.db2 inspector.db3 inspector.db4" + untrack_command(&names(5, ".lpm/"), CaptureRepository::Project), + "git rm --cached -- .lpm/inspector.db0 .lpm/inspector.db1 \ + .lpm/inspector.db2 .lpm/inspector.db3 .lpm/inspector.db4" ); assert_eq!( - untrack_command(&names(6), true, &HERE), - "git -C .lpm rm --cached -- \":(glob,icase)inspector.db*\"" + untrack_command(&names(6, ".lpm/"), CaptureRepository::Project), + "git rm --cached -- \":(glob,icase).lpm/inspector.db*\"" ); assert_eq!( - untrack_command(&names(6), false, &HERE), - "git rm --cached -- \":(glob,icase).lpm/inspector.db*\"" + untrack_command(&names(6, ""), CaptureRepository::Lpm), + "git -C .lpm rm --cached -- \":(glob,icase)inspector.db*\"" ); } @@ -2593,39 +2826,13 @@ mod tests { "", ] { assert_eq!( - untrack_command(&names(&["inspector.db", name]), true, &HERE), + untrack_command(&names(&["inspector.db", name]), CaptureRepository::Lpm), "git -C .lpm rm --cached -- \":(glob,icase)inspector.db*\"", "{name:?}" ); } } - #[test] - fn untracks_from_a_directory_below_the_project() { - let below = GitBase { levels_up: 2 }; - - assert_eq!( - untrack_command(&names(&["inspector.db"]), true, &below), - "git -C ../../.lpm rm --cached -- inspector.db" - ); - assert_eq!( - untrack_command(&names(&[".lpm/inspector.db"]), false, &below), - "git -C ../.. rm --cached -- .lpm/inspector.db" - ); - } - - #[test] - fn counts_the_levels_from_the_project_to_the_current_directory() { - let current = std::env::current_dir().unwrap(); - - assert_eq!(git_base(¤t), HERE); - assert_eq!( - git_base(current.parent().unwrap()), - GitBase { levels_up: 1 } - ); - assert_eq!(git_base(Path::new("/nowhere-near")), HERE); - } - #[cfg(unix)] #[test] fn tells_an_lpm_gitignore_git_skips_from_one_it_reads() { @@ -3017,7 +3224,7 @@ mod tests { writer.push(webhook.clone()).await; writer.flush().await.unwrap(); - let observer = project_inspector_state(project.path()).unwrap(); + let observer = project_inspector_state(project.path(), false).unwrap(); let mut observed = observer.get_all_persisted().await.unwrap(); assert_eq!( diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index 97d19925e..3a3c329cb 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -16,6 +16,8 @@ pub mod integrity; pub mod known_projects; pub mod local_target; pub mod lpm_json; +#[cfg(target_os = "macos")] +pub mod macos_acl; pub mod npmrc; pub mod package_instance; pub mod package_name; @@ -72,9 +74,10 @@ pub use paths::{ ExclusiveLockHandle, FsKind, GLOBAL_INSTALL_PATH_BUDGET, INSTALL_READY_MARKER, LpmRoot, ProjectLockDirectory, ProjectLockKind, SharedLockHandle, SingleFileExclusiveLockHandle, SingleFileSharedLockHandle, acquire_exclusive_lock, acquire_single_file_exclusive_lock, - acquire_single_file_exclusive_lock_from_file, acquire_single_file_exclusive_lock_nofollow, - acquire_single_file_shared_lock, acquire_single_file_shared_lock_from_file, as_extended_path, - check_install_path_budget, is_local_fs, parent_install_lock_environment, project_install_lock, + acquire_single_file_exclusive_lock_from_file, + acquire_single_file_exclusive_lock_from_file_within, acquire_single_file_shared_lock, + acquire_single_file_shared_lock_from_file, as_extended_path, check_install_path_budget, + is_local_fs, open_lock_file_nofollow, parent_install_lock_environment, project_install_lock, project_publish_lock, try_acquire_capability_exclusive_lock, try_acquire_exclusive_lock, try_acquire_single_file_exclusive_lock_from_file, try_with_exclusive_lock, with_capability_exclusive_lock, with_exclusive_lock, with_exclusive_lock_async, diff --git a/crates/lpm-common/src/macos_acl.rs b/crates/lpm-common/src/macos_acl.rs new file mode 100644 index 000000000..ae1b6c9fe --- /dev/null +++ b/crates/lpm-common/src/macos_acl.rs @@ -0,0 +1,155 @@ +//! Extended access control lists on macOS. Their entries grant access the +//! file mode doesn't show: an `everyone allow read` entry, inherited from a +//! directory, makes a 0600 file readable by any user. + +use std::ffi::{CString, c_char, c_int, c_void}; +use std::io; +use std::os::fd::{AsFd, AsRawFd as _}; +use std::os::unix::ffi::OsStrExt as _; +use std::path::Path; + +const ACL_TYPE_EXTENDED: c_int = 0x0000_0100; + +unsafe extern "C" { + fn acl_free(value: *mut c_void) -> c_int; + fn acl_get_fd_np(file_descriptor: c_int, acl_type: c_int) -> *mut c_void; + fn acl_get_link_np(path: *const c_char, acl_type: c_int) -> *mut c_void; + fn acl_init(entry_count: c_int) -> *mut c_void; + fn acl_set_fd_np(file_descriptor: c_int, acl: *mut c_void, acl_type: c_int) -> c_int; +} + +/// Whether the open file or directory has extended ACL entries. +pub fn has_extended_acl(handle: impl AsFd) -> io::Result { + let file_descriptor = handle.as_fd().as_raw_fd(); + // SAFETY: acl_get_fd_np reads a descriptor that stays open while + // `handle` is borrowed. + extended_acl_found(|| unsafe { acl_get_fd_np(file_descriptor, ACL_TYPE_EXTENDED) }) +} + +/// Whether the entry at `path` has extended ACL entries, read without +/// following a link there. It opens nothing, so it leaves alone the locks +/// this process holds on the file, which closing any descriptor of it would +/// drop. An entry that's gone has none. +pub fn has_extended_acl_at(path: &Path) -> io::Result { + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + // SAFETY: acl_get_link_np reads a NUL-terminated path that outlives the + // call. + extended_acl_found(|| unsafe { acl_get_link_np(path.as_ptr(), ACL_TYPE_EXTENDED) }) +} + +/// Runs an `acl_get_*_np` call and frees what it returns. Null with `ENOENT` +/// means the entry has no extended ACL, or is gone. +fn extended_acl_found(get: impl FnOnce() -> *mut c_void) -> io::Result { + // SAFETY: errno is this thread's own, and the ACL `get` returns, if not + // null, is freed once. + unsafe { + *libc::__error() = 0; + let acl = get(); + if acl.is_null() { + let error = *libc::__error(); + return if error == 0 || error == libc::ENOENT { + Ok(false) + } else { + Err(io::Error::from_raw_os_error(error)) + }; + } + if acl_free(acl) == 0 { + Ok(true) + } else { + Err(io::Error::last_os_error()) + } + } +} + +/// Removes every extended ACL entry from the open file or directory, and +/// checks that none remain. +pub fn clear_extended_acl(handle: impl AsFd) -> io::Result<()> { + let file_descriptor = handle.as_fd().as_raw_fd(); + // SAFETY: acl_init returns either null or an empty ACL this function + // frees once; acl_set_fd_np only reads it, and writes to a descriptor + // that stays open while `handle` is borrowed. + let error = unsafe { + let acl = acl_init(0); + if acl.is_null() { + return Err(io::Error::last_os_error()); + } + let set = acl_set_fd_np(file_descriptor, acl, ACL_TYPE_EXTENDED); + let error = (set != 0).then(io::Error::last_os_error); + if acl_free(acl) != 0 && error.is_none() { + return Err(io::Error::last_os_error()); + } + error + }; + if let Some(error) = error { + return Err(error); + } + if has_extended_acl(handle)? { + return Err(io::Error::other( + "extended access control remained after removal", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn add_entry(path: &std::path::Path, entry: &str) { + let status = std::process::Command::new("/bin/chmod") + .args(["+a", entry]) + .arg(path) + .status() + .unwrap(); + assert!(status.success()); + } + + #[test] + fn finds_and_clears_a_file_s_extended_entries() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("captures"); + std::fs::write(&path, b"").unwrap(); + let file = std::fs::File::open(&path).unwrap(); + assert!(!has_extended_acl(&file).unwrap()); + + add_entry(&path, "group:everyone allow read"); + assert!(has_extended_acl(&file).unwrap()); + + clear_extended_acl(&file).unwrap(); + assert!(!has_extended_acl(&file).unwrap()); + } + + #[test] + fn reads_an_entry_s_extended_entries_by_path_without_following_a_link() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("captures"); + let link = directory.path().join("link"); + std::fs::write(&path, b"").unwrap(); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert!(!has_extended_acl_at(&path).unwrap()); + + add_entry(&path, "group:everyone allow read"); + + assert!(has_extended_acl_at(&path).unwrap()); + assert!(!has_extended_acl_at(&link).unwrap()); + assert!(!has_extended_acl_at(&directory.path().join("gone")).unwrap()); + } + + #[test] + fn clears_a_directory_s_inheritable_entries() { + let directory = tempfile::tempdir().unwrap(); + add_entry( + directory.path(), + "group:everyone allow read,write,file_inherit,directory_inherit", + ); + let handle = std::fs::File::open(directory.path()).unwrap(); + + clear_extended_acl(&handle).unwrap(); + + std::fs::write(directory.path().join("created"), b"").unwrap(); + let created = std::fs::File::open(directory.path().join("created")).unwrap(); + assert!(!has_extended_acl(&handle).unwrap()); + assert!(!has_extended_acl(&created).unwrap()); + } +} diff --git a/crates/lpm-common/src/paths.rs b/crates/lpm-common/src/paths.rs index 40d244ccf..c6866da0a 100644 --- a/crates/lpm-common/src/paths.rs +++ b/crates/lpm-common/src/paths.rs @@ -718,7 +718,11 @@ fn open_lock_file(lock_path: &Path) -> std::io::Result { options.open(lock_path) } -fn open_lock_file_nofollow(lock_path: &Path) -> std::io::Result { +/// Opens the lock file at `lock_path`, created owner-only, without following +/// a link there, for a lock in a directory whose entries a repository can +/// supply, such as a project's `.lpm`. Anything but a regular file is +/// refused. +pub fn open_lock_file_nofollow(lock_path: &Path) -> std::io::Result { let mut options = std::fs::OpenOptions::new(); options.create(true).read(true).write(true).truncate(false); #[cfg(unix)] @@ -1319,16 +1323,6 @@ pub fn acquire_single_file_exclusive_lock( acquire_single_file_exclusive_lock_from_file(file) } -/// Acquire a single-file exclusive advisory lock whose file is opened without -/// following a link at `lock_path`, for a lock in a directory whose entries a -/// repository can supply, such as a project's `.lpm`. -pub fn acquire_single_file_exclusive_lock_nofollow( - lock_path: impl AsRef, -) -> Result { - let file = open_lock_file_nofollow(lock_path.as_ref())?; - acquire_single_file_exclusive_lock_from_file(file) -} - pub fn acquire_single_file_exclusive_lock_from_file( file: std::fs::File, ) -> Result { @@ -1344,6 +1338,34 @@ pub fn acquire_single_file_exclusive_lock_from_file( }) } +/// Acquires an exclusive advisory lock on `file`, waiting at most `timeout` +/// for another holder to release it, with the usual hint once the wait is +/// noticeable. `None` when the lock was still held at the deadline. +pub fn acquire_single_file_exclusive_lock_from_file_within( + file: std::fs::File, + timeout: std::time::Duration, +) -> Result, LpmError> { + let mut data = fd_lock::RwLock::new(file); + let start = std::time::Instant::now(); + let mut hinted = false; + loop { + if try_acquire(&mut data, LockMode::Exclusive)? { + return Ok(Some(SingleFileExclusiveLockHandle { + _data: HeldLock::new(data), + })); + } + let waited = start.elapsed(); + let Some(left) = timeout.checked_sub(waited).filter(|left| !left.is_zero()) else { + return Ok(None); + }; + if !hinted && waited >= LOCK_WAIT_HINT_AFTER { + hinted = true; + default_wait_hint(); + } + std::thread::sleep(LOCK_POLL_INTERVAL.min(left)); + } +} + pub fn try_acquire_single_file_exclusive_lock_from_file( file: std::fs::File, ) -> Result, LpmError> { @@ -2261,11 +2283,21 @@ mod tests { ); } + fn lock_nofollow_within( + lock_path: &Path, + timeout: std::time::Duration, + ) -> Result, LpmError> { + let file = open_lock_file_nofollow(lock_path)?; + acquire_single_file_exclusive_lock_from_file_within(file, timeout) + } + #[test] fn nofollow_single_file_lock_shares_the_lock_domain_of_its_path() { let tmp = TempDir::new().unwrap(); let lock_path = tmp.path().join("inspector.db-lock"); - let held = acquire_single_file_exclusive_lock_nofollow(&lock_path).unwrap(); + let held = lock_nofollow_within(&lock_path, std::time::Duration::ZERO) + .unwrap() + .unwrap(); assert!( try_acquire_single_file_exclusive_lock_from_file(open_lock_file(&lock_path).unwrap()) @@ -2288,10 +2320,40 @@ mod tests { let lock_path = tmp.path().join("inspector.db-lock"); std::os::unix::fs::symlink(&target, &lock_path).unwrap(); - assert!(acquire_single_file_exclusive_lock_nofollow(&lock_path).is_err()); + assert!(open_lock_file_nofollow(&lock_path).is_err()); assert!(!target.exists()); } + #[test] + fn a_lock_wait_with_a_deadline_gives_up_at_it_and_takes_a_released_lock() { + let tmp = TempDir::new().unwrap(); + let lock_path = tmp.path().join("inspector.db-lock"); + let held = lock_nofollow_within(&lock_path, std::time::Duration::ZERO) + .unwrap() + .unwrap(); + + let start = std::time::Instant::now(); + let timeout = std::time::Duration::from_millis(250); + assert!(lock_nofollow_within(&lock_path, timeout).unwrap().is_none()); + let waited = start.elapsed(); + assert!(waited >= timeout, "{waited:?}"); + assert!( + waited < timeout + std::time::Duration::from_secs(2), + "{waited:?}" + ); + + let release = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(150)); + drop(held); + }); + assert!( + lock_nofollow_within(&lock_path, std::time::Duration::from_secs(5)) + .unwrap() + .is_some() + ); + release.join().unwrap(); + } + /// Shared lock blocks an exclusive acquire until released. #[test] fn exclusive_blocks_while_shared_held() { diff --git a/crates/lpm-inspect/Cargo.toml b/crates/lpm-inspect/Cargo.toml index a742cde31..6e336ccbe 100644 --- a/crates/lpm-inspect/Cargo.toml +++ b/crates/lpm-inspect/Cargo.toml @@ -23,10 +23,11 @@ async-stream = "0.3" futures-util = "0.3" base64 = { workspace = true } reqwest = { workspace = true } -rusqlite = { version = "0.34", features = ["backup", "bundled"] } +rusqlite = { version = "0.40", features = ["backup", "bundled", "fallible_uint", "limits"] } chrono = { workspace = true } rand = { workspace = true } subtle = "2" +thiserror = { workspace = true } tower = "0.5" [dev-dependencies] diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index a2102f05b..9560740d9 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -43,6 +43,7 @@ pub struct InspectorDb { queued_capture_bytes: Arc, /// Database file path (for display/diagnostics). pub db_path: PathBuf, + open_to_others: bool, } /// A write operation to be batched. @@ -56,37 +57,71 @@ enum DbWrite { } impl InspectorDb { - /// Open (or create) the inspector database at `{project_dir}/.lpm/inspector.db`. - /// - /// Initializes the schema if the database is new. Starts the background - /// write-batching task. - pub fn open(project_dir: &Path) -> Result { + /// Opens (or creates) `{project_dir}/.lpm/inspector.db` to record + /// captures, starting the background write-batching task. The capture + /// files are made owner-only first: a database others could open is + /// replaced with an owner-only copy (see [`OpenError`] for when it can't + /// be yet). + pub fn open(project_dir: &Path) -> Result { + Self::open_project(project_dir, Access::Capture, CaptureOwner::of(project_dir)?) + } + + /// Opens (or creates) `{project_dir}/.lpm/inspector.db` to read and + /// manage the captures recorded there, without recording any. Existing + /// files keep their access, so this works while another process records + /// to the database; [`Self::open_to_others`] says whether others could + /// open it. + pub fn open_for_review(project_dir: &Path) -> Result { + Self::open_project(project_dir, Access::Review, CaptureOwner::of(project_dir)?) + } + + fn open_project( + project_dir: &Path, + access: Access, + owner: CaptureOwner, + ) -> Result { let lpm_dir = project_dir.join(".lpm"); let db_path = lpm_dir.join("inspector.db"); - prepare_lpm_dir(&lpm_dir).map_err(sqlite_io_error)?; - let _opening = lock_capture_files(&db_path)?; - if let Err(error) = ensure_captures_ignored(&lpm_dir) { + prepare_lpm_dir(&lpm_dir, owner, access)?; + let _opening = lock_capture_files(&db_path, owner)?; + remove_replacement_copy(&db_path)?; + if let Err(error) = ensure_captures_ignored(&lpm_dir, owner) { tracing::warn!("could not keep tunnel captures out of git: {error}"); } - if prepare_capture_files(&db_path).map_err(sqlite_io_error)? { - replace_with_owner_only_copy(&db_path)?; + let mut open_to_others = prepare_capture_files(&db_path, owner, access)?; + if open_to_others && access == Access::Capture { + open_to_others = !replace_with_owner_only_copy(&db_path, owner)?; } // Write connection (owned by the background flush task) - let write_conn = open_capture_connection(&db_path, "NORMAL")?; - refuse_foreign_schema(&write_conn, &db_path)?; - init_schema(&write_conn)?; + let write_conn = open_capture_connection(&db_path, "NORMAL").map_err(damaged(&db_path))?; + refuse_unknown_schema(&write_conn, &db_path)?; + init_schema(&write_conn).map_err(damaged(&db_path))?; // Read connection (shared across API handlers). The busy timeout // lets low-frequency mutations (tag updates, session renames) wait // while the write connection flushes a batch. - let read_conn = open_capture_connection(&db_path, "NORMAL")?; - let control_conn = open_control_connection(&db_path)?; + let read_conn = open_capture_connection(&db_path, "NORMAL").map_err(damaged(&db_path))?; + let control_conn = open_control_connection(&db_path).map_err(damaged(&db_path))?; + + Ok(Self::with_connections( + write_conn, + read_conn, + control_conn, + db_path, + open_to_others, + )) + } + fn with_connections( + write_conn: Connection, + read_conn: Connection, + control_conn: Connection, + db_path: PathBuf, + open_to_others: bool, + ) -> Self { let (write_tx, write_rx) = mpsc::channel(WRITE_QUEUE_CAPACITY); let queued_capture_bytes = Arc::new(AtomicUsize::new(0)); - - // Spawn background flush task let write_conn = Arc::new(Mutex::new(write_conn)); tokio::spawn(flush_task( write_conn, @@ -94,20 +129,22 @@ impl InspectorDb { Arc::clone(&queued_capture_bytes), )); - Ok(Self { + Self { read_conn: Arc::new(Mutex::new(read_conn)), control_conn: Arc::new(std::sync::Mutex::new(control_conn)), write_tx, queued_capture_bytes, db_path, - }) + open_to_others, + } } - /// Open a temporary file-based database (for testing). + /// Open a temporary file-based database. /// /// Each call creates a unique database file so tests don't interfere /// with each other when running in parallel. - pub fn open_temp() -> Result { + #[cfg(test)] + pub(crate) fn open_temp() -> Result { use std::sync::atomic::{AtomicU64, Ordering}; static COUNTER: AtomicU64 = AtomicU64::new(0); let id = COUNTER.fetch_add(1, Ordering::Relaxed); @@ -123,27 +160,25 @@ impl InspectorDb { let write_conn = Connection::open(&tmp)?; configure_connection(&write_conn, "NORMAL")?; init_schema(&write_conn)?; - let read_conn = Connection::open(&tmp)?; configure_connection(&read_conn, "NORMAL")?; let control_conn = open_control_connection(&tmp)?; - let (write_tx, write_rx) = mpsc::channel(WRITE_QUEUE_CAPACITY); - let queued_capture_bytes = Arc::new(AtomicUsize::new(0)); - let write_conn = Arc::new(Mutex::new(write_conn)); - tokio::spawn(flush_task( + Ok(Self::with_connections( write_conn, - write_rx, - Arc::clone(&queued_capture_bytes), - )); + read_conn, + control_conn, + tmp, + false, + )) + } - Ok(Self { - read_conn: Arc::new(Mutex::new(read_conn)), - control_conn: Arc::new(std::sync::Mutex::new(control_conn)), - write_tx, - queued_capture_bytes, - db_path: tmp, - }) + /// Whether other users could open the capture files when this handle + /// opened them: a database [`Self::open_for_review`] found so, or one on + /// a filesystem that shows its files open to others whatever their mode, + /// which [`Self::open`] can't make owner-only. + pub fn open_to_others(&self) -> bool { + self.open_to_others } /// Insert a captured request (non-blocking — queued for batch write). @@ -728,7 +763,12 @@ impl InspectorDb { /// Remove captured requests and completed sessions from the inspector database. /// /// Open sessions are retained so another process can continue attaching - /// captures without violating the request-to-session foreign key. + /// captures without violating the request-to-session foreign key. The + /// captures leave no trace in the file: the full-text index, which keeps + /// a deleted entry's terms until its segments merge, is rebuilt from its + /// emptied content; `secure_delete` zeroes the freed pages; and the WAL, + /// which still holds the captures' earlier pages, is checkpointed and + /// truncated, unless another connection is reading at that moment. pub async fn clear_history(&self) -> Result<(), rusqlite::Error> { let conn = self.read_conn.lock().await; // Immediate: a deferred transaction that reads before it writes gets @@ -737,9 +777,14 @@ impl InspectorDb { let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; tx.execute("DELETE FROM requests_fts", [])?; + tx.execute( + "INSERT INTO requests_fts(requests_fts) VALUES ('rebuild')", + [], + )?; tx.execute("DELETE FROM requests", [])?; tx.execute("DELETE FROM sessions WHERE ended_at IS NOT NULL", [])?; - tx.commit() + tx.commit()?; + conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(())) } /// Return SQLite's connection-local external-write revision. @@ -753,6 +798,143 @@ impl InspectorDb { } } +/// Why the capture database couldn't be opened. +#[derive(Debug, thiserror::Error)] +pub enum OpenError { + /// `.lpm` or a capture file failed a check or a file operation; the + /// message names it, and says what to do. One others could open that + /// another process has open, which [`InspectorDb::open`] can't replace + /// yet, is [`std::io::ErrorKind::ResourceBusy`]. + #[error("{0}")] + Files(#[from] std::io::Error), + /// SQLite can't read the file as a database, or finds it damaged. + #[error( + "{} is damaged or isn't a capture database ({detail}); move it aside and retry", + path.display() + )] + Damaged { path: PathBuf, detail: String }, + /// Any other SQLite failure. + #[error(transparent)] + Database(#[from] rusqlite::Error), +} + +impl OpenError { + /// The I/O failure this is, for callers that report it as one. + pub fn kind(&self) -> std::io::ErrorKind { + match self { + Self::Files(error) => error.kind(), + Self::Damaged { .. } => std::io::ErrorKind::InvalidData, + Self::Database(_) => std::io::ErrorKind::Other, + } + } +} + +/// Reads SQLite's refusal of a file it can't use as a database as such. +fn damaged(db_path: &Path) -> impl Fn(rusqlite::Error) -> OpenError + '_ { + move |error| match error.sqlite_error_code() { + Some(rusqlite::ErrorCode::NotADatabase | rusqlite::ErrorCode::DatabaseCorrupt) => { + OpenError::Damaged { + path: db_path.to_path_buf(), + detail: error.to_string(), + } + } + _ => OpenError::Database(error), + } +} + +/// What an open of the capture database may change. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Access { + /// Records captures: makes `.lpm` and the capture files safe to write + /// them to, replacing a database others could open. + Capture, + /// Reads and manages recorded captures. Changes no existing file's + /// access, and so runs while another process records to the database. + Review, +} + +/// Whose capture files are. A user's own; under root, as in a container +/// over a mounted checkout, the project directory's owner's, or root's. +/// What root creates is given to that owner, so they can open it without +/// root later; SQLite gives its sidecars to the database's owner itself. +#[cfg(unix)] +#[derive(Debug, Clone, Copy)] +struct CaptureOwner { + uid: u32, + /// The group root gives what it creates to. + gid: u32, + as_root: bool, +} + +#[cfg(unix)] +impl CaptureOwner { + fn of(project_dir: &Path) -> std::io::Result { + use std::os::unix::fs::MetadataExt; + + let uid = lpm_common::current_uid(); + if uid != 0 { + return Ok(Self { + uid, + gid: 0, + as_root: false, + }); + } + let project = + std::fs::metadata(project_dir).map_err(|error| with_path(project_dir, error))?; + Ok(Self { + uid: project.uid(), + gid: project.gid(), + as_root: true, + }) + } + + fn owns(self, metadata: &std::fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + + metadata.uid() == self.uid || (self.as_root && metadata.uid() == 0) + } + + /// Gives a file or directory this process created to the owner, when + /// running as root. + fn give(self, handle: &std::fs::File, path: &Path) -> std::io::Result<()> { + if !self.as_root { + return Ok(()); + } + std::os::unix::fs::fchown(handle, Some(self.uid), Some(self.gid)) + .map_err(|error| with_path(path, error)) + } + + /// [`Self::give`] for an entry renamed into place, by its name, which + /// no one else can swap in a directory only its owner can write. + fn give_at(self, path: &Path) -> std::io::Result<()> { + if !self.as_root { + return Ok(()); + } + std::os::unix::fs::lchown(path, Some(self.uid), Some(self.gid)) + .map_err(|error| with_path(path, error)) + } +} + +/// Access to the capture files on Windows is the project's ACLs. +#[cfg(not(unix))] +#[derive(Debug, Clone, Copy)] +struct CaptureOwner; + +#[cfg(not(unix))] +impl CaptureOwner { + fn of(_project_dir: &Path) -> std::io::Result { + Ok(Self) + } + + fn give(self, _handle: &std::fs::File, _path: &Path) -> std::io::Result<()> { + Ok(()) + } + + fn give_at(self, _path: &Path) -> std::io::Result<()> { + Ok(()) + } +} + /// The `.lpm/.gitignore` line that keeps the capture database and its /// SQLite sidecars out of git, since captures hold request headers and /// bodies. @@ -763,6 +945,19 @@ const CAPTURES_IGNORE_LINE: &str = "inspector.db*"; /// busy. const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +/// How long opening waits for another LPM process to finish opening the +/// capture files, which takes longest when one replaces a large database. +const OPEN_LOCK_TIMEOUT: std::time::Duration = if cfg!(test) { + std::time::Duration::from_secs(1) +} else { + std::time::Duration::from_secs(10) +}; + +/// The longest string, blob or row SQLite builds: a capture's two bodies of +/// at most 50 MiB each and its headers fit, and a database from elsewhere +/// can't have SQLite build anything much larger for a capture written to it. +const LONGEST_VALUE_BYTES: i32 = 128 * 1024 * 1024; + /// The suffixes of the files SQLite keeps beside the capture database. const SQLITE_SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; @@ -783,10 +978,6 @@ fn capture_files(db_path: &Path) -> [PathBuf; 4] { ] } -fn sqlite_io_error(error: std::io::Error) -> rusqlite::Error { - rusqlite::Error::ToSqlConversionFailure(Box::new(error)) -} - fn with_path(path: &Path, error: std::io::Error) -> std::io::Error { std::io::Error::new(error.kind(), format!("{}: {error}", path.display())) } @@ -826,35 +1017,37 @@ fn hard_linked(path: &Path) -> std::io::Error { ) } -/// Whether an entry is a link: a symbolic link, or on Windows a junction. -/// Other reparse points, such as a synced folder's cloud placeholders, are -/// the project's own files. -fn is_link(metadata: &std::fs::Metadata) -> bool { - metadata.file_type().is_symlink() -} - /// Makes sure `.lpm` is a directory of the project's own, creating it if -/// needed: a link, which a cloned repository could point anywhere, such as -/// into a directory it serves, is refused. On Unix it must also be the -/// user's own, and is made writable by no one else, so no other user can -/// swap the capture files' names between the checks below and SQLite's -/// opens. -fn prepare_lpm_dir(lpm_dir: &Path) -> std::io::Result<()> { - match std::fs::symlink_metadata(lpm_dir) { - Ok(metadata) if is_link(&metadata) || !metadata.is_dir() => { - return Err(linked_or_not(lpm_dir, "directory")); - } - Ok(_) => {} +/// needed. A link, which a cloned repository could point anywhere, such as +/// into a directory it serves, is refused; the metadata of a link, read +/// without following it, is never a directory's. Reparse points other than +/// links, such as a synced folder's placeholders on Windows, read as the +/// directories they stand for. See [`restrict_lpm_dir`] for Unix. +fn prepare_lpm_dir(lpm_dir: &Path, owner: CaptureOwner, access: Access) -> std::io::Result<()> { + let created = match std::fs::symlink_metadata(lpm_dir) { + Ok(metadata) if !metadata.is_dir() => return Err(linked_or_not(lpm_dir, "directory")), + Ok(_) => false, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { std::fs::create_dir_all(lpm_dir).map_err(|error| with_path(lpm_dir, error))?; + true } Err(error) => return Err(with_path(lpm_dir, error)), - } - restrict_lpm_dir(lpm_dir) + }; + restrict_lpm_dir(lpm_dir, owner, access, created) } +/// `.lpm` must be the capture owner's. A capture writer makes it writable +/// by no one else, and on macOS clears its extended ACL, whose entries can +/// grant what its mode doesn't and pass to the files created in it, so no +/// other user can swap the capture files' names between the checks below +/// and SQLite's opens, or read the files created there. #[cfg(unix)] -fn restrict_lpm_dir(lpm_dir: &Path) -> std::io::Result<()> { +fn restrict_lpm_dir( + lpm_dir: &Path, + owner: CaptureOwner, + access: Access, + created: bool, +) -> std::io::Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let directory = @@ -863,56 +1056,97 @@ fn restrict_lpm_dir(lpm_dir: &Path) -> std::io::Result<()> { if !metadata.is_dir() { return Err(linked_or_not(lpm_dir, "directory")); } - if metadata.uid() != lpm_common::current_uid() { + if !owner.owns(&metadata) { return Err(another_users(lpm_dir)); } + if created { + owner.give(&directory, lpm_dir)?; + } + if access == Access::Review { + return Ok(()); + } let mode = metadata.mode() & 0o7777; if mode & 0o022 != 0 { directory .set_permissions(std::fs::Permissions::from_mode(mode & !0o022)) .map_err(|error| with_path(lpm_dir, error))?; } + #[cfg(target_os = "macos")] + if lpm_common::macos_acl::has_extended_acl(&directory)? { + lpm_common::macos_acl::clear_extended_acl(&directory) + .map_err(|error| with_path(lpm_dir, error))?; + } Ok(()) } #[cfg(not(unix))] -fn restrict_lpm_dir(_lpm_dir: &Path) -> std::io::Result<()> { +fn restrict_lpm_dir( + _lpm_dir: &Path, + _owner: CaptureOwner, + _access: Access, + _created: bool, +) -> std::io::Result<()> { Ok(()) } /// Holds other LPM processes off the capture files while this one checks, /// replaces and opens them. A process that opened the database while it was -/// being replaced would go on writing to the old file. +/// being replaced would go on writing to the old file. Releases before the +/// lock don't take it. fn lock_capture_files( db_path: &Path, -) -> Result { + owner: CaptureOwner, +) -> std::io::Result { let lock_path = capture_file(db_path, "-lock"); - lpm_common::acquire_single_file_exclusive_lock_nofollow(&lock_path).map_err(|error| { - sqlite_io_error(std::io::Error::other(format!( + let file = lpm_common::open_lock_file_nofollow(&lock_path) + .map_err(|error| with_path(&lock_path, error))?; + owner.give(&file, &lock_path)?; + match lpm_common::acquire_single_file_exclusive_lock_from_file_within(file, OPEN_LOCK_TIMEOUT) { + Ok(Some(lock)) => Ok(lock), + Ok(None) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!( + "another LPM process held {} for {} s while opening the captures; retry \ + once it has started", + lock_path.display(), + OPEN_LOCK_TIMEOUT.as_secs() + ), + )), + Err(error) => Err(std::io::Error::other(format!( "{}: {error}", lock_path.display() - ))) - }) + ))), + } +} + +/// Removes the copy, and its journal, of a replacement that didn't finish, +/// as they hold captures. Only an open holding the lock makes one, so any +/// found under it is left over. +fn remove_replacement_copy(db_path: &Path) -> std::io::Result<()> { + let copy_path = capture_file(db_path, "-copy"); + remove_if_present(&capture_file(©_path, "-journal"))?; + remove_if_present(©_path) } -/// Makes the capture files safe to write before SQLite opens them, and says -/// whether the database must be replaced, as others could open it or a -/// sidecar. No capture file may be a link, another user's, or have other -/// hard links. Existing ones are made owner-only, as a checkout or an older -/// release may have left them readable (see [`restrict_capture_file`]). -/// Sidecars without a database, which SQLite could read into a new one, are -/// removed, and a new database is created owner-only, so it's never -/// readable by others. -fn prepare_capture_files(db_path: &Path) -> std::io::Result { +/// Checks the capture files before SQLite opens them, and says whether +/// others could open them (see [`check_capture_file`]). No capture file may +/// be a link, another user's (see [`CaptureOwner`]), or have other hard +/// links. Sidecars without +/// a database are a removed database's leftovers, which may hold its +/// captures, and are removed; SQLite would only discard them on its next +/// write. A missing database is created owner-only. +fn prepare_capture_files( + db_path: &Path, + owner: CaptureOwner, + access: Access, +) -> std::io::Result { let mut open_to_others = false; let mut database_exists = false; for path in capture_files(db_path) { match std::fs::symlink_metadata(&path) { - Ok(metadata) if is_link(&metadata) || !metadata.is_file() => { - return Err(linked_or_not(&path, "file")); - } + Ok(metadata) if !metadata.is_file() => return Err(linked_or_not(&path, "file")), Ok(metadata) => { - open_to_others |= restrict_capture_file(&path, &metadata)?; + open_to_others |= check_capture_file(&path, &metadata, owner, access)?; database_exists |= path == db_path; } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} @@ -925,8 +1159,7 @@ fn prepare_capture_files(db_path: &Path) -> std::io::Result { for suffix in SQLITE_SIDECARS { remove_if_present(&capture_file(db_path, suffix))?; } - create_owner_only(db_path)?; - Ok(false) + Ok(!create_owner_only(db_path, owner)?) } fn remove_if_present(path: &Path) -> std::io::Result<()> { @@ -936,28 +1169,43 @@ fn remove_if_present(path: &Path) -> std::io::Result<()> { } } -/// Checks an existing capture file and says whether others could open it. -/// One they couldn't is made owner-only. One they could keeps its mode, -/// made only the owner's to read and write as well, until the database is -/// replaced, so a replacement that can't run yet runs on a later open. -/// Modes are changed by name, which no other user can swap in a directory -/// only its owner can write, rather than through a handle, whose closing -/// would drop the locks SQLite holds on the file in this process, and which -/// a file without read permission can't have. A file that's gone needs -/// nothing: SQLite removes `-wal` and `-shm` when another process closes -/// its last connection. +/// Checks an existing capture file and says whether others could open it +/// to read: its mode lets its group or anyone read it, or on macOS it has +/// an extended ACL, whose entries can grant what the mode doesn't. A +/// capture writer makes one others couldn't open owner-only. One they could +/// keeps its mode, made only the owner's to read and write as well, until +/// the database is replaced, so a replacement that can't run yet runs on a +/// later open. Modes are read and changed by name, which no other user can +/// swap in a directory only its owner can write, rather than through a +/// handle, whose closing would drop the locks SQLite holds on the file in +/// this process, and which a file without read permission can't have. A +/// file that's gone needs nothing: SQLite removes `-wal` and `-shm` when +/// another process closes its last connection. #[cfg(unix)] -fn restrict_capture_file(path: &Path, metadata: &std::fs::Metadata) -> std::io::Result { +fn check_capture_file( + path: &Path, + metadata: &std::fs::Metadata, + owner: CaptureOwner, + access: Access, +) -> std::io::Result { use std::os::unix::fs::{MetadataExt, PermissionsExt}; - if metadata.uid() != lpm_common::current_uid() { + if !owner.owns(metadata) { return Err(another_users(path)); } if metadata.nlink() > 1 { return Err(hard_linked(path)); } let mode = metadata.mode() & 0o7777; - let open_to_others = mode & 0o077 != 0; + #[cfg(target_os = "macos")] + let extended_acl = + lpm_common::macos_acl::has_extended_acl_at(path).map_err(|error| with_path(path, error))?; + #[cfg(not(target_os = "macos"))] + let extended_acl = false; + let open_to_others = mode & 0o044 != 0 || extended_acl; + if access == Access::Review { + return Ok(open_to_others); + } let wanted = if open_to_others { mode | 0o600 } else { 0o600 }; if mode != wanted { match std::fs::set_permissions(path, std::fs::Permissions::from_mode(wanted)) { @@ -972,7 +1220,12 @@ fn restrict_capture_file(path: &Path, metadata: &std::fs::Metadata) -> std::io:: /// Refuses an existing capture file with other hard links, read through a /// handle that doesn't follow a link. Access to it is the project's ACLs. #[cfg(not(unix))] -fn restrict_capture_file(path: &Path, _metadata: &std::fs::Metadata) -> std::io::Result { +fn check_capture_file( + path: &Path, + _metadata: &std::fs::Metadata, + _owner: CaptureOwner, + _access: Access, +) -> std::io::Result { let file = match lpm_common::open_nofollow(path) { Ok(file) => file, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), @@ -984,81 +1237,146 @@ fn restrict_capture_file(path: &Path, _metadata: &std::fs::Metadata) -> std::io: Ok(false) } +/// Creates a capture file only its owner can open, given to the capture +/// owner, and says whether it reads that way: a filesystem without Unix +/// modes, such as FAT or a Windows drive mounted in Linux, shows the mode it +/// likes. On macOS, extended ACL entries it inherited are cleared. A file +/// that appears at the name meanwhile, made by an LPM release that doesn't +/// take the lock, is refused until a retry checks it; `O_EXCL` also refuses +/// a link there. #[cfg(unix)] -fn create_owner_only(path: &Path) -> std::io::Result<()> { - use std::os::unix::fs::OpenOptionsExt; +fn create_owner_only(path: &Path, owner: CaptureOwner) -> std::io::Result { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; - match std::fs::OpenOptions::new() + let file = match std::fs::OpenOptions::new() .write(true) .create_new(true) .mode(0o600) .open(path) { - Ok(_) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), - Err(error) => Err(with_path(path, error)), + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + return Err(std::io::Error::new( + error.kind(), + format!( + "{} appeared while LPM was creating it; retry", + path.display() + ), + )); + } + Err(error) => return Err(with_path(path, error)), + }; + owner.give(&file, path)?; + #[cfg(target_os = "macos")] + if lpm_common::macos_acl::has_extended_acl(&file)? { + lpm_common::macos_acl::clear_extended_acl(&file).map_err(|error| with_path(path, error))?; } + let mode = file.metadata()?.mode(); + #[cfg(test)] + let mode = if tests::MODELESS_FILESYSTEM.get() { + mode | 0o777 + } else { + mode + }; + Ok(mode & 0o044 == 0) } #[cfg(not(unix))] -fn create_owner_only(_path: &Path) -> std::io::Result<()> { - Ok(()) +fn create_owner_only(_path: &Path, _owner: CaptureOwner) -> std::io::Result { + Ok(true) } -/// Replaces a capture database others could open with an owner-only copy. -/// Making the file owner-only stops new opens, but not a handle another user -/// opened while it was readable, which would go on reading every capture -/// written after; the copy is a new file such a handle doesn't reach. A -/// database another process has open isn't replaced, since that process -/// would go on writing to the old file. -fn replace_with_owner_only_copy(db_path: &Path) -> Result<(), rusqlite::Error> { - let in_use = || { - sqlite_io_error(std::io::Error::new( - std::io::ErrorKind::ResourceBusy, - format!( - "{} is open to other users, and another process has it open, so it \ - can't be replaced with an owner-only copy yet; stop other `lpm tunnel` \ - and `lpm dev` runs in this project and retry", - db_path.display() - ), - )) - }; - let database = open_nofollow_connection(db_path)?; - // Leaving WAL needs the only connection, so it also proves no other - // process has the database open, and folds the WAL into the file. +/// Replaces a capture database others could open with an owner-only copy, +/// and says whether it did. Making the file owner-only stops new opens, but +/// not a handle another user opened while it was readable, which would go +/// on reading every capture written after; the copy is a new file such a +/// handle doesn't reach. As one others could open may have come from +/// anywhere, its schema and integrity are checked first. A database another +/// process has open isn't replaced, since that process would go on writing +/// to the old file. Nor is one on a filesystem that shows the copy open to +/// others too, which the copy wouldn't improve. +fn replace_with_owner_only_copy(db_path: &Path, owner: CaptureOwner) -> Result { + let database = open_nofollow_connection(db_path).map_err(damaged(db_path))?; + refuse_unknown_schema(&database, db_path)?; + // In WAL, which every LPM connection uses, leaving it needs the only + // connection, so it proves no other LPM process has the database open, + // and folds the WAL into the file. match database.query_row("PRAGMA journal_mode=DELETE", [], |row| { row.get::<_, String>(0) }) { Ok(mode) if mode.eq_ignore_ascii_case("delete") => {} - Ok(_) => return Err(in_use()), - Err(error) if is_busy(&error) => return Err(in_use()), - Err(error) => return Err(error), + Ok(_) => return Err(in_use(db_path)), + Err(error) if is_busy(&error) => return Err(in_use(db_path)), + Err(error) => return Err(damaged(db_path)(error)), } + refuse_damage(&database, db_path)?; let copy_path = capture_file(db_path, "-copy"); - for path in [©_path, &capture_file(©_path, "-journal")] { - remove_if_present(path).map_err(sqlite_io_error)?; - } - create_owner_only(©_path).map_err(sqlite_io_error)?; - { - let mut copy = open_nofollow_connection(©_path)?; - let backup = rusqlite::backup::Backup::new(&database, &mut copy)?; - let deadline = std::time::Instant::now() + BUSY_TIMEOUT; - loop { - match backup.step(-1)? { - rusqlite::backup::StepResult::Done => break, - _ if std::time::Instant::now() < deadline => { - std::thread::sleep(std::time::Duration::from_millis(10)); - } - _ => return Err(in_use()), - } - } + let copied = copy_database(&database, ©_path, owner, db_path); + if !matches!(copied, Ok(true)) { + drop(database); + let removed = remove_replacement_copy(db_path); + return match (copied, removed) { + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error.into()), + (Ok(_), Ok(())) => Ok(false), + }; } drop(database); for suffix in SQLITE_SIDECARS { - remove_if_present(&capture_file(db_path, suffix)).map_err(sqlite_io_error)?; + remove_if_present(&capture_file(db_path, suffix))?; + } + std::fs::rename(©_path, db_path).map_err(|error| with_path(db_path, error))?; + Ok(true) +} + +fn in_use(db_path: &Path) -> OpenError { + OpenError::Files(std::io::Error::new( + std::io::ErrorKind::ResourceBusy, + format!( + "{} is open to other users, and another process has it open, so it can't \ + be replaced with an owner-only copy yet; close what has it open, such as \ + other `lpm tunnel` and `lpm dev` runs in this project, and retry", + db_path.display() + ), + )) +} + +/// Copies the database into a new owner-only file, and says whether it +/// did: not when the new file doesn't read owner-only. A lock another +/// connection holds on the database is waited out for the busy timeout, +/// inside the copy's one step. +fn copy_database( + database: &Connection, + copy_path: &Path, + owner: CaptureOwner, + db_path: &Path, +) -> Result { + if !create_owner_only(copy_path, owner)? { + return Ok(false); + } + let mut copy = open_nofollow_connection(copy_path)?; + let backup = rusqlite::backup::Backup::new(database, &mut copy)?; + match backup.step(-1).map_err(damaged(db_path))? { + rusqlite::backup::StepResult::Done => Ok(true), + _ => Err(in_use(db_path)), } - std::fs::rename(©_path, db_path).map_err(|error| sqlite_io_error(with_path(db_path, error))) +} + +/// Refuses a database SQLite finds damaged, in its pages and records or in +/// its full-text index, whose damage a crafted file could aim at SQLite's +/// full-text code: `quick_check` runs the index's own check too. +fn refuse_damage(connection: &Connection, db_path: &Path) -> Result<(), OpenError> { + let pages: String = connection + .query_row("PRAGMA quick_check(1)", [], |row| row.get(0)) + .map_err(damaged(db_path))?; + if pages != "ok" { + return Err(OpenError::Damaged { + path: db_path.to_path_buf(), + detail: lpm_common::sanitize_terminal_inline(&pages).into_owned(), + }); + } + Ok(()) } /// Opens a capture database without following a link at its path, with the @@ -1081,7 +1399,9 @@ fn connect_nofollow(path: &Path) -> Result { _ => Path::new("."), }; std::fs::canonicalize(directory) - .map_err(|error| sqlite_io_error(with_path(directory, error)))? + .map_err(|error| { + rusqlite::Error::ToSqlConversionFailure(Box::new(with_path(directory, error))) + })? .join(path.file_name().unwrap_or(path.as_os_str())) }; Connection::open_with_flags( @@ -1090,15 +1410,21 @@ fn connect_nofollow(path: &Path) -> Result { ) } -/// What every capture connection has: a database from a repository or -/// another tool can't run schema code with side effects or write SQLite's -/// internal tables, and a busy database is waited for. +/// What every capture connection has. A database from a repository or +/// another tool can't run schema code with side effects, write SQLite's +/// internal tables, or have SQLite build a value larger than +/// [`LONGEST_VALUE_BYTES`]; SQLite checks each page's cells as it reads +/// them; deleted captures are overwritten, not left in free pages; and a +/// busy database is waited for. fn harden_connection(connection: &Connection) -> Result<(), rusqlite::Error> { use rusqlite::config::DbConfig; + use rusqlite::limits::Limit; connection.set_db_config(DbConfig::SQLITE_DBCONFIG_DEFENSIVE, true)?; connection.set_db_config(DbConfig::SQLITE_DBCONFIG_TRUSTED_SCHEMA, false)?; - connection.busy_timeout(BUSY_TIMEOUT) + connection.set_limit(Limit::SQLITE_LIMIT_LENGTH, LONGEST_VALUE_BYTES)?; + connection.busy_timeout(BUSY_TIMEOUT)?; + connection.execute_batch("PRAGMA cell_size_check=ON; PRAGMA secure_delete=ON;") } /// Opens a capture connection in WAL, with `synchronous` as its sync level. @@ -1141,26 +1467,88 @@ fn is_busy(error: &rusqlite::Error) -> bool { ) } -/// Refuses a database whose schema holds triggers or views, which LPM never -/// creates: one from a repository or another tool could run them on every -/// capture written. -fn refuse_foreign_schema(connection: &Connection, db_path: &Path) -> Result<(), rusqlite::Error> { - let foreign: bool = connection.query_row( - "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type IN ('trigger', 'view'))", - [], - |row| row.get(0), +/// A schema object as SQLite stores it: its type, name, table and SQL, the +/// SQL in its [`canonical_sql`] form. +type SchemaObject = (String, String, String, Option); + +/// SQL without the whitespace that separates nothing: a run of whitespace +/// is kept, as one space, only between two word characters, so the text an +/// `ALTER TABLE` splices in reads the same however it's spaced. +fn canonical_sql(sql: &str) -> String { + let is_word = |character: char| character.is_alphanumeric() || character == '_'; + let mut canonical = String::with_capacity(sql.len()); + let mut spaced = false; + for character in sql.chars() { + if character.is_whitespace() { + spaced = true; + continue; + } + if spaced && canonical.ends_with(is_word) && is_word(character) { + canonical.push(' '); + } + spaced = false; + canonical.push(character); + } + canonical +} + +/// The database's schema objects, but for the query planner's statistics, +/// which run nothing. +fn schema_objects(connection: &Connection) -> Result, rusqlite::Error> { + let mut statement = connection.prepare( + "SELECT type, name, tbl_name, sql FROM sqlite_master + WHERE substr(name, 1, 11) <> 'sqlite_stat'", )?; - if foreign { - return Err(sqlite_io_error(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!( - "{} holds triggers or views, which LPM never creates; move it aside \ - and retry", - db_path.display() - ), - ))); + statement + .query_map([], |row| { + let sql: Option = row.get(3)?; + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + sql.as_deref().map(canonical_sql), + )) + })? + .collect() +} + +/// LPM's own schema objects: those [`CAPTURE_SCHEMA`] creates, which a +/// database from an earlier release may hold as they are, and those after +/// [`ensure_capture_schema`] upgrades them, both built by this SQLite in +/// memory. +fn lpm_schema() -> Result, rusqlite::Error> { + let connection = Connection::open_in_memory()?; + connection.execute_batch(CAPTURE_SCHEMA)?; + let mut objects: std::collections::HashSet<_> = + schema_objects(&connection)?.into_iter().collect(); + ensure_capture_schema(&connection)?; + objects.extend(schema_objects(&connection)?); + Ok(objects) +} + +/// Refuses a database holding a schema object LPM never creates. One from +/// a repository or another tool could otherwise run code of its own on +/// every capture written: a trigger, or a check, default, generated column +/// or index expression that's slow by design. +fn refuse_unknown_schema(connection: &Connection, db_path: &Path) -> Result<(), OpenError> { + let known = lpm_schema()?; + let unknown: Vec = schema_objects(connection) + .map_err(damaged(db_path))? + .into_iter() + .filter(|object| !known.contains(object)) + .map(|(kind, name, _, _)| format!("{kind} {}", lpm_common::sanitize_terminal_inline(&name))) + .collect(); + if unknown.is_empty() { + return Ok(()); } - Ok(()) + Err(OpenError::Files(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "{} holds schema LPM never creates ({}); move it aside and retry", + db_path.display(), + unknown.join(", ") + ), + ))) } /// A `.gitignore` line as git reads it: without one trailing carriage @@ -1218,8 +1606,8 @@ fn could_name_a_capture_file(pattern: &[u8]) -> bool { /// project's, being a link or not a regular file, or that its owner keeps /// from being written, being read-only or on Unix another user's, is left /// as it is; the CLI's capture warning then says git doesn't ignore the -/// captures. -fn ensure_captures_ignored(lpm_dir: &Path) -> std::io::Result<()> { +/// captures. Under root, the file written is given to the capture owner. +fn ensure_captures_ignored(lpm_dir: &Path, owner: CaptureOwner) -> std::io::Result<()> { let path = lpm_dir.join(".gitignore"); let (existing, metadata) = match lpm_common::read_file_capped_unlinked( &path, @@ -1233,7 +1621,7 @@ fn ensure_captures_ignored(lpm_dir: &Path) -> std::io::Result<()> { return Ok(()); } if let Some(metadata) = &metadata { - refuse_kept_from_writing(&path, metadata)?; + refuse_kept_from_writing(&path, metadata, owner)?; } let newline: &[u8] = if existing.windows(2).any(|pair| pair == b"\r\n") { b"\r\n" @@ -1248,15 +1636,21 @@ fn ensure_captures_ignored(lpm_dir: &Path) -> std::io::Result<()> { updated.extend_from_slice(newline); updated.extend_from_slice(CAPTURES_IGNORE_LINE.as_bytes()); updated.extend_from_slice(newline); - lpm_common::write_file_atomic(&path, updated) + lpm_common::write_file_atomic(&path, updated)?; + owner.give_at(&path) } -fn refuse_kept_from_writing(path: &Path, metadata: &std::fs::Metadata) -> std::io::Result<()> { +#[cfg_attr(not(unix), allow(unused_variables))] +fn refuse_kept_from_writing( + path: &Path, + metadata: &std::fs::Metadata, + owner: CaptureOwner, +) -> std::io::Result<()> { #[cfg(unix)] let read_only = { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; + use std::os::unix::fs::PermissionsExt; - if metadata.uid() != lpm_common::current_uid() { + if !owner.owns(metadata) { return Err(std::io::Error::new( std::io::ErrorKind::PermissionDenied, format!("{} belongs to another user", path.display()), @@ -1282,8 +1676,13 @@ fn open_control_connection(path: &Path) -> Result { } fn init_schema(conn: &Connection) -> Result<(), rusqlite::Error> { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS sessions ( + conn.execute_batch(CAPTURE_SCHEMA)?; + ensure_capture_schema(conn) +} + +/// The capture tables and indexes, as every release has created them; +/// [`ensure_capture_schema`] upgrades them since. +const CAPTURE_SCHEMA: &str = "CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, name TEXT, domain TEXT, @@ -1327,10 +1726,7 @@ fn init_schema(conn: &Connection) -> Result<(), rusqlite::Error> { CREATE VIRTUAL TABLE IF NOT EXISTS requests_fts USING fts5( path, summary, request_body, response_body - );", - )?; - ensure_capture_schema(conn) -} + );"; fn ensure_capture_schema(conn: &Connection) -> Result<(), rusqlite::Error> { let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; @@ -1749,6 +2145,19 @@ mod tests { use lpm_tunnel::webhook::WebhookProvider; use std::collections::HashMap; + #[cfg(unix)] + thread_local! { + /// Makes this thread's new capture files read as a filesystem + /// without Unix modes shows them: open to anyone. + pub(super) static MODELESS_FILESYSTEM: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + + /// The capture owner of a project of this user's. + #[cfg(unix)] + fn this_user() -> CaptureOwner { + CaptureOwner::of(Path::new(".")).unwrap() + } + const CAPTURES_IGNORED: &str = "# LPM tunnel captures hold request headers and bodies.\ninspector.db*\n"; @@ -1958,7 +2367,7 @@ mod tests { return; }; - let error = refuse_kept_from_writing(&path, &metadata).unwrap_err(); + let error = refuse_kept_from_writing(&path, &metadata, this_user()).unwrap_err(); assert!( error.to_string().contains("belongs to another user"), @@ -2009,7 +2418,8 @@ mod tests { return; } - let error = restrict_lpm_dir(Path::new("/usr")).unwrap_err(); + let error = + restrict_lpm_dir(Path::new("/usr"), this_user(), Access::Review, false).unwrap_err(); assert!( error.to_string().contains("belongs to another user"), @@ -2024,7 +2434,7 @@ mod tests { return; }; - let error = restrict_capture_file(&path, &metadata).unwrap_err(); + let error = check_capture_file(&path, &metadata, this_user(), Access::Review).unwrap_err(); assert!( error.to_string().contains("belongs to another user"), @@ -2034,7 +2444,7 @@ mod tests { #[cfg(unix)] #[test] - fn restricting_a_capture_file_that_vanished_needs_nothing() { + fn checking_a_capture_file_that_vanished_needs_nothing() { let project = tempfile::tempdir().unwrap(); let path = project.path().join("inspector.db-wal"); std::fs::write(&path, b"").unwrap(); @@ -2042,7 +2452,7 @@ mod tests { let metadata = std::fs::symlink_metadata(&path).unwrap(); std::fs::remove_file(&path).unwrap(); - assert!(!restrict_capture_file(&path, &metadata).unwrap()); + assert!(!check_capture_file(&path, &metadata, this_user(), Access::Capture).unwrap()); } #[cfg(unix)] @@ -2056,7 +2466,9 @@ mod tests { set_mode(&served, 0o644); std::fs::hard_link(&served, lpm_dir.join("inspector.db")).unwrap(); - let error = prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap_err(); + let error = + prepare_capture_files(&lpm_dir.join("inspector.db"), this_user(), Access::Capture) + .unwrap_err(); assert!(error.to_string().contains("other hard links"), "{error}"); assert_eq!(mode(&served), 0o644); @@ -2098,15 +2510,18 @@ mod tests { for (name, leftover_mode) in [ ("inspector.db", 0o000), ("inspector.db-wal", 0o400), - ("inspector.db-shm", 0o700), - ("inspector.db-journal", 0o600), + ("inspector.db-shm", 0o733), + ("inspector.db-journal", 0o611), ] { let path = lpm_dir.join(name); std::fs::write(&path, b"").unwrap(); set_mode(&path, leftover_mode); } - assert!(!prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap()); + assert!( + !prepare_capture_files(&lpm_dir.join("inspector.db"), this_user(), Access::Capture) + .unwrap() + ); for name in [ "inspector.db", @@ -2130,11 +2545,34 @@ mod tests { set_mode(&path, leftover_mode); } - assert!(prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap()); + assert!( + prepare_capture_files(&lpm_dir.join("inspector.db"), this_user(), Access::Capture) + .unwrap() + ); assert_eq!(mode(&lpm_dir.join("inspector.db-wal")), 0o604); } + #[cfg(unix)] + #[test] + fn preparing_reads_a_capture_file_its_group_could_read_as_open_to_others() { + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + for (name, leftover_mode) in [("inspector.db", 0o600), ("inspector.db-shm", 0o440)] { + let path = lpm_dir.join(name); + std::fs::write(&path, b"").unwrap(); + set_mode(&path, leftover_mode); + } + + assert!( + prepare_capture_files(&lpm_dir.join("inspector.db"), this_user(), Access::Capture) + .unwrap() + ); + + assert_eq!(mode(&lpm_dir.join("inspector.db-shm")), 0o640); + } + #[cfg(unix)] #[test] fn preparing_removes_sidecars_left_without_a_database() { @@ -2151,7 +2589,10 @@ mod tests { set_mode(&path, 0o644); } - assert!(!prepare_capture_files(&lpm_dir.join("inspector.db")).unwrap()); + assert!( + !prepare_capture_files(&lpm_dir.join("inspector.db"), this_user(), Access::Capture) + .unwrap() + ); for name in [ "inspector.db-wal", @@ -2231,7 +2672,7 @@ mod tests { #[cfg(unix)] #[test] - fn opening_waits_to_replace_a_database_another_process_has_open() { + fn opening_waits_to_replace_a_database_another_connection_has_open() { let project = project_with_a_capture(); let db_path = project.path().join(".lpm/inspector.db"); set_mode(&db_path, 0o644); @@ -2252,6 +2693,8 @@ mod tests { error.to_string().contains("another process has it open"), "{error}" ); + assert_eq!(error.kind(), std::io::ErrorKind::ResourceBusy); + assert!(!error.to_string().contains("Move or repair"), "{error}"); // Leaving WAL tells at once; the copy's own wait is the busy timeout. assert!(started.elapsed() < std::time::Duration::from_secs(4)); assert_eq!(mode(&db_path), 0o644); @@ -2260,28 +2703,544 @@ mod tests { assert_eq!(mode(&db_path), 0o600); } + /// Holds the lock openers take while they check, replace and open the + /// capture files. + fn hold_capture_lock(project: &Path) -> lpm_common::SingleFileExclusiveLockHandle { + let file = + lpm_common::open_lock_file_nofollow(&project.join(".lpm/inspector.db-lock")).unwrap(); + lpm_common::acquire_single_file_exclusive_lock_from_file_within( + file, + std::time::Duration::ZERO, + ) + .unwrap() + .unwrap() + } + + /// Opens a project's capture database, as `open` would, on a thread of + /// its own with a runtime of its own, and drops it. + fn open_on_a_thread( + project: &Path, + open: fn(&Path) -> Result, + ) -> std::thread::JoinHandle> { + let project = project.to_path_buf(); + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { open(&project).map(drop) }) + }) + } + + #[cfg(unix)] + #[test] + fn reviewing_leaves_a_database_others_could_open_as_it_is_and_says_so() { + use std::os::unix::fs::MetadataExt; + + let project = project_with_a_capture(); + let lpm_dir = project.path().join(".lpm"); + let db_path = lpm_dir.join("inspector.db"); + set_mode(&db_path, 0o644); + set_mode(&lpm_dir, 0o775); + let inode = std::fs::metadata(&db_path).unwrap().ino(); + // A process recording to it, which a writer's open would refuse to + // replace it under. + let recording = Connection::open(&db_path).unwrap(); + recording + .query_row("SELECT count(*) FROM requests", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + + let reviewing = tokio::runtime::Runtime::new().unwrap(); + let (open_to_others, kept) = reviewing.block_on(async { + let db = InspectorDb::open_for_review(project.path()).unwrap(); + (db.open_to_others(), db.get_webhook("kept").await.unwrap()) + }); + // Shutting the runtime down closes the review's connections, as the + // end of its process would. + drop(reviewing); + + assert!(open_to_others); + assert!(kept.is_some()); + assert_eq!(mode(&db_path), 0o644); + assert_eq!(mode(&lpm_dir), 0o775); + assert_eq!(std::fs::metadata(&db_path).unwrap().ino(), inode); + drop(recording); + let open_to_others = tokio::runtime::Runtime::new() + .unwrap() + .block_on(async { InspectorDb::open(project.path()).unwrap().open_to_others() }); + assert!(!open_to_others); + assert_eq!(mode(&db_path), 0o600); + assert_eq!(mode(&lpm_dir), 0o755); + } + + #[cfg(unix)] #[tokio::test] - async fn opening_refuses_a_database_with_triggers_or_views() { - for statement in [ - "CREATE TRIGGER copy_requests AFTER INSERT ON requests BEGIN SELECT 1; END;", - "CREATE VIEW recent AS SELECT * FROM requests;", - ] { + async fn reviewing_a_new_project_creates_an_owner_only_database() { + let project = tempfile::tempdir().unwrap(); + + let db = InspectorDb::open_for_review(project.path()).unwrap(); + + assert!(!db.open_to_others()); + assert_eq!(mode(&project.path().join(".lpm/inspector.db")), 0o600); + assert_eq!(lpm_gitignore(project.path()), CAPTURES_IGNORED); + } + + #[cfg(unix)] + #[tokio::test] + async fn reviewing_leaves_capture_files_others_couldnt_open_as_they_are() { + let project = tempfile::tempdir().unwrap(); + drop(InspectorDb::open(project.path()).unwrap()); + let db_path = project.path().join(".lpm/inspector.db"); + set_mode(&db_path, 0o620); + + let db = InspectorDb::open_for_review(project.path()).unwrap(); + + assert!(!db.open_to_others()); + assert_eq!(mode(&db_path), 0o620); + } + + #[tokio::test] + async fn every_open_removes_a_replacement_copy_left_over() { + for open in [InspectorDb::open, InspectorDb::open_for_review] { let project = tempfile::tempdir().unwrap(); - std::fs::create_dir(project.path().join(".lpm")).unwrap(); - let db_path = project.path().join(".lpm/inspector.db"); - Connection::open(&db_path) + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + std::fs::write(lpm_dir.join("inspector.db-copy"), b"captures").unwrap(); + std::fs::write(lpm_dir.join("inspector.db-copy-journal"), b"captures").unwrap(); + + drop(open(project.path()).unwrap()); + + assert!(!lpm_dir.join("inspector.db-copy").exists()); + assert!(!lpm_dir.join("inspector.db-copy-journal").exists()); + } + } + + #[cfg(unix)] + #[test] + fn a_filesystem_without_modes_keeps_its_database_and_says_it_s_open_to_others() { + use std::os::unix::fs::MetadataExt; + + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + set_mode(&db_path, 0o644); + let inode = std::fs::metadata(&db_path).unwrap().ino(); + let fresh = tempfile::tempdir().unwrap(); + MODELESS_FILESYSTEM.set(true); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (existing, kept, new) = runtime.block_on(async { + let db = InspectorDb::open(project.path()).unwrap(); + let kept = db.get_webhook("kept").await.unwrap(); + let new = InspectorDb::open(fresh.path()).unwrap().open_to_others(); + (db.open_to_others(), kept, new) + }); + MODELESS_FILESYSTEM.set(false); + + assert!(existing); + assert!(new); + assert!(kept.is_some()); + assert_eq!(std::fs::metadata(&db_path).unwrap().ino(), inode); + assert!(!project.path().join(".lpm/inspector.db-copy").exists()); + } + + #[cfg(unix)] + #[test] + fn root_accepts_the_project_owner_s_files_and_its_own() { + let as_root = CaptureOwner { + uid: 501, + gid: 20, + as_root: true, + }; + let as_user = CaptureOwner { + uid: 501, + gid: 0, + as_root: false, + }; + let directory = tempfile::tempdir().unwrap(); + let mine = std::fs::symlink_metadata(directory.path()).unwrap(); + let roots = std::fs::symlink_metadata("/").unwrap(); + let me = CaptureOwner { + uid: lpm_common::current_uid(), + ..as_root + }; + + assert!(as_root.owns(&roots)); + assert!(!as_user.owns(&roots)); + assert!(me.owns(&mine)); + assert!( + !CaptureOwner { + uid: lpm_common::current_uid() + 1, + ..as_user + } + .owns(&mine) + ); + } + + /// A group this user is in besides the one its files get, or None. + #[cfg(unix)] + fn another_group_of_mine(file: &Path) -> Option { + use std::os::unix::fs::MetadataExt; + + let primary = std::fs::metadata(file).unwrap().gid(); + let groups = std::process::Command::new("id").arg("-G").output().ok()?; + String::from_utf8_lossy(&groups.stdout) + .split_whitespace() + .filter_map(|group| group.parse().ok()) + .find(|group| *group != primary) + } + + #[cfg(unix)] + #[test] + fn root_gives_what_it_creates_to_the_project_s_owner() { + use std::os::unix::fs::MetadataExt; + + let project = project_with_a_capture(); + let Some(group) = another_group_of_mine(project.path()) else { + return; + }; + let lpm_dir = project.path().join(".lpm"); + set_mode(&lpm_dir.join("inspector.db"), 0o644); + std::fs::remove_file(lpm_dir.join("inspector.db-lock")).unwrap(); + std::fs::remove_file(lpm_dir.join(".gitignore")).unwrap(); + let fresh = tempfile::tempdir().unwrap(); + // Root's opens give what they create to the project's owner; this + // user, chowning to itself, can stand in for root by giving files a + // group of its own instead. + let owner = CaptureOwner { + uid: lpm_common::current_uid(), + gid: group, + as_root: true, + }; + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + drop(InspectorDb::open_project(project.path(), Access::Capture, owner).unwrap()); + drop(InspectorDb::open_project(fresh.path(), Access::Review, owner).unwrap()); + }); + + for path in [ + lpm_dir.join("inspector.db"), + lpm_dir.join("inspector.db-lock"), + lpm_dir.join(".gitignore"), + fresh.path().join(".lpm"), + fresh.path().join(".lpm/inspector.db"), + fresh.path().join(".lpm/inspector.db-lock"), + fresh.path().join(".lpm/.gitignore"), + ] { + assert_eq!( + std::fs::metadata(&path).unwrap().gid(), + group, + "{}", + path.display() + ); + } + } + + #[cfg(target_os = "macos")] + fn add_acl_entry(path: &Path, entry: &str) { + let status = std::process::Command::new("/bin/chmod") + .args(["+a", entry]) + .arg(path) + .status() + .unwrap(); + assert!(status.success()); + } + + #[cfg(target_os = "macos")] + fn has_acl(path: &Path) -> bool { + lpm_common::macos_acl::has_extended_acl_at(path).unwrap() + } + + #[cfg(target_os = "macos")] + #[test] + fn a_capture_file_with_an_extended_acl_is_open_to_others() { + use std::os::unix::fs::MetadataExt; + + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + add_acl_entry(&db_path, "group:everyone allow read"); + let inode = std::fs::metadata(&db_path).unwrap().ino(); + + let reviewing = tokio::runtime::Runtime::new().unwrap(); + let reviewed = reviewing.block_on(async { + InspectorDb::open_for_review(project.path()) .unwrap() - .execute_batch(&format!("CREATE TABLE requests (id TEXT); {statement}")) - .unwrap(); - #[cfg(unix)] - set_mode(&db_path, 0o600); + .open_to_others() + }); + drop(reviewing); + assert!(reviewed); + assert!(has_acl(&db_path)); + + let (open_to_others, kept) = tokio::runtime::Runtime::new().unwrap().block_on(async { + let db = InspectorDb::open(project.path()).unwrap(); + (db.open_to_others(), db.get_webhook("kept").await.unwrap()) + }); + + assert!(!open_to_others); + assert!(kept.is_some()); + assert!(!has_acl(&db_path)); + assert_ne!(std::fs::metadata(&db_path).unwrap().ino(), inode); + } - let error = InspectorDb::open(project.path()).map(drop).unwrap_err(); + #[cfg(target_os = "macos")] + #[tokio::test] + async fn an_extended_acl_lpm_passes_on_reaches_no_capture_file() { + const INHERITED: &str = "group:everyone allow read,write,file_inherit,directory_inherit"; + + let reviewed = tempfile::tempdir().unwrap(); + let reviewed_lpm = reviewed.path().join(".lpm"); + std::fs::create_dir(&reviewed_lpm).unwrap(); + add_acl_entry(&reviewed_lpm, INHERITED); + let db = InspectorDb::open_for_review(reviewed.path()).unwrap(); + assert!(!db.open_to_others()); + assert!(has_acl(&reviewed_lpm)); + assert!(!has_acl(&reviewed_lpm.join("inspector.db"))); + + let captured = tempfile::tempdir().unwrap(); + let captured_lpm = captured.path().join(".lpm"); + std::fs::create_dir(&captured_lpm).unwrap(); + add_acl_entry(&captured_lpm, INHERITED); + let db = InspectorDb::open(captured.path()).unwrap(); + db.insert_durable_request(Arc::new(make_webhook("kept", 200)), None) + .await + .unwrap(); + assert!(!db.open_to_others()); + assert!(!has_acl(&captured_lpm)); + for name in ["inspector.db", "inspector.db-wal", "inspector.db-shm"] { + assert!(!has_acl(&captured_lpm.join(name)), "{name}"); + } + } + + /// A project whose capture database others could open, built by + /// `build` on a connection that may write SQLite's own tables. + fn project_with_a_database(build: &str) -> tempfile::TempDir { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let db_path = project.path().join(".lpm/inspector.db"); + Connection::open(&db_path) + .unwrap() + .execute_batch(build) + .unwrap(); + #[cfg(unix)] + set_mode(&db_path, 0o644); + project + } - assert!(error.to_string().contains("triggers or views"), "{error}"); + #[tokio::test] + async fn opening_refuses_a_database_with_schema_lpm_never_creates() { + let lpm_schema = format!( + "{CAPTURE_SCHEMA} ALTER TABLE requests ADD COLUMN response_body_incomplete \ + INTEGER NOT NULL DEFAULT 0;" + ); + let planted = [ + ( + "trigger copy_requests", + format!( + "{lpm_schema} CREATE TRIGGER copy_requests AFTER INSERT ON requests \ + BEGIN SELECT 1; END;" + ), + ), + ( + "view recent", + format!("{lpm_schema} CREATE VIEW recent AS SELECT * FROM requests;"), + ), + ( + "table requests", + lpm_schema.replace( + "tags TEXT,", + "tags TEXT CHECK (length(hex(zeroblob(1000))) > 0),", + ), + ), + ( + "table requests", + lpm_schema.replace( + "tags TEXT,", + "tags TEXT, slow TEXT GENERATED ALWAYS AS (hex(zeroblob(1000))) VIRTUAL,", + ), + ), + ( + "table sessions", + lpm_schema.replace("name TEXT,", "name TEXT DEFAULT (hex(zeroblob(1000))),"), + ), + ( + "index slow_requests", + format!( + "{lpm_schema} CREATE INDEX slow_requests ON requests(length(request_body));" + ), + ), + ( + "index slow_index", + format!( + "{lpm_schema} CREATE INDEX slow_index ON requests_fts_data(length(block));" + ), + ), + ( + "table extra", + format!("{lpm_schema} CREATE TABLE extra (x);"), + ), + ]; + for (named, build) in planted { + for open in [InspectorDb::open, InspectorDb::open_for_review] { + let project = project_with_a_database(&build); + + let error = open(project.path()).map(drop).unwrap_err(); + + assert!( + error.to_string().contains("holds schema LPM never creates") + && error.to_string().contains(named), + "{named}: {error}" + ); + #[cfg(unix)] + assert_eq!( + mode(&project.path().join(".lpm/inspector.db")), + 0o644, + "{named}" + ); + } } } + #[tokio::test] + async fn opening_upgrades_a_database_an_earlier_release_created() { + let earlier = CAPTURE_SCHEMA + .replace( + "CREATE TABLE IF NOT EXISTS pending_sessions ( + session_id TEXT PRIMARY KEY, + FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE + );", + "", + ) + .replace("\n ", "\n\t"); + assert_ne!(earlier, CAPTURE_SCHEMA); + // With the query planner's statistics, which a tool may have added. + let project = project_with_a_database(&format!( + "{earlier} INSERT INTO requests (id, timestamp, method, path) \ + VALUES ('kept', '2026-04-06T12:00:00Z', 'POST', '/'); ANALYZE;" + )); + + let db = InspectorDb::open(project.path()).unwrap(); + + assert_eq!(db.count().await.unwrap(), 1); + let connection = Connection::open(project.path().join(".lpm/inspector.db")).unwrap(); + let tables: i64 = connection + .query_row( + "SELECT count(*) FROM sqlite_master WHERE name = 'pending_sessions'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(tables, 1); + } + + #[test] + fn canonical_sql_keeps_only_the_spaces_between_words() { + assert_eq!( + canonical_sql("CREATE TABLE t (\n\t a TEXT ,b\tINT )"), + "CREATE TABLE t(a TEXT,b INT)" + ); + assert_ne!(canonical_sql("summary TEXT"), canonical_sql("summ aryTEXT")); + } + + #[cfg(unix)] + #[test] + fn copying_waits_out_a_lock_another_connection_holds_on_the_database() { + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("inspector.db"); + let database = Connection::open(&db_path).unwrap(); + database + .execute_batch("CREATE TABLE kept (x); INSERT INTO kept VALUES (1);") + .unwrap(); + let holder = Connection::open(&db_path).unwrap(); + holder + .execute_batch("BEGIN EXCLUSIVE; INSERT INTO kept VALUES (2);") + .unwrap(); + let released = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(200)); + holder.execute_batch("COMMIT").unwrap(); + }); + + let copied = copy_database( + &database, + &capture_file(&db_path, "-copy"), + this_user(), + &db_path, + ) + .unwrap(); + released.join().unwrap(); + + assert!(copied); + let rows: i64 = Connection::open(capture_file(&db_path, "-copy")) + .unwrap() + .query_row("SELECT count(*) FROM kept", [], |row| row.get(0)) + .unwrap(); + assert_eq!(rows, 2); + } + + #[cfg(unix)] + #[test] + fn replacing_refuses_a_database_with_a_damaged_full_text_index() { + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + let damaging = Connection::open(&db_path).unwrap(); + damaging + .execute_batch("UPDATE requests_fts_data SET block = zeroblob(64) WHERE id > 10;") + .unwrap(); + drop(damaging); + set_mode(&db_path, 0o644); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let error = runtime + .block_on(async { InspectorDb::open(project.path()).map(drop) }) + .unwrap_err(); + + assert!(matches!(error, OpenError::Damaged { .. }), "{error}"); + assert!(error.to_string().contains("move it aside"), "{error}"); + assert_eq!(mode(&db_path), 0o644); + assert!(!project.path().join(".lpm/inspector.db-copy").exists()); + } + + #[cfg(unix)] + #[test] + fn replacing_refuses_a_database_with_damaged_pages() { + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + let connection = Connection::open(&db_path).unwrap(); + let (page_size, pages): (u64, u64) = connection + .query_row( + "SELECT * FROM pragma_page_size, pragma_page_count", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + connection + .execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") + .unwrap(); + drop(connection); + assert!(pages > 3); + // The header of page 2, the `sessions` table's root. + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(&db_path) + .unwrap(); + use std::io::{Seek as _, Write as _}; + file.seek(std::io::SeekFrom::Start(page_size)).unwrap(); + file.write_all(&[0xff; 8]).unwrap(); + drop(file); + set_mode(&db_path, 0o644); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let error = runtime + .block_on(async { InspectorDb::open(project.path()).map(drop) }) + .unwrap_err(); + + assert!(matches!(error, OpenError::Damaged { .. }), "{error}"); + } + #[cfg(unix)] #[test] fn capture_connections_refuse_a_linked_database() { @@ -2313,6 +3272,50 @@ mod tests { .db_config(rusqlite::config::DbConfig::SQLITE_DBCONFIG_DEFENSIVE) .unwrap() ); + for pragma in ["cell_size_check", "secure_delete"] { + let on: i64 = connection + .query_row(&format!("PRAGMA {pragma}"), [], |row| row.get(0)) + .unwrap(); + assert_eq!(on, 1, "{pragma}"); + } + assert_eq!( + connection + .limit(rusqlite::limits::Limit::SQLITE_LIMIT_LENGTH) + .unwrap(), + LONGEST_VALUE_BYTES + ); + assert!( + connection + .query_row( + "SELECT length(zeroblob(?1))", + [LONGEST_VALUE_BYTES + 1], + |row| { row.get::<_, i64>(0) } + ) + .is_err() + ); + } + + #[test] + fn the_largest_capture_fits_the_longest_value() { + let connection = Connection::open_in_memory().unwrap(); + harden_connection(&connection).unwrap(); + init_schema(&connection).unwrap(); + let body = vec![b'x'; 50 * 1024 * 1024]; + let mut webhook = make_webhook("largest", 200); + webhook.request_body.clone_from(&body); + webhook.response_body = body; + + insert_request_row(&connection, &webhook, None).unwrap(); + } + + #[test] + fn bundled_sqlite_has_the_full_text_fix() { + // 3.50.3 fixed CVE-2025-7709, a read past a damaged FTS5 index. + assert!( + rusqlite::version_number() >= 3_050_003, + "{}", + rusqlite::version() + ); } #[test] @@ -2354,7 +3357,7 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn opening_a_new_database_from_several_processes_at_once_succeeds() { + async fn opening_a_new_database_from_several_openers_at_once_succeeds() { for _ in 0..10 { let project = tempfile::tempdir().unwrap(); let openers: Vec<_> = (0..8) @@ -2370,17 +3373,14 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn opening_waits_for_another_lpm_process_opening_the_captures() { + async fn opening_waits_while_another_opener_holds_the_capture_lock() { let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); - let lock = lpm_common::acquire_single_file_exclusive_lock_nofollow( - project.path().join(".lpm/inspector.db-lock"), - ) - .unwrap(); + let lock = hold_capture_lock(project.path()); let path = project.path().to_path_buf(); let opener = tokio::task::spawn_blocking(move || { InspectorDb::open(&path).map(drop)?; - Ok::<_, rusqlite::Error>(std::time::Instant::now()) + Ok::<_, OpenError>(std::time::Instant::now()) }); tokio::time::sleep(std::time::Duration::from_millis(300)).await; @@ -2391,8 +3391,97 @@ mod tests { assert!(opened >= released, "opened before the lock was released"); } + #[test] + fn opening_gives_up_once_another_opener_holds_the_lock_too_long() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let lock = hold_capture_lock(project.path()); + let started = std::time::Instant::now(); + + let error = open_on_a_thread(project.path(), InspectorDb::open) + .join() + .unwrap() + .unwrap_err(); + + assert!(started.elapsed() >= OPEN_LOCK_TIMEOUT); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + assert!( + error.to_string().contains("another LPM process held"), + "{error}" + ); + assert!(!project.path().join(".lpm/inspector.db").exists()); + drop(lock); + } + + #[cfg(unix)] + #[test] + fn openers_racing_to_replace_a_database_others_could_open_all_succeed() { + for round in 0..30 { + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + set_mode(&db_path, 0o644); + let start = Arc::new(std::sync::Barrier::new(4)); + let openers: Vec<_> = (0..4) + .map(|_| { + let start = Arc::clone(&start); + let project = project.path().to_path_buf(); + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + start.wait(); + runtime.block_on(async { + // Each keeps its database open while the others + // open theirs, as concurrent runs would. + let db = InspectorDb::open(&project)?; + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + Ok::<_, OpenError>(db.open_to_others()) + }) + }) + }) + .collect(); + for opener in openers { + assert!(!opener.join().unwrap().unwrap(), "round {round}"); + } + assert_eq!(mode(&db_path), 0o600, "round {round}"); + } + } + + #[tokio::test] + async fn clearing_history_leaves_no_capture_on_disk_while_another_handle_stays_open() { + const SECRET: &[u8] = b"zqxcapturedsecretzqx"; + + let project = tempfile::tempdir().unwrap(); + let recording = InspectorDb::open(project.path()).unwrap(); + let mut webhook = make_webhook("secret", 200); + webhook.request_body = SECRET.to_vec(); + webhook.summary = String::from_utf8(SECRET.to_vec()).unwrap(); + webhook.request_headers.insert( + "authorization".into(), + String::from_utf8(SECRET.to_vec()).unwrap(), + ); + recording + .insert_durable_request(Arc::new(webhook), None) + .await + .unwrap(); + let clearing = InspectorDb::open_for_review(project.path()).unwrap(); + + clearing.clear_history().await.unwrap(); + + for name in ["inspector.db", "inspector.db-wal"] { + let path = project.path().join(".lpm").join(name); + let bytes = std::fs::read(&path).unwrap_or_default(); + assert!( + !bytes.windows(SECRET.len()).any(|window| window == SECRET), + "{name} still holds the capture" + ); + } + assert_eq!(recording.count().await.unwrap(), 0); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn clearing_history_from_two_processes_at_once_succeeds() { + async fn clearing_history_from_two_handles_at_once_succeeds() { let project = tempfile::tempdir().unwrap(); let first = InspectorDb::open(project.path()).unwrap(); let second = InspectorDb::open(project.path()).unwrap(); @@ -2659,6 +3748,7 @@ mod tests { write_tx, queued_capture_bytes: Arc::clone(&queued_capture_bytes), db_path, + open_to_others: false, }; db.start_session("saturated-session".to_string(), None, 3000) @@ -3108,12 +4198,18 @@ mod tests { std::fs::create_dir_all(&lpm_dir).unwrap(); std::fs::write(lpm_dir.join("inspector.db"), b"not a sqlite database").unwrap(); - let error = match InspectorDb::open(project.path()) { - Ok(_) => panic!("corrupt database must not open"), - Err(error) => error, - }; + for open in [InspectorDb::open, InspectorDb::open_for_review] { + let error = match open(project.path()) { + Ok(_) => panic!("corrupt database must not open"), + Err(error) => error, + }; - assert!(error.to_string().contains("database")); + assert!(matches!(error, OpenError::Damaged { .. }), "{error}"); + assert!( + error.to_string().contains("move it aside and retry"), + "{error}" + ); + } } } diff --git a/crates/lpm-vault/src/storage_transaction.rs b/crates/lpm-vault/src/storage_transaction.rs index fa24d5f06..bdb5d05e6 100644 --- a/crates/lpm-vault/src/storage_transaction.rs +++ b/crates/lpm-vault/src/storage_transaction.rs @@ -5,6 +5,8 @@ use std::path::{Path, PathBuf}; use cap_fs_ext::{DirExt as _, FollowSymlinks, OpenOptionsFollowExt as _, OpenOptionsSyncExt as _}; use cap_std::ambient_authority; use cap_std::fs::{Dir, OpenOptions}; +#[cfg(target_os = "macos")] +use lpm_common::macos_acl; const TRANSACTION_LOCK_NAME: &str = ".vault-keychain.lock"; @@ -94,7 +96,7 @@ impl VaultStorageDirectory { )); } #[cfg(target_os = "macos")] - macos_acl::clear_directory(&directory).map_err(|error| { + macos_acl::clear_extended_acl(&directory).map_err(|error| { format!( "failed to remove extended access control from {}: {error}", display.display() @@ -340,7 +342,7 @@ impl VaultStorageDirectory { { use cap_std::fs::{Permissions, PermissionsExt as _}; #[cfg(target_os = "macos")] - macos_acl::clear_file(file)?; + macos_acl::clear_extended_acl(&*file)?; file.set_permissions(Permissions::from_mode(0o600))?; } file.write_all(contents)?; @@ -426,7 +428,7 @@ impl VaultStorageDirectory { Err(error) => return Err(format!("failed to create {label}: {error}")), }; #[cfg(target_os = "macos")] - macos_acl::clear_file(&file) + macos_acl::clear_extended_acl(&file) .map_err(|error| format!("failed to secure {label}: {error}"))?; self.validate_regular_file(&file, OsStr::new(name), true)?; if let Err(error) = file.write_all(contents).and_then(|()| file.sync_all()) { @@ -536,81 +538,6 @@ impl VaultStorageDirectory { } } -#[cfg(target_os = "macos")] -mod macos_acl { - use std::ffi::{c_int, c_void}; - use std::os::fd::AsRawFd as _; - - const ACL_TYPE_EXTENDED: c_int = 0x0000_0100; - - unsafe extern "C" { - fn acl_free(value: *mut c_void) -> c_int; - fn acl_get_fd_np(file_descriptor: c_int, acl_type: c_int) -> *mut c_void; - fn acl_init(entry_count: c_int) -> *mut c_void; - fn acl_set_fd_np(file_descriptor: c_int, acl: *mut c_void, acl_type: c_int) -> c_int; - } - - pub(super) fn has_extended_acl(file: &cap_std::fs::File) -> std::io::Result { - has_extended_acl_fd(file.as_raw_fd()) - } - - pub(super) fn clear_file(file: &cap_std::fs::File) -> std::io::Result<()> { - clear_extended_acl_fd(file.as_raw_fd()) - } - - pub(super) fn clear_directory(directory: &cap_std::fs::Dir) -> std::io::Result<()> { - clear_extended_acl_fd(directory.as_raw_fd()) - } - - pub(super) fn has_extended_acl_fd(file_descriptor: c_int) -> std::io::Result { - unsafe { - *libc::__error() = 0; - let acl = acl_get_fd_np(file_descriptor, ACL_TYPE_EXTENDED); - if acl.is_null() { - let error = *libc::__error(); - return if error == 0 || error == libc::ENOENT { - Ok(false) - } else { - Err(std::io::Error::from_raw_os_error(error)) - }; - } - if acl_free(acl) == 0 { - Ok(true) - } else { - Err(std::io::Error::last_os_error()) - } - } - } - - fn clear_extended_acl_fd(file_descriptor: c_int) -> std::io::Result<()> { - unsafe { - let acl = acl_init(0); - if acl.is_null() { - return Err(std::io::Error::last_os_error()); - } - let result = acl_set_fd_np(file_descriptor, acl, ACL_TYPE_EXTENDED); - let error = if result == 0 { - None - } else { - Some(std::io::Error::last_os_error()) - }; - let free_result = acl_free(acl); - if let Some(error) = error { - return Err(error); - } - if free_result != 0 { - return Err(std::io::Error::last_os_error()); - } - if has_extended_acl_fd(file_descriptor)? { - return Err(std::io::Error::other( - "extended access control remained after removal", - )); - } - Ok(()) - } - } -} - pub(crate) fn with_vault_transaction( operation: impl FnOnce(&VaultStorageDirectory) -> Result, ) -> Result { @@ -1144,8 +1071,6 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn protected_directory_and_new_files_remove_inherited_access_control() { - use std::os::fd::AsRawFd as _; - with_temp_home(|home| { let directory = home.join(".lpm"); std::fs::create_dir(&directory).expect("create .lpm directory"); @@ -1173,13 +1098,10 @@ mod tests { let created = std::fs::File::open(directory.join(".created-key")) .expect("open created protected file"); assert!( - !macos_acl::has_extended_acl_fd(opened_directory.as_raw_fd()) + !macos_acl::has_extended_acl(&opened_directory) .expect("inspect protected directory ACL") ); - assert!( - !macos_acl::has_extended_acl_fd(created.as_raw_fd()) - .expect("inspect protected file ACL") - ); + assert!(!macos_acl::has_extended_acl(&created).expect("inspect protected file ACL")); }); } } diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index 280630859..4c08b0a1b 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1360,7 +1360,8 @@ async fn tunnel_start_names_capture_files_git_tracks_and_how_to_stop_it() { assert!( stderr.contains("git tracks '.lpm/inspector.db'") - && stderr.contains("git -C .lpm rm --cached -- inspector.db") + && stderr.contains("run `git rm --cached -- .lpm/inspector.db`") + && !stderr.contains("git -C .lpm") && stderr.contains("rotate any credentials"), "tunnel startup must name tracked capture files:\n{stderr}" ); @@ -1503,7 +1504,7 @@ fn tunnel_inspect_reports_a_capture_database_it_cant_open_as_a_disk_error_under_ assert!( envelope["error"] .as_str() - .is_some_and(|message| message.contains("Move or repair it")), + .is_some_and(|message| message.contains("move it aside and retry")), "{envelope}" ); } @@ -1549,6 +1550,55 @@ fn tunnel_inspect_without_auth_reads_project_inspector_database_under_json() { assert_eq!(rows[0]["status"], serde_json::json!(402)); } +#[cfg(unix)] +#[test] +fn reading_captures_others_could_open_works_while_another_process_records_and_warns() { + use std::os::unix::fs::PermissionsExt; + + let project = TempProject::empty(r#"{"name":"tunnel","version":"1.0.0"}"#); + let db_path = project.path().join(".lpm/inspector.db"); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let recording = runtime.block_on(async { + let state = lpm_inspect::state::InspectorState::with_db( + 3000, + lpm_inspect::db::InspectorDb::open(project.path()).unwrap(), + ); + state + .push(captured_webhook("wh-kept", "2026-05-22T10:05:00Z", 200)) + .await; + state.flush().await.unwrap(); + state + }); + // As an earlier release or a checkout left it. + std::fs::set_permissions(&db_path, std::fs::Permissions::from_mode(0o644)).unwrap(); + + let human = lpm(&project).args(["tunnel", "log"]).output().unwrap(); + let json = lpm(&project) + .args(["--json", "tunnel", "log"]) + .output() + .unwrap(); + + let rendered = format!( + "{}{}", + String::from_utf8_lossy(&human.stdout), + String::from_utf8_lossy(&human.stderr) + ); + assert!(human.status.success(), "{rendered}"); + assert!(rendered.contains("is open to other users"), "{rendered}"); + assert!( + json.status.success(), + "{}", + String::from_utf8_lossy(&json.stderr) + ); + let rows: serde_json::Value = serde_json::from_slice(&json.stdout).unwrap(); + assert_eq!(rows.as_array().map(Vec::len), Some(1)); + assert_eq!( + std::fs::metadata(&db_path).unwrap().permissions().mode() & 0o777, + 0o644 + ); + drop(recording); +} + #[test] fn tunnel_inspect_filters_and_details_share_deterministic_project_history() { let project = TempProject::empty(r#"{"name":"tunnel","version":"1.0.0"}"#); @@ -1694,7 +1744,7 @@ fn tunnel_inspect_reports_an_actionable_corrupt_database_error() { String::from_utf8_lossy(&output.stderr) ); assert!(rendered.contains(".lpm/inspector.db")); - assert!(rendered.contains("Move or repair")); + assert!(rendered.contains("move it aside and retry")); assert!( !rendered.contains("network connection"), "a local database error isn't a network one:\n{rendered}" From 53d74283a1205e4532f7e150f8a94b930984bf7f Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Mon, 12 Oct 2026 16:44:18 +0100 Subject: [PATCH 08/17] fix(tunnel): let captures open past a reader where modes don't hold, and close the schema, ACL and root gaps - On a filesystem without Unix modes, the owner-only copy's file is made first, so a writer no longer stops while a reader has the database open. - The schema check no longer exempts names starting with sqlite_stat; the reference schema runs ANALYZE, so only SQLite's own statistics tables are known. Its canonical form uses SQLite's whitespace and word rules. - Review opens create missing WAL files owner-only first, so they don't inherit .lpm's ACL entries. - As root, a capture file's mode changes through a no-follow handle checked to be the file checked, and the new .gitignore is given to its owner before it's renamed into place. - Capture writes take the write lock as they begin, so a concurrent clear or tag edit no longer drops them. - A full disk names the space the copy needs; one git query timing out no longer drops what the other confirmed. - The lock wait is 10 s in tests too, but for the test of the wait. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/tunnel.rs | 81 +++- crates/lpm-common/src/paths.rs | 16 + crates/lpm-inspect/src/db.rs | 578 +++++++++++++++++++++----- tests/workflows/tests/dev_tunnel.rs | 2 + 4 files changed, 562 insertions(+), 115 deletions(-) diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index a5cdbdc3f..891dc64a2 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -773,6 +773,13 @@ fn capture_warnings( let Some(state) = state else { return warnings; }; + if state.incomplete { + warnings.push( + "git didn't answer in time for every capture file, so check that git ignores \ + and doesn't track inspector.db*" + .to_string(), + ); + } if state.lpm_unasked { warnings.push( "git couldn't be asked about .lpm, so check that git ignores and doesn't \ @@ -882,11 +889,16 @@ struct CaptureGitState { /// Whether git refused to answer for `.lpm`, as for one that looks like a /// bare repository. lpm_unasked: bool, + /// Whether one of the queries couldn't run or didn't answer in time, so + /// what the other found stands but git's ignoring the files isn't + /// confirmed. + incomplete: bool, } impl CaptureGitState { fn kept_out(&self) -> bool { !self.lpm_unasked + && !self.incomplete && self.project_tracked.is_empty() && self.lpm_tracked.is_empty() && self.unignored.is_empty() @@ -930,8 +942,9 @@ struct CaptureGitQuery { /// whatever the case of their directory; and git in `.lpm`, for those /// neither tracked nor ignored there, and for those a repository of `.lpm`'s /// own tracks, such as a submodule, whose files the project's index doesn't -/// list. None when neither is a repository, git can't be run, or it doesn't -/// answer in time. +/// list. What one query confirms stands when the other doesn't answer in +/// time. None when neither answers: neither is a repository, git can't be +/// run, or it doesn't answer in time. async fn capture_git_state(project_dir: &Path) -> Option { capture_git_state_by(project_dir, |query| { capture_git_output(capture_git_command(&query), CAPTURE_GIT_TIMEOUT) @@ -975,7 +988,10 @@ where true } Err(CaptureGitFailure::Refused) => false, - Err(CaptureGitFailure::Unavailable) => return None, + Err(CaptureGitFailure::Unavailable) => { + state.incomplete = true; + false + } }; let inside_answered = match inside { Some(Ok(files)) => { @@ -991,7 +1007,10 @@ where state.lpm_unasked = true; false } - Some(Err(CaptureGitFailure::Unavailable)) => return None, + Some(Err(CaptureGitFailure::Unavailable)) => { + state.incomplete = true; + false + } None => false, }; (project_answered || inside_answered).then_some(state) @@ -2559,6 +2578,7 @@ mod tests { lpm_tracked: Vec::new(), unignored: names(&["inspector.db-wal"]), lpm_unasked: false, + incomplete: false, }) ); } @@ -2579,17 +2599,17 @@ mod tests { } #[tokio::test] - async fn says_nothing_of_git_when_either_query_doesn_t_answer_in_time() { + async fn says_nothing_of_git_when_neither_query_answers() { let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); for (project_answer, inside_answer) in [ ( Err(CaptureGitFailure::Unavailable), - Ok(CaptureGitFiles::default()), + Err(CaptureGitFailure::Unavailable), ), ( - Ok(CaptureGitFiles::default()), + Err(CaptureGitFailure::Refused), Err(CaptureGitFailure::Unavailable), ), ( @@ -2603,6 +2623,50 @@ mod tests { } } + #[tokio::test] + async fn keeps_what_one_query_confirms_when_the_other_doesn_t_answer_in_time() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + + let (tracked, _) = state_from_answers( + project.path(), + Ok(files(&[".lpm/inspector.db"], &[])), + Err(CaptureGitFailure::Unavailable), + ) + .await; + let (quiet, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Unavailable), + Ok(CaptureGitFiles::default()), + ) + .await; + + let tracked = tracked.unwrap(); + assert_eq!(tracked.project_tracked, names(&[".lpm/inspector.db"])); + let warnings = warnings_for(Some(tracked), false); + assert!(warnings[0].ends_with(")"), "{warnings:?}"); + assert!( + warnings.iter().any( + |warning| warning.starts_with("git tracks '.lpm/inspector.db'") + && warning.contains("git rm --cached") + ), + "{warnings:?}" + ); + assert!( + warnings + .iter() + .any(|warning| warning.starts_with("git didn't answer in time")), + "{warnings:?}" + ); + let quiet = warnings_for(quiet, false); + assert!(!quiet[0].contains("git ignores it"), "{quiet:?}"); + assert!( + quiet + .iter() + .any(|warning| warning.starts_with("git didn't answer in time")) + ); + } + fn warnings_for(state: Option, ignore_file_skipped: bool) -> Vec { capture_warnings(state.as_ref(), ignore_file_skipped, CaptureStore::OwnerOnly) } @@ -2617,6 +2681,7 @@ mod tests { lpm_tracked: names(lpm_tracked), unignored: names(unignored), lpm_unasked: false, + incomplete: false, } } @@ -2627,6 +2692,7 @@ mod tests { None, Some(CaptureGitState { lpm_unasked: true, + incomplete: false, ..state(&[], &[], &[]) }), Some(state(&[".lpm/inspector.db"], &[], &[])), @@ -2645,6 +2711,7 @@ mod tests { let warnings = warnings_for( Some(CaptureGitState { lpm_unasked: true, + incomplete: false, ..state(&[], &[], &[]) }), false, diff --git a/crates/lpm-common/src/paths.rs b/crates/lpm-common/src/paths.rs index c6866da0a..6a7fd6320 100644 --- a/crates/lpm-common/src/paths.rs +++ b/crates/lpm-common/src/paths.rs @@ -2324,6 +2324,22 @@ mod tests { assert!(!target.exists()); } + #[cfg(unix)] + #[test] + fn nofollow_single_file_lock_refuses_a_path_that_is_not_a_regular_file() { + use std::os::unix::ffi::OsStrExt as _; + + let tmp = TempDir::new().unwrap(); + let lock_path = tmp.path().join("inspector.db-lock"); + let name = std::ffi::CString::new(lock_path.as_os_str().as_bytes()).unwrap(); + // SAFETY: `name` is a valid NUL-terminated path. + assert_eq!(unsafe { libc::mkfifo(name.as_ptr(), 0o600) }, 0); + + let error = open_lock_file_nofollow(&lock_path).unwrap_err(); + + assert!(error.to_string().contains("not a regular file"), "{error}"); + } + #[test] fn a_lock_wait_with_a_deadline_gives_up_at_it_and_takes_a_released_lock() { let tmp = TempDir::new().unwrap(); diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index 9560740d9..fe8ae30c0 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -92,6 +92,7 @@ impl InspectorDb { if open_to_others && access == Access::Capture { open_to_others = !replace_with_owner_only_copy(&db_path, owner)?; } + open_to_others |= !create_wal_sidecars(&db_path, owner)?; // Write connection (owned by the background flush task) let write_conn = open_capture_connection(&db_path, "NORMAL").map_err(damaged(&db_path))?; @@ -233,7 +234,7 @@ impl InspectorDb { .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); let transaction = connection - .transaction() + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) .map_err(|error| error.to_string())?; insert_request_row(&transaction, &webhook, session_id.as_deref()) .map_err(|error| error.to_string())?; @@ -289,7 +290,8 @@ impl InspectorDb { .control_conn .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let transaction = conn.transaction()?; + let transaction = + conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let inserted = transaction.execute( "INSERT OR IGNORE INTO sessions (id, name, domain, local_port) VALUES (?1, ?2, ?3, ?4)", @@ -310,7 +312,8 @@ impl InspectorDb { .control_conn .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let transaction = conn.transaction()?; + let transaction = + conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; transaction.execute( "DELETE FROM pending_sessions WHERE session_id = ?1", params![id], @@ -333,7 +336,8 @@ impl InspectorDb { .control_conn .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let transaction = conn.transaction()?; + let transaction = + conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let committed = transaction.execute( "DELETE FROM pending_sessions WHERE session_id = ?1", params![id], @@ -348,7 +352,8 @@ impl InspectorDb { .control_conn .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let transaction = conn.transaction()?; + let transaction = + conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; transaction.execute( "UPDATE sessions SET ended_at = datetime('now') WHERE id = ?1", params![&id], @@ -903,16 +908,6 @@ impl CaptureOwner { std::os::unix::fs::fchown(handle, Some(self.uid), Some(self.gid)) .map_err(|error| with_path(path, error)) } - - /// [`Self::give`] for an entry renamed into place, by its name, which - /// no one else can swap in a directory only its owner can write. - fn give_at(self, path: &Path) -> std::io::Result<()> { - if !self.as_root { - return Ok(()); - } - std::os::unix::fs::lchown(path, Some(self.uid), Some(self.gid)) - .map_err(|error| with_path(path, error)) - } } /// Access to the capture files on Windows is the project's ACLs. @@ -929,10 +924,6 @@ impl CaptureOwner { fn give(self, _handle: &std::fs::File, _path: &Path) -> std::io::Result<()> { Ok(()) } - - fn give_at(self, _path: &Path) -> std::io::Result<()> { - Ok(()) - } } /// The `.lpm/.gitignore` line that keeps the capture database and its @@ -947,11 +938,15 @@ const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); /// How long opening waits for another LPM process to finish opening the /// capture files, which takes longest when one replaces a large database. -const OPEN_LOCK_TIMEOUT: std::time::Duration = if cfg!(test) { - std::time::Duration::from_secs(1) -} else { - std::time::Duration::from_secs(10) -}; +const OPEN_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +fn open_lock_timeout() -> std::time::Duration { + #[cfg(test)] + if let Some(timeout) = tests::OPEN_LOCK_WAIT.get() { + return timeout; + } + OPEN_LOCK_TIMEOUT +} /// The longest string, blob or row SQLite builds: a capture's two bodies of /// at most 50 MiB each and its headers fit, and a database from elsewhere @@ -1101,7 +1096,8 @@ fn lock_capture_files( let file = lpm_common::open_lock_file_nofollow(&lock_path) .map_err(|error| with_path(&lock_path, error))?; owner.give(&file, &lock_path)?; - match lpm_common::acquire_single_file_exclusive_lock_from_file_within(file, OPEN_LOCK_TIMEOUT) { + let timeout = open_lock_timeout(); + match lpm_common::acquire_single_file_exclusive_lock_from_file_within(file, timeout) { Ok(Some(lock)) => Ok(lock), Ok(None) => Err(std::io::Error::new( std::io::ErrorKind::TimedOut, @@ -1109,7 +1105,7 @@ fn lock_capture_files( "another LPM process held {} for {} s while opening the captures; retry \ once it has started", lock_path.display(), - OPEN_LOCK_TIMEOUT.as_secs() + timeout.as_secs() ), )), Err(error) => Err(std::io::Error::other(format!( @@ -1162,6 +1158,27 @@ fn prepare_capture_files( Ok(!create_owner_only(db_path, owner)?) } +/// Makes the WAL files SQLite would create, when they're missing, +/// owner-only first, and says whether they read that way. A file SQLite +/// creates takes the inheritable ACL entries of its directory on macOS, +/// which a review leaves on `.lpm`. Under the open lock no LPM process has +/// the database open in WAL without them, so none is made under one in +/// use, and SQLite takes empty ones as new. +fn create_wal_sidecars(db_path: &Path, owner: CaptureOwner) -> std::io::Result { + let mut owner_only = true; + for suffix in ["-wal", "-shm"] { + let path = capture_file(db_path, suffix); + match std::fs::symlink_metadata(&path) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + owner_only &= create_owner_only(&path, owner)?; + } + Err(error) => return Err(with_path(&path, error)), + } + } + Ok(owner_only) +} + fn remove_if_present(path: &Path) -> std::io::Result<()> { match std::fs::remove_file(path) { Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(with_path(path, error)), @@ -1178,8 +1195,9 @@ fn remove_if_present(path: &Path) -> std::io::Result<()> { /// later open. Modes are read and changed by name, which no other user can /// swap in a directory only its owner can write, rather than through a /// handle, whose closing would drop the locks SQLite holds on the file in -/// this process, and which a file without read permission can't have. A -/// file that's gone needs nothing: SQLite removes `-wal` and `-shm` when +/// this process, and which a file without read permission can't have; as +/// root, whose capture owner can swap them, through a checked handle (see +/// [`set_mode_through_handle`]). A file that's gone needs nothing: SQLite removes `-wal` and `-shm` when /// another process closes its last connection. #[cfg(unix)] fn check_capture_file( @@ -1208,7 +1226,12 @@ fn check_capture_file( } let wanted = if open_to_others { mode | 0o600 } else { 0o600 }; if mode != wanted { - match std::fs::set_permissions(path, std::fs::Permissions::from_mode(wanted)) { + let changed = if owner.as_root { + set_mode_through_handle(path, metadata, wanted) + } else { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(wanted)) + }; + match changed { Ok(()) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), Err(error) => return Err(with_path(path, error)), @@ -1217,6 +1240,36 @@ fn check_capture_file( Ok(open_to_others) } +/// Changes a capture file's mode as root, through a handle that doesn't +/// follow a link, once the handle is checked to be the file `checked` +/// describes. Root's capture owner is another user, who can swap the names +/// in `.lpm` for a link to, or another name of, a file root's change would +/// open up. Root can open the file whatever its mode, and this process holds +/// no SQLite lock on it yet, which the handle's closing would drop, as each +/// LPM command opens the captures once. +#[cfg(unix)] +fn set_mode_through_handle( + path: &Path, + checked: &std::fs::Metadata, + mode: u32, +) -> std::io::Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let file = lpm_common::open_nofollow(path)?; + let opened = file.metadata()?; + if !opened.is_file() + || opened.dev() != checked.dev() + || opened.ino() != checked.ino() + || opened.nlink() > 1 + { + return Err(std::io::Error::other(format!( + "{} changed while its access was checked; retry", + path.display() + ))); + } + file.set_permissions(std::fs::Permissions::from_mode(mode)) +} + /// Refuses an existing capture file with other hard links, read through a /// handle that doesn't follow a link. Access to it is the project's ACLs. #[cfg(not(unix))] @@ -1294,8 +1347,38 @@ fn create_owner_only(_path: &Path, _owner: CaptureOwner) -> std::io::Result Result { + let copy_path = capture_file(db_path, "-copy"); + let owner_only = create_owner_only(©_path, owner); + if !matches!(owner_only, Ok(true)) { + let removed = remove_replacement_copy(db_path); + owner_only?; + removed?; + return Ok(false); + } + if let Err(error) = copy_checked_database(db_path, ©_path) { + if let Err(removal) = remove_replacement_copy(db_path) { + tracing::warn!( + "could not remove {}: {removal}", + capture_file(db_path, "-copy").display() + ); + } + return Err(error); + } + for suffix in SQLITE_SIDECARS { + remove_if_present(&capture_file(db_path, suffix))?; + } + std::fs::rename(©_path, db_path).map_err(|error| with_path(db_path, error))?; + Ok(true) +} + +/// Checks a database others could open, and copies it into the owner-only +/// file made for the copy. +fn copy_checked_database(db_path: &Path, copy_path: &Path) -> Result<(), OpenError> { let database = open_nofollow_connection(db_path).map_err(damaged(db_path))?; refuse_unknown_schema(&database, db_path)?; // In WAL, which every LPM connection uses, leaving it needs the only @@ -1310,24 +1393,7 @@ fn replace_with_owner_only_copy(db_path: &Path, owner: CaptureOwner) -> Result return Err(damaged(db_path)(error)), } refuse_damage(&database, db_path)?; - - let copy_path = capture_file(db_path, "-copy"); - let copied = copy_database(&database, ©_path, owner, db_path); - if !matches!(copied, Ok(true)) { - drop(database); - let removed = remove_replacement_copy(db_path); - return match (copied, removed) { - (Err(error), _) => Err(error), - (Ok(_), Err(error)) => Err(error.into()), - (Ok(_), Ok(())) => Ok(false), - }; - } - drop(database); - for suffix in SQLITE_SIDECARS { - remove_if_present(&capture_file(db_path, suffix))?; - } - std::fs::rename(©_path, db_path).map_err(|error| with_path(db_path, error))?; - Ok(true) + copy_database(&database, copy_path, db_path) } fn in_use(db_path: &Path) -> OpenError { @@ -1336,33 +1402,51 @@ fn in_use(db_path: &Path) -> OpenError { format!( "{} is open to other users, and another process has it open, so it can't \ be replaced with an owner-only copy yet; close what has it open, such as \ - other `lpm tunnel` and `lpm dev` runs in this project, and retry", + `lpm tunnel inspect --ui` or other `lpm tunnel` and `lpm dev` runs in this \ + project, and retry", db_path.display() ), )) } -/// Copies the database into a new owner-only file, and says whether it -/// did: not when the new file doesn't read owner-only. A lock another -/// connection holds on the database is waited out for the busy timeout, -/// inside the copy's one step. -fn copy_database( - database: &Connection, - copy_path: &Path, - owner: CaptureOwner, - db_path: &Path, -) -> Result { - if !create_owner_only(copy_path, owner)? { - return Ok(false); - } +/// Copies the database into the copy's file. A lock another connection +/// holds on the database is waited out for the busy timeout, inside the +/// copy's one step; one held past it leaves the copy unfinished, which is +/// refused rather than taken for the database. The copy takes as much +/// space as the database, free pages included, so a disk without it is +/// named as the cause. +fn copy_database(database: &Connection, copy_path: &Path, db_path: &Path) -> Result<(), OpenError> { let mut copy = open_nofollow_connection(copy_path)?; + #[cfg(test)] + if let Some(pages) = tests::COPY_PAGE_LIMIT.get() { + copy.pragma_update(None, "max_page_count", pages)?; + } let backup = rusqlite::backup::Backup::new(database, &mut copy)?; - match backup.step(-1).map_err(damaged(db_path))? { - rusqlite::backup::StepResult::Done => Ok(true), - _ => Err(in_use(db_path)), + match backup.step(-1) { + Ok(rusqlite::backup::StepResult::Done) => Ok(()), + Ok(_) => Err(in_use(db_path)), + Err(error) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DiskFull) => { + Err(no_room_to_copy(db_path)) + } + Err(error) => Err(damaged(db_path)(error)), } } +fn no_room_to_copy(db_path: &Path) -> OpenError { + let needed = std::fs::metadata(db_path).map_or_else( + |_| "as much as it takes".to_owned(), + |metadata| format!("about {}", lpm_common::format_bytes(metadata.len())), + ); + OpenError::Files(std::io::Error::new( + std::io::ErrorKind::StorageFull, + format!( + "{} is open to other users, and replacing it with an owner-only copy needs \ + {needed} free on its disk; free that much, or move the database aside, and retry", + db_path.display() + ), + )) +} + /// Refuses a database SQLite finds damaged, in its pages and records or in /// its full-text index, whose damage a crafted file could aim at SQLite's /// full-text code: `quick_check` runs the index's own check too. @@ -1473,13 +1557,19 @@ type SchemaObject = (String, String, String, Option); /// SQL without the whitespace that separates nothing: a run of whitespace /// is kept, as one space, only between two word characters, so the text an -/// `ALTER TABLE` splices in reads the same however it's spaced. +/// `ALTER TABLE` splices in reads the same however it's spaced. Whitespace +/// and word characters are SQLite's own: only ASCII space, tab, newline, +/// form feed and carriage return separate tokens, and any other character +/// past ASCII belongs to a word, so text whose tokens differ for SQLite +/// can't read the same here. fn canonical_sql(sql: &str) -> String { - let is_word = |character: char| character.is_alphanumeric() || character == '_'; + let is_word = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '_' | '$') || !character.is_ascii() + }; let mut canonical = String::with_capacity(sql.len()); let mut spaced = false; for character in sql.chars() { - if character.is_whitespace() { + if matches!(character, ' ' | '\t' | '\n' | '\x0c' | '\r') { spaced = true; continue; } @@ -1492,13 +1582,10 @@ fn canonical_sql(sql: &str) -> String { canonical } -/// The database's schema objects, but for the query planner's statistics, -/// which run nothing. +/// The database's schema objects. fn schema_objects(connection: &Connection) -> Result, rusqlite::Error> { - let mut statement = connection.prepare( - "SELECT type, name, tbl_name, sql FROM sqlite_master - WHERE substr(name, 1, 11) <> 'sqlite_stat'", - )?; + let mut statement = + connection.prepare("SELECT type, name, tbl_name, sql FROM sqlite_master")?; statement .query_map([], |row| { let sql: Option = row.get(3)?; @@ -1515,13 +1602,16 @@ fn schema_objects(connection: &Connection) -> Result, rusqlite /// LPM's own schema objects: those [`CAPTURE_SCHEMA`] creates, which a /// database from an earlier release may hold as they are, and those after /// [`ensure_capture_schema`] upgrades them, both built by this SQLite in -/// memory. +/// memory, with the statistics tables `ANALYZE` adds, which run nothing. +/// Anything else, whatever its name, is unknown: SQLite loads an object +/// named like its own from a schema it didn't write. fn lpm_schema() -> Result, rusqlite::Error> { let connection = Connection::open_in_memory()?; connection.execute_batch(CAPTURE_SCHEMA)?; let mut objects: std::collections::HashSet<_> = schema_objects(&connection)?.into_iter().collect(); ensure_capture_schema(&connection)?; + connection.execute_batch("ANALYZE")?; objects.extend(schema_objects(&connection)?); Ok(objects) } @@ -1636,8 +1726,12 @@ fn ensure_captures_ignored(lpm_dir: &Path, owner: CaptureOwner) -> std::io::Resu updated.extend_from_slice(newline); updated.extend_from_slice(CAPTURES_IGNORE_LINE.as_bytes()); updated.extend_from_slice(newline); - lpm_common::write_file_atomic(&path, updated)?; - owner.give_at(&path) + // Given to the owner through the new file's own handle, before it's + // renamed into place: under root, the owner can swap the name after. + lpm_common::write_file_atomic_with(&path, lpm_common::AtomicWriteOptions::new(), |file| { + owner.give(file, &path)?; + std::io::Write::write_all(file, &updated) + }) } #[cfg_attr(not(unix), allow(unused_variables))] @@ -1842,8 +1936,13 @@ fn finish_batch( } } +/// Writes a batch of captures in one transaction, which takes the write +/// lock as it begins: a capture's insert reads before it writes, and a +/// transaction that only took a snapshot can't write once another +/// connection commits, such as a clear or a tag edit, where one that waited +/// for the lock writes after it. fn flush_batch(conn: &Connection, batch: &[DbWrite]) -> Result<(), rusqlite::Error> { - let tx = conn.unchecked_transaction()?; + let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; for write in batch { match write { @@ -2145,6 +2244,17 @@ mod tests { use lpm_tunnel::webhook::WebhookProvider; use std::collections::HashMap; + thread_local! { + /// The most pages this thread's database copies may hold, as a + /// full disk would stop them. + pub(super) static COPY_PAGE_LIMIT: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + /// How long this thread's opens wait for the lock, in place of + /// [`OPEN_LOCK_TIMEOUT`], for a test of the wait itself. + pub(super) static OPEN_LOCK_WAIT: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + } + #[cfg(unix)] thread_local! { /// Makes this thread's new capture files read as a filesystem @@ -2716,22 +2826,6 @@ mod tests { .unwrap() } - /// Opens a project's capture database, as `open` would, on a thread of - /// its own with a runtime of its own, and drops it. - fn open_on_a_thread( - project: &Path, - open: fn(&Path) -> Result, - ) -> std::thread::JoinHandle> { - let project = project.to_path_buf(); - std::thread::spawn(move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - runtime.block_on(async { open(&project).map(drop) }) - }) - } - #[cfg(unix)] #[test] fn reviewing_leaves_a_database_others_could_open_as_it_is_and_says_so() { @@ -2848,6 +2942,28 @@ mod tests { assert!(!project.path().join(".lpm/inspector.db-copy").exists()); } + #[cfg(unix)] + #[test] + fn a_filesystem_without_modes_lets_a_capture_open_while_a_review_has_the_database_open() { + let project = project_with_a_capture(); + set_mode(&project.path().join(".lpm/inspector.db"), 0o644); + MODELESS_FILESYSTEM.set(true); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let opened = runtime.block_on(async { + let review = InspectorDb::open_for_review(project.path()).unwrap(); + let opened = InspectorDb::open(project.path()).map(|db| db.open_to_others()); + drop(review); + opened + }); + MODELESS_FILESYSTEM.set(false); + + assert!(matches!(opened, Ok(true)), "got {opened:?}"); + } + #[cfg(unix)] #[test] fn root_accepts_the_project_owner_s_files_and_its_own() { @@ -2881,6 +2997,63 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn as_root_a_capture_file_swapped_after_its_check_keeps_the_other_file_s_mode() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let me_as_root = CaptureOwner { + uid: lpm_common::current_uid(), + gid: std::fs::metadata(".").unwrap().gid(), + as_root: true, + }; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("inspector.db"); + let elsewhere = directory.path().join("elsewhere"); + for swap in ["link", "another file"] { + std::fs::write(&path, b"captures").unwrap(); + // Others can't open it, so a writer makes it 0600. + set_mode(&path, 0o700); + std::fs::write(&elsewhere, b"not captures").unwrap(); + set_mode(&elsewhere, 0o644); + let checked = std::fs::symlink_metadata(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + match swap { + "link" => std::os::unix::fs::symlink(&elsewhere, &path).unwrap(), + _ => std::fs::write(&path, b"swapped in").unwrap(), + } + + let checked_now = check_capture_file(&path, &checked, me_as_root, Access::Capture); + + assert!(checked_now.is_err(), "{swap}"); + assert_eq!( + std::fs::metadata(&elsewhere).unwrap().permissions().mode() & 0o7777, + 0o644, + "{swap}" + ); + std::fs::remove_file(&path).unwrap(); + } + } + + #[cfg(unix)] + #[test] + fn as_root_a_capture_file_s_mode_changes_through_its_checked_handle() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("inspector.db"); + std::fs::write(&path, b"captures").unwrap(); + set_mode(&path, 0o640); + let checked = std::fs::symlink_metadata(&path).unwrap(); + + set_mode_through_handle(&path, &checked, 0o600).unwrap(); + + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o7777, + 0o600 + ); + } + /// A group this user is in besides the one its files get, or None. #[cfg(unix)] fn another_group_of_mine(file: &Path) -> Option { @@ -2997,9 +3170,12 @@ mod tests { std::fs::create_dir(&reviewed_lpm).unwrap(); add_acl_entry(&reviewed_lpm, INHERITED); let db = InspectorDb::open_for_review(reviewed.path()).unwrap(); + db.rename_session("unknown", "renamed").await.unwrap(); assert!(!db.open_to_others()); assert!(has_acl(&reviewed_lpm)); - assert!(!has_acl(&reviewed_lpm.join("inspector.db"))); + for name in ["inspector.db", "inspector.db-wal", "inspector.db-shm"] { + assert!(!has_acl(&reviewed_lpm.join(name)), "{name}"); + } let captured = tempfile::tempdir().unwrap(); let captured_lpm = captured.path().join(".lpm"); @@ -3016,6 +3192,81 @@ mod tests { } } + #[tokio::test] + async fn a_capture_waits_out_another_writer_rather_than_failing() { + let project = tempfile::tempdir().unwrap(); + let db = InspectorDb::open(project.path()).unwrap(); + let db_path = project.path().join(".lpm/inspector.db"); + let (began, holding) = std::sync::mpsc::channel(); + let holder = std::thread::spawn(move || { + let connection = Connection::open(&db_path).unwrap(); + connection + .execute_batch("BEGIN IMMEDIATE; INSERT INTO sessions (id) VALUES ('held');") + .unwrap(); + began.send(()).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(300)); + connection.execute_batch("COMMIT").unwrap(); + }); + holding.recv().unwrap(); + + let inserted = db + .insert_durable_request(Arc::new(make_webhook("kept", 200)), None) + .await; + holder.join().unwrap(); + + assert_eq!(inserted, Ok(())); + assert!(db.get_webhook("kept").await.unwrap().is_some()); + } + + #[tokio::test] + async fn a_batch_of_captures_waits_out_another_writer_rather_than_failing() { + let project = tempfile::tempdir().unwrap(); + let db = InspectorDb::open(project.path()).unwrap(); + let db_path = project.path().join(".lpm/inspector.db"); + let (began, holding) = std::sync::mpsc::channel(); + let holder = std::thread::spawn(move || { + let connection = Connection::open(&db_path).unwrap(); + connection + .execute_batch("BEGIN IMMEDIATE; INSERT INTO sessions (id) VALUES ('held');") + .unwrap(); + began.send(()).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(300)); + connection.execute_batch("COMMIT").unwrap(); + }); + holding.recv().unwrap(); + + db.insert_request(make_webhook("batched", 200), None); + let flushed = db.flush_pending_writes().await; + holder.join().unwrap(); + + assert_eq!(flushed, Ok(())); + assert!(db.get_webhook("batched").await.unwrap().is_some()); + } + + #[tokio::test] + async fn opening_accepts_a_database_whose_planner_statistics_were_gathered() { + for open in [InspectorDb::open_for_review, InspectorDb::open] { + let project = project_with_a_database(&format!( + "{CAPTURE_SCHEMA} ALTER TABLE requests ADD COLUMN response_body_incomplete \ + INTEGER NOT NULL DEFAULT 0; ANALYZE;" + )); + + open(project.path()).map(drop).unwrap(); + } + } + + /// SQL renaming the schema object `planted` to `name`, which SQLite + /// reserves for its own and refuses to create, through the writable + /// schema a crafted file could have been written with. + fn named_like_sqlite(name: &str) -> String { + format!( + "PRAGMA writable_schema = ON; \ + UPDATE sqlite_master SET name = '{name}', sql = replace(sql, 'planted', '{name}') \ + WHERE name = 'planted'; \ + PRAGMA writable_schema = OFF;" + ) + } + /// A project whose capture database others could open, built by /// `build` on a connection that may write SQLite's own tables. fn project_with_a_database(build: &str) -> tempfile::TempDir { @@ -3083,6 +3334,25 @@ mod tests { "table extra", format!("{lpm_schema} CREATE TABLE extra (x);"), ), + ( + "trigger sqlite_stat_planted", + format!( + "{lpm_schema} CREATE TRIGGER planted AFTER INSERT ON requests \ + BEGIN SELECT 1; END; {}", + named_like_sqlite("sqlite_stat_planted") + ), + ), + ( + "index sqlite_stat_planted", + format!( + "{lpm_schema} CREATE INDEX planted ON requests(length(request_body)); {}", + named_like_sqlite("sqlite_stat_planted") + ), + ), + ( + "table sessions", + lpm_schema.replace("id TEXT PRIMARY KEY,", "id TEXT\u{a0}PRIMARY KEY,"), + ), ]; for (named, build) in planted { for open in [InspectorDb::open, InspectorDb::open_for_review] { @@ -3147,6 +3417,61 @@ mod tests { } #[cfg(unix)] + #[test] + fn copying_onto_a_full_disk_says_how_much_room_the_copy_needs() { + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("inspector.db"); + let database = Connection::open(&db_path).unwrap(); + database + .execute_batch("CREATE TABLE kept (x); INSERT INTO kept VALUES (zeroblob(100000));") + .unwrap(); + let copy_path = capture_file(&db_path, "-copy"); + assert!(create_owner_only(©_path, this_user()).unwrap()); + + COPY_PAGE_LIMIT.set(Some(1)); + let copied = copy_database(&database, ©_path, &db_path); + COPY_PAGE_LIMIT.set(None); + + let Err(OpenError::Files(error)) = copied else { + panic!("a full disk must stop the copy, got {copied:?}"); + }; + assert_eq!(error.kind(), std::io::ErrorKind::StorageFull); + assert!( + error.to_string().contains(&format!( + "needs about {} free on its disk", + lpm_common::format_bytes(std::fs::metadata(&db_path).unwrap().len()) + )), + "{error}" + ); + } + + #[test] + fn copying_refuses_an_unfinished_copy_when_a_lock_outlasts_the_wait() { + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("inspector.db"); + let database = Connection::open(&db_path).unwrap(); + database + .execute_batch("CREATE TABLE kept (x); INSERT INTO kept VALUES (1);") + .unwrap(); + database + .busy_timeout(std::time::Duration::from_millis(100)) + .unwrap(); + let holder = Connection::open(&db_path).unwrap(); + holder + .execute_batch("BEGIN EXCLUSIVE; INSERT INTO kept VALUES (2);") + .unwrap(); + let copy_path = capture_file(&db_path, "-copy"); + assert!(create_owner_only(©_path, this_user()).unwrap()); + + let copied = copy_database(&database, ©_path, &db_path); + holder.execute_batch("COMMIT").unwrap(); + + assert!(matches!( + copied, + Err(OpenError::Files(ref error)) if error.kind() == std::io::ErrorKind::ResourceBusy + )); + } + #[test] fn copying_waits_out_a_lock_another_connection_holds_on_the_database() { let directory = tempfile::tempdir().unwrap(); @@ -3164,16 +3489,11 @@ mod tests { holder.execute_batch("COMMIT").unwrap(); }); - let copied = copy_database( - &database, - &capture_file(&db_path, "-copy"), - this_user(), - &db_path, - ) - .unwrap(); + let copy_path = capture_file(&db_path, "-copy"); + assert!(create_owner_only(©_path, this_user()).unwrap()); + copy_database(&database, ©_path, &db_path).unwrap(); released.join().unwrap(); - assert!(copied); let rows: i64 = Connection::open(capture_file(&db_path, "-copy")) .unwrap() .query_row("SELECT count(*) FROM kept", [], |row| row.get(0)) @@ -3204,6 +3524,39 @@ mod tests { assert!(!project.path().join(".lpm/inspector.db-copy").exists()); } + #[cfg(unix)] + #[test] + fn opening_takes_a_schema_sqlite_finds_corrupt_for_damage() { + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + let connection = Connection::open(&db_path).unwrap(); + connection + .execute_batch("PRAGMA journal_mode=DELETE;") + .unwrap(); + drop(connection); + // The b-tree header of page 1, which holds the schema, past the + // 100-byte database header. + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(&db_path) + .unwrap(); + use std::io::{Seek as _, Write as _}; + file.seek(std::io::SeekFrom::Start(100)).unwrap(); + file.write_all(&[0xff; 8]).unwrap(); + drop(file); + set_mode(&db_path, 0o600); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let error = runtime + .block_on(async { InspectorDb::open_for_review(project.path()).map(drop) }) + .unwrap_err(); + + assert!( + matches!(&error, OpenError::Damaged { detail, .. } if detail.contains("malformed")), + "{error:?}" + ); + } + #[cfg(unix)] #[test] fn replacing_refuses_a_database_with_damaged_pages() { @@ -3396,14 +3749,23 @@ mod tests { let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let lock = hold_capture_lock(project.path()); + let wait = std::time::Duration::from_secs(1); let started = std::time::Instant::now(); - let error = open_on_a_thread(project.path(), InspectorDb::open) - .join() - .unwrap() - .unwrap_err(); + let path = project.path().to_path_buf(); + let error = std::thread::spawn(move || { + OPEN_LOCK_WAIT.set(Some(wait)); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { InspectorDb::open(&path).map(drop) }) + }) + .join() + .unwrap() + .unwrap_err(); - assert!(started.elapsed() >= OPEN_LOCK_TIMEOUT); + assert!(started.elapsed() >= wait); assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); assert!( error.to_string().contains("another LPM process held"), diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index 4c08b0a1b..9e1ad8690 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1592,6 +1592,8 @@ fn reading_captures_others_could_open_works_while_another_process_records_and_wa ); let rows: serde_json::Value = serde_json::from_slice(&json.stdout).unwrap(); assert_eq!(rows.as_array().map(Vec::len), Some(1)); + let json_log = String::from_utf8_lossy(&json.stderr); + assert!(json_log.contains("is open to other users"), "{json_log}"); assert_eq!( std::fs::metadata(&db_path).unwrap().permissions().mode() & 0o777, 0o644 From 93274506025aa36d926e08f3f57e061b49134639 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Mon, 12 Oct 2026 18:38:58 +0100 Subject: [PATCH 09/17] fix(tunnel): keep quoted schema text, count ownership-ignoring volumes as open, and name the git query that didn't answer - The schema check keeps quoted text as written, so a default whose string differs only in its spaces, such as datetime(' now'), is refused. - On macOS, a volume that ignores ownership, as external drives and disk images do by default, counts as open to other users: no replacement copy is made there, and the review says so too. - When the project's git query doesn't answer and `.lpm` has no repository of its own, the files git in `.lpm` finds tracked are reported as the project's, with the command to untrack them. - The warning names the repository whose query couldn't run or didn't answer in time. - A disk too full to create the replacement copy says how much room it needs. - Tests: the lock wait's upper bound, the busy error's command, and the damaged-schema refusal on both opens. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/tunnel.rs | 127 +++++++++++--- crates/lpm-common/src/macos_acl.rs | 30 +++- crates/lpm-inspect/src/db.rs | 239 +++++++++++++++++++++----- 3 files changed, 325 insertions(+), 71 deletions(-) diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index 891dc64a2..013143415 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -762,7 +762,7 @@ fn capture_warnings( let access = match store { CaptureStore::OwnerOnly => "owner-only on Unix; project ACLs on Windows", CaptureStore::OpenToOthers => { - "open to other users, as this filesystem doesn't keep Unix file modes" + "open to other users, as this filesystem's file modes don't keep them out" } }; let mut warnings = vec![format!( @@ -773,11 +773,19 @@ fn capture_warnings( let Some(state) = state else { return warnings; }; - if state.incomplete { + if let Some(repository) = state.unanswered { warnings.push( - "git didn't answer in time for every capture file, so check that git ignores \ - and doesn't track inspector.db*" - .to_string(), + match repository { + CaptureRepository::Project => { + "git couldn't run, or didn't answer in time, for the project's repository, \ + so check that it doesn't track .lpm/inspector.db*" + } + CaptureRepository::Lpm => { + "git couldn't run, or didn't answer in time, in .lpm, so check that git \ + ignores and doesn't track .lpm/inspector.db*" + } + } + .to_string(), ); } if state.lpm_unasked { @@ -854,8 +862,9 @@ fn capture_warning(message: &str, show_human_warning: bool) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum CaptureStore { OwnerOnly, - /// On a filesystem that shows every file open to others, which no - /// replacement improves. + /// On a filesystem whose file modes don't keep other users out, which + /// no replacement improves: one without Unix modes, or on macOS one + /// that ignores ownership. OpenToOthers, } @@ -889,16 +898,16 @@ struct CaptureGitState { /// Whether git refused to answer for `.lpm`, as for one that looks like a /// bare repository. lpm_unasked: bool, - /// Whether one of the queries couldn't run or didn't answer in time, so + /// The repository whose query couldn't run or didn't answer in time, so /// what the other found stands but git's ignoring the files isn't /// confirmed. - incomplete: bool, + unanswered: Option, } impl CaptureGitState { fn kept_out(&self) -> bool { !self.lpm_unasked - && !self.incomplete + && self.unanswered.is_none() && self.project_tracked.is_empty() && self.lpm_tracked.is_empty() && self.unignored.is_empty() @@ -989,16 +998,23 @@ where } Err(CaptureGitFailure::Refused) => false, Err(CaptureGitFailure::Unavailable) => { - state.incomplete = true; + state.unanswered = Some(CaptureRepository::Project); false } }; let inside_answered = match inside { Some(Ok(files)) => { // Without a repository of its own, git in `.lpm` lists the - // project's tracked files, which the project's query has. + // project's tracked files, which the project's query has when + // it answers; when it doesn't, they're what's known of them. if own_repository { state.lpm_tracked = files.tracked; + } else if !project_answered { + state.project_tracked = files + .tracked + .into_iter() + .map(|name| format!(".lpm/{name}")) + .collect(); } state.unignored = files.unignored; true @@ -1008,7 +1024,7 @@ where false } Some(Err(CaptureGitFailure::Unavailable)) => { - state.incomplete = true; + state.unanswered = Some(CaptureRepository::Lpm); false } None => false, @@ -2578,11 +2594,29 @@ mod tests { lpm_tracked: Vec::new(), unignored: names(&["inspector.db-wal"]), lpm_unasked: false, - incomplete: false, + unanswered: None, }) ); } + #[tokio::test] + async fn keeps_the_project_s_names_for_its_tracked_files_when_both_answer() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + + let (state, _) = state_from_answers( + project.path(), + Ok(files(&[".LPM/inspector.db"], &[])), + Ok(files(&["inspector.db"], &[])), + ) + .await; + + assert_eq!( + state.unwrap().project_tracked, + names(&[".LPM/inspector.db"]) + ); + } + #[tokio::test] async fn counts_what_a_repository_of_lpm_s_own_tracks() { let project = tempfile::tempdir().unwrap(); @@ -2653,18 +2687,52 @@ mod tests { "{warnings:?}" ); assert!( - warnings - .iter() - .any(|warning| warning.starts_with("git didn't answer in time")), + warnings.iter().any(|warning| warning + .starts_with("git couldn't run, or didn't answer in time, in .lpm")), "{warnings:?}" ); let quiet = warnings_for(quiet, false); assert!(!quiet[0].contains("git ignores it"), "{quiet:?}"); + assert!(quiet.iter().any(|warning| warning.starts_with( + "git couldn't run, or didn't answer in time, for the project's repository" + ))); + } + + #[tokio::test] + async fn reports_what_git_in_lpm_finds_tracked_when_the_project_s_query_doesn_t_answer() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + + let (state, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Unavailable), + Ok(files(&["inspector.db"], &[])), + ) + .await; + + let state = state.unwrap(); + assert_eq!(state.project_tracked, names(&[".lpm/inspector.db"])); + assert_eq!(state.unanswered, Some(CaptureRepository::Project)); + let warnings = warnings_for(Some(state), false); assert!( - quiet - .iter() - .any(|warning| warning.starts_with("git didn't answer in time")) + warnings.iter().any( + |warning| warning.starts_with("git tracks '.lpm/inspector.db'") + && warning.contains("`git rm --cached -- .lpm/inspector.db`") + ), + "{warnings:?}" ); + + std::fs::create_dir(project.path().join(".lpm/.git")).unwrap(); + let (own, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Unavailable), + Ok(files(&["inspector.db"], &[])), + ) + .await; + + let own = own.unwrap(); + assert!(own.project_tracked.is_empty(), "{own:?}"); + assert_eq!(own.lpm_tracked, names(&["inspector.db"])); } fn warnings_for(state: Option, ignore_file_skipped: bool) -> Vec { @@ -2681,7 +2749,7 @@ mod tests { lpm_tracked: names(lpm_tracked), unignored: names(unignored), lpm_unasked: false, - incomplete: false, + unanswered: None, } } @@ -2692,12 +2760,20 @@ mod tests { None, Some(CaptureGitState { lpm_unasked: true, - incomplete: false, + unanswered: None, ..state(&[], &[], &[]) }), Some(state(&[".lpm/inspector.db"], &[], &[])), Some(state(&[], &["inspector.db"], &[])), Some(state(&[], &[], &["inspector.db-wal"])), + Some(CaptureGitState { + unanswered: Some(CaptureRepository::Project), + ..state(&[], &[], &[]) + }), + Some(CaptureGitState { + unanswered: Some(CaptureRepository::Lpm), + ..state(&[], &[], &[]) + }), ] { assert!( !warnings_for(unconfirmed.clone(), false)[0].contains("git ignores it"), @@ -2711,7 +2787,7 @@ mod tests { let warnings = warnings_for( Some(CaptureGitState { lpm_unasked: true, - incomplete: false, + unanswered: None, ..state(&[], &[], &[]) }), false, @@ -2776,8 +2852,9 @@ mod tests { let owner_only = capture_warnings(None, false, CaptureStore::OwnerOnly); assert!( - open[0] - .contains("(open to other users, as this filesystem doesn't keep Unix file modes)"), + open[0].contains( + "(open to other users, as this filesystem's file modes don't keep them out)" + ), "{open:?}" ); assert!(!open[0].contains("owner-only"), "{open:?}"); diff --git a/crates/lpm-common/src/macos_acl.rs b/crates/lpm-common/src/macos_acl.rs index ae1b6c9fe..1420ae628 100644 --- a/crates/lpm-common/src/macos_acl.rs +++ b/crates/lpm-common/src/macos_acl.rs @@ -1,6 +1,7 @@ -//! Extended access control lists on macOS. Their entries grant access the -//! file mode doesn't show: an `everyone allow read` entry, inherited from a -//! directory, makes a 0600 file readable by any user. +//! Access on macOS that the file mode doesn't show. Extended access control +//! list entries grant it: an `everyone allow read` entry, inherited from a +//! directory, makes a 0600 file readable by any user. So does a volume that +//! ignores ownership, on which every user owns every file. use std::ffi::{CString, c_char, c_int, c_void}; use std::io; @@ -18,6 +19,21 @@ unsafe extern "C" { fn acl_set_fd_np(file_descriptor: c_int, acl: *mut c_void, acl_type: c_int) -> c_int; } +/// Whether the volume holding the open file or directory ignores ownership, +/// as one mounted `noowners` does, which external drives are by default: +/// every user is then the owner of its files, whatever their mode. +pub fn ignores_ownership(handle: impl AsFd) -> io::Result { + let mut volume = std::mem::MaybeUninit::::uninit(); + // SAFETY: fstatfs fills `volume` from a descriptor that stays open while + // `handle` is borrowed. + if unsafe { libc::fstatfs(handle.as_fd().as_raw_fd(), volume.as_mut_ptr()) } != 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: fstatfs succeeded, so it filled `volume`. + let flags = unsafe { volume.assume_init() }.f_flags; + Ok(flags & libc::MNT_IGNORE_OWNERSHIP as u32 != 0) +} + /// Whether the open file or directory has extended ACL entries. pub fn has_extended_acl(handle: impl AsFd) -> io::Result { let file_descriptor = handle.as_fd().as_raw_fd(); @@ -96,6 +112,14 @@ pub fn clear_extended_acl(handle: impl AsFd) -> io::Result<()> { mod tests { use super::*; + #[test] + fn a_volume_that_keeps_ownership_reads_as_such() { + let directory = tempfile::tempdir().unwrap(); + let handle = std::fs::File::open(directory.path()).unwrap(); + + assert!(!ignores_ownership(&handle).unwrap()); + } + fn add_entry(path: &std::path::Path, entry: &str) { let status = std::process::Command::new("/bin/chmod") .args(["+a", entry]) diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index fe8ae30c0..910cb6dc6 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -82,14 +82,17 @@ impl InspectorDb { ) -> Result { let lpm_dir = project_dir.join(".lpm"); let db_path = lpm_dir.join("inspector.db"); - prepare_lpm_dir(&lpm_dir, owner, access)?; + let ownership_ignored = prepare_lpm_dir(&lpm_dir, owner, access)?; let _opening = lock_capture_files(&db_path, owner)?; remove_replacement_copy(&db_path)?; if let Err(error) = ensure_captures_ignored(&lpm_dir, owner) { tracing::warn!("could not keep tunnel captures out of git: {error}"); } - let mut open_to_others = prepare_capture_files(&db_path, owner, access)?; - if open_to_others && access == Access::Capture { + // On a volume that ignores ownership, every user owns the files, so + // they're open to others whatever their mode, and a copy is too. + let mut open_to_others = + prepare_capture_files(&db_path, owner, access)? || ownership_ignored; + if open_to_others && access == Access::Capture && !ownership_ignored { open_to_others = !replace_with_owner_only_copy(&db_path, owner)?; } open_to_others |= !create_wal_sidecars(&db_path, owner)?; @@ -1017,8 +1020,9 @@ fn hard_linked(path: &Path) -> std::io::Error { /// into a directory it serves, is refused; the metadata of a link, read /// without following it, is never a directory's. Reparse points other than /// links, such as a synced folder's placeholders on Windows, read as the -/// directories they stand for. See [`restrict_lpm_dir`] for Unix. -fn prepare_lpm_dir(lpm_dir: &Path, owner: CaptureOwner, access: Access) -> std::io::Result<()> { +/// directories they stand for. See [`restrict_lpm_dir`] for Unix, and for +/// whether `.lpm`'s volume ignores ownership, which this returns. +fn prepare_lpm_dir(lpm_dir: &Path, owner: CaptureOwner, access: Access) -> std::io::Result { let created = match std::fs::symlink_metadata(lpm_dir) { Ok(metadata) if !metadata.is_dir() => return Err(linked_or_not(lpm_dir, "directory")), Ok(_) => false, @@ -1035,14 +1039,15 @@ fn prepare_lpm_dir(lpm_dir: &Path, owner: CaptureOwner, access: Access) -> std:: /// by no one else, and on macOS clears its extended ACL, whose entries can /// grant what its mode doesn't and pass to the files created in it, so no /// other user can swap the capture files' names between the checks below -/// and SQLite's opens, or read the files created there. +/// and SQLite's opens, or read the files created there. Says whether its +/// volume ignores ownership, as only macOS reports. #[cfg(unix)] fn restrict_lpm_dir( lpm_dir: &Path, owner: CaptureOwner, access: Access, created: bool, -) -> std::io::Result<()> { +) -> std::io::Result { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let directory = @@ -1054,11 +1059,18 @@ fn restrict_lpm_dir( if !owner.owns(&metadata) { return Err(another_users(lpm_dir)); } + #[cfg(target_os = "macos")] + let ownership_ignored = lpm_common::macos_acl::ignores_ownership(&directory) + .map_err(|error| with_path(lpm_dir, error))?; + #[cfg(not(target_os = "macos"))] + let ownership_ignored = false; + #[cfg(test)] + let ownership_ignored = ownership_ignored || tests::OWNERSHIP_IGNORED.get(); if created { owner.give(&directory, lpm_dir)?; } if access == Access::Review { - return Ok(()); + return Ok(ownership_ignored); } let mode = metadata.mode() & 0o7777; if mode & 0o022 != 0 { @@ -1071,7 +1083,7 @@ fn restrict_lpm_dir( lpm_common::macos_acl::clear_extended_acl(&directory) .map_err(|error| with_path(lpm_dir, error))?; } - Ok(()) + Ok(ownership_ignored) } #[cfg(not(unix))] @@ -1080,8 +1092,8 @@ fn restrict_lpm_dir( _owner: CaptureOwner, _access: Access, _created: bool, -) -> std::io::Result<()> { - Ok(()) +) -> std::io::Result { + Ok(false) } /// Holds other LPM processes off the capture files while this one checks, @@ -1354,9 +1366,19 @@ fn create_owner_only(_path: &Path, _owner: CaptureOwner) -> std::io::Result Result { let copy_path = capture_file(db_path, "-copy"); let owner_only = create_owner_only(©_path, owner); + #[cfg(test)] + let owner_only = tests::COPY_CREATE_ERROR + .get() + .map_or(owner_only, |kind| Err(kind.into())); if !matches!(owner_only, Ok(true)) { let removed = remove_replacement_copy(db_path); - owner_only?; + if let Err(error) = owner_only { + return Err(if error.kind() == std::io::ErrorKind::StorageFull { + no_room_to_copy(db_path) + } else { + error.into() + }); + } removed?; return Ok(false); } @@ -1560,15 +1582,17 @@ type SchemaObject = (String, String, String, Option); /// `ALTER TABLE` splices in reads the same however it's spaced. Whitespace /// and word characters are SQLite's own: only ASCII space, tab, newline, /// form feed and carriage return separate tokens, and any other character -/// past ASCII belongs to a word, so text whose tokens differ for SQLite -/// can't read the same here. +/// past ASCII belongs to a word. Quoted text, a string or a quoted name, is +/// kept as written, since its spaces are part of it. So text whose tokens +/// differ for SQLite can't read the same here. fn canonical_sql(sql: &str) -> String { let is_word = |character: char| { character.is_ascii_alphanumeric() || matches!(character, '_' | '$') || !character.is_ascii() }; let mut canonical = String::with_capacity(sql.len()); let mut spaced = false; - for character in sql.chars() { + let mut characters = sql.chars(); + while let Some(character) = characters.next() { if matches!(character, ' ' | '\t' | '\n' | '\x0c' | '\r') { spaced = true; continue; @@ -1578,6 +1602,19 @@ fn canonical_sql(sql: &str) -> String { } spaced = false; canonical.push(character); + let close = match character { + '\'' | '"' | '`' => character, + '[' => ']', + _ => continue, + }; + // A doubled quote, which stands for one, ends a span where the next + // begins, so its text is kept all the same. + for quoted in characters.by_ref() { + canonical.push(quoted); + if quoted == close { + break; + } + } } canonical } @@ -2253,6 +2290,10 @@ mod tests { /// [`OPEN_LOCK_TIMEOUT`], for a test of the wait itself. pub(super) static OPEN_LOCK_WAIT: std::cell::Cell> = const { std::cell::Cell::new(None) }; + /// The failure this thread's replacement copies are created with, as + /// a disk with no blocks left fails them. + pub(super) static COPY_CREATE_ERROR: std::cell::Cell> = + const { std::cell::Cell::new(None) }; } #[cfg(unix)] @@ -2260,6 +2301,9 @@ mod tests { /// Makes this thread's new capture files read as a filesystem /// without Unix modes shows them: open to anyone. pub(super) static MODELESS_FILESYSTEM: std::cell::Cell = const { std::cell::Cell::new(false) }; + /// Makes this thread's `.lpm` read as on a volume that ignores + /// ownership. + pub(super) static OWNERSHIP_IGNORED: std::cell::Cell = const { std::cell::Cell::new(false) }; } /// The capture owner of a project of this user's. @@ -2800,7 +2844,8 @@ mod tests { .unwrap_err(); assert!( - error.to_string().contains("another process has it open"), + error.to_string().contains("another process has it open") + && error.to_string().contains("`lpm tunnel inspect --ui`"), "{error}" ); assert_eq!(error.kind(), std::io::ErrorKind::ResourceBusy); @@ -2942,6 +2987,45 @@ mod tests { assert!(!project.path().join(".lpm/inspector.db-copy").exists()); } + #[cfg(unix)] + #[test] + fn a_volume_that_ignores_ownership_keeps_its_database_and_says_it_s_open_to_others() { + use std::os::unix::fs::MetadataExt; + + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + set_mode(&db_path, 0o644); + let inode = std::fs::metadata(&db_path).unwrap().ino(); + let fresh = tempfile::tempdir().unwrap(); + let owner_only = project_with_a_capture(); + assert_eq!(mode(&owner_only.path().join(".lpm/inspector.db")), 0o600); + OWNERSHIP_IGNORED.set(true); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (reviewed, existing, kept, new) = runtime.block_on(async { + let reviewed = InspectorDb::open_for_review(owner_only.path()) + .unwrap() + .open_to_others(); + let review = InspectorDb::open_for_review(project.path()).unwrap(); + let db = InspectorDb::open(project.path()).unwrap(); + let kept = db.get_webhook("kept").await.unwrap(); + let new = InspectorDb::open(fresh.path()).unwrap().open_to_others(); + drop(review); + (reviewed, db.open_to_others(), kept, new) + }); + OWNERSHIP_IGNORED.set(false); + + assert!(reviewed); + assert!(existing); + assert!(new); + assert!(kept.is_some()); + assert_eq!(std::fs::metadata(&db_path).unwrap().ino(), inode); + assert!(!project.path().join(".lpm/inspector.db-copy").exists()); + } + #[cfg(unix)] #[test] fn a_filesystem_without_modes_lets_a_capture_open_while_a_review_has_the_database_open() { @@ -3353,6 +3437,10 @@ mod tests { "table sessions", lpm_schema.replace("id TEXT PRIMARY KEY,", "id TEXT\u{a0}PRIMARY KEY,"), ), + ( + "table sessions", + lpm_schema.replacen("datetime('now')", "datetime(' now')", 1), + ), ]; for (named, build) in planted { for open in [InspectorDb::open, InspectorDb::open_for_review] { @@ -3416,6 +3504,36 @@ mod tests { assert_ne!(canonical_sql("summary TEXT"), canonical_sql("summ aryTEXT")); } + #[test] + fn canonical_sql_keeps_quoted_text_as_written() { + for (written, respaced) in [ + ("DEFAULT (datetime('now'))", "DEFAULT (datetime(' now'))"), + ("DEFAULT (datetime('now'))", "DEFAULT (datetime('now '))"), + ("CREATE TABLE \"a b\" (x)", "CREATE TABLE \"a b\" (x)"), + ("CREATE TABLE `a b` (x)", "CREATE TABLE `a\tb` (x)"), + ("CREATE TABLE [a b] (x)", "CREATE TABLE [a\nb] (x)"), + ("DEFAULT ('it''s a')", "DEFAULT ('it''s a')"), + ] { + assert_ne!( + canonical_sql(written), + canonical_sql(respaced), + "{respaced}" + ); + } + assert_eq!( + canonical_sql("DEFAULT ( 'it''s a' ) , b"), + "DEFAULT('it''s a'),b" + ); + assert_eq!( + canonical_sql("CHECK (x IN ('a ]', [b c] ))"), + "CHECK(x IN('a ]',[b c]))" + ); + assert_eq!( + canonical_sql("CREATE TABLE [a]] b] (x)"), + "CREATE TABLE[a]]b](x)" + ); + } + #[cfg(unix)] #[test] fn copying_onto_a_full_disk_says_how_much_room_the_copy_needs() { @@ -3445,6 +3563,38 @@ mod tests { ); } + #[cfg(unix)] + #[tokio::test] + async fn a_disk_too_full_to_create_the_copy_says_how_much_room_it_needs() { + for (kind, room) in [ + (std::io::ErrorKind::StorageFull, true), + (std::io::ErrorKind::PermissionDenied, false), + ] { + let project = project_with_a_database(&format!( + "{CAPTURE_SCHEMA} ALTER TABLE requests ADD COLUMN response_body_incomplete \ + INTEGER NOT NULL DEFAULT 0;" + )); + let db_path = project.path().join(".lpm/inspector.db"); + + COPY_CREATE_ERROR.set(Some(kind)); + let opened = InspectorDb::open(project.path()).map(drop); + COPY_CREATE_ERROR.set(None); + + let error = opened.unwrap_err(); + assert_eq!(error.kind(), kind); + assert_eq!( + error.to_string().contains(&format!( + "needs about {} free on its disk", + lpm_common::format_bytes(std::fs::metadata(&db_path).unwrap().len()) + )), + room, + "{error}" + ); + assert_eq!(mode(&db_path), 0o644); + assert!(!capture_file(&db_path, "-copy").exists()); + } + } + #[test] fn copying_refuses_an_unfinished_copy_when_a_lock_outlasts_the_wait() { let directory = tempfile::tempdir().unwrap(); @@ -3527,34 +3677,36 @@ mod tests { #[cfg(unix)] #[test] fn opening_takes_a_schema_sqlite_finds_corrupt_for_damage() { - let project = project_with_a_capture(); - let db_path = project.path().join(".lpm/inspector.db"); - let connection = Connection::open(&db_path).unwrap(); - connection - .execute_batch("PRAGMA journal_mode=DELETE;") - .unwrap(); - drop(connection); - // The b-tree header of page 1, which holds the schema, past the - // 100-byte database header. - let mut file = std::fs::OpenOptions::new() - .write(true) - .open(&db_path) - .unwrap(); - use std::io::{Seek as _, Write as _}; - file.seek(std::io::SeekFrom::Start(100)).unwrap(); - file.write_all(&[0xff; 8]).unwrap(); - drop(file); - set_mode(&db_path, 0o600); - let runtime = tokio::runtime::Runtime::new().unwrap(); - let error = runtime - .block_on(async { InspectorDb::open_for_review(project.path()).map(drop) }) - .unwrap_err(); + for open in [InspectorDb::open, InspectorDb::open_for_review] { + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + let connection = Connection::open(&db_path).unwrap(); + connection + .execute_batch("PRAGMA journal_mode=DELETE;") + .unwrap(); + drop(connection); + // The b-tree header of page 1, which holds the schema, past the + // 100-byte database header. + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(&db_path) + .unwrap(); + use std::io::{Seek as _, Write as _}; + file.seek(std::io::SeekFrom::Start(100)).unwrap(); + file.write_all(&[0xff; 8]).unwrap(); + drop(file); + set_mode(&db_path, 0o600); + + let error = runtime + .block_on(async { open(project.path()).map(drop) }) + .unwrap_err(); - assert!( - matches!(&error, OpenError::Damaged { detail, .. } if detail.contains("malformed")), - "{error:?}" - ); + assert!( + matches!(&error, OpenError::Damaged { detail, .. } if detail.contains("malformed")), + "{error:?}" + ); + } } #[cfg(unix)] @@ -3765,7 +3917,8 @@ mod tests { .unwrap() .unwrap_err(); - assert!(started.elapsed() >= wait); + let waited = started.elapsed(); + assert!(waited >= wait && waited < wait * 4, "{waited:?}"); assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); assert!( error.to_string().contains("another LPM process held"), From 9fdfbcfa504cb7edb4213bf5f9a43a15ff8229cb Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Mon, 12 Oct 2026 21:27:15 +0100 Subject: [PATCH 10/17] fix(tunnel): read schema SQL as SQLite tokenizes it, check the replacement's disk room, and read repositories from their .git entries - canonical_sql drops whitespace only next to parentheses and commas, collapses other runs, and keeps quoted text and comments as written, so text SQLite reads apart stays apart. - Replacing a database others could open checks its disk for the copy, the WAL folded in and the shared-memory file before anything is opened, and a disk that fills while the database is checked names the room needed. - A review on a volume that ignores ownership says no tunnel run can make the database owner-only there (InspectorDb::ownership_ignored). - Git: a repository is a .git directory or a gitdir: file, in the project or above it, or .lpm's own; in one, a failed query, a refusal included, is named; files only git in .lpm reports for the project get the case-insensitive pathspec command. - filesystem_available_bytes moves to lpm-common. - macOS tests on real disk images: a volume mounted without owners, and a disk that fills while the database is checked. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/cache_prune.rs | 33 +- crates/lpm-cli/src/commands/tunnel.rs | 312 +++++++++++++--- crates/lpm-common/src/lib.rs | 72 ++++ crates/lpm-common/src/macos_acl.rs | 60 +++ crates/lpm-inspect/src/db.rs | 405 ++++++++++++++++++--- tests/workflows/tests/dev_tunnel.rs | 6 +- 6 files changed, 768 insertions(+), 120 deletions(-) diff --git a/crates/lpm-cli/src/commands/cache_prune.rs b/crates/lpm-cli/src/commands/cache_prune.rs index 9e9f1d7e1..c026b854c 100644 --- a/crates/lpm-cli/src/commands/cache_prune.rs +++ b/crates/lpm-cli/src/commands/cache_prune.rs @@ -303,7 +303,7 @@ fn run_all_virtual_stores_locked( ) -> Result { let free_before = flags .apply - .then(|| filesystem_available_bytes(v3_store.paths().root())) + .then(|| lpm_common::filesystem_available_bytes(v3_store.paths().root())) .flatten(); let stores = [v2_store, v3_store]; let directories = [ @@ -353,7 +353,7 @@ fn run_all_virtual_stores_locked( } let mut summary = merge_prune_summaries(v2, v3); summary.observed_physical_bytes_freed = free_before.and_then(|before| { - filesystem_available_bytes(v3_store.paths().root())?.checked_sub(before) + lpm_common::filesystem_available_bytes(v3_store.paths().root())?.checked_sub(before) }); Ok(summary) } @@ -1498,35 +1498,6 @@ fn dir_size(dir: &Path) -> std::io::Result { Ok(total) } -#[cfg(unix)] -fn filesystem_available_bytes(path: &Path) -> Option { - use std::ffi::CString; - use std::os::unix::ffi::OsStrExt; - - let mut probe = path.to_path_buf(); - while !probe.try_exists().ok()? { - if !probe.pop() { - return None; - } - } - let path = CString::new(probe.as_os_str().as_bytes()).ok()?; - let mut stats = std::mem::MaybeUninit::::uninit(); - // SAFETY: `path` is NUL-terminated and `stats` points to writable, - // properly aligned storage that is read only after a successful call. - if unsafe { libc::statvfs(path.as_ptr(), stats.as_mut_ptr()) } != 0 { - return None; - } - // SAFETY: `statvfs` returned success and initialized the output structure. - let stats = unsafe { stats.assume_init() }; - let bytes = u128::from(stats.f_bavail).checked_mul(u128::from(stats.f_frsize))?; - u64::try_from(bytes).ok() -} - -#[cfg(not(unix))] -fn filesystem_available_bytes(_path: &Path) -> Option { - None -} - /// LRU + crash-recovery orphan detection for the compat-island cache at /// `~/.lpm/store/v2/compat/`. /// diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index 013143415..929fcd787 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -773,16 +773,16 @@ fn capture_warnings( let Some(state) = state else { return warnings; }; - if let Some(repository) = state.unanswered { + for repository in &state.unanswered { warnings.push( match repository { CaptureRepository::Project => { - "git couldn't run, or didn't answer in time, for the project's repository, \ - so check that it doesn't track .lpm/inspector.db*" + "git didn't answer for the project's repository (it couldn't run, refused, \ + or took over 2 s), so check that it doesn't track .lpm/inspector.db*" } CaptureRepository::Lpm => { - "git couldn't run, or didn't answer in time, in .lpm, so check that git \ - ignores and doesn't track .lpm/inspector.db*" + "git didn't answer in .lpm (it couldn't run or took over 2 s), so check \ + that git ignores and doesn't track .lpm/inspector.db*" } } .to_string(), @@ -803,12 +803,25 @@ fn capture_warnings( .chain(state.lpm_tracked.iter().map(|name| format!(".lpm/{name}"))) .collect(); let commands: Vec = [ - (&state.project_tracked, CaptureRepository::Project), - (&state.lpm_tracked, CaptureRepository::Lpm), + ( + &state.project_tracked, + CaptureRepository::Project, + state.project_spelling_unknown, + ), + (&state.lpm_tracked, CaptureRepository::Lpm, false), ] .into_iter() - .filter(|(names, _)| !names.is_empty()) - .map(|(names, repository)| format!("`{}`", untrack_command(names, repository))) + .filter(|(names, _, _)| !names.is_empty()) + .map(|(names, repository, spelling_unknown)| { + format!( + "`{}`", + if spelling_unknown { + untrack_every_capture(repository) + } else { + untrack_command(names, repository) + } + ) + }) .collect(); warnings.push(format!( "git tracks {}, so captures there can be committed and stay in its history; \ @@ -891,6 +904,10 @@ struct CaptureGitFiles { struct CaptureGitState { /// Those the project's repository tracks, by name in the project. project_tracked: Vec, + /// Whether `project_tracked` is spelled as git in `.lpm` lists it, when + /// the project's query didn't answer: the index may spell the directory + /// otherwise, as an upstream `.LPM/` on a case-insensitive disk. + project_spelling_unknown: bool, /// Those a repository of `.lpm`'s own tracks, by name in `.lpm`. lpm_tracked: Vec, /// Those git neither tracks nor ignores, by name in `.lpm`. @@ -898,16 +915,16 @@ struct CaptureGitState { /// Whether git refused to answer for `.lpm`, as for one that looks like a /// bare repository. lpm_unasked: bool, - /// The repository whose query couldn't run or didn't answer in time, so - /// what the other found stands but git's ignoring the files isn't - /// confirmed. - unanswered: Option, + /// The repositories whose query couldn't run, was refused in a + /// repository, or didn't answer in time, so what the other found stands + /// but git's ignoring the files isn't confirmed. + unanswered: Vec, } impl CaptureGitState { fn kept_out(&self) -> bool { !self.lpm_unasked - && self.unanswered.is_none() + && self.unanswered.is_empty() && self.project_tracked.is_empty() && self.lpm_tracked.is_empty() && self.unignored.is_empty() @@ -968,7 +985,10 @@ where { let lpm_dir = project_dir.join(".lpm"); let lpm_exists = lpm_dir.is_dir(); - let own_repository = std::fs::symlink_metadata(lpm_dir.join(".git")).is_ok(); + let own_repository = marks_a_repository(&lpm_dir.join(".git")); + let in_repository = project_dir + .ancestors() + .any(|directory| marks_a_repository(&directory.join(".git"))); let project = ask(CaptureGitQuery { directory: project_dir.to_path_buf(), pathspec: PROJECT_CAPTURES_PATHSPEC, @@ -996,11 +1016,14 @@ where state.project_tracked = files.tracked; true } - Err(CaptureGitFailure::Refused) => false, - Err(CaptureGitFailure::Unavailable) => { - state.unanswered = Some(CaptureRepository::Project); + // In a repository, a query that fails is no answer, git refusing + // included, as for one another user owns, run as root; outside one, + // there's nothing to ask. + Err(_) if in_repository => { + state.unanswered.push(CaptureRepository::Project); false } + Err(_) => false, }; let inside_answered = match inside { Some(Ok(files)) => { @@ -1015,6 +1038,7 @@ where .into_iter() .map(|name| format!(".lpm/{name}")) .collect(); + state.project_spelling_unknown = !state.project_tracked.is_empty(); } state.unignored = files.unignored; true @@ -1024,12 +1048,28 @@ where false } Some(Err(CaptureGitFailure::Unavailable)) => { - state.unanswered = Some(CaptureRepository::Lpm); + state.unanswered.push(CaptureRepository::Lpm); false } None => false, }; - (project_answered || inside_answered).then_some(state) + (project_answered || inside_answered || in_repository || own_repository).then_some(state) +} + +/// Whether a `.git` entry makes its directory a repository's: a directory, +/// or a file pointing at one (`gitdir: …`). An empty or other file doesn't, +/// as git doesn't take it for one. +fn marks_a_repository(git: &Path) -> bool { + match std::fs::symlink_metadata(git) { + Ok(metadata) if metadata.is_dir() => true, + Ok(metadata) if metadata.is_file() => { + let mut start = [0_u8; 7]; + lpm_common::open_nofollow(git) + .and_then(|mut file| std::io::Read::read_exact(&mut file, &mut start)) + .is_ok_and(|()| &start == b"gitdir:") + } + _ => false, + } } fn capture_git_command(query: &CaptureGitQuery) -> tokio::process::Command { @@ -1122,15 +1162,27 @@ fn listed(names: &[String]) -> String { /// otherwise the repository's capture pathspec, which every shell reads /// alike in double quotes. fn untrack_command(tracked: &[String], repository: CaptureRepository) -> String { - let named = tracked.len() <= LISTED_FILES && tracked.iter().all(|name| is_plain(name)); - let (command, pathspec) = match repository { - CaptureRepository::Project => ("git rm --cached --", PROJECT_CAPTURES_PATHSPEC), - CaptureRepository::Lpm => ("git -C .lpm rm --cached --", CAPTURES_PATHSPEC), - }; - if named { - format!("{command} {}", tracked.join(" ")) + if tracked.len() <= LISTED_FILES && tracked.iter().all(|name| is_plain(name)) { + format!("{} {}", untrack_prefix(repository), tracked.join(" ")) } else { - format!("{command} \"{pathspec}\"") + untrack_every_capture(repository) + } +} + +/// The command that stops `repository` tracking every capture file, by the +/// capture pathspec, matched without regard to case. +fn untrack_every_capture(repository: CaptureRepository) -> String { + let pathspec = match repository { + CaptureRepository::Project => PROJECT_CAPTURES_PATHSPEC, + CaptureRepository::Lpm => CAPTURES_PATHSPEC, + }; + format!("{} \"{pathspec}\"", untrack_prefix(repository)) +} + +fn untrack_prefix(repository: CaptureRepository) -> &'static str { + match repository { + CaptureRepository::Project => "git rm --cached --", + CaptureRepository::Lpm => "git -C .lpm rm --cached --", } } @@ -1657,15 +1709,31 @@ fn open_inspector_db_for_review( ) -> Result { let db = lpm_inspect::db::InspectorDb::open_for_review(project_dir).map_err(inspector_open_error)?; - if db.open_to_others() { - capture_warning(CAPTURES_OPEN_TO_OTHERS, show_human_warning); + if let Some(warning) = review_warning(db.open_to_others(), db.ownership_ignored()) { + capture_warning(warning, show_human_warning); } Ok(db) } +/// What a review tells of a capture database other users could open, and +/// whether a capture writer's run can change it. +fn review_warning(open_to_others: bool, ownership_ignored: bool) -> Option<&'static str> { + if !open_to_others { + None + } else if ownership_ignored { + Some(CAPTURES_ON_A_SHARED_VOLUME) + } else { + Some(CAPTURES_OPEN_TO_OTHERS) + } +} + const CAPTURES_OPEN_TO_OTHERS: &str = ".lpm/inspector.db is open to other users, so they may \ read its captures; an `lpm tunnel` or `lpm dev --tunnel` run in this project makes it \ - owner-only where the filesystem keeps Unix file modes"; + owner-only where file modes keep other users out"; + +const CAPTURES_ON_A_SHARED_VOLUME: &str = ".lpm/inspector.db is open to other users, so they \ + may read its captures: its volume ignores ownership, so every user owns its files whatever \ + their mode; keep the project on a volume that keeps ownership"; /// The capture database's open failures are local, so they're reported as /// such, not as a tunnel's; each says what to do. @@ -2591,10 +2659,11 @@ mod tests { state, Some(CaptureGitState { project_tracked: names(&[".lpm/inspector.db"]), + project_spelling_unknown: false, lpm_tracked: Vec::new(), unignored: names(&["inspector.db-wal"]), lpm_unasked: false, - unanswered: None, + unanswered: Vec::new(), }) ); } @@ -2633,7 +2702,7 @@ mod tests { } #[tokio::test] - async fn says_nothing_of_git_when_neither_query_answers() { + async fn says_nothing_of_git_outside_a_repository_when_neither_query_answers() { let project = tempfile::tempdir().unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); @@ -2655,11 +2724,138 @@ mod tests { state_from_answers(project.path(), project_answer, inside_answer).await; assert_eq!(state, None); } + + // Git in `.lpm` answered, so only the project's query went unanswered, + // and outside a repository there was nothing for it to find. + let (answered, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Unavailable), + Ok(CaptureGitFiles::default()), + ) + .await; + let answered = answered.unwrap(); + assert!(answered.unanswered.is_empty(), "{answered:?}"); + assert!(answered.kept_out()); + } + + #[tokio::test] + async fn names_lpm_s_own_repository_when_git_there_doesn_t_answer() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(project.path().join(".lpm/.git")).unwrap(); + + let (state, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::Unavailable), + ) + .await; + + let state = state.unwrap(); + assert_eq!(state.unanswered, vec![CaptureRepository::Lpm]); + let warnings = warnings_for(Some(state), false); + assert!( + warnings + .iter() + .any(|warning| warning.starts_with("git didn't answer in .lpm")), + "{warnings:?}" + ); + } + + #[tokio::test] + async fn names_each_query_a_repository_leaves_unanswered_even_when_neither_answers() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(project.path().join(".git")).unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + + // As for a repository another user owns, run as root. + let (refused, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::Refused), + ) + .await; + let refused = refused.unwrap(); + assert_eq!(refused.unanswered, vec![CaptureRepository::Project]); + assert!(refused.lpm_unasked); + let warnings = warnings_for(Some(refused), false); + assert!( + warnings.iter().any(|warning| warning + .starts_with("git didn't answer for the project's repository (it couldn't run, refused, or took over 2 s)")), + "{warnings:?}" + ); + + let (slow, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Unavailable), + Err(CaptureGitFailure::Unavailable), + ) + .await; + let slow = slow.unwrap(); + assert_eq!( + slow.unanswered, + vec![CaptureRepository::Project, CaptureRepository::Lpm] + ); + assert!(!slow.kept_out()); + let warnings = warnings_for(Some(slow), false); + assert!( + warnings + .iter() + .any(|warning| warning.starts_with("git didn't answer in .lpm")), + "{warnings:?}" + ); + } + + #[tokio::test] + async fn takes_a_project_inside_a_repository_s_subdirectory_as_in_that_repository() { + let repository = tempfile::tempdir().unwrap(); + std::fs::create_dir(repository.path().join(".git")).unwrap(); + let project = repository.path().join("packages/app"); + std::fs::create_dir_all(project.join(".lpm")).unwrap(); + + let (state, _) = state_from_answers( + &project, + Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::Refused), + ) + .await; + + assert_eq!(state.unwrap().unanswered, vec![CaptureRepository::Project]); + } + + #[tokio::test] + async fn takes_lpm_s_git_entry_for_a_repository_only_as_git_does() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let git = project.path().join(".lpm/.git"); + + for (contents, own) in [ + ("", false), + ("not a pointer\n", false), + ("gitdir: ../../elsewhere/.git\n", true), + ] { + std::fs::write(&git, contents).unwrap(); + let (state, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::Unavailable), + Ok(files(&["inspector.db"], &[])), + ) + .await; + let state = state.unwrap(); + assert_eq!( + ( + state.lpm_tracked.is_empty(), + state.project_tracked.is_empty() + ), + (!own, own), + "{contents:?}: {state:?}" + ); + } } #[tokio::test] async fn keeps_what_one_query_confirms_when_the_other_doesn_t_answer_in_time() { let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".git")).unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let (tracked, _) = state_from_answers( @@ -2687,20 +2883,24 @@ mod tests { "{warnings:?}" ); assert!( - warnings.iter().any(|warning| warning - .starts_with("git couldn't run, or didn't answer in time, in .lpm")), + warnings + .iter() + .any(|warning| warning.starts_with("git didn't answer in .lpm")), "{warnings:?}" ); let quiet = warnings_for(quiet, false); assert!(!quiet[0].contains("git ignores it"), "{quiet:?}"); - assert!(quiet.iter().any(|warning| warning.starts_with( - "git couldn't run, or didn't answer in time, for the project's repository" - ))); + assert!( + quiet.iter().any( + |warning| warning.starts_with("git didn't answer for the project's repository") + ) + ); } #[tokio::test] async fn reports_what_git_in_lpm_finds_tracked_when_the_project_s_query_doesn_t_answer() { let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".git")).unwrap(); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let (state, _) = state_from_answers( @@ -2712,12 +2912,16 @@ mod tests { let state = state.unwrap(); assert_eq!(state.project_tracked, names(&[".lpm/inspector.db"])); - assert_eq!(state.unanswered, Some(CaptureRepository::Project)); + assert_eq!(state.unanswered, vec![CaptureRepository::Project]); + assert!(state.project_spelling_unknown); let warnings = warnings_for(Some(state), false); + // Git in `.lpm` may spell the directory otherwise than the index, so + // the command matches every capture file without regard to case. + let command = format!("`git rm --cached -- \"{PROJECT_CAPTURES_PATHSPEC}\"`"); assert!( warnings.iter().any( |warning| warning.starts_with("git tracks '.lpm/inspector.db'") - && warning.contains("`git rm --cached -- .lpm/inspector.db`") + && warning.contains(&command) ), "{warnings:?}" ); @@ -2735,6 +2939,23 @@ mod tests { assert_eq!(own.lpm_tracked, names(&["inspector.db"])); } + #[test] + fn a_review_says_whether_a_tunnel_run_can_make_the_database_owner_only() { + assert_eq!(review_warning(false, false), None); + assert_eq!(review_warning(false, true), None); + assert!( + review_warning(true, false) + .unwrap() + .ends_with("makes it owner-only where file modes keep other users out") + ); + let shared = review_warning(true, true).unwrap(); + assert!( + shared.contains("its volume ignores ownership") + && !shared.contains("makes it owner-only"), + "{shared}" + ); + } + fn warnings_for(state: Option, ignore_file_skipped: bool) -> Vec { capture_warnings(state.as_ref(), ignore_file_skipped, CaptureStore::OwnerOnly) } @@ -2746,10 +2967,11 @@ mod tests { ) -> CaptureGitState { CaptureGitState { project_tracked: names(project_tracked), + project_spelling_unknown: false, lpm_tracked: names(lpm_tracked), unignored: names(unignored), lpm_unasked: false, - unanswered: None, + unanswered: Vec::new(), } } @@ -2760,18 +2982,18 @@ mod tests { None, Some(CaptureGitState { lpm_unasked: true, - unanswered: None, + unanswered: Vec::new(), ..state(&[], &[], &[]) }), Some(state(&[".lpm/inspector.db"], &[], &[])), Some(state(&[], &["inspector.db"], &[])), Some(state(&[], &[], &["inspector.db-wal"])), Some(CaptureGitState { - unanswered: Some(CaptureRepository::Project), + unanswered: vec![CaptureRepository::Project], ..state(&[], &[], &[]) }), Some(CaptureGitState { - unanswered: Some(CaptureRepository::Lpm), + unanswered: vec![CaptureRepository::Lpm], ..state(&[], &[], &[]) }), ] { @@ -2787,7 +3009,7 @@ mod tests { let warnings = warnings_for( Some(CaptureGitState { lpm_unasked: true, - unanswered: None, + unanswered: Vec::new(), ..state(&[], &[], &[]) }), false, diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index 3a3c329cb..db21d332a 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -315,6 +315,37 @@ pub fn is_loopback_host(host: &str) -> bool { false } +/// The bytes free to this user on the filesystem holding `path`, or its +/// nearest existing ancestor; None where it can't be read. +#[cfg(unix)] +pub fn filesystem_available_bytes(path: &std::path::Path) -> Option { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + + let mut probe = path.to_path_buf(); + while !probe.try_exists().ok()? { + if !probe.pop() { + return None; + } + } + let path = CString::new(probe.as_os_str().as_bytes()).ok()?; + let mut stats = std::mem::MaybeUninit::::uninit(); + // SAFETY: `path` is NUL-terminated and `stats` points to writable, + // properly aligned storage that is read only after a successful call. + if unsafe { libc::statvfs(path.as_ptr(), stats.as_mut_ptr()) } != 0 { + return None; + } + // SAFETY: `statvfs` returned success and initialized the output structure. + let stats = unsafe { stats.assume_init() }; + let bytes = u128::from(stats.f_bavail).checked_mul(u128::from(stats.f_frsize))?; + u64::try_from(bytes).ok() +} + +#[cfg(not(unix))] +pub fn filesystem_available_bytes(_path: &std::path::Path) -> Option { + None +} + /// Format bytes into a human-readable string (e.g., "1.2 KB", "3.4 MB"). pub fn format_bytes(bytes: u64) -> String { if bytes < 1024 { @@ -332,6 +363,47 @@ pub fn format_bytes(bytes: u64) -> String { mod tests { use super::*; + /// `df`'s figure for the filesystem holding `path`: its size and the + /// space free to this user, in bytes. + #[cfg(unix)] + fn df(path: &std::path::Path) -> (u64, u64) { + let output = std::process::Command::new("/bin/df") + .arg("-Pk") + .arg(path) + .output() + .unwrap(); + assert!(output.status.success()); + let output = String::from_utf8(output.stdout).unwrap(); + let fields: Vec = output + .lines() + .nth(1) + .unwrap() + .split_whitespace() + .skip(1) + .take(3) + .map(|field| field.parse().unwrap()) + .collect(); + (fields[0] * 1024, fields[2] * 1024) + } + + #[cfg(unix)] + #[test] + fn reads_the_space_free_to_this_user_as_df_does() { + let directory = tempfile::tempdir().unwrap(); + let (size, free) = df(directory.path()); + + let available = filesystem_available_bytes(directory.path()).unwrap(); + + // Other writers move the figure a little between the two reads; the + // space reserved for root, which this user can't use, is far more. + assert!( + available.abs_diff(free) <= size / 200, + "{available} against df's {free} of {size}" + ); + let unmade = filesystem_available_bytes(&directory.path().join("not/yet/made")).unwrap(); + assert!(unmade.abs_diff(available) <= size / 200); + } + // ── read_capped_state_file ──────────────────────────────────────── /// Files under the cap round-trip transparently. diff --git a/crates/lpm-common/src/macos_acl.rs b/crates/lpm-common/src/macos_acl.rs index 1420ae628..7b6ff4d52 100644 --- a/crates/lpm-common/src/macos_acl.rs +++ b/crates/lpm-common/src/macos_acl.rs @@ -120,6 +120,66 @@ mod tests { assert!(!ignores_ownership(&handle).unwrap()); } + struct AttachedImage(std::path::PathBuf); + + impl Drop for AttachedImage { + fn drop(&mut self) { + let _ = std::process::Command::new("/usr/bin/hdiutil") + .args(["detach", "-force", "-quiet"]) + .arg(&self.0) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); + } + } + + fn hdiutil(arguments: &[&std::ffi::OsStr]) { + let output = std::process::Command::new("/usr/bin/hdiutil") + .args(arguments) + .output() + .unwrap(); + assert!( + output.status.success(), + "hdiutil {arguments:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn a_volume_mounted_without_owners_reads_as_ignoring_ownership() { + let directory = tempfile::tempdir().unwrap(); + let image = directory.path().join("volume.dmg"); + let mount_point = directory.path().join("mounted"); + std::fs::create_dir(&mount_point).unwrap(); + hdiutil(&[ + "create".as_ref(), + "-quiet".as_ref(), + "-size".as_ref(), + "2m".as_ref(), + "-fs".as_ref(), + "HFS+".as_ref(), + "-volname".as_ref(), + "lpm-noowners".as_ref(), + image.as_os_str(), + ]); + hdiutil(&[ + "attach".as_ref(), + "-quiet".as_ref(), + "-nobrowse".as_ref(), + "-noautoopen".as_ref(), + "-owners".as_ref(), + "off".as_ref(), + "-mountpoint".as_ref(), + mount_point.as_os_str(), + image.as_os_str(), + ]); + let attached = AttachedImage(mount_point); + + let handle = std::fs::File::open(&attached.0).unwrap(); + + assert!(ignores_ownership(&handle).unwrap()); + } + fn add_entry(path: &std::path::Path, entry: &str) { let status = std::process::Command::new("/bin/chmod") .args(["+a", entry]) diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index 910cb6dc6..904f80512 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -44,6 +44,7 @@ pub struct InspectorDb { /// Database file path (for display/diagnostics). pub db_path: PathBuf, open_to_others: bool, + ownership_ignored: bool, } /// A write operation to be batched. @@ -114,6 +115,7 @@ impl InspectorDb { control_conn, db_path, open_to_others, + ownership_ignored, )) } @@ -123,6 +125,7 @@ impl InspectorDb { control_conn: Connection, db_path: PathBuf, open_to_others: bool, + ownership_ignored: bool, ) -> Self { let (write_tx, write_rx) = mpsc::channel(WRITE_QUEUE_CAPACITY); let queued_capture_bytes = Arc::new(AtomicUsize::new(0)); @@ -140,6 +143,7 @@ impl InspectorDb { queued_capture_bytes, db_path, open_to_others, + ownership_ignored, } } @@ -174,17 +178,26 @@ impl InspectorDb { control_conn, tmp, false, + false, )) } /// Whether other users could open the capture files when this handle - /// opened them: a database [`Self::open_for_review`] found so, or one on - /// a filesystem that shows its files open to others whatever their mode, - /// which [`Self::open`] can't make owner-only. + /// opened them: a database [`Self::open_for_review`] found so, or one + /// whose file modes don't keep others out, which [`Self::open`] can't + /// make owner-only: on a filesystem without Unix modes, or a volume that + /// ignores ownership ([`Self::ownership_ignored`]). pub fn open_to_others(&self) -> bool { self.open_to_others } + /// Whether the capture files' volume ignores ownership, as one mounted + /// `noowners` on macOS does: every user owns its files, so they're open + /// to others whatever their mode, and no open can change that. + pub fn ownership_ignored(&self) -> bool { + self.ownership_ignored + } + /// Insert a captured request (non-blocking — queued for batch write). pub fn insert_request(&self, webhook: CapturedWebhook, session_id: Option) { self.insert_shared_request(Arc::new(webhook), session_id); @@ -1401,7 +1414,8 @@ fn replace_with_owner_only_copy(db_path: &Path, owner: CaptureOwner) -> Result Result<(), OpenError> { - let database = open_nofollow_connection(db_path).map_err(damaged(db_path))?; + ensure_room_to_copy(db_path)?; + let database = open_nofollow_connection(db_path).map_err(checking(db_path))?; refuse_unknown_schema(&database, db_path)?; // In WAL, which every LPM connection uses, leaving it needs the only // connection, so it proves no other LPM process has the database open, @@ -1412,12 +1426,54 @@ fn copy_checked_database(db_path: &Path, copy_path: &Path) -> Result<(), OpenErr Ok(mode) if mode.eq_ignore_ascii_case("delete") => {} Ok(_) => return Err(in_use(db_path)), Err(error) if is_busy(&error) => return Err(in_use(db_path)), - Err(error) => return Err(damaged(db_path)(error)), + Err(error) => return Err(checking(db_path)(error)), } refuse_damage(&database, db_path)?; copy_database(&database, copy_path, db_path) } +/// Reads a full disk while the database is checked as the copy's want of +/// room, and any other failure as [`damaged`] does. +fn checking(db_path: &Path) -> impl Fn(rusqlite::Error) -> OpenError + '_ { + move |error| { + if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DiskFull) { + no_room_to_copy(db_path) + } else { + damaged(db_path)(error) + } + } +} + +/// The room SQLite's shared-memory file takes as the check opens the +/// database. +const SHARED_MEMORY_ROOM: u64 = 64 * 1024; + +/// The free space replacing a database takes: the copy, which holds the +/// database with its WAL folded in; the database's own growth, as leaving +/// WAL folds the WAL into it; and SQLite's shared-memory file. +fn room_to_copy(db_path: &Path) -> Option { + let database = std::fs::metadata(db_path).ok()?.len(); + let wal = std::fs::metadata(capture_file(db_path, "-wal")).map_or(0, |wal| wal.len()); + Some( + database + .saturating_add(wal.saturating_mul(2)) + .saturating_add(SHARED_MEMORY_ROOM), + ) +} + +/// Refuses a replacement its disk can't hold before the database is +/// opened, as SQLite's own failure on a full disk can come as a bare I/O +/// error and leave its files behind. +fn ensure_room_to_copy(db_path: &Path) -> Result<(), OpenError> { + let available = lpm_common::filesystem_available_bytes(db_path); + #[cfg(test)] + let available = tests::AVAILABLE_BYTES.get().or(available); + match (available, room_to_copy(db_path)) { + (Some(available), Some(needed)) if available < needed => Err(no_room_to_copy(db_path)), + _ => Ok(()), + } +} + fn in_use(db_path: &Path) -> OpenError { OpenError::Files(std::io::Error::new( std::io::ErrorKind::ResourceBusy, @@ -1455,9 +1511,9 @@ fn copy_database(database: &Connection, copy_path: &Path, db_path: &Path) -> Res } fn no_room_to_copy(db_path: &Path) -> OpenError { - let needed = std::fs::metadata(db_path).map_or_else( - |_| "as much as it takes".to_owned(), - |metadata| format!("about {}", lpm_common::format_bytes(metadata.len())), + let needed = room_to_copy(db_path).map_or_else( + || "as much as it takes".to_owned(), + |bytes| format!("about {}", lpm_common::format_bytes(bytes)), ); OpenError::Files(std::io::Error::new( std::io::ErrorKind::StorageFull, @@ -1577,44 +1633,54 @@ fn is_busy(error: &rusqlite::Error) -> bool { /// SQL in its [`canonical_sql`] form. type SchemaObject = (String, String, String, Option); -/// SQL without the whitespace that separates nothing: a run of whitespace -/// is kept, as one space, only between two word characters, so the text an -/// `ALTER TABLE` splices in reads the same however it's spaced. Whitespace -/// and word characters are SQLite's own: only ASCII space, tab, newline, -/// form feed and carriage return separate tokens, and any other character -/// past ASCII belongs to a word. Quoted text, a string or a quoted name, is -/// kept as written, since its spaces are part of it. So text whose tokens -/// differ for SQLite can't read the same here. +/// SQL as SQLite tokenizes it, for comparing schema text. Whitespace next +/// to a parenthesis or a comma, which are tokens of their own, is dropped, +/// so the text an `ALTER TABLE` splices in reads the same however it's +/// spaced; any other run of whitespace reads as one space, as it may part +/// two tokens that would otherwise read as one (`< =` and `<=`). Quoted +/// text, a string or a quoted name, and comments are kept as written, a +/// line comment with the newline that ends it. Whitespace is SQLite's own: +/// ASCII space, tab, newline, form feed and carriage return. So text whose +/// tokens differ for SQLite can't read the same here. fn canonical_sql(sql: &str) -> String { - let is_word = |character: char| { - character.is_ascii_alphanumeric() || matches!(character, '_' | '$') || !character.is_ascii() - }; + let separates = |character: char| matches!(character, '(' | ')' | ','); let mut canonical = String::with_capacity(sql.len()); let mut spaced = false; - let mut characters = sql.chars(); + let mut characters = sql.chars().peekable(); while let Some(character) = characters.next() { if matches!(character, ' ' | '\t' | '\n' | '\x0c' | '\r') { spaced = true; continue; } - if spaced && canonical.ends_with(is_word) && is_word(character) { + if spaced + && !canonical.is_empty() + && !canonical.ends_with(separates) + && !separates(character) + { canonical.push(' '); } spaced = false; canonical.push(character); - let close = match character { - '\'' | '"' | '`' => character, - '[' => ']', + let end = match (character, characters.peek()) { + ('\'' | '"' | '`', _) => character.to_string(), + ('[', _) => "]".to_owned(), + ('-', Some('-')) => "\n".to_owned(), + ('/', Some('*')) => { + canonical.push('*'); + characters.next(); + "*/".to_owned() + } _ => continue, }; - // A doubled quote, which stands for one, ends a span where the next - // begins, so its text is kept all the same. - for quoted in characters.by_ref() { - canonical.push(quoted); - if quoted == close { + // The span's own text, so `/*/` doesn't read as a closed comment. + let mut span = String::new(); + for kept in characters.by_ref() { + span.push(kept); + if span.ends_with(end.as_str()) { break; } } + canonical.push_str(&span); } canonical } @@ -2290,6 +2356,9 @@ mod tests { /// [`OPEN_LOCK_TIMEOUT`], for a test of the wait itself. pub(super) static OPEN_LOCK_WAIT: std::cell::Cell> = const { std::cell::Cell::new(None) }; + /// The free space this thread's replacements find on their disk. + pub(super) static AVAILABLE_BYTES: std::cell::Cell> = + const { std::cell::Cell::new(None) }; /// The failure this thread's replacement copies are created with, as /// a disk with no blocks left fails them. pub(super) static COPY_CREATE_ERROR: std::cell::Cell> = @@ -2972,15 +3041,16 @@ mod tests { .enable_all() .build() .unwrap(); - let (existing, kept, new) = runtime.block_on(async { + let (existing, ownership_ignored, kept, new) = runtime.block_on(async { let db = InspectorDb::open(project.path()).unwrap(); let kept = db.get_webhook("kept").await.unwrap(); let new = InspectorDb::open(fresh.path()).unwrap().open_to_others(); - (db.open_to_others(), kept, new) + (db.open_to_others(), db.ownership_ignored(), kept, new) }); MODELESS_FILESYSTEM.set(false); assert!(existing); + assert!(!ownership_ignored); assert!(new); assert!(kept.is_some()); assert_eq!(std::fs::metadata(&db_path).unwrap().ino(), inode); @@ -3006,26 +3076,185 @@ mod tests { .build() .unwrap(); let (reviewed, existing, kept, new) = runtime.block_on(async { - let reviewed = InspectorDb::open_for_review(owner_only.path()) - .unwrap() - .open_to_others(); + let reviewed = InspectorDb::open_for_review(owner_only.path()).unwrap(); let review = InspectorDb::open_for_review(project.path()).unwrap(); let db = InspectorDb::open(project.path()).unwrap(); let kept = db.get_webhook("kept").await.unwrap(); - let new = InspectorDb::open(fresh.path()).unwrap().open_to_others(); + let new = InspectorDb::open(fresh.path()).unwrap(); drop(review); - (reviewed, db.open_to_others(), kept, new) + let state = |db: &InspectorDb| (db.open_to_others(), db.ownership_ignored()); + (state(&reviewed), state(&db), kept, state(&new)) }); OWNERSHIP_IGNORED.set(false); - assert!(reviewed); - assert!(existing); - assert!(new); + assert_eq!(reviewed, (true, true)); + assert_eq!(existing, (true, true)); + assert_eq!(new, (true, true)); assert!(kept.is_some()); assert_eq!(std::fs::metadata(&db_path).unwrap().ino(), inode); assert!(!project.path().join(".lpm/inspector.db-copy").exists()); } + /// A disk image attached for a test, detached when dropped. + #[cfg(target_os = "macos")] + struct AttachedImage(std::path::PathBuf); + + #[cfg(target_os = "macos")] + impl Drop for AttachedImage { + fn drop(&mut self) { + let _ = std::process::Command::new("/usr/bin/hdiutil") + .args(["detach", "-force", "-quiet"]) + .arg(&self.0) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); + } + } + + /// Attaches a new HFS+ image of `size` in `directory`, keeping or + /// ignoring ownership on it. + #[cfg(target_os = "macos")] + fn attach_image(directory: &Path, size: &str, owners: bool) -> AttachedImage { + let hdiutil = |arguments: &[&std::ffi::OsStr]| { + let output = std::process::Command::new("/usr/bin/hdiutil") + .args(arguments) + .output() + .unwrap(); + assert!( + output.status.success(), + "hdiutil {arguments:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + }; + let image = directory.join("volume.dmg"); + let mount_point = directory.join("mounted"); + std::fs::create_dir(&mount_point).unwrap(); + hdiutil(&[ + "create".as_ref(), + "-quiet".as_ref(), + "-size".as_ref(), + size.as_ref(), + "-fs".as_ref(), + "HFS+".as_ref(), + "-volname".as_ref(), + "lpm-inspect-test".as_ref(), + image.as_os_str(), + ]); + hdiutil(&[ + "attach".as_ref(), + "-quiet".as_ref(), + "-nobrowse".as_ref(), + "-noautoopen".as_ref(), + "-owners".as_ref(), + if owners { "on" } else { "off" }.as_ref(), + "-mountpoint".as_ref(), + mount_point.as_os_str(), + image.as_os_str(), + ]); + AttachedImage(mount_point) + } + + #[cfg(target_os = "macos")] + #[test] + fn a_project_on_a_volume_mounted_without_owners_keeps_captures_and_says_they_re_open_to_others() + { + let directory = tempfile::tempdir().unwrap(); + let attached = attach_image(directory.path(), "8m", false); + let project = attached.0.join("project"); + std::fs::create_dir(&project).unwrap(); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (created, reopened, kept) = runtime.block_on(async { + let db = InspectorDb::open(&project).unwrap(); + db.insert_durable_request(Arc::new(make_webhook("kept", 200)), None) + .await + .unwrap(); + let created = (db.open_to_others(), db.ownership_ignored()); + drop(db); + let db = InspectorDb::open(&project).unwrap(); + let kept = db.get_webhook("kept").await.unwrap(); + (created, (db.open_to_others(), db.ownership_ignored()), kept) + }); + drop(runtime); + + assert_eq!(created, (true, true)); + assert_eq!(reopened, (true, true)); + assert!(kept.is_some()); + assert!(!project.join(".lpm/inspector.db-copy").exists()); + } + + #[cfg(target_os = "macos")] + #[test] + fn a_disk_that_fills_while_the_database_is_checked_says_how_much_room_the_copy_needs() { + let directory = tempfile::tempdir().unwrap(); + let attached = attach_image(directory.path(), "4m", true); + let project = attached.0.join("project"); + std::fs::create_dir(&project).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let db = InspectorDb::open(&project).unwrap(); + db.insert_durable_request(Arc::new(make_webhook("kept", 200)), None) + .await + .unwrap(); + }); + drop(runtime); + let db_path = project.join(".lpm/inspector.db"); + // A WAL that leaving WAL has to fold into the database, which the + // disk then has no room for. + let writer = Connection::open(&db_path).unwrap(); + writer + .set_db_config( + rusqlite::config::DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, + true, + ) + .unwrap(); + writer.pragma_update(None, "wal_autocheckpoint", 0).unwrap(); + writer + .execute( + "INSERT INTO requests (id, timestamp, method, path, request_body) \ + VALUES ('bulk', '2026-04-06T12:00:00Z', 'POST', '/', zeroblob(600000))", + [], + ) + .unwrap(); + drop(writer); + set_mode(&db_path, 0o644); + let filler = attached.0.join("filler"); + let free = lpm_common::filesystem_available_bytes(&attached.0).unwrap(); + std::fs::write( + &filler, + vec![0; usize::try_from(free - 200 * 1024).unwrap()], + ) + .unwrap(); + + AVAILABLE_BYTES.set(Some(u64::MAX)); + let full = InspectorDb::open(&project).map(drop); + AVAILABLE_BYTES.set(None); + + let error = full.unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::StorageFull, "{error}"); + assert!(error.to_string().contains("free on its disk"), "{error}"); + assert_eq!(mode(&db_path), 0o644); + assert!(!capture_file(&db_path, "-copy").exists()); + + std::fs::remove_file(&filler).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let kept = runtime.block_on(async { + let db = InspectorDb::open(&project).unwrap(); + db.get_webhook("kept").await.unwrap() + }); + assert!(kept.is_some()); + assert_eq!(mode(&db_path), 0o600); + } + #[cfg(unix)] #[test] fn a_filesystem_without_modes_lets_a_capture_open_while_a_review_has_the_database_open() { @@ -3496,12 +3725,16 @@ mod tests { } #[test] - fn canonical_sql_keeps_only_the_spaces_between_words() { + fn canonical_sql_drops_the_spaces_around_parentheses_and_commas() { assert_eq!( canonical_sql("CREATE TABLE t (\n\t a TEXT ,b\tINT )"), "CREATE TABLE t(a TEXT,b INT)" ); assert_ne!(canonical_sql("summary TEXT"), canonical_sql("summ aryTEXT")); + assert_eq!( + canonical_sql("\n CREATE TABLE t (x)\n"), + "CREATE TABLE t(x)" + ); } #[test] @@ -3530,7 +3763,41 @@ mod tests { ); assert_eq!( canonical_sql("CREATE TABLE [a]] b] (x)"), - "CREATE TABLE[a]]b](x)" + "CREATE TABLE [a]] b](x)" + ); + } + + #[test] + fn canonical_sql_keeps_apart_what_sqlite_reads_apart() { + for (written, respaced) in [ + // One string against two. + ("c 'a''b'", "c 'a' 'b'"), + // Subtraction of a negative against a comment. + ("DEFAULT (1 - -1)", "DEFAULT (1 --1\n)"), + ("x <= 1", "x < = 1"), + // A string against a blob literal. + ("x '00'", "x'00'"), + // A comment through the end against one ended by a newline. + ("DEFAULT (1 --1)", "DEFAULT (1 --1\n)"), + ("a /* x */ b", "a b"), + ] { + assert_ne!( + canonical_sql(written), + canonical_sql(respaced), + "{written:?} against {respaced:?}" + ); + } + assert_eq!( + canonical_sql("a /*/ b */ c -- d\n e"), + "a /*/ b */ c -- d\n e" + ); + assert_eq!( + canonical_sql("a /* ' */ b -- '\n c"), + "a /* ' */ b -- '\n c" + ); + assert_eq!( + canonical_sql("CREATE TABLE t (\n\t a TEXT ,b\tINT )"), + canonical_sql("CREATE TABLE t(a TEXT, b INT)") ); } @@ -3557,10 +3824,61 @@ mod tests { assert!( error.to_string().contains(&format!( "needs about {} free on its disk", - lpm_common::format_bytes(std::fs::metadata(&db_path).unwrap().len()) + lpm_common::format_bytes(room_to_copy(&db_path).unwrap()) + )), + "{error}" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn a_disk_without_room_for_the_replacement_says_so_before_anything_is_opened() { + let project = project_with_a_database(&format!( + "{CAPTURE_SCHEMA} ALTER TABLE requests ADD COLUMN response_body_incomplete \ + INTEGER NOT NULL DEFAULT 0;" + )); + let db_path = project.path().join(".lpm/inspector.db"); + let needed = room_to_copy(&db_path).unwrap(); + + AVAILABLE_BYTES.set(Some(needed - 1)); + let short = InspectorDb::open(project.path()).map(drop); + AVAILABLE_BYTES.set(None); + + let error = short.unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::StorageFull); + assert!( + error.to_string().contains(&format!( + "needs about {} free on its disk", + lpm_common::format_bytes(needed) )), "{error}" ); + assert_eq!(mode(&db_path), 0o644); + assert!(!capture_file(&db_path, "-copy").exists()); + assert!(!capture_file(&db_path, "-shm").exists()); + + AVAILABLE_BYTES.set(Some(needed)); + let enough = InspectorDb::open(project.path()).map(drop); + AVAILABLE_BYTES.set(None); + + enough.unwrap(); + assert_eq!(mode(&db_path), 0o600); + } + + #[test] + fn the_room_a_replacement_needs_counts_the_wal_twice() { + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("inspector.db"); + std::fs::write(&db_path, vec![0; 10_000]).unwrap(); + assert_eq!(room_to_copy(&db_path), Some(10_000 + SHARED_MEMORY_ROOM)); + + std::fs::write(capture_file(&db_path, "-wal"), vec![0; 3_000]).unwrap(); + assert_eq!( + room_to_copy(&db_path), + Some(10_000 + 6_000 + SHARED_MEMORY_ROOM) + ); + std::fs::remove_file(&db_path).unwrap(); + assert_eq!(room_to_copy(&db_path), None); } #[cfg(unix)] @@ -3585,7 +3903,7 @@ mod tests { assert_eq!( error.to_string().contains(&format!( "needs about {} free on its disk", - lpm_common::format_bytes(std::fs::metadata(&db_path).unwrap().len()) + lpm_common::format_bytes(room_to_copy(&db_path).unwrap()) )), room, "{error}" @@ -4264,6 +4582,7 @@ mod tests { queued_capture_bytes: Arc::clone(&queued_capture_bytes), db_path, open_to_others: false, + ownership_ignored: false, }; db.start_session("saturated-session".to_string(), None, 3000) diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index 9e1ad8690..ded4d111e 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1593,7 +1593,11 @@ fn reading_captures_others_could_open_works_while_another_process_records_and_wa let rows: serde_json::Value = serde_json::from_slice(&json.stdout).unwrap(); assert_eq!(rows.as_array().map(Vec::len), Some(1)); let json_log = String::from_utf8_lossy(&json.stderr); - assert!(json_log.contains("is open to other users"), "{json_log}"); + assert!( + json_log.contains("is open to other users") + && json_log.contains("makes it owner-only where file modes keep other users out"), + "{json_log}" + ); assert_eq!( std::fs::metadata(&db_path).unwrap().permissions().mode() & 0o777, 0o644 From 947c211c380f49a3c4c7654e246001ad96e5fb0d Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Tue, 13 Oct 2026 01:40:53 +0100 Subject: [PATCH 11/17] fix(tunnel): name the room a full disk lacks on the check's first read, and take repositories from git's own answers A disk too full for SQLite's shared-memory file failed the check's schema read with SQLITE_IOERR_SHMSIZE as a bare I/O error, leaving -wal and -shm behind. The schema is now read through the check's mapping, that error reads as the disk's want of room, and a failed check removes the WAL files it made. The room is the database and its WAL plus the shared-memory file, no longer the WAL twice. Which repositories exist is git's own answer: a failure where git says there's no repository found none, git inside .lpm names files from their repository's top, and the .git entries are read only where git couldn't run or answer. Free space on macOS is read with statfs, whose block counts are 64-bit. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/tunnel.rs | 331 ++++++++++++++++++++------ crates/lpm-common/src/lib.rs | 36 ++- crates/lpm-inspect/src/db.rs | 141 +++++++++-- 3 files changed, 408 insertions(+), 100 deletions(-) diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index 929fcd787..89e39d98c 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -892,7 +892,8 @@ impl CaptureStore { } /// Capture files a git query listed as tracked, and as neither tracked nor -/// ignored, by name relative to the directory it was asked in. +/// ignored, by name relative to the directory it was asked in, or to its +/// repository's top with `--full-name`. #[derive(Debug, Default, Clone, PartialEq, Eq)] struct CaptureGitFiles { tracked: Vec, @@ -947,19 +948,45 @@ const CAPTURE_GIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs( /// Why git didn't say how it treats the capture files. #[derive(Debug, PartialEq, Eq)] enum CaptureGitFailure { - /// Git ran and refused, as outside a repository, or for one it won't use. + /// Git ran and found no repository it would use there. + NoRepository, + /// Git ran and refused, for a repository it won't use, such as one + /// another user owns. Refused, /// Git couldn't be run, or didn't answer in time. Unavailable, } +/// What git says, with `LC_ALL=C`, where it finds no repository: none at +/// all, none before a filesystem boundary, or a `.git` file that doesn't +/// point at one, which no git command there gets past. +const NO_REPOSITORY_MESSAGES: [&str; 3] = [ + "not a git repository", + "invalid gitfile format", + "no path in gitfile", +]; + +fn capture_git_failure(stderr: &[u8]) -> CaptureGitFailure { + let message = String::from_utf8_lossy(stderr); + if NO_REPOSITORY_MESSAGES + .iter() + .any(|pattern| message.contains(pattern)) + { + CaptureGitFailure::NoRepository + } else { + CaptureGitFailure::Refused + } +} + /// A git query of the capture files: the directory it runs in, its -/// pathspec, and whether it lists files neither tracked nor ignored too. +/// pathspec, whether it lists files neither tracked nor ignored too, and +/// whether it names them from their repository's top. #[derive(Debug, Clone, PartialEq, Eq)] struct CaptureGitQuery { directory: PathBuf, pathspec: &'static str, unignored: bool, + full_name: bool, } /// How git treats the capture files in `project_dir`, matching their names @@ -968,9 +995,11 @@ struct CaptureGitQuery { /// whatever the case of their directory; and git in `.lpm`, for those /// neither tracked nor ignored there, and for those a repository of `.lpm`'s /// own tracks, such as a submodule, whose files the project's index doesn't -/// list. What one query confirms stands when the other doesn't answer in -/// time. None when neither answers: neither is a repository, git can't be -/// run, or it doesn't answer in time. +/// list. Git names the files in `.lpm` from their repository's top, so a +/// name without a directory is one a repository of `.lpm`'s own tracks. +/// What one query confirms stands when the other doesn't answer in time. +/// None when neither answers: neither is a repository, git can't be run, or +/// it doesn't answer in time. async fn capture_git_state(project_dir: &Path) -> Option { capture_git_state_by(project_dir, |query| { capture_git_output(capture_git_command(&query), CAPTURE_GIT_TIMEOUT) @@ -985,22 +1014,20 @@ where { let lpm_dir = project_dir.join(".lpm"); let lpm_exists = lpm_dir.is_dir(); - let own_repository = marks_a_repository(&lpm_dir.join(".git")); - let in_repository = project_dir - .ancestors() - .any(|directory| marks_a_repository(&directory.join(".git"))); let project = ask(CaptureGitQuery { directory: project_dir.to_path_buf(), pathspec: PROJECT_CAPTURES_PATHSPEC, unignored: false, + full_name: false, }); let inside = async { if lpm_exists { Some( ask(CaptureGitQuery { - directory: lpm_dir, + directory: lpm_dir.clone(), pathspec: CAPTURES_PATHSPEC, unignored: true, + full_name: true, }) .await, ) @@ -1011,14 +1038,22 @@ where let (project, inside) = tokio::join!(project, inside); let mut state = CaptureGitState::default(); + // Whether git found a repository the project is in; only where git + // didn't run or answer is that read from the `.git` entries. + let in_repository = match &project { + Ok(_) | Err(CaptureGitFailure::Refused) => true, + Err(CaptureGitFailure::NoRepository) => false, + Err(CaptureGitFailure::Unavailable) => project_dir + .ancestors() + .any(|directory| marks_a_repository(&directory.join(".git"))), + }; let project_answered = match project { Ok(files) => { state.project_tracked = files.tracked; true } // In a repository, a query that fails is no answer, git refusing - // included, as for one another user owns, run as root; outside one, - // there's nothing to ask. + // included, as for one another user owns, run as root. Err(_) if in_repository => { state.unanswered.push(CaptureRepository::Project); false @@ -1027,22 +1062,37 @@ where }; let inside_answered = match inside { Some(Ok(files)) => { - // Without a repository of its own, git in `.lpm` lists the - // project's tracked files, which the project's query has when - // it answers; when it doesn't, they're what's known of them. - if own_repository { - state.lpm_tracked = files.tracked; - } else if !project_answered { - state.project_tracked = files - .tracked - .into_iter() - .map(|name| format!(".lpm/{name}")) + let (own, project_files): (Vec, Vec) = files + .tracked + .into_iter() + .partition(|name| !name.contains('/')); + state.lpm_tracked = own; + // Git in `.lpm` lists what the project's repository tracks + // there, which the project's query has when it answers; when it + // doesn't, they're what's known of them. + if !project_answered { + state.project_tracked = project_files + .iter() + .map(|name| format!(".lpm/{}", file_name(name))) .collect(); state.project_spelling_unknown = !state.project_tracked.is_empty(); } - state.unignored = files.unignored; + state.unignored = files + .unignored + .iter() + .map(|name| file_name(name).to_owned()) + .collect(); true } + // Git in `.lpm` stopped short of the project's repository, as at a + // filesystem boundary, so whether it ignores the files there isn't + // known. + Some(Err(CaptureGitFailure::NoRepository)) => { + if in_repository { + state.unanswered.push(CaptureRepository::Lpm); + } + false + } Some(Err(CaptureGitFailure::Refused)) => { state.lpm_unasked = true; false @@ -1053,20 +1103,27 @@ where } None => false, }; + let own_repository = lpm_exists && marks_a_repository(&lpm_dir.join(".git")); (project_answered || inside_answered || in_repository || own_repository).then_some(state) } -/// Whether a `.git` entry makes its directory a repository's: a directory, -/// or a file pointing at one (`gitdir: …`). An empty or other file doesn't, -/// as git doesn't take it for one. +/// The part of a name git lists after its last directory. +fn file_name(name: &str) -> &str { + name.rsplit_once('/').map_or(name, |(_, file)| file) +} + +/// Whether a `.git` entry looks like a repository's, for when git itself +/// can't be asked: a directory, or a link to one, holding `HEAD`, or a file +/// pointing at one as git writes it (`gitdir: …`). An empty directory or +/// another file doesn't, as git looks past them. fn marks_a_repository(git: &Path) -> bool { - match std::fs::symlink_metadata(git) { - Ok(metadata) if metadata.is_dir() => true, + match std::fs::metadata(git) { + Ok(metadata) if metadata.is_dir() => git.join("HEAD").is_file(), Ok(metadata) if metadata.is_file() => { - let mut start = [0_u8; 7]; + let mut start = [0_u8; 8]; lpm_common::open_nofollow(git) .and_then(|mut file| std::io::Read::read_exact(&mut file, &mut start)) - .is_ok_and(|()| &start == b"gitdir:") + .is_ok_and(|()| &start == b"gitdir: ") } _ => false, } @@ -1077,6 +1134,9 @@ fn capture_git_command(query: &CaptureGitQuery) -> tokio::process::Command { if query.unignored { args.extend(["--others", "--exclude-standard"]); } + if query.full_name { + args.push("--full-name"); + } args.extend(["--", query.pathspec]); tokio::process::Command::from(crate::commands::npmrc::hardened_git_command( &query.directory, @@ -1093,14 +1153,14 @@ async fn capture_git_output( ) -> Result { command .stdin(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) .kill_on_drop(true); let output = tokio::time::timeout(timeout, command.output()) .await .map_err(|_| CaptureGitFailure::Unavailable)? .map_err(|_| CaptureGitFailure::Unavailable)?; if !output.status.success() { - return Err(CaptureGitFailure::Refused); + return Err(capture_git_failure(&output.stderr)); } Ok(parse_capture_git_files(&output.stdout)) } @@ -2519,6 +2579,36 @@ mod tests { assert!(!state.kept_out()); } + #[tokio::test] + async fn lists_a_capture_once_when_lpm_holds_an_empty_git_directory() { + let project = capture_project(&[".lpm/inspector.db"]); + git(project.path(), &["add", ".lpm/inspector.db"]); + std::fs::create_dir(project.path().join(".lpm/.git")).unwrap(); + + let state = capture_git_state(project.path()).await.unwrap(); + + assert_eq!(state.project_tracked, names(&[".lpm/inspector.db"])); + assert!(state.lpm_tracked.is_empty(), "{state:?}"); + assert!(state.unanswered.is_empty(), "{state:?}"); + } + + #[tokio::test] + async fn says_nothing_of_git_entries_git_takes_for_no_repository() { + for entry in ["empty directory", "gitdir:../elsewhere/.git\n"] { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + std::fs::write(project.path().join(".lpm/inspector.db"), b"x").unwrap(); + let git_entry = project.path().join(".git"); + if entry == "empty directory" { + std::fs::create_dir(&git_entry).unwrap(); + } else { + std::fs::write(&git_entry, entry).unwrap(); + } + + assert_eq!(capture_git_state(project.path()).await, None, "{entry:?}"); + } + } + #[cfg(unix)] #[tokio::test] async fn says_when_git_refuses_to_answer_for_lpm() { @@ -2587,7 +2677,11 @@ mod tests { .unwrap(); std::fs::write(web.join(".lpm/inspector.db"), b"x").unwrap(); - assert_eq!(capture_git_state(&web).await, None); + // Git without `safe.bareRepository=explicit` would use it, so its + // refusal here leaves the files unconfirmed. + let state = capture_git_state(&web).await.unwrap(); + assert_eq!(state.unanswered, vec![CaptureRepository::Project]); + assert!(state.lpm_unasked); assert!(!marker.exists()); } @@ -2633,7 +2727,7 @@ mod tests { let (state, mut asked) = state_from_answers( project.path(), Ok(files(&[".lpm/inspector.db"], &[])), - Ok(files(&["inspector.db"], &["inspector.db-wal"])), + Ok(files(&[".lpm/inspector.db"], &[".lpm/inspector.db-wal"])), ) .await; @@ -2645,16 +2739,18 @@ mod tests { directory: project.path().to_path_buf(), pathspec: PROJECT_CAPTURES_PATHSPEC, unignored: false, + full_name: false, }, CaptureGitQuery { directory: lpm_dir, pathspec: CAPTURES_PATHSPEC, unignored: true, + full_name: true, }, ] ); - // Without a repository of `.lpm`'s own, git there lists the - // project's files, which the project's query has. + // Git in `.lpm` names the project's files from its top, and the + // project's query has them. assert_eq!( state, Some(CaptureGitState { @@ -2676,7 +2772,7 @@ mod tests { let (state, _) = state_from_answers( project.path(), Ok(files(&[".LPM/inspector.db"], &[])), - Ok(files(&["inspector.db"], &[])), + Ok(files(&[".LPM/inspector.db"], &[])), ) .await; @@ -2689,16 +2785,20 @@ mod tests { #[tokio::test] async fn counts_what_a_repository_of_lpm_s_own_tracks() { let project = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(project.path().join(".lpm/.git")).unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); let (state, _) = state_from_answers( project.path(), - Err(CaptureGitFailure::Refused), - Ok(files(&["inspector.db"], &[])), + Err(CaptureGitFailure::NoRepository), + Ok(files(&["inspector.db"], &["inspector.db-wal"])), ) .await; - assert_eq!(state.unwrap().lpm_tracked, names(&["inspector.db"])); + let state = state.unwrap(); + assert_eq!(state.lpm_tracked, names(&["inspector.db"])); + assert!(state.project_tracked.is_empty()); + assert_eq!(state.unignored, names(&["inspector.db-wal"])); + assert!(state.unanswered.is_empty()); } #[tokio::test] @@ -2712,11 +2812,15 @@ mod tests { Err(CaptureGitFailure::Unavailable), ), ( - Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::NoRepository), Err(CaptureGitFailure::Unavailable), ), ( - Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::NoRepository), + Err(CaptureGitFailure::NoRepository), + ), + ( + Err(CaptureGitFailure::NoRepository), Err(CaptureGitFailure::Refused), ), ] { @@ -2741,11 +2845,11 @@ mod tests { #[tokio::test] async fn names_lpm_s_own_repository_when_git_there_doesn_t_answer() { let project = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(project.path().join(".lpm/.git")).unwrap(); + repository_entry(&project.path().join(".lpm")); let (state, _) = state_from_answers( project.path(), - Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::NoRepository), Err(CaptureGitFailure::Unavailable), ) .await; @@ -2764,7 +2868,7 @@ mod tests { #[tokio::test] async fn names_each_query_a_repository_leaves_unanswered_even_when_neither_answers() { let project = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(project.path().join(".git")).unwrap(); + repository_entry(project.path()); std::fs::create_dir(project.path().join(".lpm")).unwrap(); // As for a repository another user owns, run as root. @@ -2808,13 +2912,14 @@ mod tests { #[tokio::test] async fn takes_a_project_inside_a_repository_s_subdirectory_as_in_that_repository() { let repository = tempfile::tempdir().unwrap(); - std::fs::create_dir(repository.path().join(".git")).unwrap(); + repository_entry(repository.path()); let project = repository.path().join("packages/app"); std::fs::create_dir_all(project.join(".lpm")).unwrap(); + // Git didn't run, so the `.git` entries are what's known. let (state, _) = state_from_answers( &project, - Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::Unavailable), Err(CaptureGitFailure::Refused), ) .await; @@ -2823,39 +2928,130 @@ mod tests { } #[tokio::test] - async fn takes_lpm_s_git_entry_for_a_repository_only_as_git_does() { + async fn takes_whose_index_a_capture_is_in_from_the_name_git_gives_it() { let project = tempfile::tempdir().unwrap(); - std::fs::create_dir(project.path().join(".lpm")).unwrap(); - let git = project.path().join(".lpm/.git"); + // Git looks past an empty `.git` directory, whatever it holds. + std::fs::create_dir_all(project.path().join(".lpm/.git")).unwrap(); - for (contents, own) in [ - ("", false), - ("not a pointer\n", false), - ("gitdir: ../../elsewhere/.git\n", true), + for (listed, own) in [ + ("inspector.db", true), + (".lpm/inspector.db", false), + ("apps/web/.lpm/inspector.db", false), ] { - std::fs::write(&git, contents).unwrap(); let (state, _) = state_from_answers( project.path(), Err(CaptureGitFailure::Unavailable), - Ok(files(&["inspector.db"], &[])), + Ok(files(&[listed], &[])), ) .await; let state = state.unwrap(); + let (lpm_tracked, project_tracked) = if own { + (names(&["inspector.db"]), Vec::new()) + } else { + (Vec::new(), names(&[".lpm/inspector.db"])) + }; assert_eq!( - ( - state.lpm_tracked.is_empty(), - state.project_tracked.is_empty() - ), - (!own, own), - "{contents:?}: {state:?}" + (state.lpm_tracked, state.project_tracked), + (lpm_tracked, project_tracked), + "{listed}" ); } } + #[tokio::test] + async fn names_lpm_unanswered_where_git_there_stops_short_of_the_project_s_repository() { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + + // As at a filesystem boundary between the project and `.lpm`. + let (state, _) = state_from_answers( + project.path(), + Ok(CaptureGitFiles::default()), + Err(CaptureGitFailure::NoRepository), + ) + .await; + + let state = state.unwrap(); + assert_eq!(state.unanswered, vec![CaptureRepository::Lpm]); + assert!(!state.lpm_unasked); + assert!(!state.kept_out()); + } + + /// Makes `directory/.git` as `git init` leaves its `HEAD`. + fn repository_entry(directory: &Path) { + std::fs::create_dir_all(directory.join(".git")).unwrap(); + std::fs::write(directory.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); + } + + #[test] + fn takes_a_git_entry_for_a_repository_s_only_as_git_would() { + let directory = tempfile::tempdir().unwrap(); + let git = directory.path().join(".git"); + + for (contents, repository) in [ + ("", false), + ("not a pointer\n", false), + ("gitdir:../elsewhere/.git\n", false), + ("gitdir: ../elsewhere/.git\n", true), + ] { + std::fs::write(&git, contents).unwrap(); + assert_eq!(marks_a_repository(&git), repository, "{contents:?}"); + } + std::fs::remove_file(&git).unwrap(); + + std::fs::create_dir(&git).unwrap(); + assert!(!marks_a_repository(&git)); + repository_entry(directory.path()); + assert!(marks_a_repository(&git)); + + #[cfg(unix)] + { + let linked = directory.path().join("linked"); + std::fs::create_dir(&linked).unwrap(); + std::os::unix::fs::symlink(&git, linked.join(".git")).unwrap(); + assert!(marks_a_repository(&linked.join(".git"))); + } + assert!(!marks_a_repository(&directory.path().join("gone/.git"))); + } + + #[test] + fn reads_git_s_failure_as_no_repository_only_where_it_says_so() { + for (stderr, failure) in [ + ( + "fatal: not a git repository (or any of the parent directories): .git\n", + CaptureGitFailure::NoRepository, + ), + ( + "fatal: not a git repository (or any parent up to mount point /mnt)\n\ + Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).\n", + CaptureGitFailure::NoRepository, + ), + ( + "fatal: invalid gitfile format: /work/app/.git\n", + CaptureGitFailure::NoRepository, + ), + ( + "fatal: no path in gitfile: /work/app/.git\n", + CaptureGitFailure::NoRepository, + ), + ( + "fatal: detected dubious ownership in repository at '/work/app'\n", + CaptureGitFailure::Refused, + ), + ( + "fatal: cannot use bare repository '/work/app' (safe.bareRepository is 'explicit')\n", + CaptureGitFailure::Refused, + ), + ("", CaptureGitFailure::Refused), + ] { + assert_eq!(capture_git_failure(stderr.as_bytes()), failure, "{stderr}"); + } + } + #[tokio::test] async fn keeps_what_one_query_confirms_when_the_other_doesn_t_answer_in_time() { let project = tempfile::tempdir().unwrap(); - std::fs::create_dir(project.path().join(".git")).unwrap(); + repository_entry(project.path()); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let (tracked, _) = state_from_answers( @@ -2900,13 +3096,13 @@ mod tests { #[tokio::test] async fn reports_what_git_in_lpm_finds_tracked_when_the_project_s_query_doesn_t_answer() { let project = tempfile::tempdir().unwrap(); - std::fs::create_dir(project.path().join(".git")).unwrap(); + repository_entry(project.path()); std::fs::create_dir(project.path().join(".lpm")).unwrap(); let (state, _) = state_from_answers( project.path(), Err(CaptureGitFailure::Unavailable), - Ok(files(&["inspector.db"], &[])), + Ok(files(&[".lpm/inspector.db"], &[])), ) .await; @@ -2926,7 +3122,6 @@ mod tests { "{warnings:?}" ); - std::fs::create_dir(project.path().join(".lpm/.git")).unwrap(); let (own, _) = state_from_answers( project.path(), Err(CaptureGitFailure::Unavailable), diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index db21d332a..7a8edf563 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -329,15 +329,33 @@ pub fn filesystem_available_bytes(path: &std::path::Path) -> Option { } } let path = CString::new(probe.as_os_str().as_bytes()).ok()?; - let mut stats = std::mem::MaybeUninit::::uninit(); - // SAFETY: `path` is NUL-terminated and `stats` points to writable, - // properly aligned storage that is read only after a successful call. - if unsafe { libc::statvfs(path.as_ptr(), stats.as_mut_ptr()) } != 0 { - return None; - } - // SAFETY: `statvfs` returned success and initialized the output structure. - let stats = unsafe { stats.assume_init() }; - let bytes = u128::from(stats.f_bavail).checked_mul(u128::from(stats.f_frsize))?; + // macOS's `statvfs` counts blocks in 32 bits, which can't hold the free + // blocks of a volume with more than 16 TiB free; its `statfs` counts in + // 64 bits. + #[cfg(target_os = "macos")] + let bytes = { + let mut stats = std::mem::MaybeUninit::::uninit(); + // SAFETY: `path` is NUL-terminated and `stats` points to writable, + // properly aligned storage that is read only after a successful call. + if unsafe { libc::statfs(path.as_ptr(), stats.as_mut_ptr()) } != 0 { + return None; + } + // SAFETY: `statfs` returned success and initialized the output structure. + let stats = unsafe { stats.assume_init() }; + u128::from(stats.f_bavail).checked_mul(u128::from(stats.f_bsize))? + }; + #[cfg(not(target_os = "macos"))] + let bytes = { + let mut stats = std::mem::MaybeUninit::::uninit(); + // SAFETY: `path` is NUL-terminated and `stats` points to writable, + // properly aligned storage that is read only after a successful call. + if unsafe { libc::statvfs(path.as_ptr(), stats.as_mut_ptr()) } != 0 { + return None; + } + // SAFETY: `statvfs` returned success and initialized the output structure. + let stats = unsafe { stats.assume_init() }; + u128::from(stats.f_bavail).checked_mul(u128::from(stats.f_frsize))? + }; u64::try_from(bytes).ok() } diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index 904f80512..29df8466d 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -100,7 +100,7 @@ impl InspectorDb { // Write connection (owned by the background flush task) let write_conn = open_capture_connection(&db_path, "NORMAL").map_err(damaged(&db_path))?; - refuse_unknown_schema(&write_conn, &db_path)?; + refuse_unknown_schema(&write_conn, &db_path, damaged(&db_path))?; init_schema(&write_conn).map_err(damaged(&db_path))?; // Read connection (shared across API handlers). The busy timeout @@ -1412,11 +1412,33 @@ fn replace_with_owner_only_copy(db_path: &Path, owner: CaptureOwner) -> Result Result<(), OpenError> { ensure_room_to_copy(db_path)?; + let made_by_check: Vec = ["-wal", "-shm"] + .into_iter() + .map(|suffix| capture_file(db_path, suffix)) + .filter(|sidecar| { + matches!(sidecar.symlink_metadata(), Err(error) if error.kind() == std::io::ErrorKind::NotFound) + }) + .collect(); + let copied = check_then_copy(db_path, copy_path); + if copied.is_err() { + for sidecar in &made_by_check { + if let Err(error) = remove_if_present(sidecar) { + tracing::warn!("could not remove {}: {error}", sidecar.display()); + } + } + } + copied +} + +fn check_then_copy(db_path: &Path, copy_path: &Path) -> Result<(), OpenError> { let database = open_nofollow_connection(db_path).map_err(checking(db_path))?; - refuse_unknown_schema(&database, db_path)?; + refuse_unknown_schema(&database, db_path, checking(db_path))?; // In WAL, which every LPM connection uses, leaving it needs the only // connection, so it proves no other LPM process has the database open, // and folds the WAL into the file. @@ -1436,7 +1458,7 @@ fn copy_checked_database(db_path: &Path, copy_path: &Path) -> Result<(), OpenErr /// room, and any other failure as [`damaged`] does. fn checking(db_path: &Path) -> impl Fn(rusqlite::Error) -> OpenError + '_ { move |error| { - if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DiskFull) { + if out_of_room(&error) { no_room_to_copy(db_path) } else { damaged(db_path)(error) @@ -1444,20 +1466,38 @@ fn checking(db_path: &Path) -> impl Fn(rusqlite::Error) -> OpenError + '_ { } } -/// The room SQLite's shared-memory file takes as the check opens the -/// database. -const SHARED_MEMORY_ROOM: u64 = 64 * 1024; +/// Whether SQLite failed for want of disk space: a full disk, or a +/// shared-memory file it couldn't grow, which is how a disk with less free +/// than the file's next 32 KiB fails an open. +fn out_of_room(error: &rusqlite::Error) -> bool { + match error { + rusqlite::Error::SqliteFailure(failure, _) => { + failure.code == rusqlite::ErrorCode::DiskFull + || failure.extended_code == rusqlite::ffi::SQLITE_IOERR_SHMSIZE + } + _ => false, + } +} + +/// The room SQLite's shared-memory file takes while the check reads a +/// database with `wal` bytes of WAL: 32 KiB for about every 4,096 frames, +/// about 1/512 of the WAL, and a region more for rounding. +fn shared_memory_room(wal: u64) -> u64 { + (64 * 1024u64).saturating_add(wal / 512) +} -/// The free space replacing a database takes: the copy, which holds the -/// database with its WAL folded in; the database's own growth, as leaving -/// WAL folds the WAL into it; and SQLite's shared-memory file. +/// The free space replacing a database takes. Leaving WAL folds the WAL into +/// the database, which grows by the WAL at most while the shared-memory file +/// is open; then the WAL and that file are removed, and the copy takes the +/// grown database's size. So it never takes more than the database and its +/// WAL, with the shared-memory file on top. fn room_to_copy(db_path: &Path) -> Option { let database = std::fs::metadata(db_path).ok()?.len(); let wal = std::fs::metadata(capture_file(db_path, "-wal")).map_or(0, |wal| wal.len()); Some( database - .saturating_add(wal.saturating_mul(2)) - .saturating_add(SHARED_MEMORY_ROOM), + .saturating_add(wal) + .saturating_add(shared_memory_room(wal)), ) } @@ -1503,9 +1543,7 @@ fn copy_database(database: &Connection, copy_path: &Path, db_path: &Path) -> Res match backup.step(-1) { Ok(rusqlite::backup::StepResult::Done) => Ok(()), Ok(_) => Err(in_use(db_path)), - Err(error) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DiskFull) => { - Err(no_room_to_copy(db_path)) - } + Err(error) if out_of_room(&error) => Err(no_room_to_copy(db_path)), Err(error) => Err(damaged(db_path)(error)), } } @@ -1640,8 +1678,10 @@ type SchemaObject = (String, String, String, Option); /// two tokens that would otherwise read as one (`< =` and `<=`). Quoted /// text, a string or a quoted name, and comments are kept as written, a /// line comment with the newline that ends it. Whitespace is SQLite's own: -/// ASCII space, tab, newline, form feed and carriage return. So text whose -/// tokens differ for SQLite can't read the same here. +/// ASCII space, tab, newline, form feed and carriage return. So schema text +/// whose tokens differ for SQLite can't read the same here. Only a +/// parameter's Tcl form takes a parenthesis into its token (`$x(y)` against +/// `$x (y)`), and SQLite allows no parameter in a schema. fn canonical_sql(sql: &str) -> String { let separates = |character: char| matches!(character, '(' | ')' | ','); let mut canonical = String::with_capacity(sql.len()); @@ -1723,10 +1763,14 @@ fn lpm_schema() -> Result, rusqlite::Err /// a repository or another tool could otherwise run code of its own on /// every capture written: a trigger, or a check, default, generated column /// or index expression that's slow by design. -fn refuse_unknown_schema(connection: &Connection, db_path: &Path) -> Result<(), OpenError> { +fn refuse_unknown_schema( + connection: &Connection, + db_path: &Path, + failed: impl Fn(rusqlite::Error) -> OpenError, +) -> Result<(), OpenError> { let known = lpm_schema()?; let unknown: Vec = schema_objects(connection) - .map_err(damaged(db_path))? + .map_err(failed)? .into_iter() .filter(|object| !known.contains(object)) .map(|(kind, name, _, _)| format!("{kind} {}", lpm_common::sanitize_terminal_inline(&name))) @@ -2922,6 +2966,10 @@ mod tests { // Leaving WAL tells at once; the copy's own wait is the busy timeout. assert!(started.elapsed() < std::time::Duration::from_secs(4)); assert_eq!(mode(&db_path), 0o644); + // The open connection's WAL files aren't the refused check's to remove. + for suffix in ["-wal", "-shm"] { + assert!(capture_file(&db_path, suffix).exists(), "{suffix} removed"); + } drop(other); runtime.block_on(async { drop(InspectorDb::open(project.path()).unwrap()) }); assert_eq!(mode(&db_path), 0o600); @@ -3255,6 +3303,52 @@ mod tests { assert_eq!(mode(&db_path), 0o600); } + #[cfg(target_os = "macos")] + #[test] + fn a_disk_too_full_for_the_check_s_shared_memory_says_how_much_room_the_copy_needs() { + use std::io::Write as _; + + let directory = tempfile::tempdir().unwrap(); + let attached = attach_image(directory.path(), "4m", true); + let project = attached.0.join("project"); + std::fs::create_dir(&project).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let db = InspectorDb::open(&project).unwrap(); + db.insert_durable_request(Arc::new(make_webhook("kept", 200)), None) + .await + .unwrap(); + }); + drop(runtime); + let db_path = project.join(".lpm/inspector.db"); + set_mode(&db_path, 0o644); + // Room for the copy's empty file, but not for SQLite's first 32 KiB + // of shared memory, which the check's first read sizes. + let mut filler = std::fs::File::create(attached.0.join("filler")).unwrap(); + while filler.write_all(&[0; 4096]).is_ok() {} + let filled = filler.metadata().unwrap().len(); + filler.set_len(filled - 8 * 1024).unwrap(); + drop(filler); + + AVAILABLE_BYTES.set(Some(u64::MAX)); + let full = InspectorDb::open(&project).map(drop); + AVAILABLE_BYTES.set(None); + + let error = full.unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::StorageFull, "{error}"); + assert!(error.to_string().contains("free on its disk"), "{error}"); + assert_eq!(mode(&db_path), 0o644); + for suffix in ["-copy", "-wal", "-shm"] { + assert!( + !capture_file(&db_path, suffix).exists(), + "{suffix} left behind" + ); + } + } + #[cfg(unix)] #[test] fn a_filesystem_without_modes_lets_a_capture_open_while_a_review_has_the_database_open() { @@ -3866,16 +3960,17 @@ mod tests { } #[test] - fn the_room_a_replacement_needs_counts_the_wal_twice() { + fn the_room_a_replacement_needs_counts_the_wal_once_with_its_shared_memory() { let directory = tempfile::tempdir().unwrap(); let db_path = directory.path().join("inspector.db"); std::fs::write(&db_path, vec![0; 10_000]).unwrap(); - assert_eq!(room_to_copy(&db_path), Some(10_000 + SHARED_MEMORY_ROOM)); + assert_eq!(room_to_copy(&db_path), Some(10_000 + 64 * 1024)); - std::fs::write(capture_file(&db_path, "-wal"), vec![0; 3_000]).unwrap(); + let wal = std::fs::File::create(capture_file(&db_path, "-wal")).unwrap(); + wal.set_len(512 * 1024 * 1024).unwrap(); assert_eq!( room_to_copy(&db_path), - Some(10_000 + 6_000 + SHARED_MEMORY_ROOM) + Some(10_000 + 512 * 1024 * 1024 + 64 * 1024 + 1024 * 1024) ); std::fs::remove_file(&db_path).unwrap(); assert_eq!(room_to_copy(&db_path), None); From 73c743609330e71f53fcf1e3aeeea845e789da9e Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Tue, 13 Oct 2026 03:55:00 +0100 Subject: [PATCH 12/17] fix(tunnel): keep another connection's WAL files, name the room taken before opening, and fall back to .git entries where git stops - A check that fails no longer removes the -wal and -shm files: an earlier release or another tool can attach after the room is taken, without the opening lock, and would lose committed writes. They hold no captures. - The room a replacement needs is taken once, before the database is opened, and named in every refusal; it counts 8 MiB a filesystem keeps back. - Git stopping at a filesystem boundary or a broken .git file is told from finding no repository; the project's .git entries are read there, and .lpm's own only where git couldn't run in it. git's fatal line is read from its start, ignoring case, as releases before 2.17 capitalize it. - The unanswered warnings name stopping among their causes. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/tunnel.rs | 227 +++++++++++++++++++++----- crates/lpm-inspect/src/db.rs | 204 ++++++++++++++++------- 2 files changed, 336 insertions(+), 95 deletions(-) diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index 89e39d98c..0d5d58b4c 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -777,12 +777,15 @@ fn capture_warnings( warnings.push( match repository { CaptureRepository::Project => { - "git didn't answer for the project's repository (it couldn't run, refused, \ - or took over 2 s), so check that it doesn't track .lpm/inspector.db*" + "git didn't answer for the project's repository (it couldn't run, took \ + over 2 s, refused, or stopped at a filesystem boundary or a broken .git \ + file), so check that it doesn't track .lpm/inspector.db*" } CaptureRepository::Lpm => { - "git didn't answer in .lpm (it couldn't run or took over 2 s), so check \ - that git ignores and doesn't track .lpm/inspector.db*" + "git didn't answer in .lpm (it couldn't run, took over 2 s, or stopped \ + short of the project's repository, as at a filesystem boundary or a \ + broken .git file), so check that git ignores and doesn't track \ + .lpm/inspector.db*" } } .to_string(), @@ -917,8 +920,8 @@ struct CaptureGitState { /// bare repository. lpm_unasked: bool, /// The repositories whose query couldn't run, was refused in a - /// repository, or didn't answer in time, so what the other found stands - /// but git's ignoring the files isn't confirmed. + /// repository, stopped short of one, or didn't answer in time, so what + /// the other found stands but git's ignoring the files isn't confirmed. unanswered: Vec, } @@ -948,8 +951,13 @@ const CAPTURE_GIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs( /// Why git didn't say how it treats the capture files. #[derive(Debug, PartialEq, Eq)] enum CaptureGitFailure { - /// Git ran and found no repository it would use there. + /// Git ran and found no repository there or in any directory above. NoRepository, + /// Git ran and stopped looking before it found one: at a filesystem + /// boundary, or at a `.git` file that doesn't point at a repository, + /// which no git command there gets past. A repository above can still + /// hold the directory. + Stopped, /// Git ran and refused, for a repository it won't use, such as one /// another user owns. Refused, @@ -957,25 +965,38 @@ enum CaptureGitFailure { Unavailable, } -/// What git says, with `LC_ALL=C`, where it finds no repository: none at -/// all, none before a filesystem boundary, or a `.git` file that doesn't -/// point at one, which no git command there gets past. -const NO_REPOSITORY_MESSAGES: [&str; 3] = [ - "not a git repository", +/// How git's `fatal: ` line starts, with `LC_ALL=C`, where it found no +/// repository, and where it stopped looking. +const NO_REPOSITORY_MESSAGE: &str = "not a git repository (or any of the parent directories)"; +const STOPPED_MESSAGES: [&str; 4] = [ + "not a git repository (or any parent up to mount", + "not a git repository: ", "invalid gitfile format", "no path in gitfile", ]; +/// Reads git's failure from the start of its `fatal: ` line, as the rest can +/// quote a path, and without regard to case, as releases before 2.17 write +/// "Not a git repository". fn capture_git_failure(stderr: &[u8]) -> CaptureGitFailure { let message = String::from_utf8_lossy(stderr); - if NO_REPOSITORY_MESSAGES - .iter() - .any(|pattern| message.contains(pattern)) - { - CaptureGitFailure::NoRepository - } else { - CaptureGitFailure::Refused + for line in message.lines() { + let Some(reason) = line.strip_prefix("fatal: ") else { + continue; + }; + let starts = |start: &str| { + reason + .get(..start.len()) + .is_some_and(|text| text.eq_ignore_ascii_case(start)) + }; + if starts(NO_REPOSITORY_MESSAGE) { + return CaptureGitFailure::NoRepository; + } + if STOPPED_MESSAGES.into_iter().any(starts) { + return CaptureGitFailure::Stopped; + } } + CaptureGitFailure::Refused } /// A git query of the capture files: the directory it runs in, its @@ -1039,11 +1060,12 @@ where let mut state = CaptureGitState::default(); // Whether git found a repository the project is in; only where git - // didn't run or answer is that read from the `.git` entries. + // didn't run, answer, or get past a filesystem boundary or a broken + // `.git` file is that read from the `.git` entries. let in_repository = match &project { Ok(_) | Err(CaptureGitFailure::Refused) => true, Err(CaptureGitFailure::NoRepository) => false, - Err(CaptureGitFailure::Unavailable) => project_dir + Err(CaptureGitFailure::Stopped | CaptureGitFailure::Unavailable) => project_dir .ancestors() .any(|directory| marks_a_repository(&directory.join(".git"))), }; @@ -1060,6 +1082,10 @@ where } Err(_) => false, }; + // Only where git couldn't be run in `.lpm` is a repository of its own + // read from its `.git` entry; one git ran past isn't a repository. + let own_repository = matches!(inside, Some(Err(CaptureGitFailure::Unavailable))) + && marks_a_repository(&lpm_dir.join(".git")); let inside_answered = match inside { Some(Ok(files)) => { let (own, project_files): (Vec, Vec) = files @@ -1085,9 +1111,9 @@ where true } // Git in `.lpm` stopped short of the project's repository, as at a - // filesystem boundary, so whether it ignores the files there isn't - // known. - Some(Err(CaptureGitFailure::NoRepository)) => { + // filesystem boundary, or found none there, so whether it ignores + // the files there isn't known. + Some(Err(CaptureGitFailure::NoRepository | CaptureGitFailure::Stopped)) => { if in_repository { state.unanswered.push(CaptureRepository::Lpm); } @@ -1103,7 +1129,6 @@ where } None => false, }; - let own_repository = lpm_exists && marks_a_repository(&lpm_dir.join(".git")); (project_answered || inside_answered || in_repository || own_repository).then_some(state) } @@ -2884,7 +2909,7 @@ mod tests { let warnings = warnings_for(Some(refused), false); assert!( warnings.iter().any(|warning| warning - .starts_with("git didn't answer for the project's repository (it couldn't run, refused, or took over 2 s)")), + .starts_with("git didn't answer for the project's repository (it couldn't run, took over 2 s, refused,")), "{warnings:?}" ); @@ -2964,17 +2989,116 @@ mod tests { std::fs::create_dir(project.path().join(".lpm")).unwrap(); // As at a filesystem boundary between the project and `.lpm`. + for inside in [CaptureGitFailure::Stopped, CaptureGitFailure::NoRepository] { + let (state, _) = + state_from_answers(project.path(), Ok(CaptureGitFiles::default()), Err(inside)) + .await; + + let state = state.unwrap(); + assert_eq!(state.unanswered, vec![CaptureRepository::Lpm]); + assert!(!state.lpm_unasked); + assert!(!state.kept_out()); + } + } + + #[tokio::test] + async fn reads_the_git_entries_where_git_stops_short_of_the_project_s_repository() { + let repository = tempfile::tempdir().unwrap(); + repository_entry(repository.path()); + let project = repository.path().join("app"); + std::fs::create_dir_all(project.join(".lpm")).unwrap(); + let (state, _) = state_from_answers( - project.path(), - Ok(CaptureGitFiles::default()), - Err(CaptureGitFailure::NoRepository), + &project, + Err(CaptureGitFailure::Stopped), + Err(CaptureGitFailure::Stopped), ) .await; - let state = state.unwrap(); - assert_eq!(state.unanswered, vec![CaptureRepository::Lpm]); - assert!(!state.lpm_unasked); - assert!(!state.kept_out()); + assert_eq!( + state.unwrap().unanswered, + vec![CaptureRepository::Project, CaptureRepository::Lpm] + ); + let outside = tempfile::tempdir().unwrap(); + std::fs::create_dir(outside.path().join(".lpm")).unwrap(); + let (state, _) = state_from_answers( + outside.path(), + Err(CaptureGitFailure::Stopped), + Err(CaptureGitFailure::Stopped), + ) + .await; + assert_eq!(state, None); + } + + #[tokio::test] + async fn asks_for_a_check_where_a_broken_git_file_stops_git_inside_a_repository() { + let repository = capture_project(&["app/.lpm/inspector.db"]); + git(repository.path(), &["add", "app/.lpm/inspector.db"]); + let project = repository.path().join("app"); + for entry in ["not a pointer\n", "gitdir: \n", "gitdir: ../gone/.git\n"] { + std::fs::write(project.join(".git"), entry).unwrap(); + + let state = capture_git_state(&project).await.unwrap(); + + assert_eq!( + state.unanswered, + vec![CaptureRepository::Project, CaptureRepository::Lpm], + "{entry:?}" + ); + let warnings = warnings_for(Some(state), false); + assert!(!warnings[0].contains("git ignores it"), "{entry:?}"); + assert!( + warnings[1].contains("a broken .git file"), + "{entry:?}: {warnings:?}" + ); + } + } + + #[tokio::test] + async fn says_nothing_of_git_for_a_stale_lpm_git_entry_git_ran_past() { + for entry in ["gitdir: ../gone/.git\n", "HEAD only"] { + let project = tempfile::tempdir().unwrap(); + let lpm_git = project.path().join(".lpm/.git"); + std::fs::create_dir_all(lpm_git.parent().unwrap()).unwrap(); + std::fs::write(project.path().join(".lpm/inspector.db"), b"x").unwrap(); + if entry == "HEAD only" { + repository_entry(&project.path().join(".lpm")); + } else { + std::fs::write(&lpm_git, entry).unwrap(); + } + assert!(marks_a_repository(&lpm_git), "{entry:?}"); + + let state = capture_git_state(project.path()).await; + + assert_eq!(state, None, "{entry:?}"); + assert!( + !warnings_for(state, false)[0].contains("git ignores it"), + "{entry:?}" + ); + } + } + + #[tokio::test] + async fn reads_lpm_s_own_git_entry_only_where_git_couldn_t_run_there() { + let project = tempfile::tempdir().unwrap(); + repository_entry(&project.path().join(".lpm")); + + for (inside, unanswered) in [ + ( + CaptureGitFailure::Unavailable, + Some(vec![CaptureRepository::Lpm]), + ), + (CaptureGitFailure::Stopped, None), + (CaptureGitFailure::NoRepository, None), + ] { + let (state, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::NoRepository), + Err(inside), + ) + .await; + assert_eq!(state.map(|state| state.unanswered), unanswered); + } } /// Makes `directory/.git` as `git init` leaves its `HEAD`. @@ -3015,24 +3139,53 @@ mod tests { } #[test] - fn reads_git_s_failure_as_no_repository_only_where_it_says_so() { + fn reads_git_s_failure_from_the_start_of_its_fatal_line() { for (stderr, failure) in [ ( "fatal: not a git repository (or any of the parent directories): .git\n", CaptureGitFailure::NoRepository, ), + ( + "fatal: Not a git repository (or any of the parent directories): .git\n", + CaptureGitFailure::NoRepository, + ), ( "fatal: not a git repository (or any parent up to mount point /mnt)\n\ Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).\n", - CaptureGitFailure::NoRepository, + CaptureGitFailure::Stopped, + ), + ( + "fatal: Not a git repository (or any parent up to mount point /mnt)\n", + CaptureGitFailure::Stopped, ), ( "fatal: invalid gitfile format: /work/app/.git\n", - CaptureGitFailure::NoRepository, + CaptureGitFailure::Stopped, + ), + ( + "fatal: Invalid gitfile format: /work/app/.git\n", + CaptureGitFailure::Stopped, ), ( "fatal: no path in gitfile: /work/app/.git\n", - CaptureGitFailure::NoRepository, + CaptureGitFailure::Stopped, + ), + ( + "fatal: No path in gitfile: /work/app/.git\n", + CaptureGitFailure::Stopped, + ), + ( + "fatal: not a git repository: /work/gone/.git\n", + CaptureGitFailure::Stopped, + ), + ( + "fatal: detected dubious ownership in repository at \ + '/work/not a git repository (or any of the parent directories)'\n", + CaptureGitFailure::Refused, + ), + ( + "warning: not a git repository (or any of the parent directories)\n", + CaptureGitFailure::Refused, ), ( "fatal: detected dubious ownership in repository at '/work/app'\n", diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index 29df8466d..c02c1d503 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -1378,6 +1378,8 @@ fn create_owner_only(_path: &Path, _owner: CaptureOwner) -> std::io::Result Result { let copy_path = capture_file(db_path, "-copy"); + // Taken before anything is opened, as leaving WAL grows the database. + let needed = room_to_copy(db_path); let owner_only = create_owner_only(©_path, owner); #[cfg(test)] let owner_only = tests::COPY_CREATE_ERROR @@ -1387,7 +1389,7 @@ fn replace_with_owner_only_copy(db_path: &Path, owner: CaptureOwner) -> Result Result Result Result<(), OpenError> { - ensure_room_to_copy(db_path)?; - let made_by_check: Vec = ["-wal", "-shm"] - .into_iter() - .map(|suffix| capture_file(db_path, suffix)) - .filter(|sidecar| { - matches!(sidecar.symlink_metadata(), Err(error) if error.kind() == std::io::ErrorKind::NotFound) - }) - .collect(); - let copied = check_then_copy(db_path, copy_path); - if copied.is_err() { - for sidecar in &made_by_check { - if let Err(error) = remove_if_present(sidecar) { - tracing::warn!("could not remove {}: {error}", sidecar.display()); - } - } +/// file made for the copy. A check that fails leaves the WAL files its +/// connection made: they hold no captures, SQLite removes them as the last +/// connection closes, and a connection that attached meanwhile, as an +/// earlier release or another tool can without the opening lock, may be +/// using them. +fn copy_checked_database( + db_path: &Path, + copy_path: &Path, + needed: Option, +) -> Result<(), OpenError> { + ensure_room_to_copy(db_path, needed)?; + #[cfg(test)] + if let Some(attach) = tests::BEFORE_CHECK.with_borrow_mut(Option::take) { + attach(); } - copied -} - -fn check_then_copy(db_path: &Path, copy_path: &Path) -> Result<(), OpenError> { - let database = open_nofollow_connection(db_path).map_err(checking(db_path))?; - refuse_unknown_schema(&database, db_path, checking(db_path))?; + let database = open_nofollow_connection(db_path).map_err(checking(db_path, needed))?; + refuse_unknown_schema(&database, db_path, checking(db_path, needed))?; // In WAL, which every LPM connection uses, leaving it needs the only // connection, so it proves no other LPM process has the database open, // and folds the WAL into the file. @@ -1448,18 +1440,18 @@ fn check_then_copy(db_path: &Path, copy_path: &Path) -> Result<(), OpenError> { Ok(mode) if mode.eq_ignore_ascii_case("delete") => {} Ok(_) => return Err(in_use(db_path)), Err(error) if is_busy(&error) => return Err(in_use(db_path)), - Err(error) => return Err(checking(db_path)(error)), + Err(error) => return Err(checking(db_path, needed)(error)), } refuse_damage(&database, db_path)?; - copy_database(&database, copy_path, db_path) + copy_database(&database, copy_path, db_path, needed) } /// Reads a full disk while the database is checked as the copy's want of /// room, and any other failure as [`damaged`] does. -fn checking(db_path: &Path) -> impl Fn(rusqlite::Error) -> OpenError + '_ { +fn checking(db_path: &Path, needed: Option) -> impl Fn(rusqlite::Error) -> OpenError + '_ { move |error| { if out_of_room(&error) { - no_room_to_copy(db_path) + no_room_to_copy(db_path, needed) } else { damaged(db_path)(error) } @@ -1486,30 +1478,37 @@ fn shared_memory_room(wal: u64) -> u64 { (64 * 1024u64).saturating_add(wal / 512) } +/// Free space a filesystem reports but won't let a write use: APFS keeps a +/// few MiB back, about 2.5 MB on a 96 MB volume. +const UNUSABLE_FREE_ROOM: u64 = 8 * 1024 * 1024; + /// The free space replacing a database takes. Leaving WAL folds the WAL into /// the database, which grows by the WAL at most while the shared-memory file /// is open; then the WAL and that file are removed, and the copy takes the /// grown database's size. So it never takes more than the database and its -/// WAL, with the shared-memory file on top. +/// WAL, with the shared-memory file and what a filesystem keeps back on top. fn room_to_copy(db_path: &Path) -> Option { let database = std::fs::metadata(db_path).ok()?.len(); let wal = std::fs::metadata(capture_file(db_path, "-wal")).map_or(0, |wal| wal.len()); Some( database .saturating_add(wal) - .saturating_add(shared_memory_room(wal)), + .saturating_add(shared_memory_room(wal)) + .saturating_add(UNUSABLE_FREE_ROOM), ) } /// Refuses a replacement its disk can't hold before the database is /// opened, as SQLite's own failure on a full disk can come as a bare I/O -/// error and leave its files behind. -fn ensure_room_to_copy(db_path: &Path) -> Result<(), OpenError> { +/// error. +fn ensure_room_to_copy(db_path: &Path, needed: Option) -> Result<(), OpenError> { let available = lpm_common::filesystem_available_bytes(db_path); #[cfg(test)] let available = tests::AVAILABLE_BYTES.get().or(available); - match (available, room_to_copy(db_path)) { - (Some(available), Some(needed)) if available < needed => Err(no_room_to_copy(db_path)), + match (available, needed) { + (Some(available), Some(needed)) if available < needed => { + Err(no_room_to_copy(db_path, Some(needed))) + } _ => Ok(()), } } @@ -1533,7 +1532,12 @@ fn in_use(db_path: &Path) -> OpenError { /// refused rather than taken for the database. The copy takes as much /// space as the database, free pages included, so a disk without it is /// named as the cause. -fn copy_database(database: &Connection, copy_path: &Path, db_path: &Path) -> Result<(), OpenError> { +fn copy_database( + database: &Connection, + copy_path: &Path, + db_path: &Path, + needed: Option, +) -> Result<(), OpenError> { let mut copy = open_nofollow_connection(copy_path)?; #[cfg(test)] if let Some(pages) = tests::COPY_PAGE_LIMIT.get() { @@ -1543,13 +1547,15 @@ fn copy_database(database: &Connection, copy_path: &Path, db_path: &Path) -> Res match backup.step(-1) { Ok(rusqlite::backup::StepResult::Done) => Ok(()), Ok(_) => Err(in_use(db_path)), - Err(error) if out_of_room(&error) => Err(no_room_to_copy(db_path)), + Err(error) if out_of_room(&error) => Err(no_room_to_copy(db_path, needed)), Err(error) => Err(damaged(db_path)(error)), } } -fn no_room_to_copy(db_path: &Path) -> OpenError { - let needed = room_to_copy(db_path).map_or_else( +/// The refusal of a replacement its disk lacks room for, naming the room +/// taken before the database was opened. +fn no_room_to_copy(db_path: &Path, needed: Option) -> OpenError { + let needed = needed.map_or_else( || "as much as it takes".to_owned(), |bytes| format!("about {}", lpm_common::format_bytes(bytes)), ); @@ -1679,9 +1685,12 @@ type SchemaObject = (String, String, String, Option); /// text, a string or a quoted name, and comments are kept as written, a /// line comment with the newline that ends it. Whitespace is SQLite's own: /// ASCII space, tab, newline, form feed and carriage return. So schema text -/// whose tokens differ for SQLite can't read the same here. Only a -/// parameter's Tcl form takes a parenthesis into its token (`$x(y)` against -/// `$x (y)`), and SQLite allows no parameter in a schema. +/// whose tokens differ for SQLite can't read the same here, but for one +/// case: a parameter's Tcl form, `$x(y)` and its `@`, `:` and `#` kin, takes +/// a parenthesis into its token, so it reads the same as `$x (y)`. SQLite +/// creates no schema with a parameter, but a crafted file's schema can hold +/// one; LPM's schema has none of those characters outside quotes, so such +/// text never reads as LPM's. fn canonical_sql(sql: &str) -> String { let separates = |character: char| matches!(character, '(' | ')' | ','); let mut canonical = String::with_capacity(sql.len()); @@ -2407,6 +2416,10 @@ mod tests { /// a disk with no blocks left fails them. pub(super) static COPY_CREATE_ERROR: std::cell::Cell> = const { std::cell::Cell::new(None) }; + /// Runs once as this thread's next replacement starts its check, as + /// another process can open the database then. + pub(super) static BEFORE_CHECK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; } #[cfg(unix)] @@ -3238,7 +3251,8 @@ mod tests { #[test] fn a_disk_that_fills_while_the_database_is_checked_says_how_much_room_the_copy_needs() { let directory = tempfile::tempdir().unwrap(); - let attached = attach_image(directory.path(), "4m", true); + // Past the room a replacement keeps back, for the retry. + let attached = attach_image(directory.path(), "16m", true); let project = attached.0.join("project"); std::fs::create_dir(&project).unwrap(); let runtime = tokio::runtime::Builder::new_current_thread() @@ -3272,6 +3286,7 @@ mod tests { .unwrap(); drop(writer); set_mode(&db_path, 0o644); + let needed = room_to_copy(&db_path).unwrap(); let filler = attached.0.join("filler"); let free = lpm_common::filesystem_available_bytes(&attached.0).unwrap(); std::fs::write( @@ -3286,7 +3301,9 @@ mod tests { let error = full.unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::StorageFull, "{error}"); - assert!(error.to_string().contains("free on its disk"), "{error}"); + // The room taken before the WAL was folded in, which grew the database. + let named = format!("needs about {} free", lpm_common::format_bytes(needed)); + assert!(error.to_string().contains(&named), "{error}"); assert_eq!(mode(&db_path), 0o644); assert!(!capture_file(&db_path, "-copy").exists()); @@ -3309,7 +3326,8 @@ mod tests { use std::io::Write as _; let directory = tempfile::tempdir().unwrap(); - let attached = attach_image(directory.path(), "4m", true); + // Past the room a replacement keeps back, for the retry. + let attached = attach_image(directory.path(), "16m", true); let project = attached.0.join("project"); std::fs::create_dir(&project).unwrap(); let runtime = tokio::runtime::Builder::new_current_thread() @@ -3341,12 +3359,79 @@ mod tests { assert_eq!(error.kind(), std::io::ErrorKind::StorageFull, "{error}"); assert!(error.to_string().contains("free on its disk"), "{error}"); assert_eq!(mode(&db_path), 0o644); - for suffix in ["-copy", "-wal", "-shm"] { - assert!( - !capture_file(&db_path, suffix).exists(), - "{suffix} left behind" - ); + assert!(!capture_file(&db_path, "-copy").exists()); + // What the check's connection left holds no captures. + let wal = capture_file(&db_path, "-wal"); + assert!(!wal.exists() || std::fs::metadata(&wal).unwrap().len() == 0); + + std::fs::remove_file(attached.0.join("filler")).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let kept = runtime.block_on(async { + let db = InspectorDb::open(&project).unwrap(); + db.get_webhook("kept").await.unwrap() + }); + assert!(kept.is_some()); + assert_eq!(mode(&db_path), 0o600); + } + + #[cfg(unix)] + #[test] + fn a_connection_that_attaches_during_the_check_keeps_its_wal_files() { + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + set_mode(&db_path, 0o644); + // As an earlier release, which takes no opening lock, or another + // tool can, once the room was taken and before the check opens it. + let other = std::rc::Rc::new(std::cell::RefCell::new(None)); + let held = std::rc::Rc::clone(&other); + let path = db_path.clone(); + BEFORE_CHECK.with_borrow_mut(|hook| { + *hook = Some(Box::new(move || { + let connection = Connection::open(&path).unwrap(); + connection + .query_row("SELECT count(*) FROM requests", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + *held.borrow_mut() = Some(connection); + })); + }); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let error = runtime + .block_on(async { InspectorDb::open(project.path()).map(drop) }) + .unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::ResourceBusy, "{error}"); + for suffix in ["-wal", "-shm"] { + assert!(capture_file(&db_path, suffix).exists(), "{suffix} removed"); } + let connection = other.borrow_mut().take().unwrap(); + connection + .execute( + "INSERT INTO requests (id, timestamp, method, path) \ + VALUES ('later', '2026-04-06T12:00:00Z', 'POST', '/')", + [], + ) + .unwrap(); + let reader = Connection::open(&db_path).unwrap(); + let later: i64 = reader + .query_row( + "SELECT count(*) FROM requests WHERE id = 'later'", + [], + |row| row.get(0), + ) + .unwrap(); + drop((connection, reader)); + assert_eq!(later, 1); + let integrity: String = Connection::open(&db_path) + .unwrap() + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(integrity, "ok"); } #[cfg(unix)] @@ -3908,7 +3993,7 @@ mod tests { assert!(create_owner_only(©_path, this_user()).unwrap()); COPY_PAGE_LIMIT.set(Some(1)); - let copied = copy_database(&database, ©_path, &db_path); + let copied = copy_database(&database, ©_path, &db_path, room_to_copy(&db_path)); COPY_PAGE_LIMIT.set(None); let Err(OpenError::Files(error)) = copied else { @@ -3964,13 +4049,16 @@ mod tests { let directory = tempfile::tempdir().unwrap(); let db_path = directory.path().join("inspector.db"); std::fs::write(&db_path, vec![0; 10_000]).unwrap(); - assert_eq!(room_to_copy(&db_path), Some(10_000 + 64 * 1024)); + assert_eq!( + room_to_copy(&db_path), + Some(10_000 + 64 * 1024 + UNUSABLE_FREE_ROOM) + ); let wal = std::fs::File::create(capture_file(&db_path, "-wal")).unwrap(); wal.set_len(512 * 1024 * 1024).unwrap(); assert_eq!( room_to_copy(&db_path), - Some(10_000 + 512 * 1024 * 1024 + 64 * 1024 + 1024 * 1024) + Some(10_000 + 512 * 1024 * 1024 + 64 * 1024 + 1024 * 1024 + UNUSABLE_FREE_ROOM) ); std::fs::remove_file(&db_path).unwrap(); assert_eq!(room_to_copy(&db_path), None); @@ -4026,7 +4114,7 @@ mod tests { let copy_path = capture_file(&db_path, "-copy"); assert!(create_owner_only(©_path, this_user()).unwrap()); - let copied = copy_database(&database, ©_path, &db_path); + let copied = copy_database(&database, ©_path, &db_path, room_to_copy(&db_path)); holder.execute_batch("COMMIT").unwrap(); assert!(matches!( @@ -4054,7 +4142,7 @@ mod tests { let copy_path = capture_file(&db_path, "-copy"); assert!(create_owner_only(©_path, this_user()).unwrap()); - copy_database(&database, ©_path, &db_path).unwrap(); + copy_database(&database, ©_path, &db_path, room_to_copy(&db_path)).unwrap(); released.join().unwrap(); let rows: i64 = Connection::open(capture_file(&db_path, "-copy")) From 7d079d7dbcdc13f318565713fa2afce6ec1bd2ff Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Tue, 13 Oct 2026 07:19:05 +0100 Subject: [PATCH 13/17] fix(tunnel): name the files that move with the capture database, count what APFS keeps back, and read .git files as git does - Every advice to move the capture database aside names its -wal, -shm and -journal files, which an open removes when it finds them without a database, and a hard-linked sidecar names the database. - The replacement's room check counts the space a write can take: what the filesystem reports free, less what APFS holds back (about 2% of the volume plus 1 MiB, at most 33 MiB), in place of a flat 8 MiB. The shared-memory room follows the WAL's page size. - A replacement removes files another connection left beside the database before the copy takes its name, and checks its room before anything opens the database. - A .git file is read as git reads it: up to 1 MiB, the whole of it after `gitdir: ` but for its trailing line breaks, pointing at a directory holding HEAD. Git refusing a repository in .lpm, outside the project's, asks for a check. - The test of a file swapped after its check uses another name of an existing file, which never shares its inode number, as a new file on ext4 can. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/tunnel.rs | 205 ++++++++++++++-- crates/lpm-common/src/lib.rs | 178 +++++++++++++- crates/lpm-inspect/src/db.rs | 322 ++++++++++++++++++++++---- 3 files changed, 624 insertions(+), 81 deletions(-) diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index 0d5d58b4c..067229a14 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -968,11 +968,14 @@ enum CaptureGitFailure { /// How git's `fatal: ` line starts, with `LC_ALL=C`, where it found no /// repository, and where it stopped looking. const NO_REPOSITORY_MESSAGE: &str = "not a git repository (or any of the parent directories)"; -const STOPPED_MESSAGES: [&str; 4] = [ +const STOPPED_MESSAGES: [&str; 7] = [ "not a git repository (or any parent up to mount", "not a git repository: ", "invalid gitfile format", "no path in gitfile", + "error opening '", + "error reading ", + "too large to be a .git file", ]; /// Reads git's failure from the start of its `fatal: ` line, as the rest can @@ -1082,10 +1085,13 @@ where } Err(_) => false, }; - // Only where git couldn't be run in `.lpm` is a repository of its own - // read from its `.git` entry; one git ran past isn't a repository. - let own_repository = matches!(inside, Some(Err(CaptureGitFailure::Unavailable))) - && marks_a_repository(&lpm_dir.join(".git")); + // `.lpm` has a repository of its own where git refused to use one there, + // and, where git couldn't be run there, where its `.git` entry says so. + let own_repository = match &inside { + Some(Err(CaptureGitFailure::Refused)) => true, + Some(Err(CaptureGitFailure::Unavailable)) => marks_a_repository(&lpm_dir.join(".git")), + _ => false, + }; let inside_answered = match inside { Some(Ok(files)) => { let (own, project_files): (Vec, Vec) = files @@ -1114,9 +1120,7 @@ where // filesystem boundary, or found none there, so whether it ignores // the files there isn't known. Some(Err(CaptureGitFailure::NoRepository | CaptureGitFailure::Stopped)) => { - if in_repository { - state.unanswered.push(CaptureRepository::Lpm); - } + state.unanswered.push(CaptureRepository::Lpm); false } Some(Err(CaptureGitFailure::Refused)) => { @@ -1137,18 +1141,46 @@ fn file_name(name: &str) -> &str { name.rsplit_once('/').map_or(name, |(_, file)| file) } +/// The largest `.git` file git reads; it stops at a larger one. +const GITFILE_LIMIT: u64 = 1024 * 1024; + /// Whether a `.git` entry looks like a repository's, for when git itself -/// can't be asked: a directory, or a link to one, holding `HEAD`, or a file -/// pointing at one as git writes it (`gitdir: …`). An empty directory or -/// another file doesn't, as git looks past them. +/// can't be asked, or stopped: a directory, or a link to one, holding +/// `HEAD`, or a file pointing at one as git reads it: `gitdir: ` and the +/// rest of the file, but for the line breaks that end it, a path from the +/// file's directory. An empty directory, another file, or a pointer to no +/// repository doesn't, as git looks past the first two and stops at the +/// last, and at a file too large to be a pointer. fn marks_a_repository(git: &Path) -> bool { match std::fs::metadata(git) { Ok(metadata) if metadata.is_dir() => git.join("HEAD").is_file(), Ok(metadata) if metadata.is_file() => { - let mut start = [0_u8; 8]; - lpm_common::open_nofollow(git) - .and_then(|mut file| std::io::Read::read_exact(&mut file, &mut start)) - .is_ok_and(|()| &start == b"gitdir: ") + let mut contents = Vec::new(); + let read = lpm_common::open_nofollow(git).and_then(|file| { + std::io::Read::read_to_end( + &mut std::io::Read::take(file, GITFILE_LIMIT + 1), + &mut contents, + ) + }); + if read.is_err() || contents.len() as u64 > GITFILE_LIMIT { + return false; + } + let Some(pointer) = contents + .strip_prefix(b"gitdir: ") + .map(|rest| { + let end = rest + .iter() + .rposition(|byte| !matches!(byte, b'\n' | b'\r')) + .map_or(0, |last| last + 1); + &rest[..end] + }) + .and_then(|path| std::str::from_utf8(path).ok()) + .filter(|path| !path.is_empty()) + else { + return false; + }; + let directory = git.parent().unwrap_or(Path::new(".")); + directory.join(pointer).join("HEAD").is_file() } _ => false, } @@ -2846,7 +2878,11 @@ mod tests { ), ( Err(CaptureGitFailure::NoRepository), - Err(CaptureGitFailure::Refused), + Err(CaptureGitFailure::Stopped), + ), + ( + Err(CaptureGitFailure::Stopped), + Err(CaptureGitFailure::Stopped), ), ] { let (state, _) = @@ -2854,6 +2890,18 @@ mod tests { assert_eq!(state, None); } + // A repository git refused in `.lpm`, where the project's query found + // none, is `.lpm`'s own, so it's asked to be checked. + let (refused, _) = state_from_answers( + project.path(), + Err(CaptureGitFailure::NoRepository), + Err(CaptureGitFailure::Refused), + ) + .await; + let refused = refused.unwrap(); + assert!(refused.lpm_unasked, "{refused:?}"); + assert!(refused.unanswered.is_empty(), "{refused:?}"); + // Git in `.lpm` answered, so only the project's query went unanswered, // and outside a repository there was nothing for it to find. let (answered, _) = state_from_answers( @@ -2998,6 +3046,15 @@ mod tests { assert_eq!(state.unanswered, vec![CaptureRepository::Lpm]); assert!(!state.lpm_unasked); assert!(!state.kept_out()); + let warnings = warnings_for(Some(state), false); + assert!( + warnings.iter().any(|warning| warning + == "git didn't answer in .lpm (it couldn't run, took over 2 s, or stopped \ + short of the project's repository, as at a filesystem boundary or a \ + broken .git file), so check that git ignores and doesn't track \ + .lpm/inspector.db*"), + "{warnings:?}" + ); } } @@ -3055,7 +3112,7 @@ mod tests { } #[tokio::test] - async fn says_nothing_of_git_for_a_stale_lpm_git_entry_git_ran_past() { + async fn says_nothing_of_git_for_an_lpm_git_entry_git_stops_at_or_looks_past() { for entry in ["gitdir: ../gone/.git\n", "HEAD only"] { let project = tempfile::tempdir().unwrap(); let lpm_git = project.path().join(".lpm/.git"); @@ -3066,7 +3123,6 @@ mod tests { } else { std::fs::write(&lpm_git, entry).unwrap(); } - assert!(marks_a_repository(&lpm_git), "{entry:?}"); let state = capture_git_state(project.path()).await; @@ -3079,15 +3135,16 @@ mod tests { } #[tokio::test] - async fn reads_lpm_s_own_git_entry_only_where_git_couldn_t_run_there() { + async fn takes_lpm_for_a_repository_of_its_own_where_git_refused_or_couldn_t_run_there() { let project = tempfile::tempdir().unwrap(); repository_entry(&project.path().join(".lpm")); for (inside, unanswered) in [ ( CaptureGitFailure::Unavailable, - Some(vec![CaptureRepository::Lpm]), + Some((vec![CaptureRepository::Lpm], false)), ), + (CaptureGitFailure::Refused, Some((Vec::new(), true))), (CaptureGitFailure::Stopped, None), (CaptureGitFailure::NoRepository, None), ] { @@ -3097,10 +3154,71 @@ mod tests { Err(inside), ) .await; - assert_eq!(state.map(|state| state.unanswered), unanswered); + assert_eq!( + state.map(|state| (state.unanswered, state.lpm_unasked)), + unanswered + ); + } + } + + #[tokio::test] + async fn asks_for_a_check_of_an_lpm_repository_git_refuses_outside_any_project_repository() { + let project = tempfile::tempdir().unwrap(); + let lpm_dir = project.path().join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + git(&lpm_dir, &["init", "--quiet"]); + std::fs::write(lpm_dir.join("inspector.db"), b"x").unwrap(); + git(&lpm_dir, &["add", "inspector.db"]); + // A repository whose format git doesn't know, which it refuses. + git(&lpm_dir, &["config", "core.repositoryformatversion", "1"]); + git(&lpm_dir, &["config", "extensions.lpmUnknown", "true"]); + + let state = capture_git_state(project.path()).await; + + let warnings = warnings_for(state, false); + assert!(!warnings[0].contains("git ignores it"), "{warnings:?}"); + assert!( + warnings.iter().any(|warning| warning + == "git couldn't be asked about .lpm, so check that git ignores and doesn't \ + track inspector.db*"), + "{warnings:?}" + ); + } + + #[tokio::test] + async fn says_nothing_of_git_for_a_project_whose_own_git_file_points_nowhere_outside_any_repository() + { + for entry in ["gitdir: ../gone/.git\n", "gitdir: \n", "not a pointer\n"] { + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + std::fs::write(project.path().join(".lpm/inspector.db"), b"x").unwrap(); + std::fs::write(project.path().join(".git"), entry).unwrap(); + + assert_eq!(capture_git_state(project.path()).await, None, "{entry:?}"); } } + #[cfg(unix)] + #[tokio::test] + async fn says_nothing_of_git_for_a_project_whose_own_git_file_it_can_t_read_outside_any_repository() + { + use std::os::unix::fs::PermissionsExt as _; + + if lpm_common::current_uid() == 0 { + return; + } + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir(project.path().join(".lpm")).unwrap(); + let git_file = project.path().join(".git"); + std::fs::write(&git_file, "gitdir: elsewhere\n").unwrap(); + std::fs::set_permissions(&git_file, std::fs::Permissions::from_mode(0o000)).unwrap(); + + let state = capture_git_state(project.path()).await; + + std::fs::set_permissions(&git_file, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!(state, None); + } + /// Makes `directory/.git` as `git init` leaves its `HEAD`. fn repository_entry(directory: &Path) { std::fs::create_dir_all(directory.join(".git")).unwrap(); @@ -3112,15 +3230,39 @@ mod tests { let directory = tempfile::tempdir().unwrap(); let git = directory.path().join(".git"); + repository_entry(&directory.path().join("elsewhere")); + let absolute = format!( + "gitdir: {}\n", + directory.path().join("elsewhere/.git").display() + ); for (contents, repository) in [ ("", false), ("not a pointer\n", false), - ("gitdir:../elsewhere/.git\n", false), - ("gitdir: ../elsewhere/.git\n", true), + ("gitdir:elsewhere/.git\n", false), + ("gitdir: \n", false), + ("gitdir: gone/.git\n", false), + ("gitdir: elsewhere/.git\n", true), + ("gitdir: elsewhere/.git\r\n", true), + ("gitdir: elsewhere/.git\n\n\r\n", true), + (absolute.as_str(), true), + // Git reads the whole file as the path: a second line, or a + // space at its end, is part of it. + ("gitdir: elsewhere/.git\nsecond line\n", false), + ("gitdir: elsewhere/.git \n", false), + // A directory without `HEAD` isn't a repository. + ("gitdir: elsewhere\n", false), ] { std::fs::write(&git, contents).unwrap(); assert_eq!(marks_a_repository(&git), repository, "{contents:?}"); } + // Git reads a pointer of up to 1 MiB, and stops at a larger file. + let mut padded = b"gitdir: elsewhere/.git".to_vec(); + padded.resize(1024 * 1024, b'\n'); + std::fs::write(&git, &padded).unwrap(); + assert!(marks_a_repository(&git)); + padded.push(b'\n'); + std::fs::write(&git, &padded).unwrap(); + assert!(!marks_a_repository(&git)); std::fs::remove_file(&git).unwrap(); std::fs::create_dir(&git).unwrap(); @@ -3178,6 +3320,23 @@ mod tests { "fatal: not a git repository: /work/gone/.git\n", CaptureGitFailure::Stopped, ), + ( + "fatal: error opening '/work/app/.git': Permission denied\n", + CaptureGitFailure::Stopped, + ), + ( + "fatal: too large to be a .git file: '/work/app/.git'\n", + CaptureGitFailure::Stopped, + ), + ( + "fatal: error reading /work/app/.git\n", + CaptureGitFailure::Stopped, + ), + ( + "warning: unable to access '/home/user/.config/git/attributes'\n\ + fatal: not a git repository (or any of the parent directories): .git\n", + CaptureGitFailure::NoRepository, + ), ( "fatal: detected dubious ownership in repository at \ '/work/not a git repository (or any of the parent directories)'\n", diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index 7a8edf563..ac51bdde6 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -317,8 +317,27 @@ pub fn is_loopback_host(host: &str) -> bool { /// The bytes free to this user on the filesystem holding `path`, or its /// nearest existing ancestor; None where it can't be read. -#[cfg(unix)] pub fn filesystem_available_bytes(path: &std::path::Path) -> Option { + filesystem_space(path).map(|space| space.available) +} + +/// The bytes a write by this user can take on the filesystem holding +/// `path`, or its nearest existing ancestor: what it reports free, less +/// what APFS keeps back from writes (see [`apfs_held_back`]). None where it +/// can't be read. +pub fn filesystem_writable_bytes(path: &std::path::Path) -> Option { + filesystem_space(path).map(|space| space.available.saturating_sub(space.held_back)) +} + +/// What a filesystem reports free to this user, and how much of that it +/// keeps back from writes. +struct FilesystemSpace { + available: u64, + held_back: u64, +} + +#[cfg(unix)] +fn filesystem_space(path: &std::path::Path) -> Option { use std::ffi::CString; use std::os::unix::ffi::OsStrExt; @@ -333,7 +352,7 @@ pub fn filesystem_available_bytes(path: &std::path::Path) -> Option { // blocks of a volume with more than 16 TiB free; its `statfs` counts in // 64 bits. #[cfg(target_os = "macos")] - let bytes = { + let (available, held_back) = { let mut stats = std::mem::MaybeUninit::::uninit(); // SAFETY: `path` is NUL-terminated and `stats` points to writable, // properly aligned storage that is read only after a successful call. @@ -342,10 +361,25 @@ pub fn filesystem_available_bytes(path: &std::path::Path) -> Option { } // SAFETY: `statfs` returned success and initialized the output structure. let stats = unsafe { stats.assume_init() }; - u128::from(stats.f_bavail).checked_mul(u128::from(stats.f_bsize))? + let block = u128::from(stats.f_bsize); + let available = u128::from(stats.f_bavail).checked_mul(block)?; + let apfs = stats + .f_fstypename + .iter() + .take_while(|&&byte| byte != 0) + .map(|&byte| byte as u8) + .eq(*b"apfs"); + let held_back = if apfs { + let size = u128::from(stats.f_blocks).checked_mul(block)?; + apfs_held_back(u64::try_from(size).unwrap_or(u64::MAX)) + } else { + 0 + }; + (available, held_back) }; + // `f_bavail` already leaves out what's reserved for root. #[cfg(not(target_os = "macos"))] - let bytes = { + let (available, held_back) = { let mut stats = std::mem::MaybeUninit::::uninit(); // SAFETY: `path` is NUL-terminated and `stats` points to writable, // properly aligned storage that is read only after a successful call. @@ -354,16 +388,35 @@ pub fn filesystem_available_bytes(path: &std::path::Path) -> Option { } // SAFETY: `statvfs` returned success and initialized the output structure. let stats = unsafe { stats.assume_init() }; - u128::from(stats.f_bavail).checked_mul(u128::from(stats.f_frsize))? + ( + u128::from(stats.f_bavail).checked_mul(u128::from(stats.f_frsize))?, + 0, + ) }; - u64::try_from(bytes).ok() + Some(FilesystemSpace { + available: u64::try_from(available).ok()?, + held_back, + }) } #[cfg(not(unix))] -pub fn filesystem_available_bytes(_path: &std::path::Path) -> Option { +fn filesystem_space(_path: &std::path::Path) -> Option { None } +/// What APFS reports free on a volume of `size` bytes but refuses writes: +/// a write there fails with about 2% of the volume and 460 KB more still +/// reported free, and never more than 32 MiB and 80 KiB, as measured on +/// volumes of 64 MB to 8 GB. Rounded up by about half a MiB, which also +/// covers the blocks a file's own metadata takes, so a write that would +/// just fit can be refused, but never one counted as fitting. +#[cfg(any(target_os = "macos", test))] +fn apfs_held_back(size: u64) -> u64 { + (size / 50) + .saturating_add(1024 * 1024) + .min(33 * 1024 * 1024) +} + /// Format bytes into a human-readable string (e.g., "1.2 KB", "3.4 MB"). pub fn format_bytes(bytes: u64) -> String { if bytes < 1024 { @@ -422,6 +475,117 @@ mod tests { assert!(unmade.abs_diff(available) <= size / 200); } + #[test] + fn apfs_holds_back_at_least_what_measured_volumes_kept_from_writes() { + // A volume's size, and what it still reported free once a write + // failed. + for (size, kept) in [ + (104_816_640, 2_564_096), + (314_531_840, 6_758_400), + (524_247_040, 10_952_704), + (1_073_700_864, 21_946_368), + (2_147_442_688, 33_636_352), + (4_085_211_136, 33_636_352), + (8_380_178_432, 33_636_352), + ] { + let held_back = apfs_held_back(size); + assert!(held_back >= kept, "{size}: {held_back} < {kept}"); + assert!( + held_back - kept < 1024 * 1024, + "{size}: {held_back} past {kept}" + ); + } + assert_eq!(apfs_held_back(u64::MAX), 33 * 1024 * 1024); + } + + #[cfg(unix)] + #[test] + fn on_any_filesystem_but_apfs_a_write_can_take_what_it_reports_free() { + let directory = tempfile::tempdir().unwrap(); + let space = filesystem_space(directory.path()).unwrap(); + if cfg!(target_os = "macos") && space.held_back > 0 { + return; + } + + assert_eq!(space.held_back, 0); + let writable = filesystem_writable_bytes(directory.path()).unwrap(); + let available = filesystem_available_bytes(directory.path()).unwrap(); + let (size, _) = df(directory.path()); + assert!(writable.abs_diff(available) <= size / 200); + } + + #[cfg(target_os = "macos")] + #[test] + fn on_apfs_a_write_takes_what_it_counts_writable_and_little_more() { + use std::io::Write as _; + + let directory = tempfile::tempdir().unwrap(); + let image = directory.path().join("volume.sparseimage"); + let mount_point = directory.path().join("mounted"); + std::fs::create_dir(&mount_point).unwrap(); + let hdiutil = |arguments: &[&std::ffi::OsStr]| { + let output = std::process::Command::new("/usr/bin/hdiutil") + .args(arguments) + .output() + .unwrap(); + assert!( + output.status.success(), + "hdiutil {arguments:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + }; + hdiutil(&[ + "create".as_ref(), + "-quiet".as_ref(), + "-size".as_ref(), + "64m".as_ref(), + "-fs".as_ref(), + "APFS".as_ref(), + "-type".as_ref(), + "SPARSE".as_ref(), + "-volname".as_ref(), + "lpm-common-test".as_ref(), + image.as_os_str(), + ]); + hdiutil(&[ + "attach".as_ref(), + "-quiet".as_ref(), + "-nobrowse".as_ref(), + "-noautoopen".as_ref(), + "-mountpoint".as_ref(), + mount_point.as_os_str(), + image.as_os_str(), + ]); + struct Detach<'a>(&'a std::path::Path); + impl Drop for Detach<'_> { + fn drop(&mut self) { + let _ = std::process::Command::new("/usr/bin/hdiutil") + .args(["detach", "-force", "-quiet"]) + .arg(self.0) + .status(); + } + } + let _detach = Detach(&mount_point); + + let space = filesystem_space(&mount_point).unwrap(); + assert!(space.held_back > 0); + let writable = filesystem_writable_bytes(&mount_point).unwrap(); + let mut filler = std::fs::File::create(mount_point.join("filler")).unwrap(); + let mut written = 0u64; + for chunk in [64 * 1024, 4096] { + let block = vec![0; chunk]; + while filler.write_all(&block).is_ok() { + written += chunk as u64; + } + } + + assert!(written >= writable, "wrote {written} of {writable}"); + assert!( + written - writable < 1024 * 1024, + "wrote {written}, far past {writable}" + ); + } + // ── read_capped_state_file ──────────────────────────────────────── /// Files under the cap round-trip transparently. diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index c02c1d503..7b470f929 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -830,8 +830,9 @@ pub enum OpenError { Files(#[from] std::io::Error), /// SQLite can't read the file as a database, or finds it damaged. #[error( - "{} is damaged or isn't a capture database ({detail}); move it aside and retry", - path.display() + "{} is damaged or isn't a capture database ({detail}); move it aside {}, and retry", + path.display(), + with_its_sidecars(path) )] Damaged { path: PathBuf, detail: String }, /// Any other SQLite failure. @@ -1017,17 +1018,34 @@ fn another_users(path: &Path) -> std::io::Error { ) } -fn hard_linked(path: &Path) -> std::io::Error { +fn hard_linked(path: &Path, db_path: &Path) -> std::io::Error { + let database = if path == db_path { + "it".to_owned() + } else { + db_path.display().to_string() + }; std::io::Error::new( std::io::ErrorKind::InvalidInput, format!( "{} has other hard links, so captures written to it would change those \ - files too; move it aside and retry", - path.display() + files too; move {database} aside {}, and retry", + path.display(), + with_its_sidecars(db_path) ), ) } +/// Names the files SQLite keeps beside a capture database, for advice to +/// move it: they go with it, as an open removes those it finds without a +/// database, and a WAL not yet folded into the database holds part of it. +fn with_its_sidecars(db_path: &Path) -> String { + let name = db_path + .file_name() + .unwrap_or(db_path.as_os_str()) + .to_string_lossy(); + format!("together with any {name}-wal, {name}-shm and {name}-journal beside it") +} + /// Makes sure `.lpm` is a directory of the project's own, creating it if /// needed. A link, which a cloned repository could point anywhere, such as /// into a directory it serves, is refused; the metadata of a link, read @@ -1167,7 +1185,7 @@ fn prepare_capture_files( match std::fs::symlink_metadata(&path) { Ok(metadata) if !metadata.is_file() => return Err(linked_or_not(&path, "file")), Ok(metadata) => { - open_to_others |= check_capture_file(&path, &metadata, owner, access)?; + open_to_others |= check_capture_file(&path, db_path, &metadata, owner, access)?; database_exists |= path == db_path; } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} @@ -1227,6 +1245,7 @@ fn remove_if_present(path: &Path) -> std::io::Result<()> { #[cfg(unix)] fn check_capture_file( path: &Path, + db_path: &Path, metadata: &std::fs::Metadata, owner: CaptureOwner, access: Access, @@ -1237,7 +1256,7 @@ fn check_capture_file( return Err(another_users(path)); } if metadata.nlink() > 1 { - return Err(hard_linked(path)); + return Err(hard_linked(path, db_path)); } let mode = metadata.mode() & 0o7777; #[cfg(target_os = "macos")] @@ -1300,6 +1319,7 @@ fn set_mode_through_handle( #[cfg(not(unix))] fn check_capture_file( path: &Path, + db_path: &Path, _metadata: &std::fs::Metadata, _owner: CaptureOwner, _access: Access, @@ -1310,7 +1330,7 @@ fn check_capture_file( Err(error) => return Err(with_path(path, error)), }; if lpm_common::link_count(&file).map_err(|error| with_path(path, error))? > 1 { - return Err(hard_linked(path)); + return Err(hard_linked(path, db_path)); } Ok(false) } @@ -1406,6 +1426,10 @@ fn replace_with_owner_only_copy(db_path: &Path, owner: CaptureOwner) -> Result bool { } /// The room SQLite's shared-memory file takes while the check reads a -/// database with `wal` bytes of WAL: 32 KiB for about every 4,096 frames, -/// about 1/512 of the WAL, and a region more for rounding. -fn shared_memory_room(wal: u64) -> u64 { - (64 * 1024u64).saturating_add(wal / 512) +/// database with `wal` bytes of WAL in pages of `page_size`: the WAL index +/// keeps a 32 KiB region for each 4,096 frames, the first region 4,062, +/// and a region more is counted for rounding. +fn shared_memory_room(wal: u64, page_size: u64) -> u64 { + const REGION: u64 = 32 * 1024; + const WAL_HEADER: u64 = 32; + const FRAME_HEADER: u64 = 24; + let frames = wal.saturating_sub(WAL_HEADER) / (page_size + FRAME_HEADER); + let regions = 1 + frames.saturating_sub(4062).div_ceil(4096); + (regions + 1) * REGION } -/// Free space a filesystem reports but won't let a write use: APFS keeps a -/// few MiB back, about 2.5 MB on a 96 MB volume. -const UNUSABLE_FREE_ROOM: u64 = 8 * 1024 * 1024; +/// The size of the pages the WAL beside `db_path` holds, from its header, +/// or else of the database's, from its header; where neither reads as one, +/// SQLite's smallest, 512 bytes, which counts the most frames in a WAL. +fn page_size(db_path: &Path) -> u64 { + let header = |path: &Path, bytes: &mut [u8]| { + lpm_common::open_nofollow(path) + .and_then(|mut file| std::io::Read::read_exact(&mut file, bytes)) + .is_ok() + }; + let valid = |size: u64| size.is_power_of_two() && (512..=65_536).contains(&size); + let mut wal = [0; 12]; + if header(&capture_file(db_path, "-wal"), &mut wal) + && matches!( + u32::from_be_bytes([wal[0], wal[1], wal[2], wal[3]]), + 0x377f_0682 | 0x377f_0683 + ) + { + let size = u64::from(u32::from_be_bytes([wal[8], wal[9], wal[10], wal[11]])); + if valid(size) { + return size; + } + } + let mut database = [0; 18]; + if header(db_path, &mut database) { + let size = match u16::from_be_bytes([database[16], database[17]]) { + 1 => 65_536, + size => u64::from(size), + }; + if valid(size) { + return size; + } + } + 512 +} /// The free space replacing a database takes. Leaving WAL folds the WAL into /// the database, which grows by the WAL at most while the shared-memory file /// is open; then the WAL and that file are removed, and the copy takes the /// grown database's size. So it never takes more than the database and its -/// WAL, with the shared-memory file and what a filesystem keeps back on top. +/// WAL, with the shared-memory file on top. fn room_to_copy(db_path: &Path) -> Option { let database = std::fs::metadata(db_path).ok()?.len(); let wal = std::fs::metadata(capture_file(db_path, "-wal")).map_or(0, |wal| wal.len()); Some( database .saturating_add(wal) - .saturating_add(shared_memory_room(wal)) - .saturating_add(UNUSABLE_FREE_ROOM), + .saturating_add(shared_memory_room(wal, page_size(db_path))), ) } @@ -1502,7 +1562,7 @@ fn room_to_copy(db_path: &Path) -> Option { /// opened, as SQLite's own failure on a full disk can come as a bare I/O /// error. fn ensure_room_to_copy(db_path: &Path, needed: Option) -> Result<(), OpenError> { - let available = lpm_common::filesystem_available_bytes(db_path); + let available = lpm_common::filesystem_writable_bytes(db_path); #[cfg(test)] let available = tests::AVAILABLE_BYTES.get().or(available); match (available, needed) { @@ -1563,8 +1623,9 @@ fn no_room_to_copy(db_path: &Path, needed: Option) -> OpenError { std::io::ErrorKind::StorageFull, format!( "{} is open to other users, and replacing it with an owner-only copy needs \ - {needed} free on its disk; free that much, or move the database aside, and retry", - db_path.display() + {needed} free on its disk; free that much, or move it aside {}, and retry", + db_path.display(), + with_its_sidecars(db_path) ), )) } @@ -1790,9 +1851,10 @@ fn refuse_unknown_schema( Err(OpenError::Files(std::io::Error::new( std::io::ErrorKind::InvalidData, format!( - "{} holds schema LPM never creates ({}); move it aside and retry", + "{} holds schema LPM never creates ({}); move it aside {}, and retry", db_path.display(), - unknown.join(", ") + unknown.join(", "), + with_its_sidecars(db_path) ), ))) } @@ -2400,6 +2462,10 @@ mod tests { use lpm_tunnel::webhook::WebhookProvider; use std::collections::HashMap; + /// The advice for the capture database LPM can't use. + const ADVICE: &str = "move it aside together with any inspector.db-wal, \ + inspector.db-shm and inspector.db-journal beside it, and retry"; + thread_local! { /// The most pages this thread's database copies may hold, as a /// full disk would stop them. @@ -2420,6 +2486,11 @@ mod tests { /// another process can open the database then. pub(super) static BEFORE_CHECK: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; + /// Runs once after this thread's next replacement has copied the + /// database, before the copy takes its name, as a connection + /// without the opening lock can leave files beside it then. + pub(super) static AFTER_COPY: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; } #[cfg(unix)] @@ -2714,7 +2785,8 @@ mod tests { return; }; - let error = check_capture_file(&path, &metadata, this_user(), Access::Review).unwrap_err(); + let error = + check_capture_file(&path, &path, &metadata, this_user(), Access::Review).unwrap_err(); assert!( error.to_string().contains("belongs to another user"), @@ -2732,7 +2804,9 @@ mod tests { let metadata = std::fs::symlink_metadata(&path).unwrap(); std::fs::remove_file(&path).unwrap(); - assert!(!check_capture_file(&path, &metadata, this_user(), Access::Capture).unwrap()); + assert!( + !check_capture_file(&path, &path, &metadata, this_user(), Access::Capture).unwrap() + ); } #[cfg(unix)] @@ -2744,14 +2818,32 @@ mod tests { let served = project.path().join("public.txt"); std::fs::write(&served, b"").unwrap(); set_mode(&served, 0o644); - std::fs::hard_link(&served, lpm_dir.join("inspector.db")).unwrap(); + let db_path = lpm_dir.join("inspector.db"); + std::fs::hard_link(&served, &db_path).unwrap(); - let error = - prepare_capture_files(&lpm_dir.join("inspector.db"), this_user(), Access::Capture) - .unwrap_err(); + let error = prepare_capture_files(&db_path, this_user(), Access::Capture).unwrap_err(); assert!(error.to_string().contains("other hard links"), "{error}"); + assert!(error.to_string().ends_with(ADVICE), "{error}"); assert_eq!(mode(&served), 0o644); + + // A sidecar with other hard links moves with its database, which an + // open would otherwise find without it and remove. + std::fs::remove_file(&db_path).unwrap(); + std::fs::write(&db_path, b"").unwrap(); + std::fs::hard_link(&served, lpm_dir.join("inspector.db-wal")).unwrap(); + + let error = prepare_capture_files(&db_path, this_user(), Access::Capture).unwrap_err(); + + assert!( + error.to_string().ends_with(&format!( + "move {} aside together with any inspector.db-wal, inspector.db-shm and \ + inspector.db-journal beside it, and retry", + db_path.display() + )), + "{error}" + ); + assert!(lpm_dir.join("inspector.db-wal").exists()); } #[cfg(unix)] @@ -3251,7 +3343,7 @@ mod tests { #[test] fn a_disk_that_fills_while_the_database_is_checked_says_how_much_room_the_copy_needs() { let directory = tempfile::tempdir().unwrap(); - // Past the room a replacement keeps back, for the retry. + // Room for the retry, which the WAL folded in has grown. let attached = attach_image(directory.path(), "16m", true); let project = attached.0.join("project"); std::fs::create_dir(&project).unwrap(); @@ -3304,6 +3396,12 @@ mod tests { // The room taken before the WAL was folded in, which grew the database. let named = format!("needs about {} free", lpm_common::format_bytes(needed)); assert!(error.to_string().contains(&named), "{error}"); + assert!( + error + .to_string() + .ends_with(&format!("free that much, or {ADVICE}")), + "{error}" + ); assert_eq!(mode(&db_path), 0o644); assert!(!capture_file(&db_path, "-copy").exists()); @@ -3326,7 +3424,7 @@ mod tests { use std::io::Write as _; let directory = tempfile::tempdir().unwrap(); - // Past the room a replacement keeps back, for the retry. + // Room for the retry, which the WAL folded in has grown. let attached = attach_image(directory.path(), "16m", true); let project = attached.0.join("project"); std::fs::create_dir(&project).unwrap(); @@ -3502,7 +3600,10 @@ mod tests { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("inspector.db"); let elsewhere = directory.path().join("elsewhere"); - for swap in ["link", "another file"] { + // A file written in its place instead could take its inode number, + // as ext4 gives a freed one out again, and is one the owner could + // change anyway; a name of an existing file never has its number. + for swap in ["link", "another name"] { std::fs::write(&path, b"captures").unwrap(); // Others can't open it, so a writer makes it 0600. set_mode(&path, 0o700); @@ -3512,10 +3613,11 @@ mod tests { std::fs::remove_file(&path).unwrap(); match swap { "link" => std::os::unix::fs::symlink(&elsewhere, &path).unwrap(), - _ => std::fs::write(&path, b"swapped in").unwrap(), + _ => std::fs::hard_link(&elsewhere, &path).unwrap(), } - let checked_now = check_capture_file(&path, &checked, me_as_root, Access::Capture); + let checked_now = + check_capture_file(&path, &path, &checked, me_as_root, Access::Capture); assert!(checked_now.is_err(), "{swap}"); assert_eq!( @@ -3858,7 +3960,8 @@ mod tests { assert!( error.to_string().contains("holds schema LPM never creates") - && error.to_string().contains(named), + && error.to_string().contains(named) + && error.to_string().ends_with(ADVICE), "{named}: {error}" ); #[cfg(unix)] @@ -3992,8 +4095,13 @@ mod tests { let copy_path = capture_file(&db_path, "-copy"); assert!(create_owner_only(©_path, this_user()).unwrap()); + // The room taken before the database was opened, which this + // database's room now would read as less. + let needed = 3 * 1024 * 1024; + assert!(room_to_copy(&db_path).unwrap() < needed); + COPY_PAGE_LIMIT.set(Some(1)); - let copied = copy_database(&database, ©_path, &db_path, room_to_copy(&db_path)); + let copied = copy_database(&database, ©_path, &db_path, Some(needed)); COPY_PAGE_LIMIT.set(None); let Err(OpenError::Files(error)) = copied else { @@ -4001,17 +4109,16 @@ mod tests { }; assert_eq!(error.kind(), std::io::ErrorKind::StorageFull); assert!( - error.to_string().contains(&format!( - "needs about {} free on its disk", - lpm_common::format_bytes(room_to_copy(&db_path).unwrap()) - )), + error + .to_string() + .contains("needs about 3.0 MB free on its disk"), "{error}" ); } #[cfg(unix)] #[tokio::test] - async fn a_disk_without_room_for_the_replacement_says_so_before_anything_is_opened() { + async fn a_disk_without_room_for_the_replacement_says_so_before_the_database_is_opened() { let project = project_with_a_database(&format!( "{CAPTURE_SCHEMA} ALTER TABLE requests ADD COLUMN response_body_incomplete \ INTEGER NOT NULL DEFAULT 0;" @@ -4019,9 +4126,11 @@ mod tests { let db_path = project.path().join(".lpm/inspector.db"); let needed = room_to_copy(&db_path).unwrap(); + BEFORE_CHECK.set(Some(Box::new(|| {}))); AVAILABLE_BYTES.set(Some(needed - 1)); let short = InspectorDb::open(project.path()).map(drop); AVAILABLE_BYTES.set(None); + let opened = BEFORE_CHECK.with_borrow_mut(Option::take).is_none(); let error = short.unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::StorageFull); @@ -4032,9 +4141,12 @@ mod tests { )), "{error}" ); + assert!( + !opened, + "the database was opened before its room was checked" + ); assert_eq!(mode(&db_path), 0o644); assert!(!capture_file(&db_path, "-copy").exists()); - assert!(!capture_file(&db_path, "-shm").exists()); AVAILABLE_BYTES.set(Some(needed)); let enough = InspectorDb::open(project.path()).map(drop); @@ -4048,22 +4160,133 @@ mod tests { fn the_room_a_replacement_needs_counts_the_wal_once_with_its_shared_memory() { let directory = tempfile::tempdir().unwrap(); let db_path = directory.path().join("inspector.db"); - std::fs::write(&db_path, vec![0; 10_000]).unwrap(); + let mut database = vec![0; 10_000]; + database[16..18].copy_from_slice(&4096u16.to_be_bytes()); + std::fs::write(&db_path, &database).unwrap(); + assert_eq!(page_size(&db_path), 4096); + assert_eq!(room_to_copy(&db_path), Some(10_000 + 64 * 1024)); + database[16..18].copy_from_slice(&1u16.to_be_bytes()); + std::fs::write(&db_path, &database).unwrap(); + assert_eq!(page_size(&db_path), 65_536); + + // 130,308 frames of the WAL's 4 KiB pages: 32 regions of the WAL + // index, and one more. The WAL's header says what its frames hold. + let mut wal = std::fs::File::create(capture_file(&db_path, "-wal")).unwrap(); + std::io::Write::write_all(&mut wal, &[0x37, 0x7f, 0x06, 0x82, 0, 0x2d, 0xe2, 0x18]) + .unwrap(); + std::io::Write::write_all(&mut wal, &4096u32.to_be_bytes()).unwrap(); + wal.set_len(512 * 1024 * 1024).unwrap(); assert_eq!( room_to_copy(&db_path), - Some(10_000 + 64 * 1024 + UNUSABLE_FREE_ROOM) + Some(10_000 + 512 * 1024 * 1024 + 33 * 32 * 1024) ); - let wal = std::fs::File::create(capture_file(&db_path, "-wal")).unwrap(); + // A WAL without a header counts frames of SQLite's smallest pages. + wal.set_len(0).unwrap(); wal.set_len(512 * 1024 * 1024).unwrap(); + database[16..18].copy_from_slice(&[0, 0]); + std::fs::write(&db_path, &database).unwrap(); + assert_eq!(page_size(&db_path), 512); assert_eq!( room_to_copy(&db_path), - Some(10_000 + 512 * 1024 * 1024 + 64 * 1024 + 1024 * 1024 + UNUSABLE_FREE_ROOM) + Some(10_000 + 512 * 1024 * 1024 + 246 * 32 * 1024) ); + std::fs::remove_file(&db_path).unwrap(); assert_eq!(room_to_copy(&db_path), None); } + #[test] + fn the_room_check_counts_what_a_write_can_take_rather_than_what_is_reported_free() { + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("inspector.db"); + std::fs::write(&db_path, b"").unwrap(); + let available = lpm_common::filesystem_available_bytes(&db_path).unwrap(); + let writable = lpm_common::filesystem_writable_bytes(&db_path).unwrap(); + if writable >= available { + return; + } + + // Halfway, so other writers moving the free space by less than half + // of what's held back don't change the answer. + let error = + ensure_room_to_copy(&db_path, Some(writable + (available - writable) / 2)).unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::StorageFull, "{error}"); + } + + #[test] + fn the_room_for_shared_memory_follows_the_page_size_of_the_wal() { + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("inspector.db"); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch( + "PRAGMA page_size = 512; PRAGMA journal_mode = WAL; \ + PRAGMA wal_autocheckpoint = 0; CREATE TABLE kept (x); BEGIN;", + ) + .unwrap(); + for _ in 0..12_000 { + writer + .execute("INSERT INTO kept VALUES (zeroblob(400))", []) + .unwrap(); + } + writer.execute_batch("COMMIT;").unwrap(); + let wal = std::fs::metadata(capture_file(&db_path, "-wal")) + .unwrap() + .len(); + let shared_memory = std::fs::metadata(capture_file(&db_path, "-shm")) + .unwrap() + .len(); + + assert_eq!(page_size(&db_path), 512); + // Frames past the first region, which 4 KiB pages would need eight + // times the WAL for. + assert!(shared_memory > 32 * 1024, "{shared_memory}"); + assert!(shared_memory_room(wal, 4096) < shared_memory); + assert!(shared_memory_room(wal, 512) >= shared_memory); + drop(writer); + assert_eq!(page_size(&db_path), 512); + } + + #[cfg(unix)] + #[tokio::test] + async fn a_replacement_removes_files_left_beside_the_database_it_replaces() { + let project = project_with_a_database(&format!( + "{CAPTURE_SCHEMA} ALTER TABLE requests ADD COLUMN response_body_incomplete \ + INTEGER NOT NULL DEFAULT 0;" + )); + let db_path = project.path().join(".lpm/inspector.db"); + let left = [ + capture_file(&db_path, "-wal"), + capture_file(&db_path, "-shm"), + ]; + let leaving = left.clone(); + AFTER_COPY.set(Some(Box::new(move || { + for path in &leaving { + std::fs::write(path, b"left by another connection").unwrap(); + set_mode(path, 0o644); + } + }))); + + let db = InspectorDb::open(project.path()); + let copied = AFTER_COPY.with_borrow_mut(Option::take).is_none(); + + let db = db.unwrap(); + assert!(copied); + assert_eq!(mode(&db_path), 0o600); + for path in &left { + assert_eq!(mode(path), 0o600, "{}", path.display()); + assert_ne!( + std::fs::read(path).unwrap(), + b"left by another connection", + "{}", + path.display() + ); + } + drop(db); + } + #[cfg(unix)] #[tokio::test] async fn a_disk_too_full_to_create_the_copy_says_how_much_room_it_needs() { @@ -4170,7 +4393,7 @@ mod tests { .unwrap_err(); assert!(matches!(error, OpenError::Damaged { .. }), "{error}"); - assert!(error.to_string().contains("move it aside"), "{error}"); + assert!(error.to_string().ends_with(ADVICE), "{error}"); assert_eq!(mode(&db_path), 0o644); assert!(!project.path().join(".lpm/inspector.db-copy").exists()); } @@ -5222,10 +5445,7 @@ mod tests { }; assert!(matches!(error, OpenError::Damaged { .. }), "{error}"); - assert!( - error.to_string().contains("move it aside and retry"), - "{error}" - ); + assert!(error.to_string().ends_with(ADVICE), "{error}"); } } } From 4017a80a617ad9e8531ea9171a03d8883cdea692 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Tue, 13 Oct 2026 07:42:17 +0100 Subject: [PATCH 14/17] test(tunnel): expect the database advice that names its sidecars in the inspect workflows Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/workflows/tests/dev_tunnel.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index ded4d111e..0332b364b 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1504,7 +1504,10 @@ fn tunnel_inspect_reports_a_capture_database_it_cant_open_as_a_disk_error_under_ assert!( envelope["error"] .as_str() - .is_some_and(|message| message.contains("move it aside and retry")), + .is_some_and(|message| message.ends_with( + "move it aside together with any inspector.db-wal, inspector.db-shm and \ + inspector.db-journal beside it, and retry" + )), "{envelope}" ); } @@ -1750,7 +1753,10 @@ fn tunnel_inspect_reports_an_actionable_corrupt_database_error() { String::from_utf8_lossy(&output.stderr) ); assert!(rendered.contains(".lpm/inspector.db")); - assert!(rendered.contains("move it aside and retry")); + assert!(rendered.contains( + "move it aside together with any inspector.db-wal, inspector.db-shm and \ + inspector.db-journal beside it, and retry" + )); assert!( !rendered.contains("network connection"), "a local database error isn't a network one:\n{rendered}" From c223c066a3dc6a9b2ef092a635fe88246eaba205 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Tue, 13 Oct 2026 13:14:57 +0100 Subject: [PATCH 15/17] fix(tunnel): open another user's captures as that user under root, and read .git entries by git's rules - Root in a project another user owns opened the capture files by name, so that user could move .lpm while it opened them and have root write, remove or create files elsewhere. On 64-bit Linux the opening thread now takes the owner's filesystem identity for the whole open (lpm_common::thread_identity), so every step, SQLite's opens included, is the owner's access; root's connections keep their WAL files, so SQLite removes nothing by name as root later. Elsewhere, and where the identity can't be taken, root stops and says to run as the owner. - A .git directory is a repository's when its HEAD names a branch or a commit and its common directory has searchable objects and refs, and a .git file, followed through a link, points at one, as git 2.53 reads them. 'error opening' and 'error reading' read as git stopping only for a .git file. - A hard-linked sidecar without a database is told to move alone, test databases live in directories of their own, and the APFS hold-back is the least a write needs. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-cli/src/commands/tunnel.rs | 304 ++++++++++-- crates/lpm-common/src/lib.rs | 19 +- crates/lpm-common/src/thread_identity.rs | 170 +++++++ crates/lpm-inspect/src/db.rs | 604 +++++++++++++---------- 4 files changed, 787 insertions(+), 310 deletions(-) create mode 100644 crates/lpm-common/src/thread_identity.rs diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index 067229a14..23f601812 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -968,13 +968,11 @@ enum CaptureGitFailure { /// How git's `fatal: ` line starts, with `LC_ALL=C`, where it found no /// repository, and where it stopped looking. const NO_REPOSITORY_MESSAGE: &str = "not a git repository (or any of the parent directories)"; -const STOPPED_MESSAGES: [&str; 7] = [ +const STOPPED_MESSAGES: [&str; 5] = [ "not a git repository (or any parent up to mount", "not a git repository: ", "invalid gitfile format", "no path in gitfile", - "error opening '", - "error reading ", "too large to be a .git file", ]; @@ -995,13 +993,28 @@ fn capture_git_failure(stderr: &[u8]) -> CaptureGitFailure { if starts(NO_REPOSITORY_MESSAGE) { return CaptureGitFailure::NoRepository; } - if STOPPED_MESSAGES.into_iter().any(starts) { + if STOPPED_MESSAGES.into_iter().any(starts) || unreadable_git_file(reason, starts) { return CaptureGitFailure::Stopped; } } CaptureGitFailure::Refused } +/// Whether git's `fatal: ` line says it couldn't open or read a `.git` +/// file, which it names: "error opening ''" and "error reading +/// ", told from git's other errors that start the same way, such as +/// "error reading from ", by the name. +fn unreadable_git_file(reason: &str, starts: impl Fn(&str) -> bool) -> bool { + let names_a_git_file = |path: &str| path.ends_with("/.git") || path == ".git"; + if starts("error opening '") { + return reason["error opening '".len()..] + .split('\'') + .next() + .is_some_and(names_a_git_file); + } + starts("error reading ") && names_a_git_file(reason["error reading ".len()..].trim_end()) +} + /// A git query of the capture files: the directory it runs in, its /// pathspec, whether it lists files neither tracked nor ignored too, and /// whether it names them from their repository's top. @@ -1144,48 +1157,129 @@ fn file_name(name: &str) -> &str { /// The largest `.git` file git reads; it stops at a larger one. const GITFILE_LIMIT: u64 = 1024 * 1024; -/// Whether a `.git` entry looks like a repository's, for when git itself -/// can't be asked, or stopped: a directory, or a link to one, holding -/// `HEAD`, or a file pointing at one as git reads it: `gitdir: ` and the -/// rest of the file, but for the line breaks that end it, a path from the -/// file's directory. An empty directory, another file, or a pointer to no -/// repository doesn't, as git looks past the first two and stops at the -/// last, and at a file too large to be a pointer. +/// Whether a `.git` entry marks a repository as git 2.53 reads one, for when +/// git itself can't be asked, or stopped: a directory, or a link to one, +/// that's a git directory ([`is_git_directory`]), or a file, or a link to +/// one, pointing at one as git reads it: `gitdir: ` and the rest of the +/// file, but for the line breaks that end it, a path from the file's +/// directory. Git looks past a directory that isn't a git directory, and any +/// other entry, and stops at a file that doesn't point at one, or is too +/// large to be a pointer. fn marks_a_repository(git: &Path) -> bool { match std::fs::metadata(git) { - Ok(metadata) if metadata.is_dir() => git.join("HEAD").is_file(), + Ok(metadata) if metadata.is_dir() => is_git_directory(git), Ok(metadata) if metadata.is_file() => { - let mut contents = Vec::new(); - let read = lpm_common::open_nofollow(git).and_then(|file| { - std::io::Read::read_to_end( - &mut std::io::Read::take(file, GITFILE_LIMIT + 1), - &mut contents, - ) - }); - if read.is_err() || contents.len() as u64 > GITFILE_LIMIT { + let Ok((contents, _)) = + lpm_common::read_regular_file_capped_with_metadata(git, GITFILE_LIMIT) + else { return false; - } + }; let Some(pointer) = contents .strip_prefix(b"gitdir: ") - .map(|rest| { - let end = rest - .iter() - .rposition(|byte| !matches!(byte, b'\n' | b'\r')) - .map_or(0, |last| last + 1); - &rest[..end] - }) + .map(trim_line_breaks) .and_then(|path| std::str::from_utf8(path).ok()) .filter(|path| !path.is_empty()) else { return false; }; let directory = git.parent().unwrap_or(Path::new(".")); - directory.join(pointer).join("HEAD").is_file() + is_git_directory(&directory.join(pointer)) } _ => false, } } +/// `bytes` without the line breaks, CR or LF, that end it. +fn trim_line_breaks(bytes: &[u8]) -> &[u8] { + let end = bytes + .iter() + .rposition(|byte| !matches!(byte, b'\n' | b'\r')) + .map_or(0, |last| last + 1); + &bytes[..end] +} + +/// Whether `directory` is a git directory as git's `is_git_directory` +/// reads one: its `HEAD` names a branch or a commit ([`names_a_head`]), and +/// its common directory, which a worktree's `commondir` file names, has +/// `objects` and `refs` it can search. +fn is_git_directory(directory: &Path) -> bool { + if !names_a_head(&directory.join("HEAD")) { + return false; + } + let common = match lpm_common::read_regular_file_capped_with_metadata( + &directory.join("commondir"), + GITFILE_LIMIT, + ) { + Ok((contents, _)) => match std::str::from_utf8(trim_line_breaks(&contents)) { + Ok(path) => directory.join(path), + Err(_) => return false, + }, + Err(lpm_common::BoundedReadError::NotFound { .. }) => directory.to_path_buf(), + Err(_) => return false, + }; + searchable(&common.join("objects")) && searchable(&common.join("refs")) +} + +/// Whether a `HEAD` file names a branch or a commit as git's +/// `validate_headref` reads one: a link into `refs/`, or a file starting +/// with `ref:` and a name in `refs/` after any spaces, or with an object ID +/// in hex, of which git reads the first 255 bytes. +fn names_a_head(head: &Path) -> bool { + match std::fs::symlink_metadata(head) { + Ok(metadata) if metadata.file_type().is_symlink() => std::fs::read_link(head) + .is_ok_and(|target| target.as_os_str().as_encoded_bytes().starts_with(b"refs/")), + Ok(_) => { + let Some(contents) = first_bytes(head, 255) else { + return false; + }; + contents.strip_prefix(b"ref:").is_some_and(|name| { + // Git's own isspace: a space, tab, LF or CR, not a vertical + // tab or form feed. + let start = name + .iter() + .position(|byte| !matches!(byte, b' ' | b'\t' | b'\n' | b'\r')) + .unwrap_or(name.len()); + name[start..].starts_with(b"refs/") + }) || contents + .get(..40) + .is_some_and(|id| id.iter().all(u8::is_ascii_hexdigit)) + } + Err(_) => false, + } +} + +/// Up to `limit` bytes from the start of a regular file, opened without +/// blocking on a special file. +fn first_bytes(path: &Path, limit: u64) -> Option> { + let mut options = std::fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + + options.custom_flags(libc::O_NONBLOCK); + } + let file = options.open(path).ok()?; + if !file.metadata().ok()?.is_file() { + return None; + } + let mut contents = Vec::new(); + std::io::Read::read_to_end(&mut std::io::Read::take(file, limit), &mut contents).ok()?; + Some(contents) +} + +/// Whether git can search `path`, as its `access(X_OK)` checks. +fn searchable(path: &Path) -> bool { + #[cfg(unix)] + { + rustix::fs::access(path, rustix::fs::Access::EXEC_OK).is_ok() + } + #[cfg(not(unix))] + { + path.is_dir() + } +} + fn capture_git_command(query: &CaptureGitQuery) -> tokio::process::Command { let mut args = vec!["ls-files", "-z", "-t", "--cached"]; if query.unignored { @@ -3119,7 +3213,7 @@ mod tests { std::fs::create_dir_all(lpm_git.parent().unwrap()).unwrap(); std::fs::write(project.path().join(".lpm/inspector.db"), b"x").unwrap(); if entry == "HEAD only" { - repository_entry(&project.path().join(".lpm")); + head_only_entry(&project.path().join(".lpm")); } else { std::fs::write(&lpm_git, entry).unwrap(); } @@ -3219,8 +3313,18 @@ mod tests { assert_eq!(state, None); } - /// Makes `directory/.git` as `git init` leaves its `HEAD`. + /// Makes `directory/.git` a git directory as `git init` leaves one: + /// `HEAD`, `objects` and `refs`. fn repository_entry(directory: &Path) { + for name in ["objects", "refs/heads"] { + std::fs::create_dir_all(directory.join(".git").join(name)).unwrap(); + } + std::fs::write(directory.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); + } + + /// Makes `directory/.git` a directory holding only `HEAD`, which git + /// looks past. + fn head_only_entry(directory: &Path) { std::fs::create_dir_all(directory.join(".git")).unwrap(); std::fs::write(directory.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); } @@ -3249,7 +3353,7 @@ mod tests { // space at its end, is part of it. ("gitdir: elsewhere/.git\nsecond line\n", false), ("gitdir: elsewhere/.git \n", false), - // A directory without `HEAD` isn't a repository. + // A directory without `HEAD`, `objects` and `refs` isn't one. ("gitdir: elsewhere\n", false), ] { std::fs::write(&git, contents).unwrap(); @@ -3267,6 +3371,8 @@ mod tests { std::fs::create_dir(&git).unwrap(); assert!(!marks_a_repository(&git)); + head_only_entry(directory.path()); + assert!(!marks_a_repository(&git)); repository_entry(directory.path()); assert!(marks_a_repository(&git)); @@ -3276,8 +3382,126 @@ mod tests { std::fs::create_dir(&linked).unwrap(); std::os::unix::fs::symlink(&git, linked.join(".git")).unwrap(); assert!(marks_a_repository(&linked.join(".git"))); + // Git follows a link to a `.git` file too. + let pointer = directory.path().join("pointer"); + std::fs::write(&pointer, "gitdir: elsewhere/.git\n").unwrap(); + let linked_file = directory.path().join("linked-file"); + std::fs::create_dir(&linked_file).unwrap(); + std::os::unix::fs::symlink(&pointer, linked_file.join(".git")).unwrap(); + assert!(!marks_a_repository(&linked_file.join(".git"))); + std::os::unix::fs::symlink( + directory.path().join("elsewhere"), + linked_file.join("elsewhere"), + ) + .unwrap(); + assert!(marks_a_repository(&linked_file.join(".git"))); } assert!(!marks_a_repository(&directory.path().join("gone/.git"))); + + // A pointer with no path is none, even in a git directory, which the + // empty path would otherwise name. + let bare = tempfile::tempdir().unwrap(); + for name in ["objects", "refs"] { + std::fs::create_dir(bare.path().join(name)).unwrap(); + } + std::fs::write(bare.path().join("HEAD"), "ref: refs/heads/main\n").unwrap(); + std::fs::write(bare.path().join(".git"), "gitdir: \n").unwrap(); + assert!(!marks_a_repository(&bare.path().join(".git"))); + } + + #[test] + fn takes_a_git_directory_only_as_git_s_is_git_directory_would() { + let directory = tempfile::tempdir().unwrap(); + let git = directory.path().join(".git"); + repository_entry(directory.path()); + let head = git.join("HEAD"); + for (contents, repository) in [ + ("ref: refs/heads/main\n", true), + ("ref:refs/heads/main", true), + ("ref: \t\r\n refs/heads/main\n", true), + ("ref: \x0b refs/heads/main\n", false), + ("ref: \x0c refs/heads/main\n", false), + ("ref: heads/main\n", false), + ("refs/heads/main\n", false), + ("0123456789abcdef0123456789ABCDEF01234567\n", true), + ("0123456789abcdef0123456789abcdef01234567", true), + ("0123456789abcdef0123456789abcdef0123456\n", false), + ("0123456789abcdef0123456789abcdef0123456g\n", false), + ("", false), + ] { + std::fs::write(&head, contents).unwrap(); + assert_eq!(marks_a_repository(&git), repository, "{contents:?}"); + } + std::fs::write(&head, "ref: refs/heads/main\n").unwrap(); + + for missing in ["objects", "refs"] { + let aside = directory.path().join("aside"); + std::fs::rename(git.join(missing), &aside).unwrap(); + assert!(!marks_a_repository(&git), "{missing}"); + std::fs::rename(&aside, git.join(missing)).unwrap(); + } + assert!(marks_a_repository(&git)); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + + std::fs::remove_file(&head).unwrap(); + std::os::unix::fs::symlink("refs/heads/main", &head).unwrap(); + assert!(marks_a_repository(&git)); + std::fs::remove_file(&head).unwrap(); + std::os::unix::fs::symlink("../elsewhere", &head).unwrap(); + assert!(!marks_a_repository(&git)); + std::fs::remove_file(&head).unwrap(); + std::fs::write(&head, "ref: refs/heads/main\n").unwrap(); + + // Git checks it can search them, unless run as root. + if lpm_common::current_uid() != 0 { + let objects = git.join("objects"); + std::fs::set_permissions(&objects, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert!(!marks_a_repository(&git)); + std::fs::set_permissions(&objects, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + } + } + + #[test] + fn takes_a_worktree_s_git_directory_by_its_common_directory() { + let directory = tempfile::tempdir().unwrap(); + repository_entry(directory.path()); + let worktree_git = directory.path().join(".git/worktrees/feature"); + std::fs::create_dir_all(&worktree_git).unwrap(); + std::fs::write(worktree_git.join("HEAD"), "ref: refs/heads/feature\n").unwrap(); + let checkout = directory.path().join("feature"); + std::fs::create_dir(&checkout).unwrap(); + std::fs::write( + checkout.join(".git"), + format!("gitdir: {}\n", worktree_git.display()), + ) + .unwrap(); + + // Without `commondir`, the worktree's own directory has no + // `objects` or `refs`. + assert!(!marks_a_repository(&checkout.join(".git"))); + for (common, repository) in [ + ("../..\n", true), + ("../..\r\n", true), + ("../../objects\n", false), + ("../gone\n", false), + ] { + std::fs::write(worktree_git.join("commondir"), common).unwrap(); + assert_eq!( + marks_a_repository(&checkout.join(".git")), + repository, + "{common:?}" + ); + } + std::fs::write( + worktree_git.join("commondir"), + format!("{}\n", directory.path().join(".git").display()), + ) + .unwrap(); + assert!(marks_a_repository(&checkout.join(".git"))); } #[test] @@ -3332,6 +3556,22 @@ mod tests { "fatal: error reading /work/app/.git\n", CaptureGitFailure::Stopped, ), + ( + "fatal: Error reading /work/app/.git\n", + CaptureGitFailure::Stopped, + ), + ( + "fatal: error reading from /work/app/.git/index: Input/output error\n", + CaptureGitFailure::Refused, + ), + ( + "fatal: error reading directory '/work/app/src'\n", + CaptureGitFailure::Refused, + ), + ( + "fatal: error opening '/work/app/.git/index': Permission denied\n", + CaptureGitFailure::Refused, + ), ( "warning: unable to access '/home/user/.config/git/attributes'\n\ fatal: not a git repository (or any of the parent directories): .git\n", diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index ac51bdde6..d161b4dc4 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -29,6 +29,9 @@ pub mod project_glob; pub mod provenance; pub mod symlink; pub mod terminal; +// The 64-bit system call's groups are 32-bit; 32-bit systems have two calls. +#[cfg(all(target_os = "linux", target_pointer_width = "64"))] +pub mod thread_identity; mod utf8_bom; #[cfg(unix)] @@ -405,11 +408,13 @@ fn filesystem_space(_path: &std::path::Path) -> Option { } /// What APFS reports free on a volume of `size` bytes but refuses writes: -/// a write there fails with about 2% of the volume and 460 KB more still -/// reported free, and never more than 32 MiB and 80 KiB, as measured on -/// volumes of 64 MB to 8 GB. Rounded up by about half a MiB, which also -/// covers the blocks a file's own metadata takes, so a write that would -/// just fit can be refused, but never one counted as fitting. +/// a stream of writes to one file stops with about 2% of the volume and +/// 460 KB more still reported free, and never more than 32 MiB and 80 KiB, +/// as measured on volumes of 64 MB to 8 GB. Rounded up by about half a MiB, +/// which also covers the blocks a file's own metadata takes. That's the +/// least a write needs: on a fresh volume, writes flushed as they go, or +/// spread over many files, stopped up to about 16 MB sooner, so a write +/// counted as fitting can still fail, as on a full disk. #[cfg(any(target_os = "macos", test))] fn apfs_held_back(size: u64) -> u64 { (size / 50) @@ -500,7 +505,7 @@ mod tests { #[cfg(unix)] #[test] - fn on_any_filesystem_but_apfs_a_write_can_take_what_it_reports_free() { + fn on_any_filesystem_but_apfs_nothing_reported_free_counts_as_kept_back() { let directory = tempfile::tempdir().unwrap(); let space = filesystem_space(directory.path()).unwrap(); if cfg!(target_os = "macos") && space.held_back > 0 { @@ -516,7 +521,7 @@ mod tests { #[cfg(target_os = "macos")] #[test] - fn on_apfs_a_write_takes_what_it_counts_writable_and_little_more() { + fn on_apfs_a_stream_of_writes_takes_what_it_counts_writable_and_little_more() { use std::io::Write as _; let directory = tempfile::tempdir().unwrap(); diff --git a/crates/lpm-common/src/thread_identity.rs b/crates/lpm-common/src/thread_identity.rs new file mode 100644 index 000000000..68319099d --- /dev/null +++ b/crates/lpm-common/src/thread_identity.rs @@ -0,0 +1,170 @@ +//! A thread's own filesystem identity, on Linux, whose kernel keeps each +//! thread's credentials: root can reach files as another user on one thread +//! alone, so a path that user controls leads only where that user could go, +//! however it changes meanwhile. The C library's `setgroups` sets every +//! thread's groups, so the thread's own are set with the system call itself. + +use std::io; + +/// The calling thread reaching files as another user, until this is +/// dropped, which gives the thread back the identity it had. A thread it +/// starts meanwhile keeps the user's for good, as Linux gives a new thread +/// its creator's. +#[must_use = "the thread acts as the user only while this is held"] +#[derive(Debug)] +pub struct ActingAs { + fsuid: libc::uid_t, + fsgid: libc::gid_t, + groups: Vec, +} + +/// Has the calling thread reach files as `uid`, with `gid` as its only +/// group. The kernel takes root's file capabilities from a thread whose +/// filesystem user isn't root, so its access is the user's. Fails, leaving +/// the thread as it was, where it may not take the identity: without +/// `CAP_SETUID` and `CAP_SETGID`, or for an ID its user namespace doesn't +/// map. +pub fn act_as(uid: u32, gid: u32) -> io::Result { + let acting = ActingAs { + fsuid: filesystem_uid(), + fsgid: filesystem_gid(), + groups: thread_groups()?, + }; + set_thread_groups(&[gid])?; + // SAFETY: each changes only the calling thread's credentials, and takes + // no pointer. + unsafe { + libc::setfsgid(gid); + libc::setfsuid(uid); + } + if filesystem_uid() != uid || filesystem_gid() != gid { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("this process may not reach files as user {uid} and group {gid}"), + )); + } + Ok(acting) +} + +impl Drop for ActingAs { + /// A thread that could take the identity keeps the capabilities to give + /// it back: `CAP_SETUID` and `CAP_SETGID` aren't file capabilities. + fn drop(&mut self) { + // SAFETY: as in `act_as`. + unsafe { + libc::setfsuid(self.fsuid); + libc::setfsgid(self.fsgid); + } + let _ = set_thread_groups(&self.groups); + } +} + +/// The thread's filesystem user ID. `setfsuid` reports no failure, so the +/// current one is read by asking for an ID no user has, which changes +/// nothing and returns it. +pub fn filesystem_uid() -> u32 { + // SAFETY: as in `act_as`; an invalid ID changes nothing. + unsafe { libc::setfsuid(libc::uid_t::MAX) as u32 } +} + +/// The thread's filesystem group ID, read as [`filesystem_uid`] is. +pub fn filesystem_gid() -> u32 { + // SAFETY: as in `act_as`; an invalid ID changes nothing. + unsafe { libc::setfsgid(libc::gid_t::MAX) as u32 } +} + +/// The calling thread's supplementary groups. +pub fn thread_groups() -> io::Result> { + loop { + // SAFETY: a count of 0 only asks how many groups there are. + let count = unsafe { libc::getgroups(0, std::ptr::null_mut()) }; + if count < 0 { + return Err(io::Error::last_os_error()); + } + let mut groups = vec![0; count as usize]; + // SAFETY: `groups` has room for `count` IDs. + let filled = unsafe { libc::getgroups(count, groups.as_mut_ptr()) }; + if filled >= 0 { + groups.truncate(filled as usize); + return Ok(groups); + } + // The groups grew in between; ask again. + if io::Error::last_os_error().raw_os_error() != Some(libc::EINVAL) { + return Err(io::Error::last_os_error()); + } + } +} + +fn set_thread_groups(groups: &[libc::gid_t]) -> io::Result<()> { + // SAFETY: the system call reads `groups.len()` IDs from `groups`, and + // changes only the calling thread's credentials. + let result = unsafe { libc::syscall(libc::SYS_setgroups, groups.len(), groups.as_ptr()) }; + if result == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_thread_that_may_not_act_as_another_user_is_left_as_it_was() { + if crate::current_uid() == 0 { + return; + } + let before = (filesystem_uid(), filesystem_gid(), thread_groups().unwrap()); + + let error = act_as(crate::current_uid() + 1, before.1 + 1).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert_eq!( + (filesystem_uid(), filesystem_gid(), thread_groups().unwrap()), + before + ); + } + + /// Runs only as root, as in the root container the gate runs Linux + /// tests in once more; CI's runner isn't root. + #[test] + fn as_root_a_thread_reaches_files_as_the_user_and_alone_until_it_lets_go() { + if crate::current_uid() != 0 { + return; + } + let directory = tempfile::tempdir().unwrap(); + let private = directory.path().join("root-only"); + std::fs::write(&private, b"root's").unwrap(); + std::fs::set_permissions( + &private, + ::from_mode(0o600), + ) + .unwrap(); + let before = (filesystem_uid(), filesystem_gid(), thread_groups().unwrap()); + let (ask, asked) = std::sync::mpsc::channel::<()>(); + let elsewhere = private.clone(); + let another_thread = std::thread::spawn(move || { + asked.recv().unwrap(); + std::fs::read(elsewhere) + }); + + { + let _acting = act_as(65_000, 65_000).unwrap(); + assert_eq!((filesystem_uid(), filesystem_gid()), (65_000, 65_000)); + assert_eq!(thread_groups().unwrap(), vec![65_000]); + assert_eq!( + std::fs::read(&private).unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); + ask.send(()).unwrap(); + assert_eq!(another_thread.join().unwrap().unwrap(), b"root's"); + } + + assert_eq!( + (filesystem_uid(), filesystem_gid(), thread_groups().unwrap()), + before + ); + assert_eq!(std::fs::read(&private).unwrap(), b"root's"); + } +} diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index 7b470f929..7a74c8c25 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -45,6 +45,10 @@ pub struct InspectorDb { pub db_path: PathBuf, open_to_others: bool, ownership_ignored: bool, + /// The directory [`Self::open_temp`]'s database is in, removed with the + /// last handle. + #[cfg(test)] + _temp_dir: Option>, } /// A write operation to be batched. @@ -64,7 +68,8 @@ impl InspectorDb { /// replaced with an owner-only copy (see [`OpenError`] for when it can't /// be yet). pub fn open(project_dir: &Path) -> Result { - Self::open_project(project_dir, Access::Capture, CaptureOwner::of(project_dir)?) + let (owner, _identity) = CaptureOwner::of(project_dir)?; + Self::open_project(project_dir, Access::Capture, owner) } /// Opens (or creates) `{project_dir}/.lpm/inspector.db` to read and @@ -73,7 +78,8 @@ impl InspectorDb { /// to the database; [`Self::open_to_others`] says whether others could /// open it. pub fn open_for_review(project_dir: &Path) -> Result { - Self::open_project(project_dir, Access::Review, CaptureOwner::of(project_dir)?) + let (owner, _identity) = CaptureOwner::of(project_dir)?; + Self::open_project(project_dir, Access::Review, owner) } fn open_project( @@ -84,7 +90,11 @@ impl InspectorDb { let lpm_dir = project_dir.join(".lpm"); let db_path = lpm_dir.join("inspector.db"); let ownership_ignored = prepare_lpm_dir(&lpm_dir, owner, access)?; - let _opening = lock_capture_files(&db_path, owner)?; + #[cfg(test)] + if let Some(meanwhile) = tests::AFTER_LPM_DIR.with_borrow_mut(Option::take) { + meanwhile(); + } + let _opening = lock_capture_files(&db_path)?; remove_replacement_copy(&db_path)?; if let Err(error) = ensure_captures_ignored(&lpm_dir, owner) { tracing::warn!("could not keep tunnel captures out of git: {error}"); @@ -94,9 +104,9 @@ impl InspectorDb { let mut open_to_others = prepare_capture_files(&db_path, owner, access)? || ownership_ignored; if open_to_others && access == Access::Capture && !ownership_ignored { - open_to_others = !replace_with_owner_only_copy(&db_path, owner)?; + open_to_others = !replace_with_owner_only_copy(&db_path)?; } - open_to_others |= !create_wal_sidecars(&db_path, owner)?; + open_to_others |= !create_wal_sidecars(&db_path)?; // Write connection (owned by the background flush task) let write_conn = open_capture_connection(&db_path, "NORMAL").map_err(damaged(&db_path))?; @@ -108,6 +118,11 @@ impl InspectorDb { // while the write connection flushes a batch. let read_conn = open_capture_connection(&db_path, "NORMAL").map_err(damaged(&db_path))?; let control_conn = open_control_connection(&db_path).map_err(damaged(&db_path))?; + if owner.by_root { + for connection in [&write_conn, &read_conn, &control_conn] { + keep_wal_files(connection).map_err(damaged(&db_path))?; + } + } Ok(Self::with_connections( write_conn, @@ -144,26 +159,20 @@ impl InspectorDb { db_path, open_to_others, ownership_ignored, + #[cfg(test)] + _temp_dir: None, } } - /// Open a temporary file-based database. - /// - /// Each call creates a unique database file so tests don't interfere - /// with each other when running in parallel. + /// Opens a database in a directory of its own, removed with the last + /// handle, so tests running in parallel don't share one. #[cfg(test)] pub(crate) fn open_temp() -> Result { - use std::sync::atomic::{AtomicU64, Ordering}; - static COUNTER: AtomicU64 = AtomicU64::new(0); - let id = COUNTER.fetch_add(1, Ordering::Relaxed); - - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |duration| duration.as_nanos()); - let tmp = std::env::temp_dir().join(format!( - "lpm-inspect-test-{}-{nanos}-{id}.db", - std::process::id() - )); + let temp_dir = tempfile::Builder::new() + .prefix("lpm-inspect-test-") + .tempdir() + .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?; + let tmp = temp_dir.path().join("inspector.db"); let write_conn = Connection::open(&tmp)?; configure_connection(&write_conn, "NORMAL")?; @@ -172,14 +181,10 @@ impl InspectorDb { configure_connection(&read_conn, "NORMAL")?; let control_conn = open_control_connection(&tmp)?; - Ok(Self::with_connections( - write_conn, - read_conn, - control_conn, - tmp, - false, - false, - )) + Ok(Self { + _temp_dir: Some(Arc::new(temp_dir)), + ..Self::with_connections(write_conn, read_conn, control_conn, tmp, false, false) + }) } /// Whether other users could open the capture files when this handle @@ -875,71 +880,135 @@ enum Access { Review, } -/// Whose capture files are. A user's own; under root, as in a container -/// over a mounted checkout, the project directory's owner's, or root's. -/// What root creates is given to that owner, so they can open it without -/// root later; SQLite gives its sidecars to the database's owner itself. +/// Whose capture files are: the user's own. Root in a project another user +/// owns, as in a container over a mounted checkout, opens them as that user +/// ([`OwnerIdentity`]): that user can move `.lpm` and the names in it at any +/// time, and so could otherwise have what root reads, writes and removes +/// there land anywhere. #[cfg(unix)] #[derive(Debug, Clone, Copy)] struct CaptureOwner { uid: u32, - /// The group root gives what it creates to. - gid: u32, - as_root: bool, + /// Whether root opens the capture files as their owner. + by_root: bool, } +/// What root holds while it opens the capture files as their owner: on +/// Linux, the owner's filesystem identity, taken by the opening thread +/// alone; elsewhere, root doesn't open another user's. +#[cfg(all(target_os = "linux", target_pointer_width = "64"))] +type OwnerIdentity = Option; +#[cfg(not(all(target_os = "linux", target_pointer_width = "64")))] +struct OwnerIdentity; + +/// What an open holds that takes no other user's identity. +#[cfg(all(target_os = "linux", target_pointer_width = "64"))] +const OWN_IDENTITY: OwnerIdentity = None; +#[cfg(not(all(target_os = "linux", target_pointer_width = "64")))] +const OWN_IDENTITY: OwnerIdentity = OwnerIdentity; + #[cfg(unix)] impl CaptureOwner { - fn of(project_dir: &Path) -> std::io::Result { + /// The capture files' owner, and the identity root holds while it opens + /// them as that owner. + fn of(project_dir: &Path) -> std::io::Result<(Self, OwnerIdentity)> { use std::os::unix::fs::MetadataExt; let uid = lpm_common::current_uid(); if uid != 0 { - return Ok(Self { - uid, - gid: 0, - as_root: false, - }); + return Ok(( + Self { + uid, + by_root: false, + }, + OWN_IDENTITY, + )); } let project = std::fs::metadata(project_dir).map_err(|error| with_path(project_dir, error))?; - Ok(Self { - uid: project.uid(), - gid: project.gid(), - as_root: true, - }) + if project.uid() == 0 { + return Ok(( + Self { + uid: 0, + by_root: false, + }, + OWN_IDENTITY, + )); + } + let identity = act_as_project_owner(project_dir, &project)?; + Ok(( + Self { + uid: project.uid(), + by_root: true, + }, + identity, + )) } fn owns(self, metadata: &std::fs::Metadata) -> bool { use std::os::unix::fs::MetadataExt; - metadata.uid() == self.uid || (self.as_root && metadata.uid() == 0) + metadata.uid() == self.uid } +} - /// Gives a file or directory this process created to the owner, when - /// running as root. - fn give(self, handle: &std::fs::File, path: &Path) -> std::io::Result<()> { - if !self.as_root { - return Ok(()); - } - std::os::unix::fs::fchown(handle, Some(self.uid), Some(self.gid)) - .map_err(|error| with_path(path, error)) - } +#[cfg(all(target_os = "linux", target_pointer_width = "64"))] +fn act_as_project_owner( + project_dir: &Path, + project: &std::fs::Metadata, +) -> std::io::Result { + use std::os::unix::fs::MetadataExt; + + lpm_common::thread_identity::act_as(project.uid(), project.gid()) + .map(Some) + .map_err(|error| { + std::io::Error::new( + error.kind(), + format!( + "{} belongs to user {}, and root records tunnel captures there only as \ + that user, so that it can't be made to write elsewhere, which this \ + process can't do ({error}); run the tunnel as the project's owner", + project_dir.display(), + project.uid() + ), + ) + }) +} + +#[cfg(all(unix, not(all(target_os = "linux", target_pointer_width = "64"))))] +fn act_as_project_owner( + project_dir: &Path, + _project: &std::fs::Metadata, +) -> std::io::Result { + Err(project_of_another_user(project_dir)) +} + +/// Root's refusal of another user's project where it can't open the +/// capture files as that user. +#[cfg(all(unix, not(all(target_os = "linux", target_pointer_width = "64"))))] +fn project_of_another_user(project_dir: &Path) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!( + "{} belongs to another user, who could move its .lpm to have root write \ + elsewhere, so root records tunnel captures, which hold request headers and \ + bodies, only in its own projects here; run the tunnel as the project's owner", + project_dir.display() + ), + ) } /// Access to the capture files on Windows is the project's ACLs. #[cfg(not(unix))] #[derive(Debug, Clone, Copy)] -struct CaptureOwner; +struct CaptureOwner { + by_root: bool, +} #[cfg(not(unix))] impl CaptureOwner { - fn of(_project_dir: &Path) -> std::io::Result { - Ok(Self) - } - - fn give(self, _handle: &std::fs::File, _path: &Path) -> std::io::Result<()> { - Ok(()) + fn of(_project_dir: &Path) -> std::io::Result<(Self, OwnerIdentity)> { + Ok((Self { by_root: false }, OWN_IDENTITY)) } } @@ -1018,19 +1087,26 @@ fn another_users(path: &Path) -> std::io::Error { ) } +/// A sidecar moves with its database, which an open would otherwise find +/// without it and remove; one found without a database moves alone. fn hard_linked(path: &Path, db_path: &Path) -> std::io::Error { - let database = if path == db_path { - "it".to_owned() + let advice = if path == db_path { + format!("move it aside {}", with_its_sidecars(db_path)) + } else if db_path.exists() { + format!( + "move {} aside {}", + db_path.display(), + with_its_sidecars(db_path) + ) } else { - db_path.display().to_string() + "move it aside".to_owned() }; std::io::Error::new( std::io::ErrorKind::InvalidInput, format!( "{} has other hard links, so captures written to it would change those \ - files too; move {database} aside {}, and retry", - path.display(), - with_its_sidecars(db_path) + files too; {advice}, and retry", + path.display() ), ) } @@ -1054,16 +1130,15 @@ fn with_its_sidecars(db_path: &Path) -> String { /// directories they stand for. See [`restrict_lpm_dir`] for Unix, and for /// whether `.lpm`'s volume ignores ownership, which this returns. fn prepare_lpm_dir(lpm_dir: &Path, owner: CaptureOwner, access: Access) -> std::io::Result { - let created = match std::fs::symlink_metadata(lpm_dir) { + match std::fs::symlink_metadata(lpm_dir) { Ok(metadata) if !metadata.is_dir() => return Err(linked_or_not(lpm_dir, "directory")), - Ok(_) => false, + Ok(_) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => { std::fs::create_dir_all(lpm_dir).map_err(|error| with_path(lpm_dir, error))?; - true } Err(error) => return Err(with_path(lpm_dir, error)), - }; - restrict_lpm_dir(lpm_dir, owner, access, created) + } + restrict_lpm_dir(lpm_dir, owner, access) } /// `.lpm` must be the capture owner's. A capture writer makes it writable @@ -1073,12 +1148,7 @@ fn prepare_lpm_dir(lpm_dir: &Path, owner: CaptureOwner, access: Access) -> std:: /// and SQLite's opens, or read the files created there. Says whether its /// volume ignores ownership, as only macOS reports. #[cfg(unix)] -fn restrict_lpm_dir( - lpm_dir: &Path, - owner: CaptureOwner, - access: Access, - created: bool, -) -> std::io::Result { +fn restrict_lpm_dir(lpm_dir: &Path, owner: CaptureOwner, access: Access) -> std::io::Result { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let directory = @@ -1097,9 +1167,6 @@ fn restrict_lpm_dir( let ownership_ignored = false; #[cfg(test)] let ownership_ignored = ownership_ignored || tests::OWNERSHIP_IGNORED.get(); - if created { - owner.give(&directory, lpm_dir)?; - } if access == Access::Review { return Ok(ownership_ignored); } @@ -1122,7 +1189,6 @@ fn restrict_lpm_dir( _lpm_dir: &Path, _owner: CaptureOwner, _access: Access, - _created: bool, ) -> std::io::Result { Ok(false) } @@ -1133,12 +1199,10 @@ fn restrict_lpm_dir( /// lock don't take it. fn lock_capture_files( db_path: &Path, - owner: CaptureOwner, ) -> std::io::Result { let lock_path = capture_file(db_path, "-lock"); let file = lpm_common::open_lock_file_nofollow(&lock_path) .map_err(|error| with_path(&lock_path, error))?; - owner.give(&file, &lock_path)?; let timeout = open_lock_timeout(); match lpm_common::acquire_single_file_exclusive_lock_from_file_within(file, timeout) { Ok(Some(lock)) => Ok(lock), @@ -1198,7 +1262,7 @@ fn prepare_capture_files( for suffix in SQLITE_SIDECARS { remove_if_present(&capture_file(db_path, suffix))?; } - Ok(!create_owner_only(db_path, owner)?) + Ok(!create_owner_only(db_path)?) } /// Makes the WAL files SQLite would create, when they're missing, @@ -1207,14 +1271,14 @@ fn prepare_capture_files( /// which a review leaves on `.lpm`. Under the open lock no LPM process has /// the database open in WAL without them, so none is made under one in /// use, and SQLite takes empty ones as new. -fn create_wal_sidecars(db_path: &Path, owner: CaptureOwner) -> std::io::Result { +fn create_wal_sidecars(db_path: &Path) -> std::io::Result { let mut owner_only = true; for suffix in ["-wal", "-shm"] { let path = capture_file(db_path, suffix); match std::fs::symlink_metadata(&path) { Ok(_) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - owner_only &= create_owner_only(&path, owner)?; + owner_only &= create_owner_only(&path)?; } Err(error) => return Err(with_path(&path, error)), } @@ -1238,9 +1302,8 @@ fn remove_if_present(path: &Path) -> std::io::Result<()> { /// later open. Modes are read and changed by name, which no other user can /// swap in a directory only its owner can write, rather than through a /// handle, whose closing would drop the locks SQLite holds on the file in -/// this process, and which a file without read permission can't have; as -/// root, whose capture owner can swap them, through a checked handle (see -/// [`set_mode_through_handle`]). A file that's gone needs nothing: SQLite removes `-wal` and `-shm` when +/// this process, and which a file without read permission can't have. A +/// file that's gone needs nothing: SQLite removes `-wal` and `-shm` when /// another process closes its last connection. #[cfg(unix)] fn check_capture_file( @@ -1270,12 +1333,7 @@ fn check_capture_file( } let wanted = if open_to_others { mode | 0o600 } else { 0o600 }; if mode != wanted { - let changed = if owner.as_root { - set_mode_through_handle(path, metadata, wanted) - } else { - std::fs::set_permissions(path, std::fs::Permissions::from_mode(wanted)) - }; - match changed { + match std::fs::set_permissions(path, std::fs::Permissions::from_mode(wanted)) { Ok(()) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), Err(error) => return Err(with_path(path, error)), @@ -1284,36 +1342,6 @@ fn check_capture_file( Ok(open_to_others) } -/// Changes a capture file's mode as root, through a handle that doesn't -/// follow a link, once the handle is checked to be the file `checked` -/// describes. Root's capture owner is another user, who can swap the names -/// in `.lpm` for a link to, or another name of, a file root's change would -/// open up. Root can open the file whatever its mode, and this process holds -/// no SQLite lock on it yet, which the handle's closing would drop, as each -/// LPM command opens the captures once. -#[cfg(unix)] -fn set_mode_through_handle( - path: &Path, - checked: &std::fs::Metadata, - mode: u32, -) -> std::io::Result<()> { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - - let file = lpm_common::open_nofollow(path)?; - let opened = file.metadata()?; - if !opened.is_file() - || opened.dev() != checked.dev() - || opened.ino() != checked.ino() - || opened.nlink() > 1 - { - return Err(std::io::Error::other(format!( - "{} changed while its access was checked; retry", - path.display() - ))); - } - file.set_permissions(std::fs::Permissions::from_mode(mode)) -} - /// Refuses an existing capture file with other hard links, read through a /// handle that doesn't follow a link. Access to it is the project's ACLs. #[cfg(not(unix))] @@ -1335,15 +1363,15 @@ fn check_capture_file( Ok(false) } -/// Creates a capture file only its owner can open, given to the capture -/// owner, and says whether it reads that way: a filesystem without Unix +/// Creates a capture file only its owner can open, and says whether it +/// reads that way: a filesystem without Unix /// modes, such as FAT or a Windows drive mounted in Linux, shows the mode it /// likes. On macOS, extended ACL entries it inherited are cleared. A file /// that appears at the name meanwhile, made by an LPM release that doesn't /// take the lock, is refused until a retry checks it; `O_EXCL` also refuses /// a link there. #[cfg(unix)] -fn create_owner_only(path: &Path, owner: CaptureOwner) -> std::io::Result { +fn create_owner_only(path: &Path) -> std::io::Result { use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; let file = match std::fs::OpenOptions::new() @@ -1364,7 +1392,6 @@ fn create_owner_only(path: &Path, owner: CaptureOwner) -> std::io::Result } Err(error) => return Err(with_path(path, error)), }; - owner.give(&file, path)?; #[cfg(target_os = "macos")] if lpm_common::macos_acl::has_extended_acl(&file)? { lpm_common::macos_acl::clear_extended_acl(&file).map_err(|error| with_path(path, error))?; @@ -1380,7 +1407,7 @@ fn create_owner_only(path: &Path, owner: CaptureOwner) -> std::io::Result } #[cfg(not(unix))] -fn create_owner_only(_path: &Path, _owner: CaptureOwner) -> std::io::Result { +fn create_owner_only(_path: &Path) -> std::io::Result { Ok(true) } @@ -1396,11 +1423,11 @@ fn create_owner_only(_path: &Path, _owner: CaptureOwner) -> std::io::Result Result { +fn replace_with_owner_only_copy(db_path: &Path) -> Result { let copy_path = capture_file(db_path, "-copy"); // Taken before anything is opened, as leaving WAL grows the database. let needed = room_to_copy(db_path); - let owner_only = create_owner_only(©_path, owner); + let owner_only = create_owner_only(©_path); #[cfg(test)] let owner_only = tests::COPY_CREATE_ERROR .get() @@ -1434,6 +1461,10 @@ fn replace_with_owner_only_copy(db_path: &Path, owner: CaptureOwner) -> Result Result Result<(), rusqlite::Error> { harden_connection(connection)?; use_wal(connection)?; connection.execute_batch(&format!( "PRAGMA synchronous={synchronous}; PRAGMA foreign_keys=ON;" - )) + ))?; + connection.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(())) +} + +/// Keeps the WAL and its shared memory when the last connection closes, +/// the WAL emptied rather than removed, for a connection root opened as the +/// capture owner: SQLite removes them by name, which by then can lead +/// anywhere the owner points it. +fn keep_wal_files(connection: &Connection) -> Result<(), rusqlite::Error> { + let mut persist: std::ffi::c_int = 1; + // SAFETY: the handle is this open connection's, the database name is + // NUL-terminated, and SQLite reads and writes the int only during the + // call. + let code = unsafe { + rusqlite::ffi::sqlite3_file_control( + connection.handle(), + c"main".as_ptr(), + rusqlite::ffi::SQLITE_FCNTL_PERSIST_WAL, + (&raw mut persist).cast(), + ) + }; + if code != rusqlite::ffi::SQLITE_OK { + return Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(code), + Some("SQLite can't keep the WAL file".to_owned()), + )); + } + connection.execute_batch("PRAGMA journal_size_limit=0;") } /// Switches the database to WAL. Connections switching a new database at @@ -1914,7 +1976,7 @@ fn could_name_a_capture_file(pattern: &[u8]) -> bool { /// project's, being a link or not a regular file, or that its owner keeps /// from being written, being read-only or on Unix another user's, is left /// as it is; the CLI's capture warning then says git doesn't ignore the -/// captures. Under root, the file written is given to the capture owner. +/// captures. fn ensure_captures_ignored(lpm_dir: &Path, owner: CaptureOwner) -> std::io::Result<()> { let path = lpm_dir.join(".gitignore"); let (existing, metadata) = match lpm_common::read_file_capped_unlinked( @@ -1944,10 +2006,7 @@ fn ensure_captures_ignored(lpm_dir: &Path, owner: CaptureOwner) -> std::io::Resu updated.extend_from_slice(newline); updated.extend_from_slice(CAPTURES_IGNORE_LINE.as_bytes()); updated.extend_from_slice(newline); - // Given to the owner through the new file's own handle, before it's - // renamed into place: under root, the owner can swap the name after. lpm_common::write_file_atomic_with(&path, lpm_common::AtomicWriteOptions::new(), |file| { - owner.give(file, &path)?; std::io::Write::write_all(file, &updated) }) } @@ -2491,6 +2550,14 @@ mod tests { /// without the opening lock can leave files beside it then. pub(super) static AFTER_COPY: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; + /// Runs once after this thread's next replacement's copy takes the + /// database's name, before anything opens it. + pub(super) static AFTER_RENAME: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + /// Runs once after this thread's next open has checked `.lpm`, as + /// the project's owner can move it then. + pub(super) static AFTER_LPM_DIR: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; } #[cfg(unix)] @@ -2506,7 +2573,10 @@ mod tests { /// The capture owner of a project of this user's. #[cfg(unix)] fn this_user() -> CaptureOwner { - CaptureOwner::of(Path::new(".")).unwrap() + CaptureOwner { + uid: lpm_common::current_uid(), + by_root: false, + } } const CAPTURES_IGNORED: &str = @@ -2769,8 +2839,7 @@ mod tests { return; } - let error = - restrict_lpm_dir(Path::new("/usr"), this_user(), Access::Review, false).unwrap_err(); + let error = restrict_lpm_dir(Path::new("/usr"), this_user(), Access::Review).unwrap_err(); assert!( error.to_string().contains("belongs to another user"), @@ -2844,6 +2913,20 @@ mod tests { "{error}" ); assert!(lpm_dir.join("inspector.db-wal").exists()); + + // Without a database, the sidecar moves alone. + std::fs::remove_file(&db_path).unwrap(); + let error = prepare_capture_files(&db_path, this_user(), Access::Capture).unwrap_err(); + + assert!( + error.to_string().ends_with(&format!( + "{} has other hard links, so captures written to it would change those \ + files too; move it aside, and retry", + lpm_dir.join("inspector.db-wal").display() + )), + "{error}" + ); + assert!(lpm_dir.join("inspector.db-wal").exists()); } #[cfg(unix)] @@ -3556,156 +3639,125 @@ mod tests { #[cfg(unix)] #[test] - fn root_accepts_the_project_owner_s_files_and_its_own() { - let as_root = CaptureOwner { - uid: 501, - gid: 20, - as_root: true, - }; - let as_user = CaptureOwner { - uid: 501, - gid: 0, - as_root: false, - }; + fn a_capture_owner_owns_only_its_own_files_even_when_root_opens_them() { let directory = tempfile::tempdir().unwrap(); let mine = std::fs::symlink_metadata(directory.path()).unwrap(); let roots = std::fs::symlink_metadata("/").unwrap(); - let me = CaptureOwner { - uid: lpm_common::current_uid(), - ..as_root - }; - - assert!(as_root.owns(&roots)); - assert!(!as_user.owns(&roots)); - assert!(me.owns(&mine)); - assert!( - !CaptureOwner { + for by_root in [false, true] { + let me = CaptureOwner { + uid: lpm_common::current_uid(), + by_root, + }; + let another = CaptureOwner { uid: lpm_common::current_uid() + 1, - ..as_user - } - .owns(&mine) - ); - } - - #[cfg(unix)] - #[test] - fn as_root_a_capture_file_swapped_after_its_check_keeps_the_other_file_s_mode() { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - - let me_as_root = CaptureOwner { - uid: lpm_common::current_uid(), - gid: std::fs::metadata(".").unwrap().gid(), - as_root: true, - }; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("inspector.db"); - let elsewhere = directory.path().join("elsewhere"); - // A file written in its place instead could take its inode number, - // as ext4 gives a freed one out again, and is one the owner could - // change anyway; a name of an existing file never has its number. - for swap in ["link", "another name"] { - std::fs::write(&path, b"captures").unwrap(); - // Others can't open it, so a writer makes it 0600. - set_mode(&path, 0o700); - std::fs::write(&elsewhere, b"not captures").unwrap(); - set_mode(&elsewhere, 0o644); - let checked = std::fs::symlink_metadata(&path).unwrap(); - std::fs::remove_file(&path).unwrap(); - match swap { - "link" => std::os::unix::fs::symlink(&elsewhere, &path).unwrap(), - _ => std::fs::hard_link(&elsewhere, &path).unwrap(), - } - - let checked_now = - check_capture_file(&path, &path, &checked, me_as_root, Access::Capture); + by_root, + }; - assert!(checked_now.is_err(), "{swap}"); - assert_eq!( - std::fs::metadata(&elsewhere).unwrap().permissions().mode() & 0o7777, - 0o644, - "{swap}" - ); - std::fs::remove_file(&path).unwrap(); + assert!(me.owns(&mine)); + assert!(!another.owns(&mine)); + assert_eq!(me.owns(&roots), lpm_common::current_uid() == 0); } } - #[cfg(unix)] - #[test] - fn as_root_a_capture_file_s_mode_changes_through_its_checked_handle() { - use std::os::unix::fs::PermissionsExt; - - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("inspector.db"); - std::fs::write(&path, b"captures").unwrap(); - set_mode(&path, 0o640); - let checked = std::fs::symlink_metadata(&path).unwrap(); - - set_mode_through_handle(&path, &checked, 0o600).unwrap(); - - assert_eq!( - std::fs::metadata(&path).unwrap().permissions().mode() & 0o7777, - 0o600 - ); + /// A project, and a directory beside it, as root makes them: the project + /// given to [`PROJECT_OWNER`], the other root's own. + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + fn another_user_s_project_beside_root_s_directory() -> (tempfile::TempDir, PathBuf, PathBuf) { + let root = tempfile::tempdir().unwrap(); + set_mode(root.path(), 0o755); + let project = root.path().join("project"); + let elsewhere = root.path().join("elsewhere"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&elsewhere).unwrap(); + std::os::unix::fs::chown(&project, Some(PROJECT_OWNER), Some(PROJECT_OWNER)).unwrap(); + (root, project, elsewhere) } - /// A group this user is in besides the one its files get, or None. - #[cfg(unix)] - fn another_group_of_mine(file: &Path) -> Option { - use std::os::unix::fs::MetadataExt; + /// A user other than root, whose projects root's tests open. + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + const PROJECT_OWNER: u32 = 65_000; - let primary = std::fs::metadata(file).unwrap().gid(); - let groups = std::process::Command::new("id").arg("-G").output().ok()?; - String::from_utf8_lossy(&groups.stdout) - .split_whitespace() - .filter_map(|group| group.parse().ok()) - .find(|group| *group != primary) - } - - #[cfg(unix)] + /// Runs only as root, as in the root container the gate runs Linux + /// tests in once more; CI's runner isn't root. + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] #[test] - fn root_gives_what_it_creates_to_the_project_s_owner() { + fn as_root_another_user_s_captures_are_made_and_kept_as_that_user_s() { use std::os::unix::fs::MetadataExt; - let project = project_with_a_capture(); - let Some(group) = another_group_of_mine(project.path()) else { + if lpm_common::current_uid() != 0 { return; - }; - let lpm_dir = project.path().join(".lpm"); - set_mode(&lpm_dir.join("inspector.db"), 0o644); - std::fs::remove_file(lpm_dir.join("inspector.db-lock")).unwrap(); - std::fs::remove_file(lpm_dir.join(".gitignore")).unwrap(); - let fresh = tempfile::tempdir().unwrap(); - // Root's opens give what they create to the project's owner; this - // user, chowning to itself, can stand in for root by giving files a - // group of its own instead. - let owner = CaptureOwner { - uid: lpm_common::current_uid(), - gid: group, - as_root: true, - }; + } + let (_root, project, _) = another_user_s_project_beside_root_s_directory(); let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async { - drop(InspectorDb::open_project(project.path(), Access::Capture, owner).unwrap()); - drop(InspectorDb::open_project(fresh.path(), Access::Review, owner).unwrap()); + drop(InspectorDb::open(&project).unwrap()); + drop(InspectorDb::open_for_review(&project).unwrap()); }); + // The connections close as the flush task ends. + drop(runtime); - for path in [ - lpm_dir.join("inspector.db"), - lpm_dir.join("inspector.db-lock"), - lpm_dir.join(".gitignore"), - fresh.path().join(".lpm"), - fresh.path().join(".lpm/inspector.db"), - fresh.path().join(".lpm/inspector.db-lock"), - fresh.path().join(".lpm/.gitignore"), + let lpm_dir = project.join(".lpm"); + for name in [ + "", + "inspector.db", + "inspector.db-wal", + "inspector.db-shm", + "inspector.db-lock", + ".gitignore", ] { + let path = lpm_dir.join(name); + let metadata = std::fs::metadata(&path).unwrap(); assert_eq!( - std::fs::metadata(&path).unwrap().gid(), - group, + (metadata.uid(), metadata.gid()), + (PROJECT_OWNER, PROJECT_OWNER), "{}", path.display() ); } + assert_eq!( + std::fs::metadata(lpm_dir.join("inspector.db-wal")) + .unwrap() + .len(), + 0 + ); + assert_eq!(lpm_common::thread_identity::filesystem_uid(), 0); + } + + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + #[test] + fn as_root_another_user_moving_lpm_while_it_opens_can_t_have_root_write_elsewhere() { + if lpm_common::current_uid() != 0 { + return; + } + let (_root, project, elsewhere) = another_user_s_project_beside_root_s_directory(); + for name in ["inspector.db-copy", "inspector.db-wal"] { + std::fs::write(elsewhere.join(name), b"root's").unwrap(); + } + let lpm_dir = project.join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + std::os::unix::fs::chown(&lpm_dir, Some(PROJECT_OWNER), Some(PROJECT_OWNER)).unwrap(); + let moved = project.join("moved"); + let link = elsewhere.clone(); + AFTER_LPM_DIR.set(Some(Box::new(move || { + std::fs::rename(&lpm_dir, &moved).unwrap(); + std::os::unix::fs::symlink(&link, &lpm_dir).unwrap(); + }))); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let opened = runtime.block_on(async { InspectorDb::open(&project).map(drop) }); + + assert!(opened.is_err()); + let mut names: Vec<_> = std::fs::read_dir(&elsewhere) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + names.sort(); + assert_eq!(names, ["inspector.db-copy", "inspector.db-wal"]); + for name in names { + assert_eq!(std::fs::read(elsewhere.join(name)).unwrap(), b"root's"); + } + assert_eq!(lpm_common::thread_identity::filesystem_uid(), 0); } #[cfg(target_os = "macos")] @@ -4093,7 +4145,7 @@ mod tests { .execute_batch("CREATE TABLE kept (x); INSERT INTO kept VALUES (zeroblob(100000));") .unwrap(); let copy_path = capture_file(&db_path, "-copy"); - assert!(create_owner_only(©_path, this_user()).unwrap()); + assert!(create_owner_only(©_path).unwrap()); // The room taken before the database was opened, which this // database's room now would read as less. @@ -4261,19 +4313,28 @@ mod tests { capture_file(&db_path, "-wal"), capture_file(&db_path, "-shm"), ]; + let journal = capture_file(&db_path, "-journal"); let leaving = left.clone(); + let journal_left = journal.clone(); AFTER_COPY.set(Some(Box::new(move || { - for path in &leaving { + for path in leaving.iter().chain([&journal_left]) { std::fs::write(path, b"left by another connection").unwrap(); set_mode(path, 0o644); } }))); + // SQLite removes a journal it can't read as it opens the database, + // and plays back one it can, so it's looked for before that. + let journal_at_rename = std::rc::Rc::new(std::cell::Cell::new(None)); + let looked = std::rc::Rc::clone(&journal_at_rename); + AFTER_RENAME.set(Some(Box::new(move || looked.set(Some(journal.exists()))))); + let db = InspectorDb::open(project.path()); let copied = AFTER_COPY.with_borrow_mut(Option::take).is_none(); let db = db.unwrap(); assert!(copied); + assert_eq!(journal_at_rename.get(), Some(false)); assert_eq!(mode(&db_path), 0o600); for path in &left { assert_eq!(mode(path), 0o600, "{}", path.display()); @@ -4335,7 +4396,7 @@ mod tests { .execute_batch("BEGIN EXCLUSIVE; INSERT INTO kept VALUES (2);") .unwrap(); let copy_path = capture_file(&db_path, "-copy"); - assert!(create_owner_only(©_path, this_user()).unwrap()); + assert!(create_owner_only(©_path).unwrap()); let copied = copy_database(&database, ©_path, &db_path, room_to_copy(&db_path)); holder.execute_batch("COMMIT").unwrap(); @@ -4364,7 +4425,7 @@ mod tests { }); let copy_path = capture_file(&db_path, "-copy"); - assert!(create_owner_only(©_path, this_user()).unwrap()); + assert!(create_owner_only(©_path).unwrap()); copy_database(&database, ©_path, &db_path, room_to_copy(&db_path)).unwrap(); released.join().unwrap(); @@ -4989,6 +5050,7 @@ mod tests { db_path, open_to_others: false, ownership_ignored: false, + _temp_dir: None, }; db.start_session("saturated-session".to_string(), None, 3000) From 95c5dba7eee916b0c694b70b3b460d8764c92c01 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Tue, 13 Oct 2026 15:37:40 +0100 Subject: [PATCH 16/17] fix(tunnel): keep captures only where no one else can change the path, and hold the database's lock while replacing it - Refuse a project whose path a user other than its owner and root can change: another user's directory, one others may write to (unless sticky and the entry is the owner's or root's), one a shared group may write to, a Linux directory with an ACL that may let others write, or a macOS one whose ACL lets others change its entries. A group counts as the owner's alone only as their primary group, named after them, with no other member. - As root in another user's project on 64-bit Linux, take the owner's primary and supplementary groups (the project's group for an unknown owner, 65534 when that's root's), give the owner `.lpm` and the capture files root made there only while `.lpm` is root's alone, and refuse a capture file root still owns, naming it. Explain a permission refusal under the owner's identity. - Hold the checking connection's exclusive lock until the owner-only copy has taken the database's name, so a connection without the opening lock can't commit a capture the copy lacks; SQLite refuses its later writes to the moved database. - Read the page size from the WAL alone: closing a handle on the database would drop the locks SQLite holds on it in this process. - Clamp counts past i64 to the largest LIMIT instead of no limit. - Read `.git` entries by git's rules for quotes in paths, NUL bytes, non-UTF-8 paths and empty or dangling `commondir` files. - Advise moving a database aside under names that still start inspector.db, so git goes on ignoring them. - ActingAs is neither Send nor Sync, checked at compile time; act_as checks the thread's groups too. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 2 + crates/lpm-cli/src/commands/tunnel.rs | 81 ++- crates/lpm-common/Cargo.toml | 1 + crates/lpm-common/src/lib.rs | 2 + crates/lpm-common/src/macos_acl.rs | 69 ++ crates/lpm-common/src/path_trust.rs | 781 +++++++++++++++++++++++ crates/lpm-common/src/thread_identity.rs | 71 ++- crates/lpm-inspect/Cargo.toml | 3 + crates/lpm-inspect/src/db.rs | 749 ++++++++++++++++++---- tests/workflows/tests/dev_tunnel.rs | 6 +- 10 files changed, 1597 insertions(+), 168 deletions(-) create mode 100644 crates/lpm-common/src/path_trust.rs diff --git a/Cargo.lock b/Cargo.lock index cc334e91d..cd7fc4020 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3518,6 +3518,7 @@ dependencies = [ "sha1 0.10.6", "sha2", "smallvec", + "static_assertions", "subtle", "tempfile", "thiserror 2.0.18", @@ -3681,6 +3682,7 @@ dependencies = [ "base64", "chrono", "futures-util", + "libc", "lpm-common", "lpm-http", "lpm-tunnel", diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index 23f601812..140fe7151 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -1007,10 +1007,11 @@ fn capture_git_failure(stderr: &[u8]) -> CaptureGitFailure { fn unreadable_git_file(reason: &str, starts: impl Fn(&str) -> bool) -> bool { let names_a_git_file = |path: &str| path.ends_with("/.git") || path == ".git"; if starts("error opening '") { + // The path is quoted, then ": " and the system's error; the path + // itself can hold a quote. return reason["error opening '".len()..] - .split('\'') - .next() - .is_some_and(names_a_git_file); + .rsplit_once("': ") + .is_some_and(|(path, _)| names_a_git_file(path)); } starts("error reading ") && names_a_git_file(reason["error reading ".len()..].trim_end()) } @@ -1177,18 +1178,41 @@ fn marks_a_repository(git: &Path) -> bool { let Some(pointer) = contents .strip_prefix(b"gitdir: ") .map(trim_line_breaks) - .and_then(|path| std::str::from_utf8(path).ok()) + .map(until_nul) .filter(|path| !path.is_empty()) else { return false; }; let directory = git.parent().unwrap_or(Path::new(".")); - is_git_directory(&directory.join(pointer)) + is_git_directory(&directory.join(path_from_bytes(pointer))) } _ => false, } } +/// `bytes` up to a NUL, where git, reading them as a C string, stops. +fn until_nul(bytes: &[u8]) -> &[u8] { + bytes + .iter() + .position(|byte| *byte == 0) + .map_or(bytes, |end| &bytes[..end]) +} + +/// A path from the bytes of a file git reads; on Windows, where a path +/// isn't bytes, only UTF-8 reads as one. +fn path_from_bytes(bytes: &[u8]) -> PathBuf { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt as _; + + PathBuf::from(std::ffi::OsStr::from_bytes(bytes)) + } + #[cfg(not(unix))] + { + PathBuf::from(String::from_utf8_lossy(bytes).into_owned()) + } +} + /// `bytes` without the line breaks, CR or LF, that end it. fn trim_line_breaks(bytes: &[u8]) -> &[u8] { let end = bytes @@ -1206,14 +1230,22 @@ fn is_git_directory(directory: &Path) -> bool { if !names_a_head(&directory.join("HEAD")) { return false; } + // Git fails on a `commondir` file it can't read, an empty one, or one + // naming a path that doesn't resolve. let common = match lpm_common::read_regular_file_capped_with_metadata( &directory.join("commondir"), GITFILE_LIMIT, ) { - Ok((contents, _)) => match std::str::from_utf8(trim_line_breaks(&contents)) { - Ok(path) => directory.join(path), - Err(_) => return false, - }, + Ok((contents, _)) => { + let path = until_nul(trim_line_breaks(&contents)); + if path.is_empty() { + return false; + } + match std::fs::canonicalize(directory.join(path_from_bytes(path))) { + Ok(common) => common, + Err(_) => return false, + } + } Err(lpm_common::BoundedReadError::NotFound { .. }) => directory.to_path_buf(), Err(_) => return false, }; @@ -3398,6 +3430,16 @@ mod tests { } assert!(!marks_a_repository(&directory.path().join("gone/.git"))); + // Git reads a pointer as a C string, so it ends at a NUL. + let nul_dir = tempfile::tempdir().unwrap(); + repository_entry(&nul_dir.path().join("elsewhere")); + std::fs::write( + nul_dir.path().join(".git"), + b"gitdir: elsewhere/.git\0junk\n", + ) + .unwrap(); + assert!(marks_a_repository(&nul_dir.path().join(".git"))); + // A pointer with no path is none, even in a git directory, which the // empty path would otherwise name. let bare = tempfile::tempdir().unwrap(); @@ -3415,6 +3457,9 @@ mod tests { let git = directory.path().join(".git"); repository_entry(directory.path()); let head = git.join("HEAD"); + // Git reads the first 255 bytes of it. + let within = format!("ref:{}refs/heads/main\n", " ".repeat(240)); + let past = format!("ref:{}refs/heads/main\n", " ".repeat(260)); for (contents, repository) in [ ("ref: refs/heads/main\n", true), ("ref:refs/heads/main", true), @@ -3422,6 +3467,8 @@ mod tests { ("ref: \x0b refs/heads/main\n", false), ("ref: \x0c refs/heads/main\n", false), ("ref: heads/main\n", false), + (within.as_str(), true), + (past.as_str(), false), ("refs/heads/main\n", false), ("0123456789abcdef0123456789ABCDEF01234567\n", true), ("0123456789abcdef0123456789abcdef01234567", true), @@ -3488,6 +3535,9 @@ mod tests { ("../..\r\n", true), ("../../objects\n", false), ("../gone\n", false), + // Git fails on an empty one. + ("\n", false), + ("", false), ] { std::fs::write(worktree_git.join("commondir"), common).unwrap(); assert_eq!( @@ -3572,6 +3622,19 @@ mod tests { "fatal: error opening '/work/app/.git/index': Permission denied\n", CaptureGitFailure::Refused, ), + ( + "fatal: error opening '/work/it's/app/.git': Permission denied\n", + CaptureGitFailure::Stopped, + ), + ( + "fatal: error opening '/work/app/x.git': Permission denied\n", + CaptureGitFailure::Refused, + ), + ( + "fatal: error reading /work/app/x.git\n", + CaptureGitFailure::Refused, + ), + ("fatal: error reading .git\n", CaptureGitFailure::Stopped), ( "warning: unable to access '/home/user/.config/git/attributes'\n\ fatal: not a git repository (or any of the parent directories): .git\n", diff --git a/crates/lpm-common/Cargo.toml b/crates/lpm-common/Cargo.toml index 6cb0cefe1..77a829849 100644 --- a/crates/lpm-common/Cargo.toml +++ b/crates/lpm-common/Cargo.toml @@ -47,6 +47,7 @@ windows-sys = { version = "0.60", features = [ ] } [dev-dependencies] +static_assertions = "1" tempfile = "3" [lints] diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index d161b4dc4..1c4b82ec5 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -21,6 +21,8 @@ pub mod macos_acl; pub mod npmrc; pub mod package_instance; pub mod package_name; +#[cfg(unix)] +pub mod path_trust; pub mod paths; pub mod peer; pub mod platform; diff --git a/crates/lpm-common/src/macos_acl.rs b/crates/lpm-common/src/macos_acl.rs index 7b6ff4d52..9a47583cd 100644 --- a/crates/lpm-common/src/macos_acl.rs +++ b/crates/lpm-common/src/macos_acl.rs @@ -11,9 +11,20 @@ use std::path::Path; const ACL_TYPE_EXTENDED: c_int = 0x0000_0100; +const ACL_FIRST_ENTRY: c_int = 0; +const ACL_NEXT_ENTRY: c_int = -1; +const ACL_EXTENDED_ALLOW: c_int = 1; +/// What lets someone change a directory's entries, or who may: adding a +/// file or a subdirectory, deleting a child, and changing its ACL or owner. +const ENTRY_CHANGING_PERMISSIONS: [c_int; 5] = [1 << 2, 1 << 5, 1 << 6, 1 << 12, 1 << 13]; + unsafe extern "C" { fn acl_free(value: *mut c_void) -> c_int; + fn acl_get_entry(acl: *mut c_void, entry_id: c_int, entry: *mut *mut c_void) -> c_int; fn acl_get_fd_np(file_descriptor: c_int, acl_type: c_int) -> *mut c_void; + fn acl_get_perm_np(permset: *mut c_void, perm: c_int) -> c_int; + fn acl_get_permset(entry: *mut c_void, permset: *mut *mut c_void) -> c_int; + fn acl_get_tag_type(entry: *mut c_void, tag: *mut c_int) -> c_int; fn acl_get_link_np(path: *const c_char, acl_type: c_int) -> *mut c_void; fn acl_init(entry_count: c_int) -> *mut c_void; fn acl_set_fd_np(file_descriptor: c_int, acl: *mut c_void, acl_type: c_int) -> c_int; @@ -78,6 +89,64 @@ fn extended_acl_found(get: impl FnOnce() -> *mut c_void) -> io::Result { } } +/// Whether the open directory's extended ACL allows anyone to add, remove +/// or rename its entries, or to change its ACL or owner. Every allow entry +/// counts, whoever it names, as the directory's owner needs none. +pub fn allows_entry_changes(handle: impl AsFd) -> io::Result { + let file_descriptor = handle.as_fd().as_raw_fd(); + // SAFETY: errno is this thread's own; the ACL is freed once, and the + // entries and permission sets read from it are used only before that. + unsafe { + *libc::__error() = 0; + let acl = acl_get_fd_np(file_descriptor, ACL_TYPE_EXTENDED); + if acl.is_null() { + let error = *libc::__error(); + return if error == 0 || error == libc::ENOENT { + Ok(false) + } else { + Err(io::Error::from_raw_os_error(error)) + }; + } + let allows = allow_entry_changes_in(acl); + if acl_free(acl) != 0 && allows.is_ok() { + return Err(io::Error::last_os_error()); + } + allows + } +} + +/// # Safety +/// `acl` must be a live ACL from `acl_get_fd_np`. +unsafe fn allow_entry_changes_in(acl: *mut c_void) -> io::Result { + let mut entry = std::ptr::null_mut(); + let mut entry_id = ACL_FIRST_ENTRY; + // SAFETY: as the caller guarantees; each out-pointer is a local. + unsafe { + while acl_get_entry(acl, entry_id, &mut entry) == 0 { + entry_id = ACL_NEXT_ENTRY; + let mut tag = 0; + if acl_get_tag_type(entry, &mut tag) != 0 { + return Err(io::Error::last_os_error()); + } + if tag != ACL_EXTENDED_ALLOW { + continue; + } + let mut permset = std::ptr::null_mut(); + if acl_get_permset(entry, &mut permset) != 0 { + return Err(io::Error::last_os_error()); + } + for permission in ENTRY_CHANGING_PERMISSIONS { + match acl_get_perm_np(permset, permission) { + 0 => {} + 1 => return Ok(true), + _ => return Err(io::Error::last_os_error()), + } + } + } + } + Ok(false) +} + /// Removes every extended ACL entry from the open file or directory, and /// checks that none remain. pub fn clear_extended_acl(handle: impl AsFd) -> io::Result<()> { diff --git a/crates/lpm-common/src/path_trust.rs b/crates/lpm-common/src/path_trust.rs new file mode 100644 index 000000000..3ef832483 --- /dev/null +++ b/crates/lpm-common/src/path_trust.rs @@ -0,0 +1,781 @@ +//! Whether other users can change what a path leads to. A directory's +//! entries can be renamed, removed or replaced by whoever may write to it, +//! or change its mode or ACL, so files opened by name further down are only +//! as safe as every directory on the way. + +use std::ffi::{CStr, CString, OsStr}; +use std::io; +use std::os::unix::ffi::OsStrExt as _; +use std::os::unix::fs::MetadataExt as _; +use std::path::{Component, Path, PathBuf}; + +/// A directory on a path that a user other than the one trusted, or root, +/// can change the entries of, and why. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Changeable { + pub directory: PathBuf, + pub reason: &'static str, +} + +/// How many links a path may pass through, as Linux allows. +const MAX_LINKS: usize = 40; + +/// The first directory, resolving `path` a component at a time as the +/// kernel does, whose entries a user other than `uid` and root can change: +/// one another user owns; one its group may write, unless its group is +/// `uid`'s own and has no other member, as with user-private groups; one +/// anyone may write; and one whose ACL lets others add or remove entries. +/// A sticky directory, as `/tmp`, is safe even so for an entry `uid` or +/// root owns, which only they may move. A link is followed, and its target +/// resolved the same way. None when every directory consulted is safe. +pub fn first_changeable_directory(path: &Path, uid: u32) -> io::Result> { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir()?.join(path) + }; + let mut pending: Vec = Vec::new(); + push_components(&mut pending, &absolute); + let mut current = PathBuf::from("/"); + if let Some(found) = changeable(¤t, &std::fs::metadata(¤t)?, uid)? { + return Ok(Some(found)); + } + let mut links = 0; + while let Some(name) = pending.pop() { + if name == OsStr::new("/") { + current = PathBuf::from("/"); + continue; + } + if name == OsStr::new("..") { + current.pop(); + continue; + } + let parent = std::fs::metadata(¤t)?; + let next = current.join(&name); + let entry = std::fs::symlink_metadata(&next)?; + if parent.mode() & 0o1000 != 0 + && entry.uid() != uid + && entry.uid() != 0 + && others_may_add(&parent, uid, Acl::At(¤t))?.is_some() + { + return Ok(Some(Changeable { + directory: current, + reason: "others may add to it, and the entry on the path belongs to another user", + })); + } + if entry.file_type().is_symlink() { + links += 1; + if links > MAX_LINKS { + return Err(io::Error::from_raw_os_error(libc::ELOOP)); + } + let target = std::fs::read_link(&next)?; + push_components(&mut pending, &target); + continue; + } + current = next; + if entry.is_dir() + && let Some(found) = changeable(¤t, &entry, uid)? + { + return Ok(Some(found)); + } + } + Ok(None) +} + +/// Pushes `path`'s components onto `pending` so they pop in order; an +/// absolute path starts from the root again. +fn push_components(pending: &mut Vec, path: &Path) { + for component in path.components().rev() { + match component { + Component::RootDir => pending.push("/".into()), + Component::ParentDir => pending.push("..".into()), + Component::Normal(name) => pending.push(name.to_os_string()), + Component::CurDir | Component::Prefix(_) => {} + } + } +} + +/// Where a directory's ACL is read from: its path, which isn't followed +/// if it's a link, or a handle open on it. +#[derive(Clone, Copy)] +enum Acl<'a> { + At(&'a Path), + Of(&'a std::fs::File), +} + +fn changeable( + directory: &Path, + metadata: &std::fs::Metadata, + uid: u32, +) -> io::Result> { + Ok( + why_changeable(metadata, uid, Acl::At(directory))?.map(|reason| Changeable { + directory: directory.to_path_buf(), + reason, + }), + ) +} + +/// Why users other than `uid` and root can change the entries of the +/// directory `metadata` describes, if they can; in a sticky one, only an +/// entry's owner can move it. +fn why_changeable( + metadata: &std::fs::Metadata, + uid: u32, + acl: Acl<'_>, +) -> io::Result> { + if metadata.uid() != uid && metadata.uid() != 0 { + return Ok(Some("it belongs to another user")); + } + if metadata.mode() & 0o1000 == 0 { + return others_may_add(metadata, uid, acl); + } + #[cfg(target_os = "macos")] + if acl_allows_entry_changes(acl)? { + return Ok(Some(MACOS_ACL_REASON)); + } + Ok(None) +} + +/// Why users other than `uid` and root may add entries to the directory +/// `metadata` describes, sticky or not, if they may. +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(unused_variables) +)] +fn others_may_add( + metadata: &std::fs::Metadata, + uid: u32, + acl: Acl<'_>, +) -> io::Result> { + #[cfg(target_os = "macos")] + if acl_allows_entry_changes(acl)? { + return Ok(Some(MACOS_ACL_REASON)); + } + let mode = metadata.mode(); + if mode & 0o002 != 0 { + return Ok(Some("anyone may write to it")); + } + if mode & 0o020 == 0 { + return Ok(None); + } + // With an ACL, the group's bits are the most any of its entries + // grants, and other users may have entries of their own. + #[cfg(target_os = "linux")] + if has_posix_acl(acl)? { + return Ok(Some("its access control list may let others write to it")); + } + if group_is_private(metadata.gid(), uid)? { + return Ok(None); + } + Ok(Some("its group, which has other members, may write to it")) +} + +#[cfg(target_os = "macos")] +const MACOS_ACL_REASON: &str = "its access control list lets others change its entries"; + +/// Whether a directory's ACL lets others add, remove or rename its +/// entries, whatever its mode, as a macOS ACL can; a Linux ACL can grant +/// no more than its mode's group bits show. +#[cfg(target_os = "macos")] +fn acl_allows_entry_changes(acl: Acl<'_>) -> io::Result { + match acl { + Acl::At(directory) => { + crate::macos_acl::allows_entry_changes(crate::open_nofollow(directory)?) + } + Acl::Of(handle) => crate::macos_acl::allows_entry_changes(handle), + } +} + +/// Whether a directory has an access ACL beyond its mode, which Linux +/// keeps in an extended attribute; one on a filesystem without them has +/// none. +#[cfg(target_os = "linux")] +fn has_posix_acl(acl: Acl<'_>) -> io::Result { + use std::os::fd::AsRawFd as _; + + const NAME: &CStr = c"system.posix_acl_access"; + // SAFETY: the strings are NUL-terminated and outlive the calls, the + // descriptor is open for the call, and a size of 0 only asks the + // attribute's length, writing nothing. + let size = match acl { + Acl::At(directory) => { + let path = CString::new(directory.as_os_str().as_bytes()) + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + unsafe { libc::lgetxattr(path.as_ptr(), NAME.as_ptr(), std::ptr::null_mut(), 0) } + } + Acl::Of(handle) => unsafe { + libc::fgetxattr(handle.as_raw_fd(), NAME.as_ptr(), std::ptr::null_mut(), 0) + }, + }; + if size >= 0 { + return Ok(true); + } + let error = io::Error::last_os_error(); + match error.raw_os_error() { + Some(libc::ENODATA | libc::ENOTSUP) => Ok(false), + _ => Err(error), + } +} + +/// A user's name, primary group and every group they're in, from the +/// system's user database; None for a user it doesn't know, as a host's +/// user in a container. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Account { + pub name: CString, + pub gid: u32, +} + +/// The account of `uid`, or None where the user database has none. +pub fn account(uid: u32) -> io::Result> { + let mut buffer = vec![0u8; 1024]; + loop { + let mut entry: libc::passwd = unsafe { std::mem::zeroed() }; + let mut result: *mut libc::passwd = std::ptr::null_mut(); + // SAFETY: `entry`, `buffer` and `result` outlive the call, which + // writes no more than `buffer.len()` bytes into `buffer`. + let error = unsafe { + libc::getpwuid_r( + uid, + &mut entry, + buffer.as_mut_ptr().cast(), + buffer.len(), + &mut result, + ) + }; + if error == libc::ERANGE && buffer.len() < 1 << 20 { + buffer.resize(buffer.len() * 2, 0); + continue; + } + if error != 0 { + return Err(io::Error::from_raw_os_error(error)); + } + if result.is_null() { + return Ok(None); + } + // SAFETY: on success `pw_name` points at a NUL-terminated name + // within `buffer`. + let name = unsafe { CStr::from_ptr(entry.pw_name) }.to_owned(); + return Ok(Some(Account { + name, + gid: entry.pw_gid, + })); + } +} + +/// A group's name and the names in its member list. +struct Group { + name: CString, + members: Vec, +} + +/// Group `gid`, or None for a group the database doesn't know. +fn group(gid: u32) -> io::Result> { + let mut buffer = vec![0u8; 4096]; + loop { + let mut entry: libc::group = unsafe { std::mem::zeroed() }; + let mut result: *mut libc::group = std::ptr::null_mut(); + // SAFETY: as in `account`. + let error = unsafe { + libc::getgrgid_r( + gid, + &mut entry, + buffer.as_mut_ptr().cast(), + buffer.len(), + &mut result, + ) + }; + if error == libc::ERANGE && buffer.len() < 1 << 22 { + buffer.resize(buffer.len() * 2, 0); + continue; + } + if error != 0 { + return Err(io::Error::from_raw_os_error(error)); + } + if result.is_null() { + return Ok(None); + } + // SAFETY: on success `gr_name` points at a NUL-terminated name + // within `buffer`. + let name = unsafe { CStr::from_ptr(entry.gr_name) }.to_owned(); + let mut members = Vec::new(); + let mut member = entry.gr_mem; + // SAFETY: `gr_mem` is a NULL-terminated array of NUL-terminated + // names within `buffer`. + unsafe { + while !member.is_null() && !(*member).is_null() { + members.push(CStr::from_ptr(*member).to_owned()); + member = member.add(1); + } + } + return Ok(Some(Group { name, members })); + } +} + +/// Whether group `gid` is `uid`'s alone, as a user-private group is: +/// `uid`'s primary group, named after them, listing no other member. Users +/// whose primary group it is aren't listed as members, so a group many +/// users share that way, as `users`, isn't taken for one; its name isn't +/// theirs. +pub fn group_is_private(gid: u32, uid: u32) -> io::Result { + let Some(account) = account(uid)? else { + return Ok(false); + }; + if account.gid != gid { + return Ok(false); + } + Ok(group(gid)?.is_some_and(|group| { + group.name == account.name && group.members.iter().all(|member| *member == account.name) + })) +} + +/// The groups `account` is in, its primary group first, as the user +/// database lists them. +#[cfg(target_os = "linux")] +pub fn groups_of(account: &Account) -> io::Result> { + let mut count: libc::c_int = 32; + loop { + let mut groups = vec![0 as libc::gid_t; count as usize]; + let mut wanted = count; + // SAFETY: `groups` has room for `wanted` IDs; the call writes how + // many it found, or needs, into `wanted`. + let found = unsafe { + libc::getgrouplist( + account.name.as_ptr(), + account.gid, + groups.as_mut_ptr(), + &mut wanted, + ) + }; + if found >= 0 { + groups.truncate(wanted as usize); + return Ok(groups); + } + if wanted <= count || wanted > 1 << 16 { + return Err(io::Error::other("the user's groups can't be listed")); + } + count = wanted; + } +} + +/// Gives what root left in `directory` to `uid` and `gid`, as a command run +/// with `sudo` leaves what it makes root's: the regular files `names` +/// there, then `directory`. Only while `directory` is root's and no one +/// else may change its entries, so nothing in it can have been moved there +/// from elsewhere; and only files with no other hard link, each through a +/// handle that doesn't follow a link, so nothing elsewhere is given away. +/// Returns those of `names` root still owns there, other than links. A +/// missing `directory`, or a link, has nothing to give. +pub fn give_root_entries( + directory: &Path, + names: &[&OsStr], + uid: u32, + gid: u32, +) -> io::Result> { + let flags = libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK; + let path = CString::new(directory.as_os_str().as_bytes()) + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + // SAFETY: `path` is NUL-terminated and outlives the call. + let directory_fd = unsafe { libc::open(path.as_ptr(), flags | libc::O_DIRECTORY) }; + if directory_fd < 0 { + let error = io::Error::last_os_error(); + if matches!( + error.raw_os_error(), + Some(libc::ENOENT | libc::ELOOP | libc::ENOTDIR) + ) { + return Ok(Vec::new()); + } + return Err(error); + } + // SAFETY: the descriptor was just opened and is owned here. + let directory_handle = + unsafe { ::from_raw_fd(directory_fd) }; + let directory_metadata = directory_handle.metadata()?; + let roots_alone = directory_metadata.uid() == 0 + && others_may_add(&directory_metadata, 0, Acl::Of(&directory_handle))?.is_none(); + let mut kept = Vec::new(); + for name in names { + let c_name = CString::new(name.as_bytes()) + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + // SAFETY: the directory descriptor is open and `c_name` is + // NUL-terminated. + let fd = unsafe { libc::openat(directory_fd, c_name.as_ptr(), flags) }; + if fd < 0 { + let error = io::Error::last_os_error(); + // A link is left for the opening to refuse, whoever made it. + if matches!(error.raw_os_error(), Some(libc::ENOENT | libc::ELOOP)) { + continue; + } + return Err(error); + } + // SAFETY: as above. + let handle = unsafe { ::from_raw_fd(fd) }; + let metadata = handle.metadata()?; + if metadata.uid() != 0 { + continue; + } + if roots_alone && metadata.is_file() && metadata.nlink() == 1 { + std::os::unix::fs::fchown(&handle, Some(uid), Some(gid))?; + } else { + kept.push(name.to_os_string()); + } + } + if roots_alone { + std::os::unix::fs::fchown(&directory_handle, Some(uid), Some(gid))?; + } + Ok(kept) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt as _; + + fn set_mode(path: &Path, mode: u32) { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + fn me() -> u32 { + crate::current_uid() + } + + #[test] + fn a_path_through_directories_only_its_user_may_change_is_safe() { + let directory = tempfile::tempdir().unwrap(); + let project = directory.path().join("project"); + std::fs::create_dir(&project).unwrap(); + set_mode(&project, 0o755); + + assert_eq!(first_changeable_directory(&project, me()).unwrap(), None); + } + + #[test] + fn a_directory_anyone_may_write_is_changeable_unless_sticky() { + let directory = tempfile::tempdir().unwrap(); + let shared = directory.path().join("shared"); + let project = shared.join("project"); + std::fs::create_dir_all(&project).unwrap(); + set_mode(&shared, 0o777); + + let found = first_changeable_directory(&project, me()).unwrap().unwrap(); + assert_eq!(found.directory, shared.canonicalize().unwrap()); + assert_eq!(found.reason, "anyone may write to it"); + + set_mode(&shared, 0o1777); + assert_eq!(first_changeable_directory(&project, me()).unwrap(), None); + } + + #[test] + fn a_link_is_followed_and_its_target_checked() { + let directory = tempfile::tempdir().unwrap(); + let open = directory.path().join("open"); + let project = open.join("project"); + std::fs::create_dir_all(&project).unwrap(); + set_mode(&open, 0o777); + let link = directory.path().join("link"); + std::os::unix::fs::symlink(&project, &link).unwrap(); + + let found = first_changeable_directory(&link, me()).unwrap().unwrap(); + assert_eq!(found.directory, open.canonicalize().unwrap()); + } + + #[test] + fn a_link_in_a_directory_anyone_may_write_is_changeable() { + let directory = tempfile::tempdir().unwrap(); + let open = directory.path().join("open"); + let project = directory.path().join("project"); + std::fs::create_dir(&open).unwrap(); + std::fs::create_dir(&project).unwrap(); + let link = open.join("link"); + std::os::unix::fs::symlink(&project, &link).unwrap(); + set_mode(&open, 0o777); + + let found = first_changeable_directory(&link, me()).unwrap().unwrap(); + assert_eq!(found.directory, open.canonicalize().unwrap()); + } + + #[test] + fn another_user_s_directory_is_changeable() { + let directory = tempfile::tempdir().unwrap(); + let (owner, another) = if me() == 0 { + std::os::unix::fs::chown(directory.path(), Some(USER), None).unwrap(); + (USER, ANOTHER_USER) + } else { + (me(), me() + 1) + }; + let metadata = std::fs::metadata(directory.path()).unwrap(); + let acl = Acl::At(directory.path()); + + assert_eq!( + why_changeable(&metadata, another, acl).unwrap(), + Some("it belongs to another user") + ); + assert_eq!(why_changeable(&metadata, owner, acl).unwrap(), None); + } + + #[test] + fn a_group_writable_directory_is_changeable_unless_its_group_is_the_user_s_alone() { + let directory = tempfile::tempdir().unwrap(); + let project = directory.path().join("project"); + std::fs::create_dir(&project).unwrap(); + set_mode(&project, 0o775); + let gid = std::fs::metadata(&project).unwrap().gid(); + + let found = first_changeable_directory(&project, me()).unwrap(); + if group_is_private(gid, me()).unwrap() { + assert_eq!(found, None); + } else { + assert_eq!( + found.unwrap().reason, + "its group, which has other members, may write to it" + ); + } + } + + #[test] + fn a_group_is_private_only_as_the_user_s_primary_group_named_after_them_alone() { + let Some(account) = account(me()).unwrap() else { + return; + }; + let primary = group(account.gid).unwrap(); + assert_eq!( + group_is_private(account.gid, me()).unwrap(), + primary.is_some_and(|primary| primary.name == account.name + && primary.members.iter().all(|member| *member == account.name)) + ); + assert!(!group_is_private(account.gid, me() + 1_000_000).unwrap()); + assert!(!group_is_private(account.gid.wrapping_add(1), me()).unwrap()); + } + + #[cfg(target_os = "macos")] + #[test] + fn an_acl_letting_anyone_add_entries_makes_a_directory_changeable() { + let directory = tempfile::tempdir().unwrap(); + let project = directory.path().join("project"); + std::fs::create_dir(&project).unwrap(); + let chmod = |entry: &str| { + assert!( + std::process::Command::new("/bin/chmod") + .args(["+a", entry]) + .arg(&project) + .status() + .unwrap() + .success() + ); + }; + + chmod("everyone deny delete"); + assert_eq!(first_changeable_directory(&project, me()).unwrap(), None); + chmod("everyone allow add_file"); + assert_eq!( + first_changeable_directory(&project, me()) + .unwrap() + .unwrap() + .reason, + "its access control list lets others change its entries" + ); + } + + #[test] + fn a_user_s_own_entries_are_neither_given_nor_kept() { + if me() == 0 { + return; + } + let directory = tempfile::tempdir().unwrap(); + std::fs::write(directory.path().join("mine"), b"").unwrap(); + + let kept = give_root_entries( + directory.path(), + &[OsStr::new("mine"), OsStr::new("missing")], + me() + 1, + 0, + ) + .unwrap(); + + assert!(kept.is_empty(), "{kept:?}"); + assert_eq!( + std::fs::metadata(directory.path().join("mine")) + .unwrap() + .uid(), + me() + ); + assert_eq!(std::fs::metadata(directory.path()).unwrap().uid(), me()); + } + + #[test] + fn a_missing_directory_or_a_link_has_nothing_to_give() { + let directory = tempfile::tempdir().unwrap(); + let link = directory.path().join("link"); + std::os::unix::fs::symlink(directory.path(), &link).unwrap(); + + for path in [directory.path().join("missing"), link] { + assert!( + give_root_entries(&path, &[OsStr::new("any")], me() + 1, 0) + .unwrap() + .is_empty() + ); + } + } + + /// A user other than root, whose directories root's tests make. + const USER: u32 = 65_000; + const USER_GROUP: u32 = 65_001; + const ANOTHER_USER: u32 = 65_002; + + fn owners(path: &Path) -> (u32, u32) { + let metadata = std::fs::symlink_metadata(path).unwrap(); + (metadata.uid(), metadata.gid()) + } + + /// Runs only as root, as in the root container the gate runs Linux + /// tests in once more; CI's runner isn't root. + #[test] + fn as_root_what_root_alone_made_is_given_but_never_what_leads_elsewhere() { + if me() != 0 { + return; + } + let root = tempfile::tempdir().unwrap(); + set_mode(root.path(), 0o755); + let elsewhere = root.path().join("elsewhere"); + let linked = root.path().join("linked"); + std::fs::write(&elsewhere, b"root's").unwrap(); + std::fs::write(&linked, b"root's").unwrap(); + let directory = root.path().join("lpm"); + std::fs::create_dir(&directory).unwrap(); + set_mode(&directory, 0o755); + std::fs::write(directory.join("made"), b"").unwrap(); + std::fs::hard_link(&linked, directory.join("hard-link")).unwrap(); + std::os::unix::fs::symlink(&elsewhere, directory.join("link")).unwrap(); + std::fs::create_dir(directory.join("folder")).unwrap(); + std::fs::write(directory.join("another's"), b"").unwrap(); + std::os::unix::fs::chown(directory.join("another's"), Some(ANOTHER_USER), None).unwrap(); + let names = [ + "made", + "hard-link", + "link", + "folder", + "another's", + "missing", + ] + .map(OsStr::new); + + let kept = give_root_entries(&directory, &names, USER, USER_GROUP).unwrap(); + + assert_eq!(kept, ["hard-link", "folder"]); + assert_eq!(owners(&directory.join("made")), (USER, USER_GROUP)); + assert_eq!(owners(&directory), (USER, USER_GROUP)); + for unchanged in [ + &linked, + &elsewhere, + &directory.join("link"), + &directory.join("folder"), + ] { + assert_eq!(owners(unchanged), (0, 0), "{}", unchanged.display()); + } + assert_eq!(owners(&directory.join("another's")).0, ANOTHER_USER); + } + + #[test] + fn as_root_nothing_is_given_from_a_directory_others_may_change() { + if me() != 0 { + return; + } + let root = tempfile::tempdir().unwrap(); + set_mode(root.path(), 0o755); + for (owner, mode) in [(0, 0o777), (0, 0o1777), (USER, 0o755)] { + let directory = root.path().join(format!("{owner}-{mode:o}")); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("made"), b"").unwrap(); + std::os::unix::fs::chown(&directory, Some(owner), None).unwrap(); + set_mode(&directory, mode); + + let kept = + give_root_entries(&directory, &[OsStr::new("made")], USER, USER_GROUP).unwrap(); + + assert_eq!(kept, ["made"], "{}", directory.display()); + assert_eq!(owners(&directory.join("made")), (0, 0)); + assert_eq!(owners(&directory).0, owner); + } + } + + #[test] + fn as_root_a_sticky_directory_its_group_may_write_leaves_another_user_s_entry_changeable() { + if me() != 0 { + return; + } + let root = tempfile::tempdir().unwrap(); + set_mode(root.path(), 0o755); + let shared = root.path().join("shared"); + let project = shared.join("project"); + std::fs::create_dir_all(&project).unwrap(); + std::os::unix::fs::chown(&shared, Some(0), Some(USER_GROUP)).unwrap(); + set_mode(&shared, 0o1770); + + std::os::unix::fs::chown(&project, Some(ANOTHER_USER), None).unwrap(); + let found = first_changeable_directory(&project, USER).unwrap().unwrap(); + assert_eq!(found.directory, shared); + assert_eq!( + found.reason, + "others may add to it, and the entry on the path belongs to another user" + ); + + std::os::unix::fs::chown(&project, Some(USER), None).unwrap(); + assert_eq!(first_changeable_directory(&project, USER).unwrap(), None); + } + + /// Gives `path` an access ACL letting `uid` write to it, as `setfacl -m + /// u::rwx` does, or says the filesystem keeps none. + #[cfg(target_os = "linux")] + fn let_write(path: &Path, uid: u32) -> bool { + let mut acl = 2u32.to_le_bytes().to_vec(); + for (tag, permissions, id) in [ + (0x01u16, 7u16, u32::MAX), + (0x02, 7, uid), + (0x04, 5, u32::MAX), + (0x10, 7, u32::MAX), + (0x20, 5, u32::MAX), + ] { + acl.extend(tag.to_le_bytes()); + acl.extend(permissions.to_le_bytes()); + acl.extend(id.to_le_bytes()); + } + let c_path = CString::new(path.as_os_str().as_bytes()).unwrap(); + // SAFETY: the strings are NUL-terminated and the value holds + // `acl.len()` bytes, all outliving the call. + let result = unsafe { + libc::setxattr( + c_path.as_ptr(), + c"system.posix_acl_access".as_ptr(), + acl.as_ptr().cast(), + acl.len(), + 0, + ) + }; + if result == 0 { + return true; + } + let error = io::Error::last_os_error(); + assert_eq!(error.raw_os_error(), Some(libc::ENOTSUP), "{error}"); + false + } + + #[cfg(target_os = "linux")] + #[test] + fn an_acl_that_may_let_others_write_makes_a_directory_changeable_whatever_its_group() { + let directory = tempfile::tempdir().unwrap(); + let project = directory.path().join("project"); + std::fs::create_dir(&project).unwrap(); + if !let_write(&project, me() + 1) { + return; + } + + assert_eq!( + first_changeable_directory(&project, me()) + .unwrap() + .unwrap() + .reason, + "its access control list may let others write to it" + ); + } +} diff --git a/crates/lpm-common/src/thread_identity.rs b/crates/lpm-common/src/thread_identity.rs index 68319099d..5d664a2a1 100644 --- a/crates/lpm-common/src/thread_identity.rs +++ b/crates/lpm-common/src/thread_identity.rs @@ -7,37 +7,39 @@ use std::io; /// The calling thread reaching files as another user, until this is -/// dropped, which gives the thread back the identity it had. A thread it -/// starts meanwhile keeps the user's for good, as Linux gives a new thread -/// its creator's. +/// dropped, which gives the thread back the identity it had, so it can't +/// leave the thread. A thread it starts meanwhile keeps the user's for +/// good, as Linux gives a new thread its creator's. #[must_use = "the thread acts as the user only while this is held"] #[derive(Debug)] pub struct ActingAs { fsuid: libc::uid_t, fsgid: libc::gid_t, groups: Vec, + thread: std::marker::PhantomData<*const ()>, } -/// Has the calling thread reach files as `uid`, with `gid` as its only -/// group. The kernel takes root's file capabilities from a thread whose -/// filesystem user isn't root, so its access is the user's. Fails, leaving -/// the thread as it was, where it may not take the identity: without -/// `CAP_SETUID` and `CAP_SETGID`, or for an ID its user namespace doesn't -/// map. -pub fn act_as(uid: u32, gid: u32) -> io::Result { +/// Has the calling thread reach files as `uid`, with `gid` as its group and +/// `groups` as its supplementary groups. The kernel takes root's file +/// capabilities from a thread whose filesystem user isn't root, so its +/// access is the user's. Fails, leaving the thread as it was, where it may +/// not take the identity: without `CAP_SETUID` and `CAP_SETGID`, or for an +/// ID its user namespace doesn't map. +pub fn act_as(uid: u32, gid: u32, groups: &[u32]) -> io::Result { let acting = ActingAs { fsuid: filesystem_uid(), fsgid: filesystem_gid(), groups: thread_groups()?, + thread: std::marker::PhantomData, }; - set_thread_groups(&[gid])?; + set_thread_groups(groups)?; // SAFETY: each changes only the calling thread's credentials, and takes // no pointer. unsafe { libc::setfsgid(gid); libc::setfsuid(uid); } - if filesystem_uid() != uid || filesystem_gid() != gid { + if filesystem_uid() != uid || filesystem_gid() != gid || thread_groups()? != groups { return Err(io::Error::new( io::ErrorKind::PermissionDenied, format!("this process may not reach files as user {uid} and group {gid}"), @@ -110,6 +112,10 @@ fn set_thread_groups(groups: &[libc::gid_t]) -> io::Result<()> { mod tests { use super::*; + // Dropped on another thread, it would give that thread the identity + // and leave its own thread the user's. + static_assertions::assert_not_impl_any!(ActingAs: Send, Sync); + #[test] fn a_thread_that_may_not_act_as_another_user_is_left_as_it_was() { if crate::current_uid() == 0 { @@ -117,7 +123,38 @@ mod tests { } let before = (filesystem_uid(), filesystem_gid(), thread_groups().unwrap()); - let error = act_as(crate::current_uid() + 1, before.1 + 1).unwrap_err(); + let error = act_as(crate::current_uid() + 1, before.1 + 1, &[before.1 + 1]).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert_eq!( + (filesystem_uid(), filesystem_gid(), thread_groups().unwrap()), + before + ); + } + + /// Whether the calling thread may set user IDs, as root in a container + /// started with `--cap-drop SETUID` may not. + fn thread_may_set_uid() -> bool { + const CAP_SETUID: u32 = 7; + let status = std::fs::read_to_string("/proc/thread-self/status").unwrap(); + let effective = status + .lines() + .find_map(|line| line.strip_prefix("CapEff:")) + .unwrap(); + u64::from_str_radix(effective.trim(), 16).unwrap() & (1 << CAP_SETUID) != 0 + } + + /// Runs only as root without `CAP_SETUID`, as in the container the gate + /// starts with `--cap-drop SETUID`: `setfsuid` fails there without + /// saying so, while the groups can still change. + #[test] + fn as_root_without_the_capability_to_set_users_a_thread_is_left_as_it_was() { + if crate::current_uid() != 0 || thread_may_set_uid() { + return; + } + let before = (filesystem_uid(), filesystem_gid(), thread_groups().unwrap()); + + let error = act_as(65_000, 65_001, &[65_001, 65_002]).unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); assert_eq!( @@ -130,7 +167,7 @@ mod tests { /// tests in once more; CI's runner isn't root. #[test] fn as_root_a_thread_reaches_files_as_the_user_and_alone_until_it_lets_go() { - if crate::current_uid() != 0 { + if crate::current_uid() != 0 || !thread_may_set_uid() { return; } let directory = tempfile::tempdir().unwrap(); @@ -150,9 +187,9 @@ mod tests { }); { - let _acting = act_as(65_000, 65_000).unwrap(); - assert_eq!((filesystem_uid(), filesystem_gid()), (65_000, 65_000)); - assert_eq!(thread_groups().unwrap(), vec![65_000]); + let _acting = act_as(65_000, 65_001, &[65_001, 65_002]).unwrap(); + assert_eq!((filesystem_uid(), filesystem_gid()), (65_000, 65_001)); + assert_eq!(thread_groups().unwrap(), vec![65_001, 65_002]); assert_eq!( std::fs::read(&private).unwrap_err().kind(), io::ErrorKind::PermissionDenied diff --git a/crates/lpm-inspect/Cargo.toml b/crates/lpm-inspect/Cargo.toml index 6e336ccbe..b45d0e33d 100644 --- a/crates/lpm-inspect/Cargo.toml +++ b/crates/lpm-inspect/Cargo.toml @@ -35,5 +35,8 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } reqwest = { workspace = true } tempfile = "3" +[target.'cfg(unix)'.dev-dependencies] +libc = "0.2" + [lints] workspace = true diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index 7a74c8c25..fa9f4ecdb 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -49,6 +49,15 @@ pub struct InspectorDb { /// last handle. #[cfg(test)] _temp_dir: Option>, + /// The flush task's connection, for tests to look at. + #[cfg(test)] + write_conn: Arc>, +} + +/// A count as SQL takes it, the largest one where it's past `i64`'s range, +/// as a negative `LIMIT` would mean no limit. +fn sql_count(count: usize) -> i64 { + i64::try_from(count).unwrap_or(i64::MAX) } /// A write operation to be batched. @@ -70,6 +79,7 @@ impl InspectorDb { pub fn open(project_dir: &Path) -> Result { let (owner, _identity) = CaptureOwner::of(project_dir)?; Self::open_project(project_dir, Access::Capture, owner) + .map_err(|error| owner.explain(error)) } /// Opens (or creates) `{project_dir}/.lpm/inspector.db` to read and @@ -79,7 +89,7 @@ impl InspectorDb { /// open it. pub fn open_for_review(project_dir: &Path) -> Result { let (owner, _identity) = CaptureOwner::of(project_dir)?; - Self::open_project(project_dir, Access::Review, owner) + Self::open_project(project_dir, Access::Review, owner).map_err(|error| owner.explain(error)) } fn open_project( @@ -145,6 +155,8 @@ impl InspectorDb { let (write_tx, write_rx) = mpsc::channel(WRITE_QUEUE_CAPACITY); let queued_capture_bytes = Arc::new(AtomicUsize::new(0)); let write_conn = Arc::new(Mutex::new(write_conn)); + #[cfg(test)] + let kept_write_conn = Arc::clone(&write_conn); tokio::spawn(flush_task( write_conn, write_rx, @@ -161,6 +173,8 @@ impl InspectorDb { ownership_ignored, #[cfg(test)] _temp_dir: None, + #[cfg(test)] + write_conn: kept_write_conn, } } @@ -418,7 +432,7 @@ impl InspectorDb { LIMIT ?1 OFFSET ?2", )?; - let rows = stmt.query_map(params![limit as i64, offset as i64], |row| { + let rows = stmt.query_map(params![sql_count(limit), sql_count(offset)], |row| { Ok(StoredRequest { id: row.get(0)?, session_id: row.get(1)?, @@ -486,7 +500,10 @@ impl InspectorDb { ORDER BY timestamp DESC, id DESC LIMIT ?1 OFFSET ?2", )?; - let rows = stmt.query_map(params![limit as i64, offset as i64], stored_detail_from_row)?; + let rows = stmt.query_map( + params![sql_count(limit), sql_count(offset)], + stored_detail_from_row, + )?; rows.map(|row| row.and_then(stored_detail_to_webhook)) .collect() @@ -555,7 +572,7 @@ impl InspectorDb { let provider_str = provider.unwrap_or(""); - let rows = stmt.query_map(params![fts_query, provider_str, limit as i64], |row| { + let rows = stmt.query_map(params![fts_query, provider_str, sql_count(limit)], |row| { Ok(StoredRequest { id: row.get(0)?, session_id: row.get(1)?, @@ -596,7 +613,7 @@ impl InspectorDb { LIMIT ?1", )?; - let rows = stmt.query_map(params![limit as i64], |row| { + let rows = stmt.query_map(params![sql_count(limit)], |row| { Ok(StoredSession { id: row.get(0)?, name: row.get(1)?, @@ -723,23 +740,26 @@ impl InspectorDb { LIMIT ?2 OFFSET ?3", )?; - let rows = stmt.query_map(params![session_id, limit as i64, offset as i64], |row| { - Ok(StoredRequest { - id: row.get(0)?, - session_id: row.get(1)?, - timestamp: row.get(2)?, - method: row.get(3)?, - path: row.get(4)?, - status: row.get(5)?, - duration_ms: row.get(6)?, - provider: row.get(7)?, - summary: row.get(8)?, - req_size: row.get(9)?, - res_size: row.get(10)?, - has_signature_issue: row.get(11)?, - auto_acked: row.get(12)?, - }) - })?; + let rows = stmt.query_map( + params![session_id, sql_count(limit), sql_count(offset)], + |row| { + Ok(StoredRequest { + id: row.get(0)?, + session_id: row.get(1)?, + timestamp: row.get(2)?, + method: row.get(3)?, + path: row.get(4)?, + status: row.get(5)?, + duration_ms: row.get(6)?, + provider: row.get(7)?, + summary: row.get(8)?, + req_size: row.get(9)?, + res_size: row.get(10)?, + has_signature_issue: row.get(11)?, + auto_acked: row.get(12)?, + }) + }, + )?; rows.collect() } @@ -910,26 +930,20 @@ const OWN_IDENTITY: OwnerIdentity = OwnerIdentity; #[cfg(unix)] impl CaptureOwner { /// The capture files' owner, and the identity root holds while it opens - /// them as that owner. + /// them as that owner. The project's path must be one no user but its + /// owner and root can change ([`refuse_changeable_path`]). fn of(project_dir: &Path) -> std::io::Result<(Self, OwnerIdentity)> { use std::os::unix::fs::MetadataExt; let uid = lpm_common::current_uid(); - if uid != 0 { - return Ok(( - Self { - uid, - by_root: false, - }, - OWN_IDENTITY, - )); - } let project = std::fs::metadata(project_dir).map_err(|error| with_path(project_dir, error))?; - if project.uid() == 0 { + let owner = if uid == 0 { project.uid() } else { uid }; + refuse_changeable_path(project_dir, owner)?; + if uid != 0 || owner == 0 { return Ok(( Self { - uid: 0, + uid: owner, by_root: false, }, OWN_IDENTITY, @@ -938,7 +952,7 @@ impl CaptureOwner { let identity = act_as_project_owner(project_dir, &project)?; Ok(( Self { - uid: project.uid(), + uid: owner, by_root: true, }, identity, @@ -950,52 +964,145 @@ impl CaptureOwner { metadata.uid() == self.uid } + + /// Says, of a file root was refused as the project's owner, whose + /// access that was. + fn explain(self, error: OpenError) -> OpenError { + match error { + OpenError::Files(error) + if self.by_root && error.kind() == std::io::ErrorKind::PermissionDenied => + { + OpenError::Files(std::io::Error::new( + error.kind(), + format!( + "{error} (root opens the capture files of user {}'s project as \ + that user)", + self.uid + ), + )) + } + other => other, + } + } } +/// Refuses a project whose path a user other than `owner` and root could +/// change: such a user could move `.lpm`, or a directory above it, while +/// the capture files are opened by name, and have them written elsewhere, +/// or their captures read. +#[cfg(unix)] +fn refuse_changeable_path(project_dir: &Path, owner: u32) -> std::io::Result<()> { + let Some(found) = lpm_common::path_trust::first_changeable_directory(project_dir, owner) + .map_err(|error| with_path(project_dir, error))? + else { + return Ok(()); + }; + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!( + "{} can be changed by other users: {}. They could move .lpm while tunnel \ + captures, which hold request headers and bodies, are opened, so captures are \ + kept only where no one but the project's owner and root can; keep the project \ + under directories only you can change", + found.directory.display(), + found.reason + ), + )) +} + +/// The names in `.lpm` root may have left its own, which it gives to the +/// project's owner before opening them as that owner. +#[cfg(all(target_os = "linux", target_pointer_width = "64"))] +const ROOTS_CAPTURE_FILES: [&str; 8] = [ + ".gitignore", + "inspector.db", + "inspector.db-wal", + "inspector.db-shm", + "inspector.db-journal", + "inspector.db-lock", + "inspector.db-copy", + "inspector.db-copy-journal", +]; + +/// Takes the project owner's identity on this thread, with their groups, +/// once root has given them what it left its own in `.lpm`, as an +/// `lpm install` run as root makes `.lpm` root's; what root can't give +/// safely is refused, naming it. An owner the user database doesn't know, +/// as a host's in a container, gets the project's group, or none where +/// that's root's. #[cfg(all(target_os = "linux", target_pointer_width = "64"))] fn act_as_project_owner( project_dir: &Path, project: &std::fs::Metadata, ) -> std::io::Result { + use lpm_common::path_trust; use std::os::unix::fs::MetadataExt; - lpm_common::thread_identity::act_as(project.uid(), project.gid()) + const NO_GROUP: u32 = 65_534; + let uid = project.uid(); + let refused = |error: std::io::Error| { + std::io::Error::new( + error.kind(), + format!( + "{} belongs to user {uid}, and root records tunnel captures there only as \ + that user, so that it can't be made to write elsewhere, which this process \ + can't do ({error}); run the tunnel as the project's owner", + project_dir.display() + ), + ) + }; + let (gid, groups) = match path_trust::account(uid).map_err(refused)? { + Some(account) => ( + account.gid, + path_trust::groups_of(&account).map_err(refused)?, + ), + None => { + let gid = if project.gid() == 0 { + NO_GROUP + } else { + project.gid() + }; + (gid, vec![gid]) + } + }; + let lpm_dir = project_dir.join(".lpm"); + let names = ROOTS_CAPTURE_FILES.map(std::ffi::OsStr::new); + let kept = path_trust::give_root_entries(&lpm_dir, &names, uid, gid) + .map_err(|error| with_path(&lpm_dir, error))?; + // A `.gitignore` root keeps only goes without the captures' line. + if let Some(name) = kept.iter().find(|name| *name != ".gitignore") { + let path = lpm_dir.join(name); + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!( + "{} belongs to root, which records tunnel captures in user {uid}'s project only as that user, and can't give it to them safely, as others could have put it there or it leads elsewhere; give it to that user (chown {uid}) or remove it", + path.display() + ), + )); + } + lpm_common::thread_identity::act_as(uid, gid, &groups) .map(Some) - .map_err(|error| { - std::io::Error::new( - error.kind(), - format!( - "{} belongs to user {}, and root records tunnel captures there only as \ - that user, so that it can't be made to write elsewhere, which this \ - process can't do ({error}); run the tunnel as the project's owner", - project_dir.display(), - project.uid() - ), - ) - }) + .map_err(refused) } #[cfg(all(unix, not(all(target_os = "linux", target_pointer_width = "64"))))] fn act_as_project_owner( project_dir: &Path, - _project: &std::fs::Metadata, + project: &std::fs::Metadata, ) -> std::io::Result { - Err(project_of_another_user(project_dir)) -} + use std::os::unix::fs::MetadataExt; -/// Root's refusal of another user's project where it can't open the -/// capture files as that user. -#[cfg(all(unix, not(all(target_os = "linux", target_pointer_width = "64"))))] -fn project_of_another_user(project_dir: &Path) -> std::io::Error { - std::io::Error::new( + Err(std::io::Error::new( std::io::ErrorKind::PermissionDenied, format!( - "{} belongs to another user, who could move its .lpm to have root write \ - elsewhere, so root records tunnel captures, which hold request headers and \ - bodies, only in its own projects here; run the tunnel as the project's owner", - project_dir.display() + "{} belongs to user {}, and root records tunnel captures, which hold request \ + headers and bodies, only in its own projects on this system, as another \ + user could move .lpm to have root write elsewhere; run the tunnel as that \ + user, without sudo", + project_dir.display(), + project.uid() ), - ) + )) } /// Access to the capture files on Windows is the project's ACLs. @@ -1010,6 +1117,10 @@ impl CaptureOwner { fn of(_project_dir: &Path) -> std::io::Result<(Self, OwnerIdentity)> { Ok((Self { by_root: false }, OWN_IDENTITY)) } + + fn explain(self, error: OpenError) -> OpenError { + error + } } /// The `.lpm/.gitignore` line that keeps the capture database and its @@ -1114,12 +1225,17 @@ fn hard_linked(path: &Path, db_path: &Path) -> std::io::Error { /// Names the files SQLite keeps beside a capture database, for advice to /// move it: they go with it, as an open removes those it finds without a /// database, and a WAL not yet folded into the database holds part of it. +/// Names that still start as the database's keep them in what +/// `.lpm/.gitignore` ignores. fn with_its_sidecars(db_path: &Path) -> String { let name = db_path .file_name() .unwrap_or(db_path.as_os_str()) .to_string_lossy(); - format!("together with any {name}-wal, {name}-shm and {name}-journal beside it") + format!( + "together with any {name}-wal, {name}-shm and {name}-journal beside it, under \ + names that still start {name} so git goes on ignoring them" + ) } /// Makes sure `.lpm` is a directory of the project's own, creating it if @@ -1444,15 +1560,18 @@ fn replace_with_owner_only_copy(db_path: &Path) -> Result { removed?; return Ok(false); } - if let Err(error) = copy_checked_database(db_path, ©_path, needed) { - if let Err(removal) = remove_replacement_copy(db_path) { - tracing::warn!( - "could not remove {}: {removal}", - capture_file(db_path, "-copy").display() - ); + let checked = match copy_checked_database(db_path, ©_path, needed) { + Ok(checked) => checked, + Err(error) => { + if let Err(removal) = remove_replacement_copy(db_path) { + tracing::warn!( + "could not remove {}: {removal}", + capture_file(db_path, "-copy").display() + ); + } + return Err(error); } - return Err(error); - } + }; #[cfg(test)] if let Some(attach) = tests::AFTER_COPY.with_borrow_mut(Option::take) { attach(); @@ -1461,6 +1580,10 @@ fn replace_with_owner_only_copy(db_path: &Path) -> Result { remove_if_present(&capture_file(db_path, suffix))?; } std::fs::rename(©_path, db_path).map_err(|error| with_path(db_path, error))?; + // Only now may another connection, as an earlier release's or another + // tool's that opened the database meanwhile, take a lock on it; SQLite + // then refuses its writes to a database moved aside. + drop(checked); #[cfg(test)] if let Some(look) = tests::AFTER_RENAME.with_borrow_mut(Option::take) { look(); @@ -1469,16 +1592,18 @@ fn replace_with_owner_only_copy(db_path: &Path) -> Result { } /// Checks a database others could open, and copies it into the owner-only -/// file made for the copy. A check that fails leaves the WAL files its -/// connection made: they hold no captures, SQLite removes them as the last -/// connection closes, and a connection that attached meanwhile, as an -/// earlier release or another tool can without the opening lock, may be -/// using them. +/// file made for the copy, returning the connection that checked it, which +/// holds the database's exclusive lock until it's dropped: a connection +/// that attaches meanwhile, as an earlier release or another tool can +/// without the opening lock, can't commit a capture the copy lacks. A check +/// that fails leaves the WAL files its connection made: they hold no +/// captures, SQLite removes them as the last connection closes, and such a +/// connection may be using them. fn copy_checked_database( db_path: &Path, copy_path: &Path, needed: Option, -) -> Result<(), OpenError> { +) -> Result { ensure_room_to_copy(db_path, needed)?; #[cfg(test)] if let Some(attach) = tests::BEFORE_CHECK.with_borrow_mut(Option::take) { @@ -1488,7 +1613,11 @@ fn copy_checked_database( refuse_unknown_schema(&database, db_path, checking(db_path, needed))?; // In WAL, which every LPM connection uses, leaving it needs the only // connection, so it proves no other LPM process has the database open, - // and folds the WAL into the file. + // and folds the WAL into the file. In exclusive locking mode, the lock + // that takes is kept until the connection closes. + database + .pragma_update(None, "locking_mode", "EXCLUSIVE") + .map_err(checking(db_path, needed))?; match database.query_row("PRAGMA journal_mode=DELETE", [], |row| { row.get::<_, String>(0) }) { @@ -1498,7 +1627,8 @@ fn copy_checked_database( Err(error) => return Err(checking(db_path, needed)(error)), } refuse_damage(&database, db_path)?; - copy_database(&database, copy_path, db_path, needed) + copy_database(&database, copy_path, db_path, needed)?; + Ok(database) } /// Reads a full disk while the database is checked as the copy's want of @@ -1539,35 +1669,24 @@ fn shared_memory_room(wal: u64, page_size: u64) -> u64 { (regions + 1) * REGION } -/// The size of the pages the WAL beside `db_path` holds, from its header, -/// or else of the database's, from its header; where neither reads as one, -/// SQLite's smallest, 512 bytes, which counts the most frames in a WAL. +/// The size of the pages the WAL beside `db_path` holds, from its header; +/// where it doesn't read as one, SQLite's smallest, 512 bytes, which counts +/// the most frames in a WAL. The database's own header isn't read: an +/// empty or missing WAL holds no frames, so its page size counts for +/// nothing, and closing a handle of this process's on the database would +/// release the locks SQLite holds on it here. fn page_size(db_path: &Path) -> u64 { - let header = |path: &Path, bytes: &mut [u8]| { - lpm_common::open_nofollow(path) - .and_then(|mut file| std::io::Read::read_exact(&mut file, bytes)) - .is_ok() - }; - let valid = |size: u64| size.is_power_of_two() && (512..=65_536).contains(&size); let mut wal = [0; 12]; - if header(&capture_file(db_path, "-wal"), &mut wal) + let read = lpm_common::open_nofollow(&capture_file(db_path, "-wal")) + .and_then(|mut file| std::io::Read::read_exact(&mut file, &mut wal)); + if read.is_ok() && matches!( u32::from_be_bytes([wal[0], wal[1], wal[2], wal[3]]), 0x377f_0682 | 0x377f_0683 ) { let size = u64::from(u32::from_be_bytes([wal[8], wal[9], wal[10], wal[11]])); - if valid(size) { - return size; - } - } - let mut database = [0; 18]; - if header(db_path, &mut database) { - let size = match u16::from_be_bytes([database[16], database[17]]) { - 1 => 65_536, - size => u64::from(size), - }; - if valid(size) { + if size.is_power_of_two() && (512..=65_536).contains(&size) { return size; } } @@ -1732,17 +1851,15 @@ fn open_capture_connection(path: &Path, synchronous: &str) -> Result Result<(), rusqlite::Error> { harden_connection(connection)?; use_wal(connection)?; connection.execute_batch(&format!( "PRAGMA synchronous={synchronous}; PRAGMA foreign_keys=ON;" - ))?; - connection.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(())) + )) } /// Keeps the WAL and its shared memory when the last connection closes, @@ -2523,7 +2640,9 @@ mod tests { /// The advice for the capture database LPM can't use. const ADVICE: &str = "move it aside together with any inspector.db-wal, \ - inspector.db-shm and inspector.db-journal beside it, and retry"; + inspector.db-shm and inspector.db-journal beside it, under names \ + that still start inspector.db so git goes on ignoring them, and \ + retry"; thread_local! { /// The most pages this thread's database copies may hold, as a @@ -2907,7 +3026,8 @@ mod tests { assert!( error.to_string().ends_with(&format!( "move {} aside together with any inspector.db-wal, inspector.db-shm and \ - inspector.db-journal beside it, and retry", + inspector.db-journal beside it, under names that still start inspector.db \ + so git goes on ignoring them, and retry", db_path.display() )), "{error}" @@ -3615,6 +3735,124 @@ mod tests { assert_eq!(integrity, "ok"); } + /// Commits a capture `id` as a connection of an earlier release, which + /// takes no opening lock, would, giving up at once on a lock. + fn commit_unlocked(connection: &Connection, id: &str) -> rusqlite::Result<()> { + connection.busy_timeout(std::time::Duration::ZERO)?; + let _ = connection.query_row("PRAGMA journal_mode=WAL", [], |row| row.get::<_, String>(0)); + connection + .execute( + "INSERT INTO requests (id, timestamp, method, path) \ + VALUES (?1, '2026-04-06T12:00:00Z', 'POST', '/')", + [id], + ) + .map(drop) + } + + #[cfg(unix)] + fn capture_ids(db_path: &Path) -> Vec { + let reader = Connection::open(db_path).unwrap(); + let mut statement = reader + .prepare("SELECT id FROM requests ORDER BY id") + .unwrap(); + statement + .query_map([], |row| row.get(0)) + .unwrap() + .map(Result::unwrap) + .collect() + } + + #[test] + fn a_user_s_own_captures_leave_no_wal_files_once_closed() { + let project = tempfile::tempdir().unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let db = InspectorDb::open(project.path()).unwrap(); + assert!(!keeps_wal_files(&*db.write_conn.lock().await)); + assert!(!keeps_wal_files(&*db.read_conn.lock().await)); + assert!(!keeps_wal_files(&db.control_conn.lock().unwrap())); + }); + drop(runtime); + + let db_path = project.path().join(".lpm/inspector.db"); + assert!(db_path.exists()); + assert!(!capture_file(&db_path, "-wal").exists()); + } + + #[cfg(unix)] + #[test] + fn a_capture_committed_while_the_database_is_replaced_is_refused_not_lost() { + let project = project_with_a_capture(); + let db_path = project.path().join(".lpm/inspector.db"); + set_mode(&db_path, 0o644); + let during = std::rc::Rc::new(std::cell::Cell::new(None)); + let committed = std::rc::Rc::clone(&during); + let other = std::rc::Rc::new(std::cell::RefCell::new(None)); + let held = std::rc::Rc::clone(&other); + let path = db_path.clone(); + AFTER_COPY.set(Some(Box::new(move || { + let connection = Connection::open(&path).unwrap(); + committed.set(Some(commit_unlocked(&connection, "during").is_ok())); + *held.borrow_mut() = Some(connection); + }))); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + runtime + .block_on(async { InspectorDb::open(project.path()).map(drop) }) + .unwrap(); + drop(runtime); + // The connection opened while the copy took the database's place + // still has the database it replaced. + let late = commit_unlocked(&other.borrow_mut().take().unwrap(), "late"); + + let ids = capture_ids(&db_path); + let during = during.get().unwrap(); + assert!(!during || ids.contains(&"during".to_owned()), "{ids:?}"); + assert!(late.is_err() || ids.contains(&"late".to_owned()), "{ids:?}"); + assert!(ids.contains(&"kept".to_owned()), "{ids:?}"); + assert_eq!(mode(&db_path), 0o600); + } + + #[cfg(unix)] + #[test] + fn a_project_others_can_change_the_path_to_keeps_no_captures() { + let directory = tempfile::tempdir().unwrap(); + let shared = directory.path().join("shared"); + let project = shared.join("project"); + std::fs::create_dir_all(&project).unwrap(); + set_mode(&shared, 0o777); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + for opened in [ + runtime.block_on(async { InspectorDb::open(&project).map(drop) }), + runtime.block_on(async { InspectorDb::open_for_review(&project).map(drop) }), + ] { + let error = opened.unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + let message = error.to_string(); + assert!( + message.contains(&shared.canonicalize().unwrap().display().to_string()) + && message.contains("anyone may write to it"), + "{message}" + ); + } + assert!(!project.join(".lpm").exists()); + + set_mode(&shared, 0o1777); + runtime + .block_on(async { InspectorDb::open(&project).map(drop) }) + .unwrap(); + set_mode(&shared, 0o755); + set_mode(&project, 0o777); + let error = runtime + .block_on(async { InspectorDb::open(&project).map(drop) }) + .unwrap_err(); + assert!( + error.to_string().contains("anyone may write to it"), + "{error}" + ); + } + #[cfg(unix)] #[test] fn a_filesystem_without_modes_lets_a_capture_open_while_a_review_has_the_database_open() { @@ -3669,21 +3907,61 @@ mod tests { let elsewhere = root.path().join("elsewhere"); std::fs::create_dir(&project).unwrap(); std::fs::create_dir(&elsewhere).unwrap(); - std::os::unix::fs::chown(&project, Some(PROJECT_OWNER), Some(PROJECT_OWNER)).unwrap(); + std::os::unix::fs::chown(&project, Some(PROJECT_OWNER), Some(PROJECT_GROUP)).unwrap(); (root, project, elsewhere) } - /// A user other than root, whose projects root's tests open. + /// A user other than root, whose projects root's tests open, and the + /// group of those projects: the user database knows neither, as a + /// container's doesn't know the host's users. #[cfg(all(target_os = "linux", target_pointer_width = "64"))] const PROJECT_OWNER: u32 = 65_000; + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + const PROJECT_GROUP: u32 = 65_001; + + /// Whether SQLite keeps `connection`'s WAL files when it closes. + fn keeps_wal_files(connection: &Connection) -> bool { + let mut persist: std::ffi::c_int = -1; + // SAFETY: as in `keep_wal_files`; -1 only asks the setting. + let code = unsafe { + rusqlite::ffi::sqlite3_file_control( + connection.handle(), + c"main".as_ptr(), + rusqlite::ffi::SQLITE_FCNTL_PERSIST_WAL, + (&raw mut persist).cast(), + ) + }; + assert_eq!(code, rusqlite::ffi::SQLITE_OK); + persist == 1 + } + + /// The owner and group of each of `project`'s capture files. + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + fn capture_owners(project: &Path) -> Vec<(String, (u32, u32))> { + use std::os::unix::fs::MetadataExt; + + let lpm_dir = project.join(".lpm"); + [ + "", + "inspector.db", + "inspector.db-wal", + "inspector.db-shm", + "inspector.db-lock", + ".gitignore", + ] + .into_iter() + .map(|name| { + let metadata = std::fs::metadata(lpm_dir.join(name)).unwrap(); + (name.to_owned(), (metadata.uid(), metadata.gid())) + }) + .collect() + } /// Runs only as root, as in the root container the gate runs Linux /// tests in once more; CI's runner isn't root. #[cfg(all(target_os = "linux", target_pointer_width = "64"))] #[test] fn as_root_another_user_s_captures_are_made_and_kept_as_that_user_s() { - use std::os::unix::fs::MetadataExt; - if lpm_common::current_uid() != 0 { return; } @@ -3691,29 +3969,19 @@ mod tests { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async { - drop(InspectorDb::open(&project).unwrap()); + let db = InspectorDb::open(&project).unwrap(); + assert!(keeps_wal_files(&*db.write_conn.lock().await)); + assert!(keeps_wal_files(&*db.read_conn.lock().await)); + assert!(keeps_wal_files(&db.control_conn.lock().unwrap())); + drop(db); drop(InspectorDb::open_for_review(&project).unwrap()); }); // The connections close as the flush task ends. drop(runtime); let lpm_dir = project.join(".lpm"); - for name in [ - "", - "inspector.db", - "inspector.db-wal", - "inspector.db-shm", - "inspector.db-lock", - ".gitignore", - ] { - let path = lpm_dir.join(name); - let metadata = std::fs::metadata(&path).unwrap(); - assert_eq!( - (metadata.uid(), metadata.gid()), - (PROJECT_OWNER, PROJECT_OWNER), - "{}", - path.display() - ); + for (name, owners) in capture_owners(&project) { + assert_eq!(owners, (PROJECT_OWNER, PROJECT_GROUP), "{name}"); } assert_eq!( std::fs::metadata(lpm_dir.join("inspector.db-wal")) @@ -3760,6 +4028,122 @@ mod tests { assert_eq!(lpm_common::thread_identity::filesystem_uid(), 0); } + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + #[test] + fn as_root_a_project_of_root_s_group_has_its_captures_in_no_group() { + if lpm_common::current_uid() != 0 { + return; + } + let (_root, project, _) = another_user_s_project_beside_root_s_directory(); + std::os::unix::fs::chown(&project, None, Some(0)).unwrap(); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { drop(InspectorDb::open(&project).unwrap()) }); + drop(runtime); + + for (name, owners) in capture_owners(&project) { + assert_eq!(owners, (PROJECT_OWNER, 65_534), "{name}"); + } + } + + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + #[test] + fn as_root_the_capture_files_root_made_in_its_own_lpm_are_given_to_the_project_s_owner() { + if lpm_common::current_uid() != 0 { + return; + } + let (_root, project, _) = another_user_s_project_beside_root_s_directory(); + // As root makes them in a project of its own, before the project + // is given to its user. + std::os::unix::fs::chown(&project, Some(0), Some(0)).unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let db = InspectorDb::open(&project).unwrap(); + db.insert_durable_request(Arc::new(make_webhook("kept", 200)), None) + .await + .unwrap(); + }); + drop(runtime); + std::os::unix::fs::chown(&project, Some(PROJECT_OWNER), Some(PROJECT_GROUP)).unwrap(); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let kept = runtime.block_on(async { + InspectorDb::open(&project) + .unwrap() + .get_webhook("kept") + .await + .unwrap() + }); + drop(runtime); + + assert!(kept.is_some()); + for (name, owners) in capture_owners(&project) { + assert_eq!(owners, (PROJECT_OWNER, PROJECT_GROUP), "{name}"); + } + } + + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + #[test] + fn as_root_a_capture_file_of_root_s_in_the_owner_s_lpm_is_refused_by_name() { + use std::os::unix::fs::MetadataExt; + + if lpm_common::current_uid() != 0 { + return; + } + let (_root, project, _) = another_user_s_project_beside_root_s_directory(); + let lpm_dir = project.join(".lpm"); + std::fs::create_dir(&lpm_dir).unwrap(); + set_mode(&lpm_dir, 0o700); + std::fs::write(lpm_dir.join("inspector.db"), b"").unwrap(); + set_mode(&lpm_dir.join("inspector.db"), 0o600); + std::os::unix::fs::chown(&lpm_dir, Some(PROJECT_OWNER), Some(PROJECT_GROUP)).unwrap(); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let error = runtime + .block_on(async { InspectorDb::open(&project).map(drop) }) + .unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + let message = error.to_string(); + assert!( + message.contains(&format!( + "{} belongs to root", + lpm_dir.join("inspector.db").display() + )) && message.contains("chown 65000"), + "{message}" + ); + assert_eq!( + std::fs::metadata(lpm_dir.join("inspector.db")) + .unwrap() + .uid(), + 0 + ); + assert_eq!(lpm_common::thread_identity::filesystem_uid(), 0); + } + + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + #[test] + fn as_root_its_own_project_others_may_write_to_keeps_no_captures() { + if lpm_common::current_uid() != 0 { + return; + } + let (_root, project, _) = another_user_s_project_beside_root_s_directory(); + std::os::unix::fs::chown(&project, Some(0), Some(0)).unwrap(); + set_mode(&project, 0o777); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let error = runtime + .block_on(async { InspectorDb::open(&project).map(drop) }) + .unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + assert!( + error.to_string().contains("anyone may write to it"), + "{error}" + ); + assert!(!project.join(".lpm").exists()); + } + #[cfg(target_os = "macos")] fn add_acl_entry(path: &Path, entry: &str) { let status = std::process::Command::new("/bin/chmod") @@ -4215,11 +4599,8 @@ mod tests { let mut database = vec![0; 10_000]; database[16..18].copy_from_slice(&4096u16.to_be_bytes()); std::fs::write(&db_path, &database).unwrap(); - assert_eq!(page_size(&db_path), 4096); + // Without a WAL, the shared memory holds its first regions alone. assert_eq!(room_to_copy(&db_path), Some(10_000 + 64 * 1024)); - database[16..18].copy_from_slice(&1u16.to_be_bytes()); - std::fs::write(&db_path, &database).unwrap(); - assert_eq!(page_size(&db_path), 65_536); // 130,308 frames of the WAL's 4 KiB pages: 32 regions of the WAL // index, and one more. The WAL's header says what its frames hold. @@ -4233,11 +4614,10 @@ mod tests { Some(10_000 + 512 * 1024 * 1024 + 33 * 32 * 1024) ); - // A WAL without a header counts frames of SQLite's smallest pages. + // A WAL without a header counts frames of SQLite's smallest pages, + // whatever the database's own are. wal.set_len(0).unwrap(); wal.set_len(512 * 1024 * 1024).unwrap(); - database[16..18].copy_from_slice(&[0, 0]); - std::fs::write(&db_path, &database).unwrap(); assert_eq!(page_size(&db_path), 512); assert_eq!( room_to_copy(&db_path), @@ -4301,6 +4681,97 @@ mod tests { assert_eq!(page_size(&db_path), 512); } + #[test] + fn the_page_size_comes_from_the_wal_alone() { + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("inspector.db"); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch( + "PRAGMA page_size = 4096; PRAGMA journal_mode = WAL; \ + PRAGMA wal_autocheckpoint = 0; CREATE TABLE kept (x); \ + INSERT INTO kept VALUES (1);", + ) + .unwrap(); + + assert_eq!(page_size(&db_path), 4096); + writer + .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(())) + .unwrap(); + assert_eq!( + std::fs::metadata(capture_file(&db_path, "-wal")) + .unwrap() + .len(), + 0 + ); + // An empty WAL holds no frames; the database's 4 KiB pages aren't + // read. + assert_eq!(page_size(&db_path), 512); + } + + /// Set in the child process [`reading_the_page_size_keeps_this_process_s_locks_on_the_database`] + /// starts, to the database it tries to lock. + #[cfg(unix)] + const LOCKING_CHILD: &str = "LPM_INSPECT_TEST_LOCKING_CHILD"; + + #[cfg(unix)] + #[test] + fn reading_the_page_size_keeps_this_process_s_locks_on_the_database() { + if let Some(db_path) = std::env::var_os(LOCKING_CHILD) { + use std::os::fd::AsRawFd as _; + + // SQLite's locks are on the bytes from 1 GiB: its exclusive + // lock is a write lock on all 512 of them. + const PENDING_BYTE: libc::off_t = 0x4000_0000; + let file = std::fs::File::open(db_path).unwrap(); + // SAFETY: an all-zero `flock` is a valid value to fill in. + let mut lock: libc::flock = unsafe { std::mem::zeroed() }; + lock.l_type = libc::F_WRLCK as _; + lock.l_whence = libc::SEEK_SET as _; + lock.l_start = PENDING_BYTE; + lock.l_len = 512; + // SAFETY: the descriptor is open, and F_GETLK only fills `lock`. + let asked = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETLK, &raw mut lock) }; + assert_eq!(asked, 0, "{}", std::io::Error::last_os_error()); + assert_ne!( + i64::from(lock.l_type), + i64::from(libc::F_UNLCK), + "nothing holds a lock on the database" + ); + return; + } + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("inspector.db"); + let held = Connection::open(&db_path).unwrap(); + held.execute_batch( + "PRAGMA journal_mode = WAL; CREATE TABLE kept (x); INSERT INTO kept VALUES (1);", + ) + .unwrap(); + held.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(())) + .unwrap(); + held.query_row("SELECT count(*) FROM kept", [], |row| row.get::<_, i64>(0)) + .unwrap(); + + page_size(&db_path); + + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "db::tests::reading_the_page_size_keeps_this_process_s_locks_on_the_database", + "--exact", + "--nocapture", + ]) + .env(LOCKING_CHILD, &db_path) + .output() + .unwrap(); + assert!( + child.status.success(), + "another process took the lock:\n{}{}", + String::from_utf8_lossy(&child.stdout), + String::from_utf8_lossy(&child.stderr) + ); + drop(held); + } + #[cfg(unix)] #[tokio::test] async fn a_replacement_removes_files_left_beside_the_database_it_replaces() { @@ -5042,6 +5513,7 @@ mod tests { let control_conn = open_control_connection(&db_path).unwrap(); let (write_tx, write_rx) = mpsc::channel(1); let queued_capture_bytes = Arc::new(AtomicUsize::new(0)); + let write_conn = Arc::new(Mutex::new(write_conn)); let db = InspectorDb { read_conn: Arc::new(Mutex::new(read_conn)), control_conn: Arc::new(std::sync::Mutex::new(control_conn)), @@ -5051,6 +5523,7 @@ mod tests { open_to_others: false, ownership_ignored: false, _temp_dir: None, + write_conn: Arc::clone(&write_conn), }; db.start_session("saturated-session".to_string(), None, 3000) @@ -5062,11 +5535,7 @@ mod tests { assert_eq!(db.write_tx.capacity(), 0); db.end_session("saturated-session".to_string()).unwrap(); - let writer = tokio::spawn(flush_task( - Arc::new(Mutex::new(write_conn)), - write_rx, - queued_capture_bytes, - )); + let writer = tokio::spawn(flush_task(write_conn, write_rx, queued_capture_bytes)); db.flush_pending_writes().await.unwrap(); let session = db.get_session("saturated-session").await.unwrap().unwrap(); diff --git a/tests/workflows/tests/dev_tunnel.rs b/tests/workflows/tests/dev_tunnel.rs index 0332b364b..456d95fa3 100644 --- a/tests/workflows/tests/dev_tunnel.rs +++ b/tests/workflows/tests/dev_tunnel.rs @@ -1506,7 +1506,8 @@ fn tunnel_inspect_reports_a_capture_database_it_cant_open_as_a_disk_error_under_ .as_str() .is_some_and(|message| message.ends_with( "move it aside together with any inspector.db-wal, inspector.db-shm and \ - inspector.db-journal beside it, and retry" + inspector.db-journal beside it, under names that still start inspector.db \ + so git goes on ignoring them, and retry" )), "{envelope}" ); @@ -1755,7 +1756,8 @@ fn tunnel_inspect_reports_an_actionable_corrupt_database_error() { assert!(rendered.contains(".lpm/inspector.db")); assert!(rendered.contains( "move it aside together with any inspector.db-wal, inspector.db-shm and \ - inspector.db-journal beside it, and retry" + inspector.db-journal beside it, under names that still start inspector.db so git \ + goes on ignoring them, and retry" )); assert!( !rendered.contains("network connection"), From 1198e7c9db163db07e2e4464bb5e06c01ea898e6 Mon Sep 17 00:00:00 2001 From: Tolga Ergin Date: Wed, 14 Oct 2026 00:32:42 +0100 Subject: [PATCH 17/17] test(common): check APFS's reserve where it holds on every release macOS 26 passed the live APFS test twice on CI, then once let a stream of writes take 0.7 MB more of the 64 MB volume than macOS 27 ever does, which broke the test's 1 MiB bound on how little more than the count a write takes. The estimate stayed on the safe side. The live test now checks what holds on every release: a write takes at least what's counted writable, APFS refuses writes before its reported free space runs out, and the writable count is the free space less the modeled reserve. The reserve's precision stays checked against measured volumes, and the doc notes that APFS sometimes keeps back less. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/lpm-common/src/lib.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index 1c4b82ec5..4c8f0ba18 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -416,7 +416,9 @@ fn filesystem_space(_path: &std::path::Path) -> Option { /// which also covers the blocks a file's own metadata takes. That's the /// least a write needs: on a fresh volume, writes flushed as they go, or /// spread over many files, stopped up to about 16 MB sooner, so a write -/// counted as fitting can still fail, as on a full disk. +/// counted as fitting can still fail, as on a full disk. APFS sometimes +/// keeps back less: one run on macOS 26 let a stream take 0.7 MB more of a +/// 64 MB volume, so a write can also fit in a little more than this counts. #[cfg(any(target_os = "macos", test))] fn apfs_held_back(size: u64) -> u64 { (size / 50) @@ -523,7 +525,7 @@ mod tests { #[cfg(target_os = "macos")] #[test] - fn on_apfs_a_stream_of_writes_takes_what_it_counts_writable_and_little_more() { + fn on_apfs_a_stream_of_writes_takes_what_it_counts_writable_but_not_all_reported_free() { use std::io::Write as _; let directory = tempfile::tempdir().unwrap(); @@ -586,10 +588,15 @@ mod tests { } } + // How much of its reserve APFS lets a stream take varies by release + // and run, so how closely the reserve matches is checked against + // measured volumes in `apfs_holds_back_at_least_what_measured_volumes_kept_from_writes`. + assert_eq!(writable, space.available - space.held_back); assert!(written >= writable, "wrote {written} of {writable}"); assert!( - written - writable < 1024 * 1024, - "wrote {written}, far past {writable}" + written < space.available, + "wrote {written}, all of the {} reported free", + space.available ); }