From 48153192e8827c286acc9d94a29431b2418f7589 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:05:54 +0200 Subject: [PATCH 1/9] perf(v3): compile the diagnostic patterns with ASCII classes Every hook is a new process, so the compact-v3 patterns are compiled on every compressed call. Unicode whitespace, digits and case folding made them several times larger to compile than their ASCII vocabulary needs. Indentation, digits and case folding are now ASCII; a Java frame keeps Unicode \w, since Java identifiers may be Unicode, and a test pins it. Outputs are byte-identical on the fixture corpus and stress inputs in v1, v2 and v3. Refs #12 --- src/clean.rs | 3 ++- src/compress.rs | 15 +++++++++------ tests/compact_v3.rs | 2 ++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/clean.rs b/src/clean.rs index 48f08c6..8a05200 100644 --- a/src/clean.rs +++ b/src/clean.rs @@ -212,9 +212,10 @@ fn uuid_at(bytes: &[u8], start: usize) -> Option { word_ends(bytes, i).then_some(i) } +/// ASCII digits: Unicode `\d` costs compiling a larger engine on every call. static TIMESTAMP: std::sync::LazyLock = std::sync::LazyLock::new(|| { regex::Regex::new( - r"(?-u:\b)\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:[.,]\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?|(?-u:\b)\d{2}:\d{2}:\d{2}(?:[.,]\d+)?", + r"(?-u:\b)[0-9]{4}-[0-9]{2}-[0-9]{2}(?:[T ][0-9]{2}:[0-9]{2}(?::[0-9]{2}(?:[.,][0-9]+)?)?(?:Z|[+-][0-9]{2}:?[0-9]{2})?)?|(?-u:\b)[0-9]{2}:[0-9]{2}:[0-9]{2}(?:[.,][0-9]+)?", ) .unwrap() }); diff --git a/src/compress.rs b/src/compress.rs index f42f280..d155095 100644 --- a/src/compress.rs +++ b/src/compress.rs @@ -60,23 +60,26 @@ static SIGNAL: LazyLock = LazyLock::new(|| { /// matching them loosely turns any line starting with `e ` into a diagnostic. /// So do error and exception type names (`TypeError`, `KeyError`, /// `NullPointerException`), whose convention is a capitalized identifier. -/// Word boundaries are ASCII: the vocabulary is ASCII, and Unicode boundaries -/// cost a slower engine on every line. +/// Word boundaries, case folding and indentation are ASCII: the vocabulary is +/// ASCII, Unicode boundaries cost a slower engine on every line, and Unicode +/// classes and folding cost compiling a larger one on every call (every hook +/// is a new process). static STRONG: LazyLock = LazyLock::new(|| { - regex::Regex::new(r"(?i:(?-u:\b)(error|errors|failed|failure|panic|panicked|exception|traceback|fatal|assertionerror|caused by|test result|tests? passed|tests? failed)(?-u:\b))|(?-u:\b)[A-Z][A-Za-z]*(Error|Exception)(?-u:\b)|npm ERR!|^\s*(FAIL|FAILED|--- FAIL|E {2,}|FATAL|×|✕|✗|✖)").unwrap() + regex::Regex::new(r"(?i-u:\b(error|errors|failed|failure|panic|panicked|exception|traceback|fatal|assertionerror|caused by|test result|tests? passed|tests? failed)\b)|(?-u:\b[A-Z][A-Za-z]*(Error|Exception)\b)|npm ERR!|^(?-u:\s*)(FAIL|FAILED|--- FAIL|E {2,}|FATAL|×|✕|✗|✖)").unwrap() }); /// Advisory lines: shown once per template, never ahead of a diagnostic. /// A passing test is not an advisory. Listing it here grouped every `PASS` /// line by exact text, which stopped a suite of passes from folding and let /// them crowd out the failure's own detail. static WEAK: LazyLock = LazyLock::new(|| { - regex::Regex::new(r"(?i:(?-u:\b)(warning|warn|deprecated|deprecation)(?-u:\b))").unwrap() + regex::Regex::new(r"(?i-u:\b(warning|warn|deprecated|deprecation)\b)").unwrap() }); /// Stack frames: context for a diagnostic, wherever they are. JavaScript and /// Python frames, Java's `at pkg.Class.method(File.java:N)`, Go's -/// `file.go:N +0x1d` and rustc's `--> file:line:col` locations. +/// `file.go:N +0x1d` and rustc's `--> file:line:col` locations. Indentation +/// and line numbers are ASCII; a Java identifier keeps Unicode `\w`. static FRAME: LazyLock = LazyLock::new(|| { - regex::Regex::new(r#"^\s+at .*\(.*:\d+:\d+\)|^\s+File ".*", line \d+|^\s+at [\w$.<>/-]+\([^()]*\)\s*$|\.go:\d+ \+0x[0-9a-f]+|^\s*--> \S+:\d+:\d+"#).unwrap() + regex::Regex::new(r#"^(?-u:\s+)at .*\(.*:[0-9]+:[0-9]+\)|^(?-u:\s+)File ".*", line [0-9]+|^(?-u:\s+)at [\w$.<>/-]+\([^()]*\)(?-u:\s*)$|\.go:[0-9]+ \+0x[0-9a-f]+|^(?-u:\s*)--> \S+:[0-9]+:[0-9]+"#).unwrap() }); /// A file extension at the end of a token: `.rs`, `.py`, `.java`. static EXTENSION: LazyLock = diff --git a/tests/compact_v3.rs b/tests/compact_v3.rs index c8e5a4e..117487e 100644 --- a/tests/compact_v3.rs +++ b/tests/compact_v3.rs @@ -235,6 +235,8 @@ fn diagnostic_vocabulary_and_frames_survive_noise() { " TypeError: Cannot read properties of undefined (reading 'id')", "Caused by: java.lang.NullPointerException: name is null", "\tat com.acme.billing.Invoice.total(Invoice.java:88)", + // Java identifiers may be Unicode: a frame stays a frame. + "\tat com.acme.société.Facture.total(Facture.java:91)", "--- FAIL: TestRefresh (0.00s)", "\t/work/app/auth/session.go:88 +0x1d", " --> src/cache.rs:17:9", From 33868a44fec8980d7d1e2f77eebcc4794e85f1fa Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:10:54 +0200 Subject: [PATCH 2/9] perf: install the termination handler with sigaction, not ctrlc On Apple platforms ctrlc links Foundation and the Objective-C runtime through dispatch2. Loading them cost about 0.6 ms on every invocation, hooks included, a quarter of a hook that compresses nothing. SIGINT, SIGTERM and SIGHUP now set the cancellation flag through a direct sigaction with SA_RESTART, replacing any disposition the caller left, as ctrlc's set_handler did. ctrlc remains for non-Unix builds. A test signals the binary, with and without ignored signals, and expects run --auto to stop its command and exit 130. Refs #12 --- Cargo.toml | 5 +++- src/main.rs | 3 +-- src/process.rs | 43 ++++++++++++++++++++++++++++++++++ tests/process_contracts.rs | 48 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 76f1bdd..ccb772d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ repository = "https://github.com/maxgfr/scopelet" [dependencies] anyhow = "1" clap = { version = "4", features = ["derive"] } -ctrlc = { version = "3", features = ["termination"] } ignore = "0.4" libc = "0.2" memchr = "2" @@ -21,6 +20,10 @@ serde_json = { version = "1", features = ["arbitrary_precision"] } sha2 = { version = "0.10", features = ["asm"] } tempfile = "3" +# Unix installs its termination handler with sigaction (`process.rs`). +[target.'cfg(not(unix))'.dependencies] +ctrlc = { version = "3", features = ["termination"] } + [dev-dependencies] assert_cmd = "2" diff --git a/src/main.rs b/src/main.rs index fdafe83..2d47d74 100644 --- a/src/main.rs +++ b/src/main.rs @@ -351,8 +351,7 @@ fn execute(cli: Cli) -> Result { compress::Version::default() }; let cancel = Arc::new(AtomicBool::new(false)); - let signal = cancel.clone(); - ctrlc::set_handler(move || signal.store(true, std::sync::atomic::Ordering::SeqCst))?; + scopelet::process::cancel_on_termination(cancel.clone())?; if matches!(cli.command, Cmd::Doctor) { let mut status = scopelet::adapters::doctor(cancel); status["integration"] = integration::doctor(); diff --git a/src/process.rs b/src/process.rs index f835310..fb0a907 100644 --- a/src/process.rs +++ b/src/process.rs @@ -84,6 +84,49 @@ fn nonblocking(pipe: &impl std::os::fd::AsRawFd) -> Result<()> { Ok(()) } +/// Turn SIGINT, SIGTERM and SIGHUP into `cancel`, replacing whatever the +/// caller left (an ignored SIGHUP under `nohup`, say), so a wrapped command is +/// stopped and its status reported (130) instead of this process dying first. +/// +/// A direct `sigaction` rather than a signal-handling crate: on Apple +/// platforms that crate links the Foundation framework, whose loading cost +/// most of a millisecond on every invocation, every hook included. +#[cfg(unix)] +pub fn cancel_on_termination(cancel: Arc) -> Result<()> { + static CANCEL: std::sync::OnceLock> = std::sync::OnceLock::new(); + extern "C" fn handle(_: libc::c_int) { + // An atomic load and an atomic store: async-signal-safe. + if let Some(cancel) = CANCEL.get() { + cancel.store(true, Ordering::SeqCst); + } + } + ensure!( + CANCEL.set(cancel).is_ok(), + "termination handler already installed" + ); + for signal in [libc::SIGINT, libc::SIGTERM, libc::SIGHUP] { + // SAFETY: a zeroed sigaction is a valid starting value, and the + // handler only touches an initialized atomic. + let installed = unsafe { + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = handle as extern "C" fn(libc::c_int) as libc::sighandler_t; + action.sa_flags = libc::SA_RESTART; + libc::sigemptyset(&mut action.sa_mask); + libc::sigaction(signal, &action, std::ptr::null_mut()) + }; + if installed != 0 { + return Err(std::io::Error::last_os_error()).context("install termination handler"); + } + } + Ok(()) +} + +#[cfg(not(unix))] +pub fn cancel_on_termination(cancel: Arc) -> Result<()> { + ctrlc::set_handler(move || cancel.store(true, Ordering::SeqCst))?; + Ok(()) +} + #[cfg(unix)] fn signal_group(pid: u32, signal: i32) { // The child starts a dedicated group; this never targets the caller's group. diff --git a/tests/process_contracts.rs b/tests/process_contracts.rs index a2f84a6..d120586 100644 --- a/tests/process_contracts.rs +++ b/tests/process_contracts.rs @@ -168,3 +168,51 @@ fn large_output_is_captured_at_pipe_speed() { started.elapsed() ); } + +/// The binary turns SIGINT, SIGTERM and SIGHUP into a cancellation, even when +/// its caller ignored them (`nohup`, a background job): the wrapped command's +/// group is stopped and `run --auto` exits 130 instead of dying or hanging. +#[test] +fn termination_signals_cancel_a_wrapped_command() { + let binary = env!("CARGO_BIN_EXE_scopelet"); + for (signal, ignored) in [ + (libc::SIGINT, false), + (libc::SIGTERM, false), + (libc::SIGHUP, false), + (libc::SIGINT, true), + (libc::SIGHUP, true), + ] { + let dir = tempfile::tempdir().unwrap(); + let trap = if ignored { "trap '' INT HUP; " } else { "" }; + let mut child = Command::new("sh") + .args([ + "-c", + &format!("{trap}exec \"$0\" --cache-dir \"$1\" run --auto -- sleep 30"), + binary, + dir.path().to_str().unwrap(), + ]) + .env("SCOPELET_EVENTS", "0") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + thread::sleep(Duration::from_millis(500)); + unsafe { libc::kill(child.id() as i32, signal) }; + let start = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if start.elapsed() > Duration::from_secs(5) { + let _ = child.kill(); + panic!("signal {signal} (ignored: {ignored}) did not cancel"); + } + thread::sleep(Duration::from_millis(20)); + }; + assert_eq!( + status.code(), + Some(130), + "signal {signal} (ignored: {ignored})" + ); + } +} From 5e386b6037edae72c290ede7729e75f7a44397af Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:14:19 +0200 Subject: [PATCH 3/9] fix(v3): keep hunk headers and commit messages with their changes Git's default three lines of context put a hunk header four lines above its first change, one past the glue reach: the header ranked last and a diff showed changes without the line numbers locating them. A hunk header now glues to the first change below it at any distance and ranks with it. Commit messages of git log -p folded by template, so messages differing by a number collapsed into one similar= line, contrary to design.md: they now fold only with identical text, like changes. On the git log -p fixture each commit's subject replaces index metadata lines. Refs #12 --- src/compress.rs | 21 +++++++++++++---- tests/compact_v3.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/compress.rs b/src/compress.rs index d155095..35ac4c3 100644 --- a/src/compress.rs +++ b/src/compress.rs @@ -919,9 +919,10 @@ fn text_units_v3<'a>( let origin = first[i] as usize; let slot = if slot_of_first[origin] != NONE { slot_of_first[origin] - } else if matches!(role(i), Diff::File | Diff::Change) { - // Headers and changes are evidence: they never fold by template, - // only with the very same text (a file every commit touches). + } else if matches!(role(i), Diff::File | Diff::Change | Diff::Message) { + // Headers, changes and commit messages are evidence: they never + // fold by template, only with the very same text (a file every + // commit touches). slot_of_first[i] = groups.len() as u32; groups.len() as u32 } else if strong.get(i) || weak.get(i) { @@ -979,7 +980,10 @@ fn text_units_v3<'a>( _ if !trivial.get(i) => reach = in_place(i).then_some(0), _ => { reach = reach.map(|r| r + 1); - if reach.is_some_and(|r| r <= GLUE_REACH) { + // A hunk header locates the first change below it, past + // however much context Git shows before that change. + let locates = !order && role(i) == Diff::Hunk; + if reach.is_some_and(|r| locates || r <= GLUE_REACH) { glued.set(i); } } @@ -1147,7 +1151,14 @@ fn text_units_v3<'a>( for (line, &priority) in line_priority.iter().enumerate().rev() { while k > 0 && trivial_units[k - 1].2 == line { k -= 1; - let (unit, ..) = trivial_units[k]; + let (unit, first_line, _) = trivial_units[k]; + // A hunk header ranks with the change it locates, at any distance. + if role(first_line) == Diff::Hunk + && let Some((priority, _)) = carry + { + units[unit].priority = priority; + continue; + } let after = carry .filter(|&(_, at)| at - line <= GLUE_REACH) .map(|(p, _)| p); diff --git a/tests/compact_v3.rs b/tests/compact_v3.rs index 117487e..23102f7 100644 --- a/tests/compact_v3.rs +++ b/tests/compact_v3.rs @@ -355,6 +355,63 @@ fn unified_diffs_keep_file_headers_and_changes_in_place() { assert_eq!(original(dir.path(), &text), raw.as_bytes()); } +/// Git's default three lines of context put a hunk header four lines above +/// its first change: the header still locates the changes shown. A tier is +/// filled from both ends, so where the budget runs out one hunk may be cut +/// between its header and its change. +#[test] +fn hunk_headers_stay_with_their_changes_at_the_default_context() { + let dir = tempfile::tempdir().unwrap(); + let mut raw = String::new(); + for f in 0..40 { + raw.push_str(&format!( + "diff --git a/src/m{f}.py b/src/m{f}.py\nindex 1111111..2222222 100644\n--- a/src/m{f}.py\n+++ b/src/m{f}.py\n@@ -{0},7 +{0},8 @@ def handler_{f}():\n x = load()\n y = parse(x)\n z = check(y)\n- return old_{f}(z)\n+ value = new_{f}(z)\n+ return value\n # end\n pass\n done()\n", + 10 * f + 5 + )); + } + let text = compress_v3(dir.path(), raw.as_bytes(), 4096); + let shown: Vec = (0..40) + .filter(|f| text.contains(&format!("+ value = new_{f}(z)\n"))) + .collect(); + assert!( + shown.len() >= 5, + "{} changes shown in:\n{text}", + shown.len() + ); + let headless: Vec<_> = shown + .iter() + .filter(|f| !text.contains(&format!("@@ def handler_{f}():\n"))) + .collect(); + assert!( + headless.len() <= 1, + "{headless:?} without header in:\n{text}" + ); + assert_eq!(original(dir.path(), &text), raw.as_bytes()); +} + +/// Commit messages fold only with identical text, like changes: messages +/// that differ by a number are distinct evidence, not a template. +#[test] +fn commit_messages_differing_by_numbers_stay_distinct() { + let dir = tempfile::tempdir().unwrap(); + let mut raw = String::new(); + for c in 0..40 { + raw.push_str(&format!( + "commit {c:040x}\nAuthor: A \nDate: Mon Jan 1 10:00:00 2024 +0000\n\n Fix parser case {c}\n\ndiff --git a/src/m{c}.rs b/src/m{c}.rs\nindex 1111111..2222222 100644\n--- a/src/m{c}.rs\n+++ b/src/m{c}.rs\n@@ -1,2 +1,2 @@\n-old {c}\n+new {c}\n" + )); + } + let text = compress_v3(dir.path(), raw.as_bytes(), 4096); + let messages = text.matches(" Fix parser case ").count(); + assert!(messages >= 5, "{messages} messages in:\n{text}"); + assert!( + !text + .lines() + .any(|l| l.contains("similar=") && l.contains("Fix parser case")), + "{text}" + ); + assert_eq!(original(dir.path(), &text), raw.as_bytes()); +} + #[test] fn warnings_do_not_evict_errors_from_a_small_budget() { let dir = tempfile::tempdir().unwrap(); From a1cf107f1a28eebd9af8b2cd49477b95a090c123 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:16:32 +0200 Subject: [PATCH 4/9] feat(codex): accept literal tildes and backslashes in double quotes The shell subset rejected any ~ and any backslash inside double quotes, so git diff HEAD~1 and rg "\d+" src stayed native. sh, bash and zsh only expand a tilde at the start of a word or after an assignment's = or :, and inside double quotes keep a backslash literal before an ordinary character. Those forms are now parsed to the words every shell gives them; a tilde that would expand and a backslash before " or \ are still rejected. Refs #12 --- docs/design.md | 11 +++++++---- src/shell.rs | 50 +++++++++++++++++++++++++++++++++++------------ tests/profiles.rs | 13 ++++++++++++ 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/docs/design.md b/docs/design.md index 346a615..576fb9c 100644 --- a/docs/design.md +++ b/docs/design.md @@ -195,10 +195,13 @@ system prompt on every step rather than through per-session hook context. A Codex command is parsed in a small POSIX subset (`src/shell.rs`) that reads the same in `sh`, `bash` and `zsh`: words may be single quoted, double quoted -without `$`, backticks or backslashes inside, or backslash-escaped; literal -`NAME=value` prefixes, `cd DIR &&`, `&&`, a final `|| true`, pipes, `2>&1` -and output or errors to `/dev/null` are accepted. Expansions (`$`, backticks, -globs, `~`, braces), grouping, `;`, a lone `&`, `|&`, `&>`, `>|`, input and +without `$` or backticks inside (a backslash there only before an ordinary +character, where it stays literal: `"\d+"`), or backslash-escaped; a `~` +inside a word (`HEAD~1`) is literal; literal `NAME=value` prefixes, +`cd DIR &&`, `&&`, a final `|| true`, pipes, `2>&1` and output or errors to +`/dev/null` are accepted. Expansions (`$`, backticks, globs, a `~` starting a +word or following an assignment's `=` or `:`, braces), grouping, `;`, a lone +`&`, `|&`, `&>`, `>|`, input and other redirections, heredocs, newlines, and words starting with `=` or `#` leave the command native. Every command in it must be recognized; after a pipe only read-only filters are allowed (`grep`, `rg`, `sort` without `-o`, `uniq`, diff --git a/src/shell.rs b/src/shell.rs index f5d816e..3ab7652 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -2,12 +2,14 @@ //! //! The hook may only wrap a command whose meaning it fully understands, so //! this grammar accepts what reads the same in `sh`, `bash` and `zsh` and -//! rejects everything else: no expansion (`$`, backticks, globs, `~`, braces, -//! history), no grouping, no input redirection, no background job, no -//! sequencing other than `&&`, `|| true` and pipes. Words may be single -//! quoted, double quoted (without `$`, backticks or backslashes inside) or -//! escaped with a backslash. The only redirections are `2>&1` and output or -//! error to `/dev/null`. A rejected command is left to the host untouched. +//! rejects everything else: no expansion (`$`, backticks, globs, a `~` that +//! would expand, braces, history), no grouping, no input redirection, no +//! background job, no sequencing other than `&&`, `|| true` and pipes. Words +//! may be single quoted, double quoted (without `$` or backticks inside, and +//! with a backslash only before an ordinary character, where it is literal) +//! or escaped with a backslash. A `~` inside a word (`HEAD~1`) is literal. The +//! only redirections are `2>&1` and output or error to `/dev/null`. A +//! rejected command is left to the host untouched. /// A redirection the grammar accepts. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -110,10 +112,15 @@ fn lex(command: &str) -> Option> { '"' => { let end = chars[i + 1..].iter().position(|&c| c == '"')? + i + 1; let inner = &chars[i + 1..end]; - if inner - .iter() - .any(|c| matches!(c, '$' | '`' | '\\' | '\n' | '\r')) - { + if inner.iter().any(|c| matches!(c, '$' | '`' | '\n' | '\r')) { + return None; + } + // A backslash stays literal before an ordinary character + // (`"\d+"`); before `"` or `\` it escapes, which this + // scanner does not model. + if inner.iter().enumerate().any(|(k, &c)| { + c == '\\' && inner.get(k + 1).is_none_or(|n| matches!(n, '"' | '\\')) + }) { return None; } word.extend(inner); @@ -200,8 +207,12 @@ fn lex(command: &str) -> Option> { } tokens.push(Token::Redirect(redirect)); } - '\n' | '\r' | ';' | '<' | '(' | ')' | '{' | '}' | '$' | '`' | '*' | '?' | '[' | '~' - | '!' => return None, + // A tilde expands at the start of a word and after the `=` or `:` + // of an assignment-like word; anywhere else (`HEAD~1`) it is literal. + '~' if !started || matches!(word.chars().last(), Some('=' | ':')) => return None, + '\n' | '\r' | ';' | '<' | '(' | ')' | '{' | '}' | '$' | '`' | '*' | '?' | '[' | '!' => { + return None; + } '#' | '=' if !started => return None, c => { word.push(c); @@ -345,6 +356,13 @@ mod tests { &[&["rg", "-e", "a|b", "--glob", "*.rs"]], ), ("rg 'x#y' a#b", &[&["rg", "x#y", "a#b"]]), + // A tilde inside a word is literal; only a leading one expands. + ("git diff HEAD~1", &[&["git", "diff", "HEAD~1"]]), + ("git log main~3..HEAD~", &[&["git", "log", "main~3..HEAD~"]]), + // Inside double quotes a backslash before an ordinary character + // is itself literal. + ("rg \"\\d+\" src", &[&["rg", "\\d+", "src"]]), + ("grep -E \"a\\.b\" x", &[&["grep", "-E", "a\\.b", "x"]]), ]; for (command, expected) in cases { let script = parse(command).unwrap_or_else(|| panic!("rejected: {command}")); @@ -390,6 +408,14 @@ mod tests { "ls file?.txt", "ls [ab].txt", "ls ~/x", + "cd ~", + "ls ~user", + "echo a=~/x", + "PATH=a:~/b cmd", + "echo \"a\\$b\"", + "echo \"a\\`b\"", + "echo \"a\\\\b\"", + "echo \"a\\\"", "echo {a,b}", "(cd x && make)", "! cargo test", diff --git a/tests/profiles.rs b/tests/profiles.rs index 39eeb59..4f44370 100644 --- a/tests/profiles.rs +++ b/tests/profiles.rs @@ -173,6 +173,19 @@ fn codex_rewrites_carry_the_profile_budget_in_v3() { ); // A 10 KB source read is below the FileRead threshold: it stays native. assert_eq!(command("cat mid.rs"), None); + // A literal tilde inside a revision and a backslash in a double-quoted + // pattern read the same in every shell: both are wrapped, words intact. + let revision = command("git diff HEAD~1").unwrap(); + assert!( + revision.ends_with("--profile git -- 'git' 'diff' 'HEAD~1'"), + "{revision}" + ); + let pattern = command("rg \"\\d+\" src").unwrap(); + assert!( + pattern.ends_with("--profile search -- 'rg' '\\d+' 'src'"), + "{pattern}" + ); + assert_eq!(command("git diff ~/x"), None); } #[test] From 9350786b65bc943235a966717c99eb39063476a6 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:18:37 +0200 Subject: [PATCH 5/9] fix(store): recovering a single-line original stores no second copy Paging or searching a single-line blob shows its only line as one record, and the view's schema-1 artifact stored that line again: recovering a 6 MB JSON line twice stored 12 MB. A recovery whose only record is a whole text original now names the blob (schema 2), as compact-v3 artifacts do; a hash check settles that the text is the whole blob. Two recoveries of a 2 MB line now store kilobytes and their artifacts still expand to the exact line. Refs #12 --- docs/design.md | 5 ++-- skills/scopelet/references/queries.md | 7 +++-- src/main.rs | 8 ++---- src/recovery.rs | 35 +++++++++++++++++++++-- tests/recovery_ids.rs | 40 +++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 14 deletions(-) diff --git a/docs/design.md b/docs/design.md index 576fb9c..818a56e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -7,8 +7,9 @@ call another model to compress data. `Source -> Dataset -> operations -> immutable artifact -> bounded View` is the single pipeline. The Rust library exposes the same types used by the CLI. Request version is 1. Saved artifacts are schema 1 (records stored) or, for -compact-v3 automatic compression, schema 2 (records named by their original -blob). Unknown request fields fail. +compact-v3 automatic compression and for a recovery whose only record is a +whole text original (a single-line blob paged or searched), schema 2 (records +named by their original blob). Unknown request fields fail. Source loading owns scope and freshness. A dataset carries full records, snapshots, the number of examined files/streams and observed skip counts. diff --git a/skills/scopelet/references/queries.md b/skills/scopelet/references/queries.md index 9f1e965..abc87e7 100644 --- a/skills/scopelet/references/queries.md +++ b/skills/scopelet/references/queries.md @@ -233,6 +233,7 @@ not a computed aggregate. Complete cells, exact numbers and missing/null semantics survive. A partial table cannot establish an exhaustive count. V2 prioritizes distinct diagnostics before repetitions, and links directly to saved-source search. Version 1 remains selectable with `--compact-version 1`. -The query schema is version 1. Artifacts are schema 1, except compact-v3 -automatic compression's schema 2, which names the original blob instead of -copying its records; `expand` and queries read both. +The query schema is version 1. Artifacts are schema 1, except schema 2 for +compact-v3 automatic compression and for a recovery that shows a whole text +original as its only record, which names the original blob instead of copying +it; `expand` and queries read both. diff --git a/src/main.rs b/src/main.rs index 2d47d74..f251b14 100644 --- a/src/main.rs +++ b/src/main.rs @@ -725,12 +725,8 @@ fn execute(cli: Cli) -> Result { context, source.as_deref(), )?; - print(&render::render( - &data, - &store, - Mode::Default, - max_bytes, - offset, + print(&scopelet::recovery::render( + &data, &store, max_bytes, offset, )?)?; return Ok(if cancel.load(std::sync::atomic::Ordering::SeqCst) { 130 diff --git a/src/recovery.rs b/src/recovery.rs index 9ec1f26..49a7aa5 100644 --- a/src/recovery.rs +++ b/src/recovery.rs @@ -95,6 +95,35 @@ pub fn search( Ok(data) } +/// Store a recovery result and render its page at `offset`. A result that is +/// one whole text original (a single-line blob paged or searched shows its +/// only line) names that blob, as a compact-v3 artifact does (schema 2), +/// instead of storing the line a second time for every recovery. +pub fn render(data: &Dataset, store: &Store, budget: usize, offset: usize) -> Result { + if let [record] = data.records.as_slice() + && let Some(blob) = record.blob.as_deref() + && blob.starts_with("blob:") + && record.value.is_none() + && record.start_line == Some(1) + && record.end_line == Some(crate::clean::line_count(&record.text)) + && record.omitted_lines.is_none() + && !record.text_truncated + // Hashing settles that the text is the whole blob, not a prefix. + && blob.ends_with(&digest(record.text.as_bytes())) + { + let artifact = store.put_json(&DatasetRef::new( + data, + RecordsRef { + blob: blob.into(), + format: Format::Text, + source: record.source.clone(), + }, + ))?; + return crate::render::render_stored(data, artifact, Mode::Default, budget, offset); + } + crate::render::render(data, store, Mode::Default, budget, offset) +} + /// Lines `start..=end` of a blob, or as many of them as fit `budget`: a view /// that cannot show them all shows the first ones and sets `next_start`. /// Only a single line larger than the budget is still a blocked record. @@ -119,7 +148,7 @@ pub fn page(store: &Store, id: &str, start: usize, end: usize, budget: usize) -> (serde_json::to_vec(&view).ok()?.len() < budget).then_some(view) }; if data.records.is_empty() || measure(&data, None).is_some() { - return crate::render::render(&data, store, Mode::Default, budget, 0); + return render(&data, store, budget, 0); } let record = &data.records[0]; let lines: Vec<&str> = record.text.split_inclusive('\n').collect(); @@ -144,9 +173,9 @@ pub fn page(store: &Store, id: &str, start: usize, end: usize, budget: usize) -> } if fits == 0 { // A single line larger than the budget: report it as blocked. - return crate::render::render(&data, store, Mode::Default, budget, 0); + return render(&data, store, budget, 0); } - let mut view = crate::render::render(&take(fits), store, Mode::Default, budget, 0)?; + let mut view = render(&take(fits), store, budget, 0)?; view.next_start = Some(first + fits); view.stop_early(NOTE); Ok(view) diff --git a/tests/recovery_ids.rs b/tests/recovery_ids.rs index d83ab79..01a4b9b 100644 --- a/tests/recovery_ids.rs +++ b/tests/recovery_ids.rs @@ -174,3 +174,43 @@ fn offset_with_a_line_range_is_rejected_instead_of_ignored() { assert_eq!(output.status.code(), Some(2)); assert!(String::from_utf8_lossy(&output.stderr).contains("cannot be used with")); } + +/// Total bytes of stored artifacts. +fn artifact_bytes(cache: &std::path::Path) -> u64 { + std::fs::read_dir(cache.join("artifacts")) + .unwrap() + .map(|entry| entry.unwrap().metadata().unwrap()) + .filter(|meta| meta.is_file()) + .map(|meta| meta.len()) + .sum() +} + +/// Paging or searching a single-line original shows that whole line as one +/// record. Its artifact names the blob instead of storing the line again, so +/// recovering a 2 MB line twice stores kilobytes, not two more copies, and +/// the artifact still expands to the exact line. +#[test] +fn recovering_a_single_line_original_stores_no_second_copy() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(Some(dir.path().into())).unwrap(); + let line = format!( + "{{\"payload\":\"{}\",\"key\":\"k39\"}}", + "x".repeat(2_000_000) + ); + let blob = store.put("blob", line.as_bytes()).unwrap(); + let before = artifact_bytes(dir.path()); + let page = json(&cli(dir.path(), &["expand", &blob])); + let found = json(&cli(dir.path(), &["expand", &blob, "--find", "k39"])); + let stored = artifact_bytes(dir.path()) - before; + assert!( + stored < 64 * 1024, + "{stored} artifact bytes for two recoveries" + ); + for view in [page, found] { + let artifact = view["artifact"].as_str().unwrap(); + let data = store.dataset(artifact).unwrap(); + assert_eq!(data.records.len(), 1); + assert_eq!(data.records[0].text, line); + assert_eq!(data.records[0].blob.as_deref(), Some(blob.as_str())); + } +} From 2b782df5e441255e6e84cf04258353baf7de4952 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:40:00 +0200 Subject: [PATCH 6/9] fix(store): name only stored blobs in recovery references A repository scan keeps only the originals of files that matched, and a later search reads an unmatched file from its unchanged local copy. That record still names the file's blob, so the schema-2 shortcut wrote an artifact referring to a blob the store never had, which then failed to expand. The shortcut now requires the blob to be stored, and compares the record with what the text parser reads back from it instead of repeating that parser's fields. Refs #12 --- src/recovery.rs | 22 ++++++++++++---------- tests/repo_scan.rs | 7 +++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/recovery.rs b/src/recovery.rs index 49a7aa5..fe2f7cf 100644 --- a/src/recovery.rs +++ b/src/recovery.rs @@ -95,21 +95,23 @@ pub fn search( Ok(data) } -/// Store a recovery result and render its page at `offset`. A result that is -/// one whole text original (a single-line blob paged or searched shows its -/// only line) names that blob, as a compact-v3 artifact does (schema 2), -/// instead of storing the line a second time for every recovery. +/// Store a recovery result and render its page at `offset`, as +/// `render::render` does. A result whose only record is one whole stored text +/// original (a single-line blob paged or searched shows its only line) names +/// that blob, as a compact-v3 artifact does (schema 2), instead of storing the +/// original a second time for every recovery. pub fn render(data: &Dataset, store: &Store, budget: usize, offset: usize) -> Result { if let [record] = data.records.as_slice() && let Some(blob) = record.blob.as_deref() + // A scanned file whose original was not kept is searched locally: + // its blob is named but absent, so the result keeps its own copy. && blob.starts_with("blob:") - && record.value.is_none() - && record.start_line == Some(1) - && record.end_line == Some(crate::clean::line_count(&record.text)) - && record.omitted_lines.is_none() - && !record.text_truncated - // Hashing settles that the text is the whole blob, not a prefix. + && store.contains(blob) + // Hashing settles that the text is the whole blob, not a prefix... && blob.ends_with(&digest(record.text.as_bytes())) + // ...and reading it back as text must give exactly this record. + && crate::sources::Parsed::Text(record.text.clone()).records(&record.source, blob.into()) + == data.records { let artifact = store.put_json(&DatasetRef::new( data, diff --git a/tests/repo_scan.rs b/tests/repo_scan.rs index 03c09b9..79b9c38 100644 --- a/tests/repo_scan.rs +++ b/tests/repo_scan.rs @@ -103,6 +103,13 @@ fn only_matching_files_keep_their_original_and_recovery_reads_the_rest_locally() None, )); assert_eq!(found["total_records"], 1); + // Its original was never stored, so the result keeps its own copy and + // expands after the local file changes. + let kept = found["artifact"].as_str().unwrap().to_owned(); + std::fs::write(root.join("d0/f0007.txt"), "rewritten\n").unwrap(); + let again = ok(cli(&cache, "4", &["expand", &kept], None)); + assert!(again.to_string().contains("hay only 7"), "{again}"); + std::fs::write(root.join("d0/f0007.txt"), "file 7\nhay only 7\n").unwrap(); // Once it changes, it is counted as not retained instead. std::fs::write(root.join("d0/f0007.txt"), "rewritten\n").unwrap(); let found = ok(cli( From a6762bd8b2978a0f730862110dfe797e32765429 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:40:00 +0200 Subject: [PATCH 7/9] test: signal the binary once its command runs, not after a delay A fixed 500 ms wait could signal the binary before its handler was installed on a loaded machine. The test now waits until the wrapped sleep is a child of the binary, which happens after the handler is set. Refs #12 --- tests/process_contracts.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/process_contracts.rs b/tests/process_contracts.rs index d120586..aeac62b 100644 --- a/tests/process_contracts.rs +++ b/tests/process_contracts.rs @@ -196,7 +196,22 @@ fn termination_signals_cancel_a_wrapped_command() { .stderr(std::process::Stdio::null()) .spawn() .unwrap(); - thread::sleep(Duration::from_millis(500)); + // The handler is installed before the command starts: once `sleep` + // runs under the binary, a signal can no longer kill it unhandled. + let ready = Instant::now(); + while !Command::new("pgrep") + .args(["-P", &child.id().to_string(), "sleep"]) + .output() + .unwrap() + .status + .success() + { + assert!( + ready.elapsed() < Duration::from_secs(10), + "the wrapped command never started" + ); + thread::sleep(Duration::from_millis(20)); + } unsafe { libc::kill(child.id() as i32, signal) }; let start = Instant::now(); let status = loop { From 4c23d14b1fcb609bf246324716e6451dac188340 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:40:00 +0200 Subject: [PATCH 8/9] docs: hunk header glue, schema-2 recoveries and the tilde rule design.md still described hunk headers as glued within three lines and limited schema-2 recoveries to single-line blobs; the shell subset rejects a tilde after any = or :, not only an assignment's. Refs #12 --- docs/design.md | 17 ++++++++++------- src/shell.rs | 3 ++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/design.md b/docs/design.md index 818a56e..ddc513e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -8,8 +8,9 @@ call another model to compress data. single pipeline. The Rust library exposes the same types used by the CLI. Request version is 1. Saved artifacts are schema 1 (records stored) or, for compact-v3 automatic compression and for a recovery whose only record is a -whole text original (a single-line blob paged or searched), schema 2 (records -named by their original blob). Unknown request fields fail. +whole stored text original (a blob paged or searched whose view is all of it, +such as a single long line), schema 2 (records named by their original blob). +Unknown request fields fail. Source loading owns scope and freshness. A dataset carries full records, snapshots, the number of examined files/streams and observed skip counts. @@ -201,7 +202,7 @@ character, where it stays literal: `"\d+"`), or backslash-escaped; a `~` inside a word (`HEAD~1`) is literal; literal `NAME=value` prefixes, `cd DIR &&`, `&&`, a final `|| true`, pipes, `2>&1` and output or errors to `/dev/null` are accepted. Expansions (`$`, backticks, globs, a `~` starting a -word or following an assignment's `=` or `:`, braces), grouping, `;`, a lone +word or following any `=` or `:`, braces), grouping, `;`, a lone `&`, `|&`, `&>`, `>|`, input and other redirections, heredocs, newlines, and words starting with `=` or `#` leave the command native. Every command in it must be recognized; after a pipe @@ -380,10 +381,12 @@ mentions `error` is a change. File headers (`diff --git`, the `---`/`+++` pair of a plain diff, and `git log -p`'s `commit `) never fold by template, only with identical text, and rank as context does (tier 2). Changed lines and commit messages fold only with identical text; a distinct -one ranks in tier 2, one repeated across files or commits in tier 1. Hunk -headers and context lines are glued like trivial lines, so a hunk is shown -with the change it locates; a file header ends what can glue and metadata -lines (`index`, modes, renames, `Author:`, `Date:`) are transparent to it. +one ranks in tier 2, one repeated across files or commits in tier 1. Context +lines are glued like trivial lines. A hunk header glues to the first change +below it however much context lies between (Git's default three lines put it +four lines away) and ranks with that change, so a hunk is shown with the +change it locates; a file header ends what can glue and metadata lines +(`index`, modes, renames, `Author:`, `Date:`) are transparent to it. V3 ranks units in five tiers: the first and last line of a record; the first occurrence of each strong diagnostic template (`error`, `failed`, `panic`, diff --git a/src/shell.rs b/src/shell.rs index 3ab7652..54cc2a0 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -208,7 +208,8 @@ fn lex(command: &str) -> Option> { tokens.push(Token::Redirect(redirect)); } // A tilde expands at the start of a word and after the `=` or `:` - // of an assignment-like word; anywhere else (`HEAD~1`) it is literal. + // of an assignment-like word, so one after any `=` or `:` is + // rejected; anywhere else (`HEAD~1`) it is literal. '~' if !started || matches!(word.chars().last(), Some('=' | ':')) => return None, '\n' | '\r' | ';' | '<' | '(' | ')' | '{' | '}' | '$' | '`' | '*' | '?' | '[' | '!' => { return None; From bbe5b2a4dcfe45e2a9ee57299c31dc4fea15203e Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:40:42 +0200 Subject: [PATCH 9/9] feat(commands): recognize npx -y and yarn running a known tool npx -y only skips the install prompt, and yarn runs a package binary as npx does, so npx -y tsc and yarn jest finish and report an outcome like their recognized forms. Unknown tools, npx -p and yarn subcommands such as add stay native. Refs #12 --- docs/design.md | 3 ++- skills/scopelet/references/setup.md | 3 ++- src/commands.rs | 8 ++++++++ tests/profiles.rs | 9 +++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/design.md b/docs/design.md index ddc513e..092f9e2 100644 --- a/docs/design.md +++ b/docs/design.md @@ -301,7 +301,8 @@ The shared command classifier recognizes builds, tests and checks (`cargo test|c `npm`/`pnpm`/`yarn` test/build/lint/typecheck, `bun test`, `make test|check|build|lint`, `go test|build|vet`, `ruff`, `mypy`, Gradle/Maven/.NET test or build, `node --test`, and `jest`, `vitest run`, `tsc`, `eslint`, -`prettier --check` or `playwright test` through `npx`, `bunx`, `pnpm exec` or +`prettier --check` or `playwright test` through `npx` (also `-y`), `bunx`, +`yarn`, `pnpm exec` or `uv run`); searches and listings (`rg`, `grep`, `find` without `-delete`/`-exec`/`-ok`/`-fprint`, `ls -R`, `tree` without `-o`); Git `diff`, `log` (with `-p`), `show`, `status`, `blame` and `grep` with no option before the diff --git a/skills/scopelet/references/setup.md b/skills/scopelet/references/setup.md index 413fc0e..9bc276f 100644 --- a/skills/scopelet/references/setup.md +++ b/skills/scopelet/references/setup.md @@ -202,7 +202,8 @@ Hooks recognize builds, tests and checks (`cargo test|check|clippy|build|nextest `npm`/`pnpm`/`yarn` test/build/lint/typecheck, `bun test`, `make test|check|build|lint`, `go test|build|vet`, `ruff`, `mypy`, Gradle/Maven/.NET test or build, `node --test`, and `jest`, `vitest run`, `tsc`, `eslint`, -`prettier --check` or `playwright test` through `npx`, `bunx`, `pnpm exec` or +`prettier --check` or `playwright test` through `npx` (also `-y`), `bunx`, +`yarn`, `pnpm exec` or `uv run`); searches and listings (`rg`, `grep`, `find` without `-delete`/`-exec`/`-ok`/`-fprint`, `ls -R`, `tree` without `-o`); Git `diff`, `log` (with `-p`), `show`, `status`, `blame` and `grep` with no option before the diff --git a/src/commands.rs b/src/commands.rs index 85a2dd6..ff98475 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -146,6 +146,10 @@ pub fn profile(args: &[String]) -> Option { if name == "pnpm" && second == "exec" { return known_tool(&args[2..]).then_some(Profile::Tests); } + // `yarn ` runs a package binary, like `npx`. + if name == "yarn" && known_tool(&args[1..]) { + return tests; + } let script = if second == "run" { args.get(2).map(String::as_str).unwrap_or("") } else { @@ -153,6 +157,10 @@ pub fn profile(args: &[String]) -> Option { }; matches!(script, "test" | "build" | "lint" | "typecheck").then_some(Profile::Tests) } + // `-y`/`--yes` only skips npx's install prompt. + "npx" if matches!(second, "-y" | "--yes") => { + known_tool(&args[2..]).then_some(Profile::Tests) + } "npx" | "bunx" => known_tool(&args[1..]).then_some(Profile::Tests), "uv" if second == "run" => known_tool(&args[2..]).then_some(Profile::Tests), "bun" if second == "test" => tests, diff --git a/tests/profiles.rs b/tests/profiles.rs index 4f44370..d5fe7ed 100644 --- a/tests/profiles.rs +++ b/tests/profiles.rs @@ -20,6 +20,10 @@ fn safe_families_are_recognized_with_their_profile() { ("npx playwright test", Profile::Tests), ("pnpm exec eslint .", Profile::Tests), ("bunx jest", Profile::Tests), + ("npx -y tsc --noEmit", Profile::Tests), + ("npx --yes jest", Profile::Tests), + ("yarn jest", Profile::Tests), + ("yarn tsc --noEmit", Profile::Tests), ("bun test", Profile::Tests), ("make test", Profile::Tests), ("make lint", Profile::Tests), @@ -81,6 +85,11 @@ fn writing_waiting_or_unknown_commands_are_not_recognized() { "git log --ext-diff", "cargo fmt", "npx some-tool", + "npx -y some-tool", + "npx -p typescript tsc", + "yarn add jest", + "yarn some-tool", + "yarn jest --watch", "uv run ./script", "python3 -m http.server", "make install",