perf: faster hooks, diff, Codex and recovery fixes (follow-up to #12)

Cargo.toml

@@ -10,7 +10,6 @@ repository = "https://github.com/maxgfr/scopelet"
 [dependencies]
 anyhow = "1"
 clap = { version = "4", features = ["derive"] }
-ctrlc = { version = "3", features = ["termination"] }
 ignore = "0.4"
 libc = "0.2"
 memchr = "2"
@@ -21,6 +20,10 @@ serde_json = { version = "1", features = ["arbitrary_precision"] }
 sha2 = { version = "0.10", features = ["asm"] }
 tempfile = "3"
 
+# Unix installs its termination handler with sigaction (`process.rs`).
+[target.'cfg(not(unix))'.dependencies]
+ctrlc = { version = "3", features = ["termination"] }
+
 [dev-dependencies]
 assert_cmd = "2"
 

docs/design.md

@@ -7,8 +7,10 @@ call another model to compress data.
 `Source -> Dataset -> operations -> immutable artifact -> bounded View` is the
 single pipeline. The Rust library exposes the same types used by the CLI.
 Request version is 1. Saved artifacts are schema 1 (records stored) or, for
-compact-v3 automatic compression, schema 2 (records named by their original
-blob). Unknown request fields fail.
+compact-v3 automatic compression and for a recovery whose only record is a
+whole stored text original (a blob paged or searched whose view is all of it,
+such as a single long line), schema 2 (records named by their original blob).
+Unknown request fields fail.
 
 Source loading owns scope and freshness. A dataset carries full records,
 snapshots, the number of examined files/streams and observed skip counts.
@@ -195,10 +197,13 @@ system prompt on every step rather than through per-session hook context.
 
 A Codex command is parsed in a small POSIX subset (`src/shell.rs`) that reads
 the same in `sh`, `bash` and `zsh`: words may be single quoted, double quoted
-without `$`, backticks or backslashes inside, or backslash-escaped; literal
-`NAME=value` prefixes, `cd DIR &&`, `&&`, a final `|| true`, pipes, `2>&1`
-and output or errors to `/dev/null` are accepted. Expansions (`$`, backticks,
-globs, `~`, braces), grouping, `;`, a lone `&`, `|&`, `&>`, `>|`, input and
+without `$` or backticks inside (a backslash there only before an ordinary
+character, where it stays literal: `"\d+"`), or backslash-escaped; a `~`
+inside a word (`HEAD~1`) is literal; literal `NAME=value` prefixes,
+`cd DIR &&`, `&&`, a final `|| true`, pipes, `2>&1` and output or errors to
+`/dev/null` are accepted. Expansions (`$`, backticks, globs, a `~` starting a
+word or following any `=` or `:`, braces), grouping, `;`, a lone
+`&`, `|&`, `&>`, `>|`, input and
 other redirections, heredocs, newlines, and words starting with `=` or `#`
 leave the command native. Every command in it must be recognized; after a pipe
 only read-only filters are allowed (`grep`, `rg`, `sort` without `-o`, `uniq`,
@@ -296,7 +301,8 @@ The shared command classifier recognizes builds, tests and checks (`cargo test|c
 `npm`/`pnpm`/`yarn` test/build/lint/typecheck, `bun test`, `make
 test|check|build|lint`, `go test|build|vet`, `ruff`, `mypy`, Gradle/Maven/.NET
 test or build, `node --test`, and `jest`, `vitest run`, `tsc`, `eslint`,
-`prettier --check` or `playwright test` through `npx`, `bunx`, `pnpm exec` or
+`prettier --check` or `playwright test` through `npx` (also `-y`), `bunx`,
+`yarn`, `pnpm exec` or
 `uv run`); searches and listings (`rg`, `grep`, `find` without
 `-delete`/`-exec`/`-ok`/`-fprint`, `ls -R`, `tree` without `-o`); Git `diff`, `log` (with
 `-p`), `show`, `status`, `blame` and `grep` with no option before the
@@ -376,10 +382,12 @@ mentions `error` is a change. File headers (`diff --git`, the `---`/`+++`
 pair of a plain diff, and `git log -p`'s `commit <hash>`) never fold by
 template, only with identical text, and rank as context does (tier 2).
 Changed lines and commit messages fold only with identical text; a distinct
-one ranks in tier 2, one repeated across files or commits in tier 1. Hunk
-headers and context lines are glued like trivial lines, so a hunk is shown
-with the change it locates; a file header ends what can glue and metadata
-lines (`index`, modes, renames, `Author:`, `Date:`) are transparent to it.
+one ranks in tier 2, one repeated across files or commits in tier 1. Context
+lines are glued like trivial lines. A hunk header glues to the first change
+below it however much context lies between (Git's default three lines put it
+four lines away) and ranks with that change, so a hunk is shown with the
+change it locates; a file header ends what can glue and metadata lines
+(`index`, modes, renames, `Author:`, `Date:`) are transparent to it.
 
 V3 ranks units in five tiers: the first and last line of a record; the first
 occurrence of each strong diagnostic template (`error`, `failed`, `panic`,

skills/scopelet/references/queries.md

@@ -233,6 +233,7 @@ not a computed aggregate. Complete cells, exact numbers and missing/null
 semantics survive. A partial table cannot establish an exhaustive count.
 V2 prioritizes distinct diagnostics before repetitions, and links directly to
 saved-source search. Version 1 remains selectable with `--compact-version 1`.
-The query schema is version 1. Artifacts are schema 1, except compact-v3
-automatic compression's schema 2, which names the original blob instead of
-copying its records; `expand` and queries read both.
+The query schema is version 1. Artifacts are schema 1, except schema 2 for
+compact-v3 automatic compression and for a recovery that shows a whole text
+original as its only record, which names the original blob instead of copying
+it; `expand` and queries read both.

skills/scopelet/references/setup.md

@@ -202,7 +202,8 @@ Hooks recognize builds, tests and checks (`cargo test|check|clippy|build|nextest
 `npm`/`pnpm`/`yarn` test/build/lint/typecheck, `bun test`, `make
 test|check|build|lint`, `go test|build|vet`, `ruff`, `mypy`, Gradle/Maven/.NET
 test or build, `node --test`, and `jest`, `vitest run`, `tsc`, `eslint`,
-`prettier --check` or `playwright test` through `npx`, `bunx`, `pnpm exec` or
+`prettier --check` or `playwright test` through `npx` (also `-y`), `bunx`,
+`yarn`, `pnpm exec` or
 `uv run`); searches and listings (`rg`, `grep`, `find` without
 `-delete`/`-exec`/`-ok`/`-fprint`, `ls -R`, `tree` without `-o`); Git `diff`, `log` (with
 `-p`), `show`, `status`, `blame` and `grep` with no option before the

src/clean.rs

@@ -212,9 +212,10 @@ fn uuid_at(bytes: &[u8], start: usize) -> Option<usize> {
     word_ends(bytes, i).then_some(i)
 }
 
+/// ASCII digits: Unicode `\d` costs compiling a larger engine on every call.
 static TIMESTAMP: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
     regex::Regex::new(
-        r"(?-u:\b)\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:[.,]\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?|(?-u:\b)\d{2}:\d{2}:\d{2}(?:[.,]\d+)?",
+        r"(?-u:\b)[0-9]{4}-[0-9]{2}-[0-9]{2}(?:[T ][0-9]{2}:[0-9]{2}(?::[0-9]{2}(?:[.,][0-9]+)?)?(?:Z|[+-][0-9]{2}:?[0-9]{2})?)?|(?-u:\b)[0-9]{2}:[0-9]{2}:[0-9]{2}(?:[.,][0-9]+)?",
     )
     .unwrap()
 });

src/commands.rs

@@ -146,13 +146,21 @@ pub fn profile(args: &[String]) -> Option<Profile> {
             if name == "pnpm" && second == "exec" {
                 return known_tool(&args[2..]).then_some(Profile::Tests);
             }
+            // `yarn <bin>` runs a package binary, like `npx`.
+            if name == "yarn" && known_tool(&args[1..]) {
+                return tests;
+            }
             let script = if second == "run" {
                 args.get(2).map(String::as_str).unwrap_or("")
             } else {
                 second
             };
             matches!(script, "test" | "build" | "lint" | "typecheck").then_some(Profile::Tests)
         }
+        // `-y`/`--yes` only skips npx's install prompt.
+        "npx" if matches!(second, "-y" | "--yes") => {
+            known_tool(&args[2..]).then_some(Profile::Tests)
+        }
         "npx" | "bunx" => known_tool(&args[1..]).then_some(Profile::Tests),
         "uv" if second == "run" => known_tool(&args[2..]).then_some(Profile::Tests),
         "bun" if second == "test" => tests,

src/compress.rs

@@ -60,23 +60,26 @@ static SIGNAL: LazyLock<regex::Regex> = LazyLock::new(|| {
 /// matching them loosely turns any line starting with `e ` into a diagnostic.
 /// So do error and exception type names (`TypeError`, `KeyError`,
 /// `NullPointerException`), whose convention is a capitalized identifier.
-/// Word boundaries are ASCII: the vocabulary is ASCII, and Unicode boundaries
-/// cost a slower engine on every line.
+/// Word boundaries, case folding and indentation are ASCII: the vocabulary is
+/// ASCII, Unicode boundaries cost a slower engine on every line, and Unicode
+/// classes and folding cost compiling a larger one on every call (every hook
+/// is a new process).
 static STRONG: LazyLock<regex::Regex> = LazyLock::new(|| {
-    regex::Regex::new(r"(?i:(?-u:\b)(error|errors|failed|failure|panic|panicked|exception|traceback|fatal|assertionerror|caused by|test result|tests? passed|tests? failed)(?-u:\b))|(?-u:\b)[A-Z][A-Za-z]*(Error|Exception)(?-u:\b)|npm ERR!|^\s*(FAIL|FAILED|--- FAIL|E {2,}|FATAL|×|✕|✗|✖)").unwrap()
+    regex::Regex::new(r"(?i-u:\b(error|errors|failed|failure|panic|panicked|exception|traceback|fatal|assertionerror|caused by|test result|tests? passed|tests? failed)\b)|(?-u:\b[A-Z][A-Za-z]*(Error|Exception)\b)|npm ERR!|^(?-u:\s*)(FAIL|FAILED|--- FAIL|E {2,}|FATAL|×|✕|✗|✖)").unwrap()
 });
 /// Advisory lines: shown once per template, never ahead of a diagnostic.
 /// A passing test is not an advisory. Listing it here grouped every `PASS`
 /// line by exact text, which stopped a suite of passes from folding and let
 /// them crowd out the failure's own detail.
 static WEAK: LazyLock<regex::Regex> = LazyLock::new(|| {
-    regex::Regex::new(r"(?i:(?-u:\b)(warning|warn|deprecated|deprecation)(?-u:\b))").unwrap()
+    regex::Regex::new(r"(?i-u:\b(warning|warn|deprecated|deprecation)\b)").unwrap()
 });
 /// Stack frames: context for a diagnostic, wherever they are. JavaScript and
 /// Python frames, Java's `at pkg.Class.method(File.java:N)`, Go's
-/// `file.go:N +0x1d` and rustc's `--> file:line:col` locations.
+/// `file.go:N +0x1d` and rustc's `--> file:line:col` locations. Indentation
+/// and line numbers are ASCII; a Java identifier keeps Unicode `\w`.
 static FRAME: LazyLock<regex::Regex> = LazyLock::new(|| {
-    regex::Regex::new(r#"^\s+at .*\(.*:\d+:\d+\)|^\s+File ".*", line \d+|^\s+at [\w$.<>/-]+\([^()]*\)\s*$|\.go:\d+ \+0x[0-9a-f]+|^\s*--> \S+:\d+:\d+"#).unwrap()
+    regex::Regex::new(r#"^(?-u:\s+)at .*\(.*:[0-9]+:[0-9]+\)|^(?-u:\s+)File ".*", line [0-9]+|^(?-u:\s+)at [\w$.<>/-]+\([^()]*\)(?-u:\s*)$|\.go:[0-9]+ \+0x[0-9a-f]+|^(?-u:\s*)--> \S+:[0-9]+:[0-9]+"#).unwrap()
 });
 /// A file extension at the end of a token: `.rs`, `.py`, `.java`.
 static EXTENSION: LazyLock<regex::Regex> =
@@ -916,9 +919,10 @@ fn text_units_v3<'a>(
         let origin = first[i] as usize;
         let slot = if slot_of_first[origin] != NONE {
             slot_of_first[origin]
-        } else if matches!(role(i), Diff::File | Diff::Change) {
-            // Headers and changes are evidence: they never fold by template,
-            // only with the very same text (a file every commit touches).
+        } else if matches!(role(i), Diff::File | Diff::Change | Diff::Message) {
+            // Headers, changes and commit messages are evidence: they never
+            // fold by template, only with the very same text (a file every
+            // commit touches).
             slot_of_first[i] = groups.len() as u32;
             groups.len() as u32
         } else if strong.get(i) || weak.get(i) {
@@ -976,7 +980,10 @@ fn text_units_v3<'a>(
                 _ if !trivial.get(i) => reach = in_place(i).then_some(0),
                 _ => {
                     reach = reach.map(|r| r + 1);
-                    if reach.is_some_and(|r| r <= GLUE_REACH) {
+                    // A hunk header locates the first change below it, past
+                    // however much context Git shows before that change.
+                    let locates = !order && role(i) == Diff::Hunk;
+                    if reach.is_some_and(|r| locates || r <= GLUE_REACH) {
                         glued.set(i);
                     }
                 }
@@ -1144,7 +1151,14 @@ fn text_units_v3<'a>(
     for (line, &priority) in line_priority.iter().enumerate().rev() {
         while k > 0 && trivial_units[k - 1].2 == line {
             k -= 1;
-            let (unit, ..) = trivial_units[k];
+            let (unit, first_line, _) = trivial_units[k];
+            // A hunk header ranks with the change it locates, at any distance.
+            if role(first_line) == Diff::Hunk
+                && let Some((priority, _)) = carry
+            {
+                units[unit].priority = priority;
+                continue;
+            }
             let after = carry
                 .filter(|&(_, at)| at - line <= GLUE_REACH)
                 .map(|(p, _)| p);

src/main.rs

@@ -351,8 +351,7 @@ fn execute(cli: Cli) -> Result<i32> {
         compress::Version::default()
     };
     let cancel = Arc::new(AtomicBool::new(false));
-    let signal = cancel.clone();
-    ctrlc::set_handler(move || signal.store(true, std::sync::atomic::Ordering::SeqCst))?;
+    scopelet::process::cancel_on_termination(cancel.clone())?;
     if matches!(cli.command, Cmd::Doctor) {
         let mut status = scopelet::adapters::doctor(cancel);
         status["integration"] = integration::doctor();
@@ -726,12 +725,8 @@ fn execute(cli: Cli) -> Result<i32> {
                     context,
                     source.as_deref(),
                 )?;
-                print(&render::render(
-                    &data,
-                    &store,
-                    Mode::Default,
-                    max_bytes,
-                    offset,
+                print(&scopelet::recovery::render(
+                    &data, &store, max_bytes, offset,
                 )?)?;
                 return Ok(if cancel.load(std::sync::atomic::Ordering::SeqCst) {
                     130

src/process.rs

@@ -84,6 +84,49 @@ fn nonblocking(pipe: &impl std::os::fd::AsRawFd) -> Result<()> {
     Ok(())
 }
 
+/// Turn SIGINT, SIGTERM and SIGHUP into `cancel`, replacing whatever the
+/// caller left (an ignored SIGHUP under `nohup`, say), so a wrapped command is
+/// stopped and its status reported (130) instead of this process dying first.
+///
+/// A direct `sigaction` rather than a signal-handling crate: on Apple
+/// platforms that crate links the Foundation framework, whose loading cost
+/// most of a millisecond on every invocation, every hook included.
+#[cfg(unix)]
+pub fn cancel_on_termination(cancel: Arc<AtomicBool>) -> Result<()> {
+    static CANCEL: std::sync::OnceLock<Arc<AtomicBool>> = std::sync::OnceLock::new();
+    extern "C" fn handle(_: libc::c_int) {
+        // An atomic load and an atomic store: async-signal-safe.
+        if let Some(cancel) = CANCEL.get() {
+            cancel.store(true, Ordering::SeqCst);
+        }
+    }
+    ensure!(
+        CANCEL.set(cancel).is_ok(),
+        "termination handler already installed"
+    );
+    for signal in [libc::SIGINT, libc::SIGTERM, libc::SIGHUP] {
+        // SAFETY: a zeroed sigaction is a valid starting value, and the
+        // handler only touches an initialized atomic.
+        let installed = unsafe {
+            let mut action: libc::sigaction = std::mem::zeroed();
+            action.sa_sigaction = handle as extern "C" fn(libc::c_int) as libc::sighandler_t;
+            action.sa_flags = libc::SA_RESTART;
+            libc::sigemptyset(&mut action.sa_mask);
+            libc::sigaction(signal, &action, std::ptr::null_mut())
+        };
+        if installed != 0 {
+            return Err(std::io::Error::last_os_error()).context("install termination handler");
+        }
+    }
+    Ok(())
+}
+
+#[cfg(not(unix))]
+pub fn cancel_on_termination(cancel: Arc<AtomicBool>) -> Result<()> {
+    ctrlc::set_handler(move || cancel.store(true, Ordering::SeqCst))?;
+    Ok(())
+}
+
 #[cfg(unix)]
 fn signal_group(pid: u32, signal: i32) {
     // The child starts a dedicated group; this never targets the caller's group.

src/recovery.rs

@@ -95,6 +95,37 @@ pub fn search(
     Ok(data)
 }
 
+/// Store a recovery result and render its page at `offset`, as
+/// `render::render` does. A result whose only record is one whole stored text
+/// original (a single-line blob paged or searched shows its only line) names
+/// that blob, as a compact-v3 artifact does (schema 2), instead of storing the
+/// original a second time for every recovery.
+pub fn render(data: &Dataset, store: &Store, budget: usize, offset: usize) -> Result<View> {
+    if let [record] = data.records.as_slice()
+        && let Some(blob) = record.blob.as_deref()
+        // A scanned file whose original was not kept is searched locally:
+        // its blob is named but absent, so the result keeps its own copy.
+        && blob.starts_with("blob:")
+        && store.contains(blob)
+        // Hashing settles that the text is the whole blob, not a prefix...
+        && blob.ends_with(&digest(record.text.as_bytes()))
+        // ...and reading it back as text must give exactly this record.
+        && crate::sources::Parsed::Text(record.text.clone()).records(&record.source, blob.into())
+            == data.records
+    {
+        let artifact = store.put_json(&DatasetRef::new(
+            data,
+            RecordsRef {
+                blob: blob.into(),
+                format: Format::Text,
+                source: record.source.clone(),
+            },
+        ))?;
+        return crate::render::render_stored(data, artifact, Mode::Default, budget, offset);
+    }
+    crate::render::render(data, store, Mode::Default, budget, offset)
+}
+
 /// Lines `start..=end` of a blob, or as many of them as fit `budget`: a view
 /// that cannot show them all shows the first ones and sets `next_start`.
 /// Only a single line larger than the budget is still a blocked record.
@@ -119,7 +150,7 @@ pub fn page(store: &Store, id: &str, start: usize, end: usize, budget: usize) ->
         (serde_json::to_vec(&view).ok()?.len() < budget).then_some(view)
     };
     if data.records.is_empty() || measure(&data, None).is_some() {
-        return crate::render::render(&data, store, Mode::Default, budget, 0);
+        return render(&data, store, budget, 0);
     }
     let record = &data.records[0];
     let lines: Vec<&str> = record.text.split_inclusive('\n').collect();
@@ -144,9 +175,9 @@ pub fn page(store: &Store, id: &str, start: usize, end: usize, budget: usize) ->
     }
     if fits == 0 {
         // A single line larger than the budget: report it as blocked.
-        return crate::render::render(&data, store, Mode::Default, budget, 0);
+        return render(&data, store, budget, 0);
     }
-    let mut view = crate::render::render(&take(fits), store, Mode::Default, budget, 0)?;
+    let mut view = render(&take(fits), store, budget, 0)?;
     view.next_start = Some(first + fits);
     view.stop_early(NOTE);
     Ok(view)