diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 856aafb5d4..8c52a46a97 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -107,7 +107,7 @@ actual backing: | **findutils** (`find`,`xargs`) | TODO | our hand-rolled on `regex`/shims | real GNU findutils, or `uutils/findutils` | | **tree** | DONE | our hand-rolled, zero deps | real `tree`, or an established one | | **grep** | DONE | our `secureexec-grep` on raw `regex` (**not** an established grep pkg) | real **GNU grep** | -| **ripgrep** (`rg`) | TODO | our `secureexec-grep` recursive search shim, not real ripgrep | real upstream **ripgrep** | +| **ripgrep** (`rg`) | DONE | our `secureexec-grep` recursive search shim, not real ripgrep | real upstream **ripgrep** | | **zip** | DONE | our 203-line `zip.c` over zlib/minizip (not Info-ZIP) | real Info-ZIP, or an established lib's CLI | | **unzip** | DONE | our 669-line `unzip.c` over zlib/minizip | real Info-ZIP unzip | | **sqlite3 CLI** | DONE | our 558-line `sqlite3_cli.c` (engine is real SQLite; the shell is ours) | real SQLite `shell.c` (its official CLI) | @@ -267,8 +267,20 @@ works (`wasi-spawn` broker), so `xargs` is not a blocker. and `2026-07-08T06-42-18-0700-grep-check-types-final.log`; e2e grep tests pass 8/8 in `2026-07-08T06-43-59-0700-grep-vitest-after-wrapper-cache-repair.log`. Rev: `uyukolvr` — `fix(grep): build upstream GNU grep`. - NB: `secureexec-grep` remains only because `@agentos-software/ripgrep` still - uses it for `rg`; replace `rg` with real ripgrep in a separate rev. +- **ripgrep — DONE.** Replaced the `@agentos-software/ripgrep` package's custom + `secureexec-grep` recursive search shim with upstream ripgrep 15.1.0 from the + canonical `BurntSushi/ripgrep` crate/release. The local `cmd-rg` crate is now + only a build trigger, and `toolchain/Makefile` builds ripgrep's own `rg` bin + directly. The old `secureexec-grep` library is gone. Proof: latest upstream + release captured in `2026-07-08T06-50-40-0700-ripgrep-github-latest-release.json`; + crate metadata captured in `2026-07-08T06-52-00-0700-ripgrep-cargo-info.log`; + upstream WASM build passes in + `2026-07-08T06-56-00-0700-ripgrep-upstream-wasm-build-after-grep-dir-remove-fixed.log`; + package build and check-types pass in + `2026-07-08T06-58-55-0700-ripgrep-package-build-after-install.log` and + `2026-07-08T06-58-56-0700-ripgrep-check-types-after-install.log`; e2e ripgrep + tests pass 8/8 in `2026-07-08T07-02-00-0700-ripgrep-vitest-after-git-fixture.log`. + Rev: `msypkqmo` — `fix(ripgrep): build upstream ripgrep`. - **zip / unzip — moderate.** Real **Info-ZIP** source (fetch+pin like zlib/sqlite); zlib is already vendored. Filesystem + `isatty`/`utime`/`chmod`/perms stubs; no sockets/threads/spawn. Friction is Info-ZIP's crufty build, not syscalls. @@ -472,12 +484,12 @@ real e2e tests that prove Linux-parity behavior — not smoke tests. - **Proof:** each un-skipped suite passes with real behavior. - **rev:** one per command, e.g. `test(duckdb): real analytical-SQL e2e; un-skip` -### 12. No tests at all — 12 software + 5 agents -- **Broken:** zero e2e coverage: `gawk, sed, grep, tar, gzip, jq, ripgrep, yq, - diffutils, file, vim`; agents `claude, codex, opencode, pi, pi-cli`. +### 12. No tests at all — 9 software + 5 agents +- **Broken:** zero e2e coverage: `gawk, sed, tar, gzip, jq, yq, diffutils, + file, vim`; agents `claude, codex, opencode, pi, pi-cli`. - **Objective:** write real e2e tests proving each behaves like its Linux - counterpart (jq processes real JSON, sed edits streams, tar round-trips archives, - grep/rg search real trees, gzip round-trips, etc.); agents exercise the real ACP + counterpart (jq processes real JSON, sed edits streams, tar round-trips archives, + gzip round-trips, etc.); agents exercise the real ACP adapter against the upstream SDK. - **Proof:** `software//test/` exists and passes for each; coverage gate green. - **rev:** one per package, e.g. `test(jq): add real JSON-processing e2e` diff --git a/packages/runtime-core/commands/rg b/packages/runtime-core/commands/rg index 76a63b75ea..d3d4306565 100644 Binary files a/packages/runtime-core/commands/rg and b/packages/runtime-core/commands/rg differ diff --git a/software/ripgrep/bin/rg b/software/ripgrep/bin/rg index 76a63b75ea..d3d4306565 100644 Binary files a/software/ripgrep/bin/rg and b/software/ripgrep/bin/rg differ diff --git a/software/ripgrep/native/crates/cmd-rg/Cargo.toml b/software/ripgrep/native/crates/cmd-rg/Cargo.toml index f702453e13..f6184892c7 100644 --- a/software/ripgrep/native/crates/cmd-rg/Cargo.toml +++ b/software/ripgrep/native/crates/cmd-rg/Cargo.toml @@ -5,10 +5,10 @@ version.workspace = true edition.workspace = true license.workspace = true description = "rg (ripgrep) standalone binary for secure-exec VM" - -[[bin]] -name = "rg" -path = "src/main.rs" +publish = false [dependencies] -secureexec-grep = { path = "../../../../../toolchain/crates/libs/grep" } +# ripgrep is a bin-only crate. `toolchain/Makefile` builds its upstream `rg` +# binary directly; this local package exists only so `cmd/rg` is discoverable +# and the upstream package is present in Cargo's resolved/vendor set. +ripgrep = { version = "15.1.0", default-features = false } diff --git a/software/ripgrep/native/crates/cmd-rg/src/lib.rs b/software/ripgrep/native/crates/cmd-rg/src/lib.rs new file mode 100644 index 0000000000..1bd8717252 --- /dev/null +++ b/software/ripgrep/native/crates/cmd-rg/src/lib.rs @@ -0,0 +1 @@ +//! Build trigger for the upstream `ripgrep` command package. diff --git a/software/ripgrep/native/crates/cmd-rg/src/main.rs b/software/ripgrep/native/crates/cmd-rg/src/main.rs deleted file mode 100644 index e44c45f021..0000000000 --- a/software/ripgrep/native/crates/cmd-rg/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_grep::rg(args)); -} diff --git a/software/ripgrep/test/ripgrep.test.ts b/software/ripgrep/test/ripgrep.test.ts new file mode 100644 index 0000000000..af8d77885e --- /dev/null +++ b/software/ripgrep/test/ripgrep.test.ts @@ -0,0 +1,128 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { createWasmVmRuntime } from "@agentos/test-harness"; +import { + COMMANDS_DIR, + NodeFileSystem, + createKernel, + describeIf, + hasWasmBinaries, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, describe, expect, it } from "vitest"; + +let tempRoot: string | undefined; + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-ripgrep-")); + + await writeFixture( + "/project/src/main.rs", + ["fn main() {", ' println!("needle");', "}"].join("\n") + "\n", + ); + await writeFixture( + "/project/src/lib.rs", + ["pub fn helper() {", " // Needle in a comment", "}"].join("\n") + "\n", + ); + await writeFixture("/project/docs/readme.md", "needle in docs\n"); + await writeFixture("/project/vendor/generated.rs", "needle in vendor\n"); + await writeFixture("/project/.hidden.txt", "needle hidden\n"); + await writeFixture("/project/.gitignore", "vendor/\n"); + await writeFixture("/project/.git/HEAD", "ref: refs/heads/main\n"); + + return new NodeFileSystem({ root: tempRoot }); +} + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +function lines(stdout: string): string[] { + return stdout + .split("\n") + .filter((line) => line.length > 0) + .sort(); +} + +describeIf(hasWasmBinaries, "ripgrep command", { timeout: 10_000 }, () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + } + + it("reports the upstream ripgrep version", async () => { + await mountFixture(); + + const result = await kernel.exec("rg --version", {}); + expect(result.stdout).toContain("ripgrep 15.1.0"); + expect(result.stdout).not.toContain("secure-exec"); + }); + + it("searches recursively and respects .gitignore by default", async () => { + await mountFixture(); + + const result = await kernel.exec("rg needle /project", {}); + const output = lines(result.stdout); + + expect(output).toContain('/project/src/main.rs: println!("needle");'); + expect(output).toContain("/project/docs/readme.md:needle in docs"); + expect(output).not.toContain("/project/vendor/generated.rs:needle in vendor"); + expect(output).not.toContain("/project/.hidden.txt:needle hidden"); + }); + + it("supports case-insensitive search", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -i needle /project/src", {}); + expect(lines(result.stdout)).toContain("/project/src/lib.rs: // Needle in a comment"); + }); + + it("supports fixed-string search", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -F 'println!(\"needle\")' /project/src", {}); + expect(result.stdout.trim()).toBe('/project/src/main.rs: println!("needle");'); + }); + + it("supports glob filtering", async () => { + await mountFixture(); + + const result = await kernel.exec("rg needle /project -g '*.md'", {}); + expect(result.stdout.trim()).toBe("/project/docs/readme.md:needle in docs"); + }); + + it("can include hidden and ignored files when requested", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -uu needle /project", {}); + const output = lines(result.stdout); + + expect(output).toContain("/project/.hidden.txt:needle hidden"); + expect(output).toContain("/project/vendor/generated.rs:needle in vendor"); + }); + + it("emits JSON search records", async () => { + await mountFixture(); + + const result = await kernel.exec("rg --json needle /project/docs", {}); + const records = lines(result.stdout).map((line) => JSON.parse(line)); + + expect(records.some((record) => record.type === "match")).toBe(true); + expect(records.some((record) => record.type === "summary")).toBe(true); + }); +}); diff --git a/toolchain/Cargo.lock b/toolchain/Cargo.lock index a6ec2c0a2b..4c6e2c57d6 100644 --- a/toolchain/Cargo.lock +++ b/toolchain/Cargo.lock @@ -1131,7 +1131,7 @@ dependencies = [ name = "cmd-rg" version = "0.1.0" dependencies = [ - "secureexec-grep", + "ripgrep", ] [[package]] @@ -1857,6 +1857,24 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" +dependencies = [ + "encoding_rs", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2268,6 +2286,85 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "grep" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "309217bc53e2c691c314389c7fa91f9cd1a998cda19e25544ea47d94103880c3" +dependencies = [ + "grep-cli", + "grep-matcher", + "grep-printer", + "grep-regex", + "grep-searcher", +] + +[[package]] +name = "grep-cli" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf32d263c5d5cc2a23ce587097f5ddafdb188492ba2e6fb638eaccdc22453631" +dependencies = [ + "bstr", + "globset", + "libc", + "log", + "termcolor", + "winapi-util", +] + +[[package]] +name = "grep-matcher" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36d7b71093325ab22d780b40d7df3066ae4aebb518ba719d38c697a8228a8023" +dependencies = [ + "memchr", +] + +[[package]] +name = "grep-printer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd76035e87871f51c1ee5b793e32122b3ccf9c692662d9622ef1686ff5321acb" +dependencies = [ + "bstr", + "grep-matcher", + "grep-searcher", + "log", + "serde", + "serde_json", + "termcolor", +] + +[[package]] +name = "grep-regex" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef" +dependencies = [ + "bstr", + "grep-matcher", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep-searcher" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac63295322dc48ebb20a25348147905d816318888e64f531bfc2a2bc0577dc34" +dependencies = [ + "bstr", + "encoding_rs", + "encoding_rs_io", + "grep-matcher", + "log", + "memchr", + "memmap2", +] + [[package]] name = "half" version = "2.7.1" @@ -2947,6 +3044,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexopt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" + [[package]] name = "libc" version = "0.2.186" @@ -3776,6 +3879,24 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "ripgrep" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f388c4955f85477c28a8667355819844a06614b083c23517f0e86bd1d6d82b73" +dependencies = [ + "anyhow", + "bstr", + "grep", + "ignore", + "lexopt", + "log", + "serde_json", + "termcolor", + "textwrap", + "tikv-jemallocator", +] + [[package]] name = "rlimit" version = "0.10.2" @@ -3928,13 +4049,6 @@ dependencies = [ "sha1", ] -[[package]] -name = "secureexec-grep" -version = "0.1.0" -dependencies = [ - "regex", -] - [[package]] name = "secureexec-gzip" version = "0.1.0" @@ -4385,6 +4499,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.4.3" @@ -4407,6 +4530,12 @@ dependencies = [ "phf_codegen 0.11.3", ] +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + [[package]] name = "thiserror" version = "1.0.69" @@ -4456,6 +4585,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tinystr" version = "0.8.2" diff --git a/toolchain/Makefile b/toolchain/Makefile index 309fc0b273..1654bfa784 100644 --- a/toolchain/Makefile +++ b/toolchain/Makefile @@ -242,6 +242,9 @@ wasm-cmd: vendor patch-vendor patch-std wasm-opt-check if [ "$(CMD)" = "fd" ]; then \ cargo_pkg="fd-find"; \ cargo_bin="fd"; \ + elif [ "$(CMD)" = "rg" ]; then \ + cargo_pkg="ripgrep"; \ + cargo_bin="rg"; \ fi; \ RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ cargo build --target $(WASM_TARGET) \ diff --git a/toolchain/crates/libs/grep/Cargo.toml b/toolchain/crates/libs/grep/Cargo.toml deleted file mode 100644 index 3cf619a9c4..0000000000 --- a/toolchain/crates/libs/grep/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-grep" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "grep/egrep/fgrep/rg implementations for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/toolchain/crates/libs/grep/src/lib.rs b/toolchain/crates/libs/grep/src/lib.rs deleted file mode 100644 index c42db0e8ac..0000000000 --- a/toolchain/crates/libs/grep/src/lib.rs +++ /dev/null @@ -1,623 +0,0 @@ -//! grep implementation using the regex crate (ripgrep's pure Rust regex engine). -//! -//! Supports grep, egrep (grep -E), and fgrep (grep -F) modes. -//! Dispatches on argv[0] basename for standalone binary usage. - -mod rg_cmd; - -use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; -use std::path::Path; - -use regex::Regex; - -const MAX_INPUT_LINE_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERN_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERNS: usize = 100_000; - -/// Unified grep entry point. Dispatches on argv[0]: -/// - "egrep" -> Extended mode -/// - "fgrep" -> Fixed mode -/// - default -> Basic mode -pub fn main(args: Vec) -> i32 { - let mode = match args.first().and_then(|a| Path::new(a).file_name()) { - Some(name) if name == "egrep" => GrepMode::Extended, - Some(name) if name == "fgrep" => GrepMode::Fixed, - _ => GrepMode::Basic, - }; - run_grep(args, mode) -} - -/// Entry point for grep command (Basic mode). -pub fn grep(args: Vec) -> i32 { - run_grep(args, GrepMode::Basic) -} - -/// Entry point for egrep command (Extended regex). -pub fn egrep(args: Vec) -> i32 { - run_grep(args, GrepMode::Extended) -} - -/// Entry point for fgrep command (Fixed strings). -pub fn fgrep(args: Vec) -> i32 { - run_grep(args, GrepMode::Fixed) -} - -/// Entry point for rg command. -pub fn rg(args: Vec) -> i32 { - rg_cmd::rg(args) -} - -/// grep mode determines how patterns are interpreted. -#[derive(Clone, Copy, PartialEq)] -enum GrepMode { - /// Basic regular expressions (default grep) - Basic, - /// Extended regular expressions (egrep / grep -E) - Extended, - /// Fixed strings (fgrep / grep -F) - Fixed, -} - -struct GrepOptions { - mode: GrepMode, - ignore_case: bool, - invert_match: bool, - count_only: bool, - files_with_matches: bool, - files_without_matches: bool, - line_numbers: bool, - word_regexp: bool, - line_regexp: bool, - max_count: Option, - quiet: bool, - patterns: Vec, - pattern_bytes: usize, - files: Vec, -} - -impl GrepOptions { - fn new(mode: GrepMode) -> Self { - Self { - mode, - ignore_case: false, - invert_match: false, - count_only: false, - files_with_matches: false, - files_without_matches: false, - line_numbers: false, - word_regexp: false, - line_regexp: false, - max_count: None, - quiet: false, - patterns: Vec::new(), - pattern_bytes: 0, - files: Vec::new(), - } - } -} - -fn run_grep(args: Vec, default_mode: GrepMode) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - let opts = match parse_args(&str_args, default_mode) { - Ok(opts) => opts, - Err(msg) => { - eprintln!("grep: {}", msg); - return 2; - } - }; - - if opts.patterns.is_empty() { - eprintln!("grep: no pattern specified"); - return 2; - } - - let regex = match build_regex(&opts) { - Ok(r) => r, - Err(msg) => { - eprintln!("grep: {}", msg); - return 2; - } - }; - - let multiple_files = opts.files.len() > 1; - let mut any_match = false; - let mut had_error = false; - - if opts.files.is_empty() { - // Read from stdin - let stdin = io::stdin(); - let reader = stdin.lock(); - match search_reader(reader, None, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}", e); - had_error = true; - } - } - } else { - for file in &opts.files { - if file == "-" { - let stdin = io::stdin(); - let reader = stdin.lock(); - let label = if multiple_files { - Some("(standard input)") - } else { - None - }; - match search_reader(reader, label, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } else { - match std::fs::File::open(file) { - Ok(f) => { - let reader = io::BufReader::new(f); - let label = if multiple_files { - Some(file.as_str()) - } else { - None - }; - match search_reader(reader, label, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } - } - } - - if had_error { - 2 - } else if any_match { - 0 - } else { - 1 - } -} - -fn parse_args(args: &[String], default_mode: GrepMode) -> Result { - let mut opts = GrepOptions::new(default_mode); - let mut i = 0; - let mut pattern_from_args = false; - - while i < args.len() { - let arg = &args[i]; - - if arg == "--" { - i += 1; - // Remaining args are files (or first is pattern if none yet) - break; - } - - if arg.starts_with('-') && arg.len() > 1 && !arg.starts_with("--") { - let chars: Vec = arg[1..].chars().collect(); - let mut j = 0; - while j < chars.len() { - match chars[j] { - 'E' => opts.mode = GrepMode::Extended, - 'F' => opts.mode = GrepMode::Fixed, - 'G' => opts.mode = GrepMode::Basic, - 'i' | 'y' => opts.ignore_case = true, - 'v' => opts.invert_match = true, - 'c' => opts.count_only = true, - 'l' => opts.files_with_matches = true, - 'L' => opts.files_without_matches = true, - 'n' => opts.line_numbers = true, - 'w' => opts.word_regexp = true, - 'x' => opts.line_regexp = true, - 'q' | 's' => opts.quiet = true, - 'h' => {} // suppress filename (handled by multiple_files logic) - 'H' => {} // force filename - 'e' => { - // -e PATTERN (rest of this flag group or next arg) - let rest: String = chars[j + 1..].iter().collect(); - if !rest.is_empty() { - push_pattern(&mut opts, rest)?; - pattern_from_args = true; - j = chars.len(); // consumed rest - continue; - } else { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'e'".to_string()); - } - push_pattern(&mut opts, args[i].clone())?; - pattern_from_args = true; - j = chars.len(); - continue; - } - } - 'f' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'f'".to_string()); - } - read_patterns_from_file(&mut opts, &args[i])?; - pattern_from_args = true; - j = chars.len(); - continue; - } - 'm' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'm'".to_string()); - } - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid max count '{}'", args[i]))?, - ); - j = chars.len(); - continue; - } - _ => { - return Err(format!("invalid option -- '{}'", chars[j])); - } - } - j += 1; - } - } else if arg.starts_with("--") { - match arg.as_str() { - "--extended-regexp" => opts.mode = GrepMode::Extended, - "--fixed-strings" => opts.mode = GrepMode::Fixed, - "--basic-regexp" => opts.mode = GrepMode::Basic, - "--ignore-case" => opts.ignore_case = true, - "--invert-match" => opts.invert_match = true, - "--count" => opts.count_only = true, - "--files-with-matches" => opts.files_with_matches = true, - "--files-without-match" => opts.files_without_matches = true, - "--line-number" => opts.line_numbers = true, - "--word-regexp" => opts.word_regexp = true, - "--line-regexp" => opts.line_regexp = true, - "--quiet" | "--silent" => opts.quiet = true, - _ if arg.starts_with("--regexp=") => { - push_pattern(&mut opts, arg[9..].to_string())?; - pattern_from_args = true; - } - _ if arg.starts_with("--max-count=") => { - opts.max_count = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid max count '{}'", &arg[12..]))?, - ); - } - _ => { - return Err(format!("unrecognized option '{}'", arg)); - } - } - } else { - // Positional argument: first is pattern (if no -e), rest are files - if !pattern_from_args && opts.patterns.is_empty() { - push_pattern(&mut opts, arg.clone())?; - pattern_from_args = true; - } else { - opts.files.push(arg.clone()); - } - } - i += 1; - } - - // Remaining args after -- - while i < args.len() { - if !pattern_from_args && opts.patterns.is_empty() { - push_pattern(&mut opts, args[i].clone())?; - pattern_from_args = true; - } else { - opts.files.push(args[i].clone()); - } - i += 1; - } - - Ok(opts) -} - -fn push_pattern(opts: &mut GrepOptions, pattern: String) -> Result<(), String> { - if opts.patterns.len() >= MAX_PATTERNS { - return Err("too many patterns".to_string()); - } - let next_bytes = opts - .pattern_bytes - .checked_add(pattern.len()) - .ok_or_else(|| "pattern data too large".to_string())?; - if next_bytes > MAX_PATTERN_BYTES { - return Err("pattern data exceeds size limit".to_string()); - } - opts.pattern_bytes = next_bytes; - opts.patterns.push(pattern); - Ok(()) -} - -fn read_patterns_from_file(opts: &mut GrepOptions, path: &str) -> Result<(), String> { - let metadata = std::fs::metadata(path).map_err(|e| format!("{}: {}", path, e))?; - if metadata.len() > MAX_PATTERN_BYTES as u64 { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - let file = std::fs::File::open(path).map_err(|e| format!("{}: {}", path, e))?; - let limit = MAX_PATTERN_BYTES - .checked_add(1) - .ok_or_else(|| "pattern file size limit is too large".to_string())?; - let mut content = String::new(); - file.take(limit as u64) - .read_to_string(&mut content) - .map_err(|e| format!("{}: {}", path, e))?; - if content.len() > MAX_PATTERN_BYTES { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - for line in content.lines() { - if !line.is_empty() { - push_pattern(opts, line.to_string())?; - } - } - Ok(()) -} - -/// Build a compiled regex from the grep options. -fn build_regex(opts: &GrepOptions) -> Result { - let pattern = if opts.patterns.len() == 1 { - build_single_pattern(&opts.patterns[0], opts) - } else { - // Multiple patterns: combine with alternation - let parts: Vec = opts - .patterns - .iter() - .map(|p| format!("(?:{})", build_single_pattern(p, opts))) - .collect(); - parts.join("|") - }; - - let mut builder = regex::RegexBuilder::new(&pattern); - builder.case_insensitive(opts.ignore_case); - - builder - .build() - .map_err(|e| format!("invalid pattern: {}", e)) -} - -/// Convert a single pattern string to a regex pattern based on mode. -fn build_single_pattern(pattern: &str, opts: &GrepOptions) -> String { - let base = match opts.mode { - GrepMode::Fixed => regex::escape(pattern), - GrepMode::Basic => convert_bre_to_ere(pattern), - GrepMode::Extended => pattern.to_string(), - }; - - let wrapped = if opts.word_regexp { - format!(r"\b(?:{})\b", base) - } else if opts.line_regexp { - format!("^(?:{})$", base) - } else { - base - }; - - wrapped -} - -/// Convert POSIX Basic Regular Expression to Extended (Rust regex syntax). -/// In BRE: \(, \), \{, \}, \+, \?, \| are special; unescaped versions are literal. -/// In ERE (and Rust regex): (, ), {, }, +, ?, | are special without backslash. -fn convert_bre_to_ere(bre: &str) -> String { - let mut result = String::with_capacity(bre.len()); - let chars: Vec = bre.chars().collect(); - let mut i = 0; - - while i < chars.len() { - if chars[i] == '\\' && i + 1 < chars.len() { - match chars[i + 1] { - '(' => { - result.push('('); - i += 2; - } - ')' => { - result.push(')'); - i += 2; - } - '{' => { - result.push('{'); - i += 2; - } - '}' => { - result.push('}'); - i += 2; - } - '+' => { - result.push('+'); - i += 2; - } - '?' => { - result.push('?'); - i += 2; - } - '|' => { - result.push('|'); - i += 2; - } - '1'..='9' => { - // Backreference - not supported in Rust regex, pass through - result.push('\\'); - result.push(chars[i + 1]); - i += 2; - } - _ => { - result.push('\\'); - result.push(chars[i + 1]); - i += 2; - } - } - } else { - match chars[i] { - // In BRE, unescaped (, ), {, }, +, ? are literal - '(' => { - result.push_str("\\("); - i += 1; - } - ')' => { - result.push_str("\\)"); - i += 1; - } - '{' => { - result.push_str("\\{"); - i += 1; - } - '}' => { - result.push_str("\\}"); - i += 1; - } - _ => { - result.push(chars[i]); - i += 1; - } - } - } - } - - result -} - -/// Search a reader for matching lines. Returns true if any match was found. -fn search_reader( - reader: R, - filename: Option<&str>, - regex: &Regex, - opts: &GrepOptions, - show_filename: bool, -) -> io::Result { - let mut buf_reader = io::BufReader::new(reader); - let mut match_count: usize = 0; - let mut line_num: usize = 0; - let stdout = io::stdout(); - let mut out = stdout.lock(); - let mut line_buf = Vec::new(); - - while let Some(line) = read_line_bounded(&mut buf_reader, &mut line_buf)? { - line_num += 1; - - let is_match = regex.is_match(&line); - let is_match = if opts.invert_match { - !is_match - } else { - is_match - }; - - if is_match { - match_count += 1; - - if opts.quiet { - return Ok(true); - } - - if opts.files_with_matches { - if let Some(name) = filename { - writeln!(out, "{}", name)?; - } else { - writeln!(out, "(standard input)")?; - } - out.flush()?; - return Ok(true); - } - - if !opts.count_only && !opts.files_without_matches { - let prefix = match (show_filename, filename, opts.line_numbers) { - (true, Some(name), true) => format!("{}:{}:", name, line_num), - (true, Some(name), false) => format!("{}:", name), - (_, _, true) => format!("{}:", line_num), - _ => String::new(), - }; - writeln!(out, "{}{}", prefix, line)?; - } - - if let Some(max) = opts.max_count { - if match_count >= max { - break; - } - } - } - } - - if opts.count_only && !opts.quiet { - if show_filename { - if let Some(name) = filename { - writeln!(out, "{}:{}", name, match_count)?; - } else { - writeln!(out, "{}", match_count)?; - } - } else { - writeln!(out, "{}", match_count)?; - } - } - - if opts.files_without_matches && match_count == 0 { - if let Some(name) = filename { - writeln!(out, "{}", name)?; - } else { - writeln!(out, "(standard input)")?; - } - } - - out.flush()?; - - Ok(match_count > 0) -} - -fn read_line_bounded( - reader: &mut R, - line_buf: &mut Vec, -) -> io::Result> { - line_buf.clear(); - - loop { - let available = reader.fill_buf()?; - if available.is_empty() { - if line_buf.is_empty() { - return Ok(None); - } - break; - } - - let newline = available.iter().position(|&b| b == b'\n'); - let take = newline.map_or(available.len(), |pos| pos + 1); - let next_len = line_buf - .len() - .checked_add(take) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "input line too long"))?; - if next_len > MAX_INPUT_LINE_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "input line exceeds size limit", - )); - } - - line_buf.extend_from_slice(&available[..take]); - reader.consume(take); - if newline.is_some() { - break; - } - } - - if line_buf.ends_with(b"\n") { - line_buf.pop(); - if line_buf.ends_with(b"\r") { - line_buf.pop(); - } - } - - String::from_utf8(line_buf.clone()) - .map(Some) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) -} diff --git a/toolchain/crates/libs/grep/src/rg_cmd.rs b/toolchain/crates/libs/grep/src/rg_cmd.rs deleted file mode 100644 index 9449f3c678..0000000000 --- a/toolchain/crates/libs/grep/src/rg_cmd.rs +++ /dev/null @@ -1,1142 +0,0 @@ -//! rg (ripgrep) implementation using the regex crate (ripgrep's pure Rust regex engine). -//! -//! Provides ripgrep-compatible search. Uses the same regex engine as ripgrep. -//! POSIX grep/egrep/fgrep remain in lib.rs for BRE/ERE/fixed string compatibility. - -use std::collections::{HashSet, VecDeque}; -use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; -use std::path::{Path, PathBuf}; - -use regex::{Regex, RegexBuilder}; - -const MAX_CONTEXT_LINES: usize = 100_000; -const MAX_CONTEXT_BYTES: usize = 16 * 1024 * 1024; -const MAX_FILE_RESULTS: usize = 1_000_000; -const MAX_INPUT_LINE_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERN_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERNS: usize = 100_000; - -/// Entry point for rg command. -pub fn rg(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run(&str_args) { - Ok(code) => code, - Err(msg) => { - eprintln!("rg: {}", msg); - 2 - } - } -} - -struct Options { - patterns: Vec, - paths: Vec, - files_mode: bool, - ignore_case: bool, - smart_case: bool, - invert_match: bool, - count_only: bool, - files_with_matches: bool, - files_without_matches: bool, - line_numbers: Option, - word_regexp: bool, - line_regexp: bool, - fixed_strings: bool, - max_count: Option, - quiet: bool, - only_matching: bool, - after_context: usize, - before_context: usize, - show_filename: Option, - hidden: bool, - max_depth: Option, - sort_modified: bool, - glob_patterns: Vec, - pattern_bytes: usize, - type_include: Vec, - type_exclude: Vec, -} - -impl Options { - fn new() -> Self { - Self { - patterns: Vec::new(), - paths: Vec::new(), - files_mode: false, - ignore_case: false, - smart_case: true, - invert_match: false, - count_only: false, - files_with_matches: false, - files_without_matches: false, - line_numbers: None, - word_regexp: false, - line_regexp: false, - fixed_strings: false, - max_count: None, - quiet: false, - only_matching: false, - after_context: 0, - before_context: 0, - show_filename: None, - hidden: false, - max_depth: None, - sort_modified: false, - glob_patterns: Vec::new(), - pattern_bytes: 0, - type_include: Vec::new(), - type_exclude: Vec::new(), - } - } - - fn show_line_numbers(&self) -> bool { - self.line_numbers.unwrap_or(true) - } - - fn resolve_show_filename(&self, multi: bool) -> bool { - self.show_filename.unwrap_or(multi) - } - - fn has_context(&self) -> bool { - self.before_context > 0 || self.after_context > 0 - } -} - -fn run(args: &[String]) -> Result { - if args.len() == 1 && (args[0] == "--version" || args[0] == "-V") { - let stdout = io::stdout(); - let mut out = stdout.lock(); - writeln!(out, "ripgrep 14.1.0 (secure-exec)").map_err(|e| e.to_string())?; - out.flush().map_err(|e| e.to_string())?; - return Ok(0); - } - - let opts = parse_args(args)?; - - if opts.files_mode { - let paths = if opts.paths.is_empty() { - vec![".".to_string()] - } else { - opts.paths.clone() - }; - - let files = collect_files_from_paths(&paths, &opts).map_err(|e| e.to_string())?; - let stdout = io::stdout(); - let mut out = stdout.lock(); - for path in files { - writeln!(out, "{}", path.to_string_lossy()).map_err(|e| e.to_string())?; - } - out.flush().map_err(|e| e.to_string())?; - return Ok(0); - } - - if opts.patterns.is_empty() { - return Err("no pattern provided".to_string()); - } - - let regex = build_regex(&opts)?; - - if opts.paths.is_empty() { - // No paths: read from stdin - let stdin = io::stdin(); - let stdout = io::stdout(); - let mut out = stdout.lock(); - let result = search_stream(stdin.lock(), ®ex, &opts, None, false, &mut out) - .map_err(|e| e.to_string())?; - if opts.quiet { - return Ok(if result.matches > 0 { 0 } else { 1 }); - } - print_file_result(None, &result, &opts, &mut out).map_err(|e| e.to_string())?; - out.flush().map_err(|e| e.to_string())?; - return Ok(if result.matches > 0 { 0 } else { 1 }); - } - - let files = collect_files_from_paths(&opts.paths, &opts).map_err(|e| e.to_string())?; - let multi = files.len() > 1; - let show_fn = opts.resolve_show_filename(multi); - let mut any_match = false; - let mut had_error = false; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - for path in &files { - match std::fs::File::open(path) { - Ok(f) => { - let reader = io::BufReader::new(f); - let fname = if show_fn { - Some(path.to_string_lossy().to_string()) - } else { - None - }; - let result = - match search_stream(reader, ®ex, &opts, fname.as_deref(), show_fn, &mut out) - { - Ok(result) => result, - Err(e) => { - eprintln!("rg: {}: {}", path.display(), e); - had_error = true; - continue; - } - }; - if result.matches > 0 { - any_match = true; - } - if opts.quiet && any_match { - return Ok(0); - } - if !opts.quiet { - print_file_result(fname.as_deref(), &result, &opts, &mut out) - .map_err(|e| e.to_string())?; - } - } - Err(e) => { - eprintln!("rg: {}: {}", path.display(), e); - had_error = true; - } - } - } - out.flush().map_err(|e| e.to_string())?; - - if had_error { - Ok(2) - } else if any_match { - Ok(0) - } else { - Ok(1) - } -} - -// --- Argument parsing --- - -fn parse_args(args: &[String]) -> Result { - let mut opts = Options::new(); - let mut i = 0; - let mut explicit_pattern = false; - - while i < args.len() { - let arg = &args[i]; - - if arg == "--" { - i += 1; - break; - } - - // Long options - if arg.starts_with("--") { - match arg.as_str() { - "--ignore-case" => opts.ignore_case = true, - "--case-sensitive" => { - opts.ignore_case = false; - opts.smart_case = false; - } - "--smart-case" => opts.smart_case = true, - "--invert-match" => opts.invert_match = true, - "--count" => opts.count_only = true, - "--files" => opts.files_mode = true, - "--files-with-matches" => opts.files_with_matches = true, - "--files-without-match" => opts.files_without_matches = true, - "--line-number" => opts.line_numbers = Some(true), - "--no-line-number" => opts.line_numbers = Some(false), - "--word-regexp" => opts.word_regexp = true, - "--line-regexp" => opts.line_regexp = true, - "--fixed-strings" => opts.fixed_strings = true, - "--quiet" | "--silent" => opts.quiet = true, - "--only-matching" => opts.only_matching = true, - "--hidden" | "--no-ignore" => opts.hidden = true, - "--follow" | "--no-ignore-vcs" | "--no-ignore-parent" => {} - "--with-filename" => opts.show_filename = Some(true), - "--no-filename" => opts.show_filename = Some(false), - "--no-heading" | "--heading" => {} // no-op (we always use inline format) - "--color=auto" | "--color=always" | "--color=never" => {} // no-op in WASI - "--no-color" => {} - _ if arg.starts_with("--color=") => {} - _ if arg.starts_with("--max-depth=") => { - opts.max_depth = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid number: '{}'", &arg[12..]))?, - ); - } - _ if arg.starts_with("--sort=") => match &arg[7..] { - "modified" => opts.sort_modified = true, - value => return Err(format!("unsupported sort: '{}'", value)), - }, - _ if arg.starts_with("--threads=") => {} - _ if arg.starts_with("--regexp=") => { - push_pattern(&mut opts, arg[9..].to_string())?; - explicit_pattern = true; - } - _ if arg.starts_with("--max-count=") => { - opts.max_count = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid number: '{}'", &arg[12..]))?, - ); - } - _ if arg.starts_with("--after-context=") => { - opts.after_context = parse_context_count(&arg[16..])?; - } - _ if arg.starts_with("--before-context=") => { - opts.before_context = parse_context_count(&arg[17..])?; - } - _ if arg.starts_with("--context=") => { - let n = parse_context_count(&arg[10..])?; - opts.before_context = n; - opts.after_context = n; - } - _ if arg.starts_with("--glob=") => { - opts.glob_patterns.push(arg[7..].to_string()); - } - _ if arg.starts_with("--type=") => { - opts.type_include.push(arg[7..].to_string()); - } - _ if arg.starts_with("--type-not=") => { - opts.type_exclude.push(arg[11..].to_string()); - } - "--regexp" | "--max-count" | "--after-context" | "--before-context" - | "--context" | "--glob" | "--type" | "--type-not" | "--file" | "--color" - | "--max-depth" | "--threads" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - match arg.as_str() { - "--regexp" => { - push_pattern(&mut opts, args[i].clone())?; - explicit_pattern = true; - } - "--max-count" => { - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - } - "--after-context" => { - opts.after_context = parse_context_count(&args[i])?; - } - "--before-context" => { - opts.before_context = parse_context_count(&args[i])?; - } - "--context" => { - let n = parse_context_count(&args[i])?; - opts.before_context = n; - opts.after_context = n; - } - "--glob" => opts.glob_patterns.push(args[i].clone()), - "--type" => opts.type_include.push(args[i].clone()), - "--type-not" => opts.type_exclude.push(args[i].clone()), - "--file" => { - read_patterns_from_file(&mut opts, &args[i])?; - explicit_pattern = true; - } - "--color" => {} // no-op - "--max-depth" => { - opts.max_depth = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - } - "--threads" => {} // no-op - _ => unreachable!(), - } - } - _ => return Err(format!("unrecognized option '{}'", arg)), - } - i += 1; - continue; - } - - // Short options - if arg.starts_with('-') && arg.len() > 1 { - let chars: Vec = arg[1..].chars().collect(); - let mut j = 0; - while j < chars.len() { - match chars[j] { - 'i' => opts.ignore_case = true, - 's' => { - opts.ignore_case = false; - opts.smart_case = false; - } - 'S' => opts.smart_case = true, - 'v' => opts.invert_match = true, - 'c' => opts.count_only = true, - 'l' => opts.files_with_matches = true, - 'n' => opts.line_numbers = Some(true), - 'N' => opts.line_numbers = Some(false), - 'w' => opts.word_regexp = true, - 'x' => opts.line_regexp = true, - 'F' => opts.fixed_strings = true, - 'q' => opts.quiet = true, - 'o' => opts.only_matching = true, - 'H' => opts.show_filename = Some(true), - '.' => opts.hidden = true, - 'e' => { - let rest: String = chars[j + 1..].iter().collect(); - if !rest.is_empty() { - push_pattern(&mut opts, rest)?; - } else { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'e'".to_string()); - } - push_pattern(&mut opts, args[i].clone())?; - } - explicit_pattern = true; - j = chars.len(); - continue; - } - 'f' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'f'".to_string()); - } - read_patterns_from_file(&mut opts, &args[i])?; - explicit_pattern = true; - j = chars.len(); - continue; - } - 'm' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'm'".to_string()); - } - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - j = chars.len(); - continue; - } - 'j' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'j'".to_string()); - } - j = chars.len(); - continue; - } - 'A' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'A'".to_string()); - } - opts.after_context = parse_context_count(&args[i])?; - j = chars.len(); - continue; - } - 'B' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'B'".to_string()); - } - opts.before_context = parse_context_count(&args[i])?; - j = chars.len(); - continue; - } - 'C' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'C'".to_string()); - } - let n = parse_context_count(&args[i])?; - opts.before_context = n; - opts.after_context = n; - j = chars.len(); - continue; - } - 'g' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'g'".to_string()); - } - opts.glob_patterns.push(args[i].clone()); - j = chars.len(); - continue; - } - 't' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 't'".to_string()); - } - opts.type_include.push(args[i].clone()); - j = chars.len(); - continue; - } - 'T' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'T'".to_string()); - } - opts.type_exclude.push(args[i].clone()); - j = chars.len(); - continue; - } - _ => return Err(format!("invalid option -- '{}'", chars[j])), - } - j += 1; - } - i += 1; - continue; - } - - // Positional argument - if opts.files_mode { - opts.paths.push(arg.clone()); - } else if !explicit_pattern && opts.patterns.is_empty() { - push_pattern(&mut opts, arg.clone())?; - explicit_pattern = true; - } else { - opts.paths.push(arg.clone()); - } - i += 1; - } - - // Remaining args after -- - while i < args.len() { - if opts.files_mode { - opts.paths.push(args[i].clone()); - } else if !explicit_pattern && opts.patterns.is_empty() { - push_pattern(&mut opts, args[i].clone())?; - explicit_pattern = true; - } else { - opts.paths.push(args[i].clone()); - } - i += 1; - } - - Ok(opts) -} - -fn parse_context_count(value: &str) -> Result { - let count: usize = value - .parse() - .map_err(|_| format!("invalid number: '{}'", value))?; - if count > MAX_CONTEXT_LINES { - return Err(format!("context count '{}' exceeds size limit", value)); - } - Ok(count) -} - -fn push_pattern(opts: &mut Options, pattern: String) -> Result<(), String> { - if opts.patterns.len() >= MAX_PATTERNS { - return Err("too many patterns".to_string()); - } - let next_bytes = opts - .pattern_bytes - .checked_add(pattern.len()) - .ok_or_else(|| "pattern data too large".to_string())?; - if next_bytes > MAX_PATTERN_BYTES { - return Err("pattern data exceeds size limit".to_string()); - } - opts.pattern_bytes = next_bytes; - opts.patterns.push(pattern); - Ok(()) -} - -fn read_patterns_from_file(opts: &mut Options, path: &str) -> Result<(), String> { - let metadata = std::fs::metadata(path).map_err(|e| format!("{}: {}", path, e))?; - if metadata.len() > MAX_PATTERN_BYTES as u64 { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - let file = std::fs::File::open(path).map_err(|e| format!("{}: {}", path, e))?; - let limit = MAX_PATTERN_BYTES - .checked_add(1) - .ok_or_else(|| "pattern file size limit is too large".to_string())?; - let mut content = String::new(); - file.take(limit as u64) - .read_to_string(&mut content) - .map_err(|e| format!("{}: {}", path, e))?; - if content.len() > MAX_PATTERN_BYTES { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - for line in content.lines() { - if !line.is_empty() { - push_pattern(opts, line.to_string())?; - } - } - Ok(()) -} - -// --- Pattern building --- - -fn build_regex(opts: &Options) -> Result { - let combined = if opts.patterns.len() == 1 { - prepare_pattern(&opts.patterns[0], opts) - } else { - let parts: Vec = opts - .patterns - .iter() - .map(|p| format!("(?:{})", prepare_pattern(p, opts))) - .collect(); - parts.join("|") - }; - - let case_insensitive = if opts.ignore_case { - true - } else if opts.smart_case { - // Smart case: insensitive unless pattern has uppercase - !combined.chars().any(|c| c.is_uppercase()) - } else { - false - }; - - RegexBuilder::new(&combined) - .case_insensitive(case_insensitive) - .build() - .map_err(|e| format!("regex error: {}", e)) -} - -fn prepare_pattern(pattern: &str, opts: &Options) -> String { - let base = if opts.fixed_strings { - regex::escape(pattern) - } else { - pattern.to_string() - }; - - if opts.word_regexp { - format!(r"\b(?:{})\b", base) - } else if opts.line_regexp { - format!("^(?:{})$", base) - } else { - base - } -} - -// --- File collection --- - -fn collect_files_from_paths(paths: &[String], opts: &Options) -> io::Result> { - let mut files = Vec::new(); - for path_str in paths { - let path = Path::new(path_str); - let metadata = std::fs::symlink_metadata(path)?; - let file_type = metadata.file_type(); - if file_type.is_dir() { - let mut active_dirs = HashSet::new(); - walk_dir(path, path, opts, &mut files, 0, &mut active_dirs)?; - } else if file_type.is_file() { - if should_include(path, path, false, opts) { - push_collected_file(&mut files, path.to_path_buf())?; - } - } else { - continue; - } - } - if opts.sort_modified { - files.sort_by_key(|path| { - std::fs::metadata(path) - .and_then(|meta| meta.modified()) - .ok() - }); - } else { - files.sort(); - } - Ok(files) -} - -fn push_collected_file(out: &mut Vec, path: PathBuf) -> io::Result<()> { - if out.len() >= MAX_FILE_RESULTS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "file result count exceeds size limit", - )); - } - out.push(path); - Ok(()) -} - -fn walk_dir( - root: &Path, - dir: &Path, - opts: &Options, - out: &mut Vec, - depth: usize, - active_dirs: &mut HashSet, -) -> io::Result<()> { - let canonical = std::fs::canonicalize(dir)?; - if !active_dirs.insert(canonical.clone()) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("recursive directory cycle at {}", dir.display()), - )); - } - - let result = (|| { - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - let file_type = entry.file_type()?; - let is_dir = file_type.is_dir(); - - // Skip hidden files/dirs unless --hidden - if !opts.hidden && name_str.starts_with('.') { - continue; - } - - if !should_include(root, &path, is_dir, opts) { - continue; - } - - if is_dir { - if opts.max_depth.map(|max| depth < max).unwrap_or(true) { - walk_dir(root, &path, opts, out, depth + 1, active_dirs)?; - } - } else if file_type.is_file() { - push_collected_file(out, path)?; - } - } - Ok(()) - })(); - - active_dirs.remove(&canonical); - result -} - -fn should_include(root: &Path, path: &Path, is_dir: bool, opts: &Options) -> bool { - let ext = path - .extension() - .map(|e| e.to_string_lossy().to_string()) - .unwrap_or_default(); - let relative_path = path - .strip_prefix(root) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/"); - let file_name = path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - - // Type include filters - if !opts.type_include.is_empty() { - let included = opts.type_include.iter().any(|t| { - type_extensions(t) - .map(|exts| exts.iter().any(|e| ext == *e)) - .unwrap_or(false) - }); - if !included { - return false; - } - } - - // Type exclude filters - if !opts.type_exclude.is_empty() { - let excluded = opts.type_exclude.iter().any(|t| { - type_extensions(t) - .map(|exts| exts.iter().any(|e| ext == *e)) - .unwrap_or(false) - }); - if excluded { - return false; - } - } - - // Glob filters - if !opts.glob_patterns.is_empty() { - for pattern in &opts.glob_patterns { - let (negated, pat) = if let Some(rest) = pattern.strip_prefix('!') { - (true, rest) - } else { - (false, pattern.as_str()) - }; - let matches = glob_matches(pat, &relative_path, &file_name, is_dir); - if negated && matches { - return false; - } - if !negated && !matches { - return false; - } - } - } - - true -} - -fn type_extensions(type_name: &str) -> Option<&'static [&'static str]> { - match type_name { - "rust" | "rs" => Some(&["rs"]), - "py" | "python" => Some(&["py", "pyi"]), - "js" | "javascript" => Some(&["js", "jsx", "mjs"]), - "ts" | "typescript" => Some(&["ts", "tsx", "mts"]), - "c" => Some(&["c", "h"]), - "cpp" | "c++" => Some(&["cpp", "cxx", "cc", "hpp", "hxx", "h"]), - "java" => Some(&["java"]), - "go" => Some(&["go"]), - "html" => Some(&["html", "htm"]), - "css" => Some(&["css"]), - "json" => Some(&["json"]), - "yaml" | "yml" => Some(&["yml", "yaml"]), - "toml" => Some(&["toml"]), - "md" | "markdown" => Some(&["md", "markdown"]), - "txt" | "text" => Some(&["txt"]), - "sh" | "shell" | "bash" => Some(&["sh", "bash"]), - "xml" => Some(&["xml"]), - "sql" => Some(&["sql"]), - "lua" => Some(&["lua"]), - "ruby" | "rb" => Some(&["rb"]), - "php" => Some(&["php"]), - "swift" => Some(&["swift"]), - "kotlin" | "kt" => Some(&["kt", "kts"]), - _ => None, - } -} - -fn glob_matches(pattern: &str, relative_path: &str, file_name: &str, is_dir: bool) -> bool { - let normalized = pattern.trim_end_matches('/'); - if pattern.ends_with('/') { - return is_dir - && relative_path - .split('/') - .any(|segment| segment == normalized || segment == file_name); - } - - let target = if pattern.contains('/') { - relative_path - } else { - file_name - }; - - let mut regex_pattern = String::from("^"); - let chars: Vec = normalized.chars().collect(); - let mut i = 0; - while i < chars.len() { - match chars[i] { - '*' => { - if i + 1 < chars.len() && chars[i + 1] == '*' { - regex_pattern.push_str(".*"); - i += 2; - } else { - regex_pattern.push_str("[^/]*"); - i += 1; - } - } - '?' => { - regex_pattern.push_str("[^/]"); - i += 1; - } - '{' => { - if let Some(end) = chars[i + 1..].iter().position(|c| *c == '}') { - let group: String = chars[i + 1..i + 1 + end].iter().collect(); - regex_pattern.push('('); - regex_pattern.push_str( - &group - .split(',') - .map(regex::escape) - .collect::>() - .join("|"), - ); - regex_pattern.push(')'); - i += end + 2; - } else { - regex_pattern.push_str("\\{"); - i += 1; - } - } - '.' | '+' | '(' | ')' | '|' | '^' | '$' | '[' | ']' | '\\' => { - regex_pattern.push('\\'); - regex_pattern.push(chars[i]); - i += 1; - } - other => { - regex_pattern.push(other); - i += 1; - } - } - } - regex_pattern.push('$'); - - Regex::new(®ex_pattern) - .map(|regex| regex.is_match(target)) - .unwrap_or(false) -} - -// --- Search --- - -struct FileResult { - matches: usize, - is_binary: bool, -} - -enum ResultLine { - Match(usize, String), - Context(usize, String), - Separator, -} - -fn search_stream( - mut reader: R, - regex: &Regex, - opts: &Options, - filename: Option<&str>, - show_filename: bool, - out: &mut W, -) -> io::Result { - let mut result = FileResult { - matches: 0, - is_binary: false, - }; - - let collect_lines = - !opts.quiet && !opts.files_with_matches && !opts.files_without_matches && !opts.count_only; - - let mut before_buf: VecDeque<(usize, String)> = VecDeque::new(); - let mut before_buf_bytes: usize = 0; - let mut after_remaining: usize = 0; - let mut last_printed: usize = 0; - let mut line_buf = Vec::new(); - let mut lineno: usize = 0; - - while let Some(line) = read_line_bounded(&mut reader, &mut line_buf)? { - lineno = lineno - .checked_add(1) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "line number overflow"))?; - - // Binary detection: null bytes in line data - if line.as_bytes().contains(&0) { - result.is_binary = true; - break; - } - - let is_match = regex.is_match(&line) != opts.invert_match; - - if is_match { - result.matches += 1; - - if opts.quiet || opts.files_with_matches { - break; - } - - if collect_lines { - // Separator for non-contiguous match groups - if opts.has_context() && last_printed > 0 { - let first_before = before_buf.front().map(|(n, _)| *n).unwrap_or(lineno); - if first_before > last_printed + 1 { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Separator, - )?; - } - } - - // Flush before-context buffer - for (bno, btext) in before_buf.drain(..) { - if bno > last_printed { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Context(bno, btext), - )?; - last_printed = bno; - } - } - before_buf_bytes = 0; - - // Emit match - if opts.only_matching && !opts.invert_match { - for mat in regex.find_iter(&line) { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Match(lineno, mat.as_str().to_string()), - )?; - } - } else { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Match(lineno, line), - )?; - } - last_printed = lineno; - after_remaining = opts.after_context; - } - - if let Some(max) = opts.max_count { - if result.matches >= max { - break; - } - } - } else if collect_lines { - if after_remaining > 0 { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Context(lineno, line), - )?; - last_printed = lineno; - after_remaining -= 1; - } else if opts.before_context > 0 { - before_buf_bytes = before_buf_bytes.checked_add(line.len()).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "context buffer too large") - })?; - if before_buf_bytes > MAX_CONTEXT_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "context buffer exceeds size limit", - )); - } - before_buf.push_back((lineno, line)); - if before_buf.len() > opts.before_context { - if let Some((_, removed)) = before_buf.pop_front() { - before_buf_bytes = before_buf_bytes.saturating_sub(removed.len()); - } - } - } - } - } - - Ok(result) -} - -// --- Output --- - -fn print_file_result( - filename: Option<&str>, - result: &FileResult, - opts: &Options, - out: &mut W, -) -> io::Result<()> { - if result.is_binary { - if result.matches > 0 { - if let Some(name) = filename { - writeln!(out, "Binary file {} matches.", name)?; - } - } - return Ok(()); - } - - if opts.files_with_matches { - if result.matches > 0 { - let name = filename.unwrap_or("(standard input)"); - writeln!(out, "{}", name)?; - } - return Ok(()); - } - - if opts.files_without_matches { - if result.matches == 0 { - let name = filename.unwrap_or("(standard input)"); - writeln!(out, "{}", name)?; - } - return Ok(()); - } - - if opts.count_only { - if let Some(name) = filename { - writeln!(out, "{}:{}", name, result.matches)?; - } else { - writeln!(out, "{}", result.matches)?; - } - return Ok(()); - } - - Ok(()) -} - -fn print_result_line( - out: &mut W, - filename: Option<&str>, - opts: &Options, - show_filename: bool, - line: &ResultLine, -) -> io::Result<()> { - match line { - ResultLine::Match(lineno, text) => { - let mut prefix = String::new(); - if show_filename { - if let Some(name) = filename { - prefix.push_str(name); - prefix.push(':'); - } - } - if opts.show_line_numbers() { - prefix.push_str(&lineno.to_string()); - prefix.push(':'); - } - writeln!(out, "{}{}", prefix, text) - } - ResultLine::Context(lineno, text) => { - let mut prefix = String::new(); - if show_filename { - if let Some(name) = filename { - prefix.push_str(name); - prefix.push('-'); - } - } - if opts.show_line_numbers() { - prefix.push_str(&lineno.to_string()); - prefix.push('-'); - } - writeln!(out, "{}{}", prefix, text) - } - ResultLine::Separator => writeln!(out, "--"), - } -} - -fn read_line_bounded( - reader: &mut R, - line_buf: &mut Vec, -) -> io::Result> { - line_buf.clear(); - - loop { - let available = reader.fill_buf()?; - if available.is_empty() { - if line_buf.is_empty() { - return Ok(None); - } - break; - } - - let newline = available.iter().position(|&b| b == b'\n'); - let take = newline.map_or(available.len(), |pos| pos + 1); - let next_len = line_buf - .len() - .checked_add(take) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "input line too long"))?; - if next_len > MAX_INPUT_LINE_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "input line exceeds size limit", - )); - } - - line_buf.extend_from_slice(&available[..take]); - reader.consume(take); - if newline.is_some() { - break; - } - } - - if line_buf.ends_with(b"\n") { - line_buf.pop(); - if line_buf.ends_with(b"\r") { - line_buf.pop(); - } - } - - String::from_utf8(line_buf.clone()) - .map(Some) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) -}