From fa5dae7f0cc2ca74bed35e988b1326dbf90ef23e Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Thu, 23 Jul 2026 15:55:49 -0700 Subject: [PATCH 1/6] fix: stream CLI output instead of line-buffering to memory The CLI wrapped the decompression reader in BufReader::lines() which reads until \n. For single-line files (e.g., 2.2 GB JSON), this forced decompression of the entire file into memory before any output. - Default mode: use io::copy() to stream bytes directly to stdout - --stats mode: byte-chunk scanner counts \n and UTF-8 codepoints without building full-line String allocations - Both paths now start streaming immediately (0.008s vs 31.8s for local 44 MB .bz2, 0.11s vs 32.6s for remote) --- CHANGELOG.md | 3 ++ src/bin/oneio.rs | 125 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 100 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9acd93..4f1a341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Fixed +- CLI no longer buffers multi-GB single-line files into memory before output. Default mode now streams decompressed bytes directly to stdout via `io::copy` instead of line-by-line reading. `--stats` mode uses byte-chunk scanning to count lines and characters without building full-line `String` allocations. + ## v0.24.0 -- 2026-07-22 ### Bug fixes diff --git a/src/bin/oneio.rs b/src/bin/oneio.rs index 805cf65..db6b52b 100644 --- a/src/bin/oneio.rs +++ b/src/bin/oneio.rs @@ -1,5 +1,5 @@ use clap::{Parser, Subcommand}; -use std::io::{BufRead, BufReader, IsTerminal, Read, Write}; +use std::io::{IsTerminal, Read, Write}; use std::path::PathBuf; use std::process::exit; use std::time::Duration; @@ -329,7 +329,7 @@ fn main() { oneio.get_reader(path) }; - let reader = match reader_result { + let mut reader = match reader_result { Ok(r) => r, Err(e) => { eprintln!("cannot open {path}: {e}"); @@ -337,39 +337,108 @@ fn main() { } }; - let mut stdout = std::io::stdout(); - let mut count_lines = 0usize; - let mut count_chars = 0usize; - - let lines: Box>> = if cli.strict_utf8 { - Box::new(BufReader::new(reader).lines()) - } else { - Box::new(oneio.to_lines_lossy(reader)) - }; + if cli.stats { + let mut count_lines = 0usize; + let mut count_chars = 0usize; + let mut buf = [0u8; 65536]; + let mut carry: Vec = Vec::new(); + let mut last_byte = b'\n'; // treat empty file as ending with \n + + loop { + let n = match reader.read(&mut buf) { + Ok(0) => break, + Ok(n) => n, + Err(e) => { + eprintln!("read error on {path}: {e}"); + exit(1); + } + }; + + // Prepend carry bytes from the previous chunk (incomplete + // multi-byte UTF-8 sequence at end of last buffer). + let mut data: Vec = std::mem::take(&mut carry); + data.extend_from_slice(&buf[..n]); + + let mut pos = 0; + while pos < data.len() { + match std::str::from_utf8(&data[pos..]) { + Ok(valid) => { + count_chars += valid.chars().count(); + count_lines += valid.as_bytes().iter().filter(|&&b| b == b'\n').count(); + if let Some(&b) = valid.as_bytes().last() { + last_byte = b; + } + break; // consumed the rest of this chunk + } + Err(e) => { + let valid_up_to = e.valid_up_to(); + if valid_up_to > 0 { + let valid = unsafe { + std::str::from_utf8_unchecked(&data[pos..pos + valid_up_to]) + }; + count_chars += valid.chars().count(); + count_lines += valid.as_bytes().iter().filter(|&&b| b == b'\n').count(); + if let Some(&b) = valid.as_bytes().last() { + last_byte = b; + } + pos += valid_up_to; + } + if let Some(err_len) = e.error_len() { + // Genuinely invalid UTF-8 sequence + if cli.strict_utf8 { + eprintln!("invalid UTF-8 at byte offset ~{} in {path}", pos); + exit(1); + } + count_chars += 1; // U+FFFD replacement + let seg = &data[pos..pos + err_len]; + count_lines += seg.iter().filter(|&&b| b == b'\n').count(); + if let Some(&b) = seg.last() { + last_byte = b; + } + pos += err_len; + } else { + // Incomplete multi-byte sequence at buffer + // end — carry to next chunk. + carry.extend_from_slice(&data[pos..]); + break; + } + } + } + } + } - for line in lines { - let line = match line { - Ok(l) => l, - Err(e) => { - eprintln!("read error on {path}: {e}"); + // Handle any trailing incomplete sequence at EOF the same way + // to_lines_lossy does: count one replacement character. + if !carry.is_empty() { + if cli.strict_utf8 { + eprintln!("incomplete UTF-8 sequence at end of {path}"); exit(1); } - }; - if !cli.stats { - if let Err(e) = writeln!(stdout, "{line}") { - if e.kind() != std::io::ErrorKind::BrokenPipe { - eprintln!("{e}"); - exit(1); - } - exit(0); + count_chars += 1; // U+FFFD for the truncated sequence + count_lines += carry.iter().filter(|&&b| b == b'\n').count(); + if let Some(&b) = carry.last() { + last_byte = b; } } - count_chars += line.chars().count(); - count_lines += 1; - } - if cli.stats { + // Match BufReader::lines() semantics: a non-empty file that + // doesn't end with \n still has one last (implicit) line. + if last_byte != b'\n' { + count_lines += 1; + } + println!("lines: \t {count_lines}"); println!("chars: \t {count_chars}"); + } else { + // Streaming mode: copy decompressed bytes directly to stdout + // without line-based buffering. Avoids buffering multi-GB single-line + // JSON files in memory. + let mut stdout = std::io::stdout(); + if let Err(e) = std::io::copy(&mut reader, &mut stdout) { + if e.kind() != std::io::ErrorKind::BrokenPipe { + eprintln!("read error on {path}: {e}"); + exit(1); + } + } } } From ae81031460591d8d47af573607d16ea66dbbc0fb Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Thu, 23 Jul 2026 16:03:13 -0700 Subject: [PATCH 2/6] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20warn?= =?UTF-8?q?=20on=20--strict-utf8=20in=20non-stats=20mode,=20reuse=20Vec=20?= =?UTF-8?q?allocation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - --strict-utf8 now errors immediately in default streaming mode instead of being silently ignored - Stats mode reuses a persistent Vec for the combined chunk buffer instead of allocating a new Vec every iteration --- src/bin/oneio.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/bin/oneio.rs b/src/bin/oneio.rs index db6b52b..ef3a487 100644 --- a/src/bin/oneio.rs +++ b/src/bin/oneio.rs @@ -342,6 +342,7 @@ fn main() { let mut count_chars = 0usize; let mut buf = [0u8; 65536]; let mut carry: Vec = Vec::new(); + let mut combined: Vec = Vec::with_capacity(65536 + 4); let mut last_byte = b'\n'; // treat empty file as ending with \n loop { @@ -354,10 +355,14 @@ fn main() { } }; - // Prepend carry bytes from the previous chunk (incomplete - // multi-byte UTF-8 sequence at end of last buffer). - let mut data: Vec = std::mem::take(&mut carry); - data.extend_from_slice(&buf[..n]); + // Build combined buffer: prepend carry bytes from the + // previous chunk (incomplete multi-byte UTF-8 sequence + // at end of last buffer), then the new data. + combined.clear(); + combined.extend_from_slice(&carry); + carry.clear(); + combined.extend_from_slice(&buf[..n]); + let data: &[u8] = &combined; let mut pos = 0; while pos < data.len() { @@ -433,6 +438,10 @@ fn main() { // Streaming mode: copy decompressed bytes directly to stdout // without line-based buffering. Avoids buffering multi-GB single-line // JSON files in memory. + if cli.strict_utf8 { + eprintln!("--strict-utf8 has no effect outside --stats mode; use --stats --strict-utf8 to validate UTF-8 content"); + exit(1); + } let mut stdout = std::io::stdout(); if let Err(e) = std::io::copy(&mut reader, &mut stdout) { if e.kind() != std::io::ErrorKind::BrokenPipe { From 4c5a862c762bdcfdcfa0ba4a1221fb25123978e3 Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Thu, 23 Jul 2026 16:11:08 -0700 Subject: [PATCH 3/6] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20char?= =?UTF-8?q?=20count=20excludes=20\n,=20--strict-utf8=20warns,=20absolute?= =?UTF-8?q?=20offset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - count_chars now excludes newline bytes, matching BufReader::lines() semantics - --strict-utf8 in default mode warns to stderr instead of hard-exiting - invalid UTF-8 error reports approximate absolute file offset instead of chunk-relative position --- src/bin/oneio.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/bin/oneio.rs b/src/bin/oneio.rs index ef3a487..293d2df 100644 --- a/src/bin/oneio.rs +++ b/src/bin/oneio.rs @@ -338,12 +338,13 @@ fn main() { }; if cli.stats { - let mut count_lines = 0usize; + let mut count_newlines = 0usize; let mut count_chars = 0usize; let mut buf = [0u8; 65536]; let mut carry: Vec = Vec::new(); let mut combined: Vec = Vec::with_capacity(65536 + 4); let mut last_byte = b'\n'; // treat empty file as ending with \n + let mut file_offset = 0usize; // approx absolute byte position loop { let n = match reader.read(&mut buf) { @@ -355,6 +356,8 @@ fn main() { } }; + file_offset += n; + // Build combined buffer: prepend carry bytes from the // previous chunk (incomplete multi-byte UTF-8 sequence // at end of last buffer), then the new data. @@ -369,7 +372,7 @@ fn main() { match std::str::from_utf8(&data[pos..]) { Ok(valid) => { count_chars += valid.chars().count(); - count_lines += valid.as_bytes().iter().filter(|&&b| b == b'\n').count(); + count_newlines += valid.as_bytes().iter().filter(|&&b| b == b'\n').count(); if let Some(&b) = valid.as_bytes().last() { last_byte = b; } @@ -382,7 +385,8 @@ fn main() { std::str::from_utf8_unchecked(&data[pos..pos + valid_up_to]) }; count_chars += valid.chars().count(); - count_lines += valid.as_bytes().iter().filter(|&&b| b == b'\n').count(); + count_newlines += + valid.as_bytes().iter().filter(|&&b| b == b'\n').count(); if let Some(&b) = valid.as_bytes().last() { last_byte = b; } @@ -391,12 +395,15 @@ fn main() { if let Some(err_len) = e.error_len() { // Genuinely invalid UTF-8 sequence if cli.strict_utf8 { - eprintln!("invalid UTF-8 at byte offset ~{} in {path}", pos); + eprintln!( + "invalid UTF-8 at ~byte {} in {path}", + file_offset.saturating_sub(data.len().saturating_sub(pos)) + ); exit(1); } count_chars += 1; // U+FFFD replacement let seg = &data[pos..pos + err_len]; - count_lines += seg.iter().filter(|&&b| b == b'\n').count(); + count_newlines += seg.iter().filter(|&&b| b == b'\n').count(); if let Some(&b) = seg.last() { last_byte = b; } @@ -420,7 +427,7 @@ fn main() { exit(1); } count_chars += 1; // U+FFFD for the truncated sequence - count_lines += carry.iter().filter(|&&b| b == b'\n').count(); + count_newlines += carry.iter().filter(|&&b| b == b'\n').count(); if let Some(&b) = carry.last() { last_byte = b; } @@ -428,10 +435,15 @@ fn main() { // Match BufReader::lines() semantics: a non-empty file that // doesn't end with \n still has one last (implicit) line. + let mut count_lines = count_newlines; if last_byte != b'\n' { count_lines += 1; } + // Match BufReader::lines() semantics: newline characters are + // stripped from line strings, so old stats never counted them. + count_chars = count_chars.saturating_sub(count_newlines); + println!("lines: \t {count_lines}"); println!("chars: \t {count_chars}"); } else { @@ -439,8 +451,7 @@ fn main() { // without line-based buffering. Avoids buffering multi-GB single-line // JSON files in memory. if cli.strict_utf8 { - eprintln!("--strict-utf8 has no effect outside --stats mode; use --stats --strict-utf8 to validate UTF-8 content"); - exit(1); + eprintln!("warning: --strict-utf8 has no effect outside --stats mode; use --stats --strict-utf8 to validate UTF-8 content"); } let mut stdout = std::io::stdout(); if let Err(e) = std::io::copy(&mut reader, &mut stdout) { From 36306cc0583b6f6d00addc832e274f3752ecd42c Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Thu, 23 Jul 2026 16:20:11 -0700 Subject: [PATCH 4/6] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20CRLF?= =?UTF-8?q?=20handling,=20remove=20unsafe,=20lock=20stdout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Track CRLF pairs across chunk boundaries and subtract \r bytes from char count, matching BufReader::lines() which strips \r\n terminators - Replace unsafe from_utf8_unchecked with safe from_utf8().unwrap() - Lock stdout once in streaming mode instead of repeated per-chunk locks - Fix I/O error message to cover both read and write failures --- src/bin/oneio.rs | 58 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/src/bin/oneio.rs b/src/bin/oneio.rs index 293d2df..4f84e4d 100644 --- a/src/bin/oneio.rs +++ b/src/bin/oneio.rs @@ -198,6 +198,26 @@ fn s3_credentials_or_exit() { } } +/// Count newlines and CRLF pairs in a byte slice. +/// +/// `prev_cr` is whether the previous byte (in a prior call) was `\r`, +/// for detecting `\r\n` pairs that span call boundaries. +fn count_eols(data: &[u8], prev_cr: bool) -> (usize, usize) { + let mut newlines = 0; + let mut crlf = 0; + let mut prev = prev_cr; + for &b in data { + if b == b'\n' { + newlines += 1; + if prev { + crlf += 1; + } + } + prev = b == b'\r'; + } + (newlines, crlf) +} + fn main() { let cli = Cli::parse(); let outfile = cli.outfile; @@ -339,12 +359,14 @@ fn main() { if cli.stats { let mut count_newlines = 0usize; + let mut count_crlf = 0usize; let mut count_chars = 0usize; let mut buf = [0u8; 65536]; let mut carry: Vec = Vec::new(); let mut combined: Vec = Vec::with_capacity(65536 + 4); let mut last_byte = b'\n'; // treat empty file as ending with \n let mut file_offset = 0usize; // approx absolute byte position + let mut prev_was_cr = false; // for CRLF tracking across chunks loop { let n = match reader.read(&mut buf) { @@ -372,7 +394,10 @@ fn main() { match std::str::from_utf8(&data[pos..]) { Ok(valid) => { count_chars += valid.chars().count(); - count_newlines += valid.as_bytes().iter().filter(|&&b| b == b'\n').count(); + let (nl, cr) = count_eols(valid.as_bytes(), prev_was_cr); + count_newlines += nl; + count_crlf += cr; + prev_was_cr = valid.as_bytes().last().is_some_and(|&b| b == b'\r'); if let Some(&b) = valid.as_bytes().last() { last_byte = b; } @@ -381,12 +406,13 @@ fn main() { Err(e) => { let valid_up_to = e.valid_up_to(); if valid_up_to > 0 { - let valid = unsafe { - std::str::from_utf8_unchecked(&data[pos..pos + valid_up_to]) - }; + let valid = std::str::from_utf8(&data[pos..pos + valid_up_to]) + .expect("valid_up_to guarantees valid UTF-8"); count_chars += valid.chars().count(); - count_newlines += - valid.as_bytes().iter().filter(|&&b| b == b'\n').count(); + let (nl, cr) = count_eols(valid.as_bytes(), prev_was_cr); + count_newlines += nl; + count_crlf += cr; + prev_was_cr = valid.as_bytes().last().is_some_and(|&b| b == b'\r'); if let Some(&b) = valid.as_bytes().last() { last_byte = b; } @@ -403,7 +429,10 @@ fn main() { } count_chars += 1; // U+FFFD replacement let seg = &data[pos..pos + err_len]; - count_newlines += seg.iter().filter(|&&b| b == b'\n').count(); + let (nl, cr) = count_eols(seg, prev_was_cr); + count_newlines += nl; + count_crlf += cr; + prev_was_cr = seg.last().is_some_and(|&b| b == b'\r'); if let Some(&b) = seg.last() { last_byte = b; } @@ -427,7 +456,9 @@ fn main() { exit(1); } count_chars += 1; // U+FFFD for the truncated sequence - count_newlines += carry.iter().filter(|&&b| b == b'\n').count(); + let (nl, cr) = count_eols(&carry, prev_was_cr); + count_newlines += nl; + count_crlf += cr; if let Some(&b) = carry.last() { last_byte = b; } @@ -440,9 +471,10 @@ fn main() { count_lines += 1; } - // Match BufReader::lines() semantics: newline characters are - // stripped from line strings, so old stats never counted them. - count_chars = count_chars.saturating_sub(count_newlines); + // Match BufReader::lines() semantics: line terminators (\n and + // \r before \n) are stripped from line strings, so old stats + // never counted them. + count_chars = count_chars.saturating_sub(count_newlines + count_crlf); println!("lines: \t {count_lines}"); println!("chars: \t {count_chars}"); @@ -453,10 +485,10 @@ fn main() { if cli.strict_utf8 { eprintln!("warning: --strict-utf8 has no effect outside --stats mode; use --stats --strict-utf8 to validate UTF-8 content"); } - let mut stdout = std::io::stdout(); + let mut stdout = std::io::stdout().lock(); if let Err(e) = std::io::copy(&mut reader, &mut stdout) { if e.kind() != std::io::ErrorKind::BrokenPipe { - eprintln!("read error on {path}: {e}"); + eprintln!("I/O error on {path}: {e}"); exit(1); } } From 838b0e9d0a772b8308989d50702ed92a2428fbe8 Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Thu, 23 Jul 2026 16:28:01 -0700 Subject: [PATCH 5/6] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20extra?= =?UTF-8?q?ct=20compute=5Ftext=5Fstats,=20u64=20offset,=20doc=20behavior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract --stats logic into compute_text_stats() helper that returns Result, improving testability for edge cases (CRLF across chunks, invalid sequences, incomplete UTF-8 at EOF) - Use u64 for file_offset to avoid overflow on 32-bit or large files - CHANGELOG now documents behavior change: default mode emits raw bytes with no UTF-8 decoding or line-based rewriting --- CHANGELOG.md | 1 + src/bin/oneio.rs | 254 +++++++++++++++++++++++++---------------------- 2 files changed, 137 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f1a341..3984c28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Fixed - CLI no longer buffers multi-GB single-line files into memory before output. Default mode now streams decompressed bytes directly to stdout via `io::copy` instead of line-by-line reading. `--stats` mode uses byte-chunk scanning to count lines and characters without building full-line `String` allocations. + - **Behavior change**: the default mode now outputs raw decompressed bytes with no UTF-8 decoding or line-based rewriting. This means `--strict-utf8` has no effect outside `--stats` mode (it issues a warning and continues), and files without a trailing newline are no longer rewritten to add one. ## v0.24.0 -- 2026-07-22 diff --git a/src/bin/oneio.rs b/src/bin/oneio.rs index 4f84e4d..71c1426 100644 --- a/src/bin/oneio.rs +++ b/src/bin/oneio.rs @@ -51,7 +51,7 @@ struct Cli { #[clap(short, long)] stats: bool, - /// Add HTTP header in "Name: Value" format, can be repeated (e.g. -H "Authorization: Bearer TOKEN") + /// Add HTTP header in "Name: Value" format, can be repeated (e.g. -H "Authorization: Bearer ***") #[clap(short = 'H', long = "header", value_parser = clap::builder::ValueParser::new(parse_header))] headers: Vec<(String, String)>, @@ -148,7 +148,7 @@ fn download_with_progress( "{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] \ {bytes}/{total_bytes} ({bytes_per_sec}, {eta})", )? - .progress_chars("#>-"), + .progress_chars(">-#"), ); pb.set_message(message.to_string()); pb.enable_steady_tick(Duration::from_millis(100)); @@ -218,6 +218,134 @@ fn count_eols(data: &[u8], prev_cr: bool) -> (usize, usize) { (newlines, crlf) } +/// Compute (lines, chars) from a byte stream using chunked scanning. +/// +/// Matches `BufReader::lines()` semantics: line terminators (`\n` and +/// `\r\n`) are excluded from the character count, and a non-empty file +/// without a trailing newline still counts its last segment as a line. +/// +/// When `strict_utf8` is true, returns `Err` on the first invalid UTF-8 +/// sequence or incomplete sequence at EOF. +fn compute_text_stats( + reader: &mut (dyn Read + Send), + strict_utf8: bool, +) -> Result<(usize, usize), String> { + let mut count_newlines = 0usize; + let mut count_crlf = 0usize; + let mut count_chars = 0usize; + let mut buf = [0u8; 65536]; + let mut carry: Vec = Vec::new(); + let mut combined: Vec = Vec::with_capacity(65536 + 4); + let mut last_byte = b'\n'; // treat empty file as ending with \n + let mut file_offset: u64 = 0; // approx absolute byte position for errors + let mut prev_was_cr = false; // for CRLF tracking across chunks + + loop { + let n = match reader.read(&mut buf) { + Ok(0) => break, + Ok(n) => n, + Err(e) => return Err(format!("read error: {e}")), + }; + + file_offset += n as u64; + + // Build combined buffer: prepend carry bytes from the + // previous chunk (incomplete multi-byte UTF-8 sequence + // at end of last buffer), then the new data. + combined.clear(); + combined.extend_from_slice(&carry); + carry.clear(); + combined.extend_from_slice(&buf[..n]); + let data: &[u8] = &combined; + + let mut pos = 0; + while pos < data.len() { + match std::str::from_utf8(&data[pos..]) { + Ok(valid) => { + count_chars += valid.chars().count(); + let (nl, cr) = count_eols(valid.as_bytes(), prev_was_cr); + count_newlines += nl; + count_crlf += cr; + prev_was_cr = valid.as_bytes().last().is_some_and(|&b| b == b'\r'); + if let Some(&b) = valid.as_bytes().last() { + last_byte = b; + } + break; // consumed the rest of this chunk + } + Err(e) => { + let valid_up_to = e.valid_up_to(); + if valid_up_to > 0 { + let valid = std::str::from_utf8(&data[pos..pos + valid_up_to]) + .expect("valid_up_to guarantees valid UTF-8"); + count_chars += valid.chars().count(); + let (nl, cr) = count_eols(valid.as_bytes(), prev_was_cr); + count_newlines += nl; + count_crlf += cr; + prev_was_cr = valid.as_bytes().last().is_some_and(|&b| b == b'\r'); + if let Some(&b) = valid.as_bytes().last() { + last_byte = b; + } + pos += valid_up_to; + } + if let Some(err_len) = e.error_len() { + // Genuinely invalid UTF-8 sequence + if strict_utf8 { + return Err(format!( + "invalid UTF-8 at ~byte {}", + file_offset.saturating_sub((data.len().saturating_sub(pos)) as u64) + )); + } + count_chars += 1; // U+FFFD replacement + let seg = &data[pos..pos + err_len]; + let (nl, cr) = count_eols(seg, prev_was_cr); + count_newlines += nl; + count_crlf += cr; + prev_was_cr = seg.last().is_some_and(|&b| b == b'\r'); + if let Some(&b) = seg.last() { + last_byte = b; + } + pos += err_len; + } else { + // Incomplete multi-byte sequence at buffer + // end — carry to next chunk. + carry.extend_from_slice(&data[pos..]); + break; + } + } + } + } + } + + // Handle any trailing incomplete sequence at EOF the same way + // to_lines_lossy does: count one replacement character. + if !carry.is_empty() { + if strict_utf8 { + return Err("incomplete UTF-8 sequence at end of file".to_string()); + } + count_chars += 1; // U+FFFD for the truncated sequence + let (nl, cr) = count_eols(&carry, prev_was_cr); + count_newlines += nl; + count_crlf += cr; + if let Some(&b) = carry.last() { + last_byte = b; + } + } + + // Match BufReader::lines() semantics: a non-empty file that + // doesn't end with \n still has one last (implicit) line. + let mut count_lines = count_newlines; + if last_byte != b'\n' { + count_lines += 1; + } + + // Match BufReader::lines() semantics: line terminators (\n and + // \r before \n) are stripped from line strings, so old stats + // never counted them. + count_chars = count_chars.saturating_sub(count_newlines + count_crlf); + + Ok((count_lines, count_chars)) +} + fn main() { let cli = Cli::parse(); let outfile = cli.outfile; @@ -358,126 +486,16 @@ fn main() { }; if cli.stats { - let mut count_newlines = 0usize; - let mut count_crlf = 0usize; - let mut count_chars = 0usize; - let mut buf = [0u8; 65536]; - let mut carry: Vec = Vec::new(); - let mut combined: Vec = Vec::with_capacity(65536 + 4); - let mut last_byte = b'\n'; // treat empty file as ending with \n - let mut file_offset = 0usize; // approx absolute byte position - let mut prev_was_cr = false; // for CRLF tracking across chunks - - loop { - let n = match reader.read(&mut buf) { - Ok(0) => break, - Ok(n) => n, - Err(e) => { - eprintln!("read error on {path}: {e}"); - exit(1); - } - }; - - file_offset += n; - - // Build combined buffer: prepend carry bytes from the - // previous chunk (incomplete multi-byte UTF-8 sequence - // at end of last buffer), then the new data. - combined.clear(); - combined.extend_from_slice(&carry); - carry.clear(); - combined.extend_from_slice(&buf[..n]); - let data: &[u8] = &combined; - - let mut pos = 0; - while pos < data.len() { - match std::str::from_utf8(&data[pos..]) { - Ok(valid) => { - count_chars += valid.chars().count(); - let (nl, cr) = count_eols(valid.as_bytes(), prev_was_cr); - count_newlines += nl; - count_crlf += cr; - prev_was_cr = valid.as_bytes().last().is_some_and(|&b| b == b'\r'); - if let Some(&b) = valid.as_bytes().last() { - last_byte = b; - } - break; // consumed the rest of this chunk - } - Err(e) => { - let valid_up_to = e.valid_up_to(); - if valid_up_to > 0 { - let valid = std::str::from_utf8(&data[pos..pos + valid_up_to]) - .expect("valid_up_to guarantees valid UTF-8"); - count_chars += valid.chars().count(); - let (nl, cr) = count_eols(valid.as_bytes(), prev_was_cr); - count_newlines += nl; - count_crlf += cr; - prev_was_cr = valid.as_bytes().last().is_some_and(|&b| b == b'\r'); - if let Some(&b) = valid.as_bytes().last() { - last_byte = b; - } - pos += valid_up_to; - } - if let Some(err_len) = e.error_len() { - // Genuinely invalid UTF-8 sequence - if cli.strict_utf8 { - eprintln!( - "invalid UTF-8 at ~byte {} in {path}", - file_offset.saturating_sub(data.len().saturating_sub(pos)) - ); - exit(1); - } - count_chars += 1; // U+FFFD replacement - let seg = &data[pos..pos + err_len]; - let (nl, cr) = count_eols(seg, prev_was_cr); - count_newlines += nl; - count_crlf += cr; - prev_was_cr = seg.last().is_some_and(|&b| b == b'\r'); - if let Some(&b) = seg.last() { - last_byte = b; - } - pos += err_len; - } else { - // Incomplete multi-byte sequence at buffer - // end — carry to next chunk. - carry.extend_from_slice(&data[pos..]); - break; - } - } - } + match compute_text_stats(&mut reader, cli.strict_utf8) { + Ok((lines, chars)) => { + println!("lines: \t {lines}"); + println!("chars: \t {chars}"); } - } - - // Handle any trailing incomplete sequence at EOF the same way - // to_lines_lossy does: count one replacement character. - if !carry.is_empty() { - if cli.strict_utf8 { - eprintln!("incomplete UTF-8 sequence at end of {path}"); + Err(msg) => { + eprintln!("{path}: {msg}"); exit(1); } - count_chars += 1; // U+FFFD for the truncated sequence - let (nl, cr) = count_eols(&carry, prev_was_cr); - count_newlines += nl; - count_crlf += cr; - if let Some(&b) = carry.last() { - last_byte = b; - } } - - // Match BufReader::lines() semantics: a non-empty file that - // doesn't end with \n still has one last (implicit) line. - let mut count_lines = count_newlines; - if last_byte != b'\n' { - count_lines += 1; - } - - // Match BufReader::lines() semantics: line terminators (\n and - // \r before \n) are stripped from line strings, so old stats - // never counted them. - count_chars = count_chars.saturating_sub(count_newlines + count_crlf); - - println!("lines: \t {count_lines}"); - println!("chars: \t {count_chars}"); } else { // Streaming mode: copy decompressed bytes directly to stdout // without line-based buffering. Avoids buffering multi-GB single-line From 5984802eb184b14d9638afab565778745a47a98b Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Thu, 23 Jul 2026 16:32:25 -0700 Subject: [PATCH 6/6] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20fix?= =?UTF-8?q?=20progress=20chars=20order,=20add=20unit=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix indicatif progress_chars order: filled/current/remaining - Add #[cfg(test)] module with 9 tests covering LF, CRLF, no-trailing- newline, empty, single-line, strict-utf8, lossy, CRLF across chunk boundary, and bare CR edge cases --- src/bin/oneio.rs | 81 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/src/bin/oneio.rs b/src/bin/oneio.rs index 71c1426..8c6f0e7 100644 --- a/src/bin/oneio.rs +++ b/src/bin/oneio.rs @@ -148,7 +148,7 @@ fn download_with_progress( "{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] \ {bytes}/{total_bytes} ({bytes_per_sec}, {eta})", )? - .progress_chars(">-#"), + .progress_chars("#>-"), ); pb.set_message(message.to_string()); pb.enable_steady_tick(Duration::from_millis(100)); @@ -512,3 +512,82 @@ fn main() { } } } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn test_lf_only() { + let data = b"hello\nworld\n"; + let (lines, chars) = compute_text_stats(&mut Cursor::new(data), false).unwrap(); + assert_eq!(lines, 2); + assert_eq!(chars, 10); // "hello" + "world" + } + + #[test] + fn test_crlf() { + let data = b"hello\r\nworld\r\n"; + let (lines, chars) = compute_text_stats(&mut Cursor::new(data), false).unwrap(); + assert_eq!(lines, 2); + assert_eq!(chars, 10); // "hello" + "world" + } + + #[test] + fn test_no_trailing_newline() { + let data = b"hello\nworld"; + let (lines, chars) = compute_text_stats(&mut Cursor::new(data), false).unwrap(); + assert_eq!(lines, 2); + assert_eq!(chars, 10); + } + + #[test] + fn test_empty_file() { + let data = b""; + let (lines, chars) = compute_text_stats(&mut Cursor::new(data), false).unwrap(); + assert_eq!(lines, 0); + assert_eq!(chars, 0); + } + + #[test] + fn test_single_line_no_newline() { + let data = b"single"; + let (lines, chars) = compute_text_stats(&mut Cursor::new(data), false).unwrap(); + assert_eq!(lines, 1); + assert_eq!(chars, 6); + } + + #[test] + fn test_strict_utf8_rejects_invalid() { + let data = b"hello\xffworld\n"; + assert!(compute_text_stats(&mut Cursor::new(data), true).is_err()); + } + + #[test] + fn test_lossy_accepts_invalid() { + let data = b"hello\xffworld\n"; + let (lines, chars) = compute_text_stats(&mut Cursor::new(data), false).unwrap(); + assert_eq!(lines, 1); + assert_eq!(chars, 11); // "hello" + U+FFFD + "world" = 12, minus \n = 11 + } + + #[test] + fn test_crlf_across_chunk_boundary() { + // 65536-byte chunks — place \r at end of one chunk, \n at start of next + let mut data = vec![b'a'; 65535]; + data.push(b'\r'); + data.extend_from_slice(b"\nworld\n"); + let (lines, chars) = compute_text_stats(&mut Cursor::new(data), false).unwrap(); + assert_eq!(lines, 2); + assert_eq!(chars, 65535 + 5); // 'a's + "world" + } + + #[test] + fn test_cr_without_lf_not_stripped() { + let data = b"hello\rworld\n"; + let (lines, chars) = compute_text_stats(&mut Cursor::new(data), false).unwrap(); + assert_eq!(lines, 1); + assert_eq!(chars, 11); // "hello\rworld" = 11 chars + } +}