From d09942072b81fc834b3b056de3ab75ed1435c43f Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 5 Oct 2026 00:04:20 +0100 Subject: [PATCH] feat(tui): highlight fenced code with terminal theme colors --- Cargo.lock | 53 ++++++++++++- Cargo.toml | 4 + src/tui/markdown.rs | 143 ++++++++++++++++++++++++++++++++-- src/tui/markdown/syntax.rs | 153 +++++++++++++++++++++++++++++++++++++ 4 files changed, 344 insertions(+), 9 deletions(-) create mode 100644 src/tui/markdown/syntax.rs diff --git a/Cargo.lock b/Cargo.lock index 104c42c..f6d0700 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -957,6 +957,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -1987,6 +1996,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fancy-regex" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fancy-regex" version = "0.19.0" @@ -2965,7 +2985,7 @@ dependencies = [ "bytecount", "data-encoding", "email_address", - "fancy-regex", + "fancy-regex 0.19.0", "fraction", "getrandom 0.3.4", "idna", @@ -3124,6 +3144,7 @@ dependencies = [ "shlex", "str0m", "subtle", + "syntect", "tar", "tempfile", "tokio", @@ -3135,6 +3156,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "two-face", "unicode-segmentation", "unicode-width", "url", @@ -5366,6 +5388,24 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "syntect" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" +dependencies = [ + "bincode", + "fancy-regex 0.16.2", + "flate2", + "fnv", + "once_cell", + "regex-syntax", + "serde", + "serde_derive", + "thiserror 2.0.18", + "walkdir", +] + [[package]] name = "tap" version = "1.0.1" @@ -5867,6 +5907,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "two-face" +version = "0.5.2+bat-0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915be7adc2ff6f4338acbf71f3eda0a146f46003b992a1669c674308610efdac" +dependencies = [ + "serde", + "serde_derive", + "syntect", +] + [[package]] name = "typed-path" version = "0.12.3" diff --git a/Cargo.toml b/Cargo.toml index 18eeb12..e5bbcaa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,12 +20,16 @@ tui = [ "dep:ratatui-image", "dep:shlex", "dep:str0m", + "dep:syntect", + "dep:two-face", "dep:tokio-tungstenite", "dep:unicode-segmentation", "dep:unicode-width", ] [dependencies] +syntect = { version = "=5.3.0", default-features = false, features = ["parsing", "regex-fancy"], optional = true } +two-face = { version = "=0.5.2", default-features = false, features = ["syntect-fancy"], optional = true } opus-pure = { version = "=0.2.2", default-features = false, optional = true } str0m = { version = "=0.24.0", default-features = false, features = ["aws-lc-rs"], optional = true } tokio-tungstenite = { version = "=0.30.0", default-features = false, features = ["handshake"], optional = true } diff --git a/src/tui/markdown.rs b/src/tui/markdown.rs index 3f2237e..167684e 100644 --- a/src/tui/markdown.rs +++ b/src/tui/markdown.rs @@ -5,6 +5,8 @@ //! actually show up in agent output and deliberately stops there: block //! quotes, rules, fenced and inline code, bullets, tables, and emphasis. +mod syntax; + use std::ops::Range; use super::{ @@ -45,6 +47,8 @@ pub(super) fn render_copyable_with_sources( .collect(); let mut lines = Vec::new(); let mut fence: Option<(String, char, usize, Range)> = None; + let mut code_syntax = syntax::CodeSyntax::plain(); + let mut syntax_budget = syntax::DOCUMENT_BUDGET; let mut table_end = 0; for (index, (offset, raw)) in raw_lines.iter().copied().enumerate() { if index < table_end { @@ -69,7 +73,11 @@ pub(super) fn render_copyable_with_sources( )); fence = None; } else { - lines.push((code_line(raw), Some(content.clone()), source_range)); + lines.push(( + code_line(raw, &mut code_syntax, &mut syntax_budget), + Some(content.clone()), + source_range, + )); } continue; } @@ -102,6 +110,7 @@ pub(super) fn render_copyable_with_sources( Some(content.clone()), source_range, )); + code_syntax = syntax::CodeSyntax::new(&language); fence = Some((language, marker, length, content)); continue; } @@ -469,12 +478,10 @@ fn bullet(trimmed: &str) -> Option<&str> { .or_else(|| trimmed.strip_prefix("* ")) } -fn code_line(raw: &str) -> LinkedLine { - plain_line(Line::from(vec![ - Span::styled("│ ", theme::faint()), - Span::styled(raw.replace('\t', " "), theme::code()), - ])) - .with_leading_gutter() +fn code_line(raw: &str, syntax: &mut syntax::CodeSyntax, budget: &mut usize) -> LinkedLine { + let mut spans = vec![Span::styled("│ ", theme::faint())]; + spans.extend(syntax.line(raw, budget)); + plain_line(Line::from(spans)).with_leading_gutter() } fn code_frame(label: &str) -> LinkedLine { @@ -1614,10 +1621,130 @@ mod tests { assert_eq!(lines.len(), 5); assert!(lines[1].spans[0].content.contains("rust")); assert_eq!(lines[2].spans[0].content, "│ "); - assert!(lines[2].spans[1].content.contains("let x = 1;")); + assert_eq!(line_text(&lines[2]), "│ let x = 1;"); assert_eq!(lines[2].spans[1].style.bg, None); } + #[test] + fn typescript_uses_terminal_styles_and_aliases() { + for language in ["ts", "typescript", "TypeScript", "tsx", "ts title=example"] { + let lines = render(&format!("```{language}\nconst answer: number = 42;\n```")); + let spans = &lines[1].spans; + assert!( + spans.iter().any(|s| s.content.contains("const") + && s.style == theme::bold(theme::accent_color())), + "{language}: {spans:?}" + ); + assert!( + spans + .iter() + .any(|s| s.content.contains("42") && s.style.fg == Some(theme::warn_color())) + ); + assert!(spans.iter().all(|s| s.style.bg.is_none() + && !matches!( + s.style.fg, + Some(ratatui::style::Color::Rgb(..) | ratatui::style::Color::Indexed(_)) + ))); + } + } + + #[test] + fn multiline_code_state_survives_lines_but_not_fences_or_renders() { + let lines = render( + "```ts\n/* comment\nstill comment */\nconst s = `first\nsecond`;\n```\n```ts\nconst fresh = 1;\n```", + ); + assert!( + lines[2] + .spans + .iter() + .skip(1) + .all(|s| s.style == theme::dim()) + ); + assert!(lines[4].spans.iter().any(|s| s.content.contains("second") + && s.style.fg == Some(theme::success_color()))); + assert!( + lines[7] + .spans + .iter() + .any(|s| s.content.contains("const") + && s.style == theme::bold(theme::accent_color())) + ); + let open = render("```ts\n/* open"); + assert!( + open[1] + .spans + .iter() + .skip(1) + .all(|s| s.style == theme::dim()) + ); + let fresh = render("```ts\nconst fresh = 1;"); + assert!( + fresh[1] + .spans + .iter() + .any(|s| s.content.contains("const") + && s.style == theme::bold(theme::accent_color())) + ); + } + + #[test] + fn highlighting_preserves_unicode_tabs_and_copy_ranges() { + for language in ["ts", "text", "unknown-language", ""] { + for closing in ["\n```", ""] { + let source = format!("```{language}\n\tconst 界 = \"é\"; {closing}"); + let lines = render_copyable(&source); + assert_eq!(line_text(&lines[1].0.line()), "│ const 界 = \"é\"; "); + assert!( + lines.iter().all( + |(_, range)| &source[range.clone().unwrap()] == "\tconst 界 = \"é\"; " + ) + ); + let wrapped = super::super::wrap::wrap_linked_tagged(&lines, 12); + assert!(wrapped.iter().all(|(line, range, _, _)| { + line.width() <= 12 && &source[range.clone().unwrap()] == "\tconst 界 = \"é\"; " + })); + assert!( + wrapped + .iter() + .any(|(line, _, _, _)| line_text(line).contains('界')) + ); + if language != "ts" { + assert!( + lines[1] + .0 + .spans + .iter() + .skip(1) + .all(|s| s.span.style == theme::code()) + ); + } + } + } + } + + #[test] + fn oversized_code_abandons_fence_without_resuming_stale_state() { + let source = format!( + "```ts\n/*\n{} */\nconst plain = 1;\n```\n```ts\nconst colored = 2;\n```", + "x".repeat(4097) + ); + let lines = render(&source); + assert!( + lines[3] + .spans + .iter() + .skip(1) + .all(|s| s.style == theme::code()) + ); + assert!( + lines[6] + .spans + .iter() + .any(|s| s.content.contains("const") + && s.style == theme::bold(theme::accent_color())) + ); + } + #[test] fn tags_fenced_rows_with_exact_code_content() { let source = "before\n```rust\n\tlet x = 1; \n\n```\nafter"; diff --git a/src/tui/markdown/syntax.rs b/src/tui/markdown/syntax.rs new file mode 100644 index 0000000..c0dac8a --- /dev/null +++ b/src/tui/markdown/syntax.rs @@ -0,0 +1,153 @@ +//! Scope-based code colors: terminal palette only, never a bundled RGB theme. + +use std::sync::OnceLock; + +use ratatui::{style::Style, text::Span}; +use syntect::parsing::{ParseState, Scope, ScopeStack, SyntaxSet}; + +use super::super::theme; + +// Bound input to the regex parser, not display/copy content. Once a line cannot +// be parsed, abandon that fence: resuming would use an invalid multiline state. +pub(super) const DOCUMENT_BUDGET: usize = 128 * 1024; +const LINE_LIMIT: usize = 4096; + +fn syntaxes() -> &'static SyntaxSet { + static SYNTAXES: OnceLock = OnceLock::new(); + SYNTAXES.get_or_init(two_face::syntax::extra_newlines) +} + +pub(super) struct CodeSyntax { + parser: Option, + scopes: ScopeStack, +} + +impl CodeSyntax { + pub(super) fn plain() -> Self { + Self { + parser: None, + scopes: ScopeStack::new(), + } + } + + pub(super) fn new(info: &str) -> Self { + let token = info + .split_whitespace() + .next() + .unwrap_or("") + .to_ascii_lowercase(); + if matches!(token.as_str(), "" | "text" | "txt" | "plain" | "plaintext") { + return Self::plain(); + } + let set = syntaxes(); + let syntax = match token.as_str() { + "ts" | "typescript" => set.find_syntax_by_name("TypeScript"), + "tsx" => set.find_syntax_by_extension("tsx"), + _ => set.find_syntax_by_token(&token), + }; + Self { + parser: syntax.map(ParseState::new), + scopes: ScopeStack::new(), + } + } + + pub(super) fn line(&mut self, raw: &str, budget: &mut usize) -> Vec> { + let plain = || vec![Span::styled(raw.replace('\t', " "), theme::code())]; + if raw.len() > LINE_LIMIT || raw.len() + 1 > *budget { + self.parser = None; + } + let Some(parser) = self.parser.as_mut() else { + return plain(); + }; + *budget -= raw.len() + 1; + // The newline-aware grammar must see line endings, including the current + // unterminated streaming line. Its synthetic newline is never displayed. + let line = format!("{raw}\n"); + let Ok(operations) = parser.parse_line(&line, syntaxes()) else { + self.parser = None; + return plain(); + }; + let mut spans = Vec::new(); + let mut start = 0; + for (offset, operation) in operations { + let end = offset.min(raw.len()); + if end > start { + push_span(&mut spans, &raw[start..end], scope_style(&self.scopes)); + } + if self.scopes.apply(&operation).is_err() { + self.parser = None; + return plain(); + } + start = end; + } + if start < raw.len() { + push_span(&mut spans, &raw[start..], scope_style(&self.scopes)); + } + if spans.is_empty() { plain() } else { spans } + } +} + +fn push_span(spans: &mut Vec>, text: &str, style: Style) { + let text = text.replace('\t', " "); + if let Some(last) = spans.last_mut() + && last.style == style + { + last.content.to_mut().push_str(&text); + } else { + spans.push(Span::styled(text, style)); + } +} + +fn scope_style(stack: &ScopeStack) -> Style { + static STYLES: OnceLock> = OnceLock::new(); + let styles = STYLES.get_or_init(|| { + [ + ("comment", theme::dim()), + ("string", theme::code().fg(theme::success_color())), + ("constant", theme::code().fg(theme::warn_color())), + ("keyword", theme::bold(theme::accent_color())), + ("storage", theme::bold(theme::accent_color())), + ("entity.name.function", theme::accent()), + ("entity.name.type", theme::code().fg(theme::user_color())), + ("support", theme::code().fg(theme::user_color())), + ("variable.language", theme::accent()), + ("invalid", theme::code().fg(theme::error_color())), + ] + .into_iter() + .filter_map(|(name, style)| Scope::new(name).ok().map(|scope| (scope, style))) + .collect() + }); + stack + .as_slice() + .iter() + .rev() + .find_map(|scope| { + styles + .iter() + .find(|(prefix, _)| prefix.is_prefix_of(*scope)) + .map(|(_, style)| *style) + }) + .unwrap_or_else(theme::code) +} + +#[cfg(test)] +#[allow(clippy::disallowed_macros)] +mod tests { + use super::*; + + #[test] + fn exhausted_budget_preserves_text_and_disables_current_fence() { + let mut syntax = CodeSyntax::new("ts"); + let mut budget = "/*\n".len(); + let comment = syntax.line("/*", &mut budget); + assert!(comment.iter().all(|span| span.style == theme::dim())); + let fallback = syntax.line("*/ const 界 = 1;", &mut budget); + assert_eq!(fallback.len(), 1); + assert_eq!(fallback[0].content, "*/ const 界 = 1;"); + assert_eq!(fallback[0].style, theme::code()); + // Even replenishing a caller's budget cannot resume a skipped state. + let mut replenished = DOCUMENT_BUDGET; + let later = syntax.line("const plain = 2;", &mut replenished); + assert!(later.iter().all(|span| span.style == theme::code())); + } +}