From c7f51d342cfef48d7526a948f3f1e67f51b43cbc Mon Sep 17 00:00:00 2001 From: konojunya Date: Thu, 3 Sep 2026 13:38:06 +0900 Subject: [PATCH] Add lossless source model --- README.md | 9 +- .../0005-add-a-lossless-source-model.md | 56 +++++ docs/specs/compiler-frontend.md | 7 +- src/lib.rs | 135 ++++++++++- src/lossless.rs | 228 ++++++++++++++++++ src/parser.rs | 6 +- tests/compiler.rs | 30 ++- tests/specification-revision | 2 +- 8 files changed, 458 insertions(+), 15 deletions(-) create mode 100644 docs/decisions/0005-add-a-lossless-source-model.md create mode 100644 src/lossless.rs diff --git a/README.md b/README.md index e060e79..e88d015 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Development and primary CI follow the latest stable Rust and Cargo releases thro ```text Stack source - -> lexical analysis + -> lexical analysis -----> lossless tokens and trivia -> syntax AST -> semantic validation -> normalized Diagram IR @@ -46,12 +46,19 @@ STACK_SPECIFICATION_DIR=../specification cargo test --features conformance --tes JSON support is development-only and does not add a runtime dependency to the compiler library. CI checks out the recorded specification revision before running the suite. +## Lossless Source Model + +`parse_lossless` and `parse_lossless_bytes` expose every authored token, whitespace segment, line comment, original string spelling, CRLF sequence, and end-exclusive source span. Concatenating token text through `lossless::Document::reconstruct` reproduces the input byte-for-byte. This source-oriented API is separate from normalized IR and performs no filesystem access. + +Lossless parsing succeeds for syntactically valid source even when semantic validation would later reject it. Lexical and syntax errors return diagnostics without a partial document. + ## Architecture - [`docs/decisions/0001-build-a-portable-rust-compiler-core.md`](./docs/decisions/0001-build-a-portable-rust-compiler-core.md) - [`docs/decisions/0002-separate-syntax-ast-from-normalized-ir.md`](./docs/decisions/0002-separate-syntax-ast-from-normalized-ir.md) - [`docs/decisions/0003-use-a-handwritten-parser.md`](./docs/decisions/0003-use-a-handwritten-parser.md) - [`docs/decisions/0004-consume-a-pinned-conformance-suite.md`](./docs/decisions/0004-consume-a-pinned-conformance-suite.md) +- [`docs/decisions/0005-add-a-lossless-source-model.md`](./docs/decisions/0005-add-a-lossless-source-model.md) - [`docs/specs/compiler-frontend.md`](./docs/specs/compiler-frontend.md) ## License diff --git a/docs/decisions/0005-add-a-lossless-source-model.md b/docs/decisions/0005-add-a-lossless-source-model.md new file mode 100644 index 0000000..62d133a --- /dev/null +++ b/docs/decisions/0005-add-a-lossless-source-model.md @@ -0,0 +1,56 @@ +# ADR-0005: Add a Lossless Source Model + +## Status + +Accepted + +## Date + +2026-09-03 + +## Context + +The canonical formatter must preserve comments and their token gaps while normalizing whitespace and authored string escapes. The existing syntax AST intentionally stores decoded semantic values and declaration spans, while the lexer discards whitespace and comments. Normalized IR excludes all source trivia by contract. + +Implementing another parser in the formatter would let syntax handling drift from the compiler. Adding trivia to normalized IR would make a portable semantic representation depend on source spelling. Mutating every existing AST node to carry trivia would also expand an API whose current responsibility is syntax and validation. + +## Decision + +Add a separate public lossless source model. `parse_lossless` and `parse_lossless_bytes` run the same lexer and recursive-descent parser used by the established parse path, then expose a flat source-order sequence containing: + +- language tokens with their exact authored text and end-exclusive span; +- decoded values for string tokens while retaining the original escape spelling; +- whitespace segments, including tabs, LF, and CRLF; +- complete line-comment lexemes without absorbing their line endings; +- a zero-width end token. + +The source text of all tokens and trivia segments concatenates to the original UTF-8 input byte-for-byte. Lossless parsing requires lexical and syntactic validity but is independent of semantic validation, so a syntactically valid document with name or layout errors remains inspectable. Lexical or syntax errors return the existing portable diagnostics without a partial lossless document. + +The AST and normalized IR types remain unchanged. The source model does not read files, consult the network or clock, or add a runtime dependency. + +## Alternatives Considered + +### Put trivia in the normalized IR + +- Pros: Downstream tools would consume one representation. +- Cons: Source spelling and comments are not portable diagram meaning and would destabilize semantic interchange. +- Rejected: Normalized IR must remain renderer-independent and formatting-free. + +### Add trivia fields throughout the existing AST + +- Pros: Formatting structure and source text would live in one tree. +- Cons: Broadly changes established syntax types and makes semantic consumers carry formatter-only data. +- Rejected: A separate token model is additive and keeps existing responsibilities intact. + +### Tokenize again in each formatter + +- Pros: No compiler API addition. +- Cons: Duplicates string, comment, operator, and position behavior and can drift from compiler diagnostics. +- Rejected: The compiler must own one lexical interpretation of Stack source. + +## Consequences + +- Formatters and editors can preserve authored bytes without reimplementing lexical rules. +- Callers use the lossless tokens alongside the existing AST when structural context is required. +- The model owns each token's exact text and decoded strings, trading modest allocation for a simple lifetime-free public API. +- Invalid UTF-8, byte order marks, and syntax errors remain diagnostic-only results. diff --git a/docs/specs/compiler-frontend.md b/docs/specs/compiler-frontend.md index 55f9be0..d708c2d 100644 --- a/docs/specs/compiler-frontend.md +++ b/docs/specs/compiler-frontend.md @@ -29,6 +29,7 @@ The primary users are Stack CLI, browser, editor, layout, and hosted-service imp src/ast.rs Syntax-oriented AST and spans src/diagnostic.rs Portable diagnostic types and codes src/lexer.rs UTF-8 text tokenization +src/lossless.rs Exact tokens, trivia, authored text, and spans src/parser.rs Recursive-descent grammar implementation src/validation.rs Semantic validation and normalization src/validation/ Focused validation unit tests @@ -45,12 +46,16 @@ docs/decisions/ Architectural decisions ```rust pub fn parse(source: &str) -> ParseOutput; pub fn parse_bytes(source: &[u8]) -> ParseOutput; +pub fn parse_lossless(source: &str) -> LosslessParseOutput; +pub fn parse_lossless_bytes(source: &[u8]) -> LosslessParseOutput; pub fn compile(source: &str) -> CompileOutput; pub fn compile_bytes(source: &[u8]) -> CompileOutput; ``` `ParseOutput` contains an AST only when decoding, lexing, and parsing succeed. `CompileOutput` contains normalized IR only when all compiler-stage errors are absent. Both outputs contain diagnostics. Semantic warnings may accompany successful IR. +`LosslessParseOutput` contains an exact token-and-trivia document only when decoding, lexing, and syntax parsing succeed. Its tokens retain original text and spans, while string tokens additionally expose decoded values. Reconstructing the document concatenates every token and trivia segment without normalization. Semantic validation remains a separate stage. + ## Code Style Prefer explicit domain types and exhaustive matches: @@ -115,7 +120,7 @@ match operator { - WebAssembly bindings - Native CLI commands -- Formatter and comment-preserving concrete syntax tree +- Canonical formatter implementation - Theme and icon resolution - Layout and renderer integration - Multi-error syntax recovery diff --git a/src/lib.rs b/src/lib.rs index 43d604b..e7de7f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod ast; pub mod diagnostic; pub mod ir; +pub mod lossless; mod lexer; mod parser; @@ -34,6 +35,15 @@ pub struct CompileOutput { pub diagnostics: Vec, } +/// Output of syntactic parsing into the lossless source model. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LosslessParseOutput { + /// Lossless document, present only when lexical and syntax parsing succeeds. + pub document: Option, + /// Lexical and syntax diagnostics. + pub diagnostics: Vec, +} + /// Parses UTF-8 Stack source into a source-oriented AST. pub fn parse(source: &str) -> ParseOutput { match parser::parse(source) { @@ -52,17 +62,45 @@ pub fn parse(source: &str) -> ParseOutput { pub fn parse_bytes(source: &[u8]) -> ParseOutput { match std::str::from_utf8(source) { Ok(source) => parse(source), - Err(error) => { - let position = position_after_valid_prefix(&source[..error.valid_up_to()]); - ParseOutput { + Err(error) => ParseOutput { + document: None, + diagnostics: vec![invalid_utf8_diagnostic(source, error)], + }, + } +} + +/// Parses UTF-8 Stack source into exact authored tokens and trivia. +pub fn parse_lossless(source: &str) -> LosslessParseOutput { + let tokens = match lexer::tokenize(source) { + Ok(tokens) => tokens, + Err(diagnostic) => { + return LosslessParseOutput { document: None, - diagnostics: vec![Diagnostic::error( - "STK1001", - "Input is not valid UTF-8.", - diagnostic::Span::point(position), - )], - } + diagnostics: vec![*diagnostic], + }; } + }; + + match parser::parse_tokens(tokens.clone()) { + Ok(_) => LosslessParseOutput { + document: Some(lossless::Document::from_lexer_tokens(source, tokens)), + diagnostics: Vec::new(), + }, + Err(diagnostic) => LosslessParseOutput { + document: None, + diagnostics: vec![*diagnostic], + }, + } +} + +/// Decodes and parses Stack source bytes into exact authored tokens and trivia. +pub fn parse_lossless_bytes(source: &[u8]) -> LosslessParseOutput { + match std::str::from_utf8(source) { + Ok(source) => parse_lossless(source), + Err(error) => LosslessParseOutput { + document: None, + diagnostics: vec![invalid_utf8_diagnostic(source, error)], + }, } } @@ -122,10 +160,22 @@ fn position_after_valid_prefix(prefix: &[u8]) -> diagnostic::SourcePosition { position } +fn invalid_utf8_diagnostic(source: &[u8], error: std::str::Utf8Error) -> Diagnostic { + let position = position_after_valid_prefix(&source[..error.valid_up_to()]); + Diagnostic::error( + "STK1001", + "Input is not valid UTF-8.", + diagnostic::Span::point(position), + ) +} + #[cfg(test)] mod tests { + use crate::lossless::TokenKind; + use super::{ - compile, compile_bytes, parse, parse_bytes, position_after_valid_prefix, validate, + compile, compile_bytes, parse, parse_bytes, parse_lossless, parse_lossless_bytes, + position_after_valid_prefix, validate, }; #[test] @@ -165,4 +215,69 @@ mod tests { crate::diagnostic::SourcePosition::start() ); } + + #[test] + fn lossless_entry_points_preserve_trivia_escapes_and_crlf() { + let source = concat!( + "// leading\r\n", + "stack 1.0\r\n", + "diagram \"\\u56F3\" {\r\n", + "\tnode api \"API\" // trailing\r\n", + "}\r\n", + ); + let output = parse_lossless(source); + assert!(output.diagnostics.is_empty()); + let Some(document) = output.document else { + return; + }; + + assert_eq!(document.reconstruct(), source); + assert!(document.tokens().iter().any(|token| { + matches!(&token.kind, TokenKind::String(value) if value == "図") + && token.text == "\"\\u56F3\"" + })); + assert!( + document.tokens().iter().any(|token| { + token.kind == TokenKind::Whitespace && token.text.contains("\r\n") + }) + ); + assert!( + document.tokens().iter().any(|token| { + token.kind == TokenKind::LineComment && token.text == "// trailing" + }) + ); + + let bytes_output = parse_lossless_bytes(source.as_bytes()); + assert_eq!(bytes_output.document, Some(document)); + } + + #[test] + fn lossless_entry_points_report_lexical_syntax_and_encoding_errors() { + let bom = parse_lossless("\u{feff}stack 1.0"); + assert!(bom.document.is_none()); + assert_eq!(bom.diagnostics[0].code, "STK1002"); + + let syntax = parse_lossless("stack 1.0 diagram \"x\" {"); + assert!(syntax.document.is_none()); + assert_eq!(syntax.diagnostics[0].code, "STK2003"); + + let encoding = parse_lossless_bytes(b"stack 1.0\r\n\xff"); + assert!(encoding.document.is_none()); + assert_eq!(encoding.diagnostics[0].code, "STK1001"); + assert_eq!(encoding.diagnostics[0].span.start.line, 2); + } + + #[test] + fn lossless_syntax_model_keeps_semantically_invalid_source() { + let source = concat!( + "stack 1.0\n", + "diagram \"Duplicate\" {\n", + " node api \"First\"\n", + " node api \"Second\"\n", + "}\n", + ); + + assert!(parse_lossless(source).document.is_some()); + assert!(compile(source).diagram.is_none()); + } } diff --git a/src/lossless.rs b/src/lossless.rs new file mode 100644 index 0000000..25d8a1d --- /dev/null +++ b/src/lossless.rs @@ -0,0 +1,228 @@ +//! Lossless lexical source model for formatters and editor tooling. + +use crate::diagnostic::{SourcePosition, Span}; +use crate::lexer; + +/// A syntactically valid Stack document represented as authored lexemes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Document { + tokens: Vec, +} + +impl Document { + pub(crate) fn from_lexer_tokens(source: &str, tokens: Vec) -> Self { + let mut lossless_tokens = Vec::new(); + let mut cursor = SourcePosition::start(); + + for token in tokens { + append_trivia(source, cursor, token.span.start, &mut lossless_tokens); + + let span = token.span; + let text = source[span.start.byte_offset..span.end.byte_offset].to_owned(); + let kind = match token.kind { + lexer::TokenKind::Bare(_) => TokenKind::Bare, + lexer::TokenKind::String(value) => TokenKind::String(value), + lexer::TokenKind::LeftBrace => TokenKind::LeftBrace, + lexer::TokenKind::RightBrace => TokenKind::RightBrace, + lexer::TokenKind::LeftBracket => TokenKind::LeftBracket, + lexer::TokenKind::RightBracket => TokenKind::RightBracket, + lexer::TokenKind::Comma => TokenKind::Comma, + lexer::TokenKind::Dot => TokenKind::Dot, + lexer::TokenKind::ForwardArrow => TokenKind::ForwardArrow, + lexer::TokenKind::BidirectionalArrow => TokenKind::BidirectionalArrow, + lexer::TokenKind::Association => TokenKind::Association, + lexer::TokenKind::End => TokenKind::End, + }; + lossless_tokens.push(Token { kind, text, span }); + cursor = span.end; + } + + Self { + tokens: lossless_tokens, + } + } + + /// Returns every authored token and trivia segment in source order. + pub fn tokens(&self) -> &[Token] { + &self.tokens + } + + /// Reconstructs the original UTF-8 source byte-for-byte. + pub fn reconstruct(&self) -> String { + let mut source = String::new(); + for token in &self.tokens { + source.push_str(&token.text); + } + source + } +} + +/// One authored lexeme or trivia segment with its exact source text and span. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Token { + /// Lexical category. String tokens also expose their decoded value. + pub kind: TokenKind, + /// Exact authored text, including escapes and original line endings. + pub text: String, + /// End-exclusive span in the original source. + pub span: Span, +} + +/// Lexical category in a lossless Stack document. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TokenKind { + /// One or more spaces, tabs, LF line endings, or CRLF line endings. + Whitespace, + /// A `//` comment without its following line ending. + LineComment, + /// An identifier, contextual keyword, integer, or unknown bare token. + Bare, + /// A source string and its decoded value. + String(String), + /// `{`. + LeftBrace, + /// `}`. + RightBrace, + /// `[`. + LeftBracket, + /// `]`. + RightBracket, + /// `,`. + Comma, + /// `.`. + Dot, + /// `->`. + ForwardArrow, + /// `<->`. + BidirectionalArrow, + /// `--`. + Association, + /// The zero-width end of the source. + End, +} + +fn append_trivia( + source: &str, + mut position: SourcePosition, + end: SourcePosition, + tokens: &mut Vec, +) { + while position.byte_offset < end.byte_offset { + let start = position; + let is_comment = source[position.byte_offset..end.byte_offset].starts_with("//"); + + if is_comment { + while position.byte_offset < end.byte_offset + && next_character(source, position.byte_offset) + .is_some_and(|character| !matches!(character, '\n' | '\r')) + { + advance_position(source, &mut position); + } + } else { + while position.byte_offset < end.byte_offset + && !source[position.byte_offset..end.byte_offset].starts_with("//") + { + advance_position(source, &mut position); + } + } + + let text = source[start.byte_offset..position.byte_offset].to_owned(); + tokens.push(Token { + kind: if is_comment { + TokenKind::LineComment + } else { + TokenKind::Whitespace + }, + text, + span: Span { + start, + end: position, + }, + }); + } +} + +fn next_character(source: &str, offset: usize) -> Option { + source[offset..].chars().next() +} + +fn advance_position(source: &str, position: &mut SourcePosition) { + let Some(character) = next_character(source, position.byte_offset) else { + return; + }; + + if character == '\r' && source[position.byte_offset..].starts_with("\r\n") { + position.byte_offset += 2; + position.line += 1; + position.column = 1; + } else { + position.byte_offset += character.len_utf8(); + if matches!(character, '\n' | '\r') { + position.line += 1; + position.column = 1; + } else { + position.column += 1; + } + } +} + +#[cfg(test)] +mod tests { + use super::{Document, TokenKind}; + use crate::lexer::tokenize; + + #[test] + fn preserves_every_lexeme_and_decoded_string_value() { + let source = " \t// lead\r\nword \"\\u0041\" {}[],. a->b a<->b a--b // tail"; + let result = tokenize(source); + assert!(result.is_ok(), "{result:?}"); + let tokens: Vec<_> = result.into_iter().flatten().collect(); + let document = Document::from_lexer_tokens(source, tokens); + + assert_eq!(document.reconstruct(), source); + assert_eq!(document.tokens[0].kind, TokenKind::Whitespace); + assert_eq!(document.tokens[0].text, " \t"); + assert_eq!(document.tokens[1].kind, TokenKind::LineComment); + assert_eq!(document.tokens[1].text, "// lead"); + assert_eq!(document.tokens[1].span.start.line, 1); + assert_eq!(document.tokens[2].text, "\r\n"); + assert_eq!(document.tokens[2].span.end.line, 2); + assert!(document.tokens.iter().any( + |token| matches!(&token.kind, TokenKind::String(value) if value == "A") + && token.text == "\"\\u0041\"" + )); + + for expected in [ + TokenKind::LeftBrace, + TokenKind::RightBrace, + TokenKind::LeftBracket, + TokenKind::RightBracket, + TokenKind::Comma, + TokenKind::Dot, + TokenKind::ForwardArrow, + TokenKind::BidirectionalArrow, + TokenKind::Association, + TokenKind::End, + ] { + assert!(document.tokens.iter().any(|token| token.kind == expected)); + } + + let mut byte_offset = 0; + for token in document.tokens() { + assert_eq!(token.span.start.byte_offset, byte_offset); + byte_offset = token.span.end.byte_offset; + } + assert_eq!(byte_offset, source.len()); + } + + #[test] + fn defensive_position_advance_stops_at_end_of_source() { + let mut position = crate::diagnostic::SourcePosition { + byte_offset: 0, + line: 1, + column: 1, + }; + super::advance_position("", &mut position); + assert_eq!(position.byte_offset, 0); + } +} diff --git a/src/parser.rs b/src/parser.rs index 4904d71..26fde4c 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -8,7 +8,11 @@ use crate::lexer::{Token, TokenKind, tokenize}; type ParseResult = Result>; pub(crate) fn parse(source: &str) -> ParseResult { - Parser::new(tokenize(source)?).parse_document() + parse_tokens(tokenize(source)?) +} + +pub(crate) fn parse_tokens(tokens: Vec) -> ParseResult { + Parser::new(tokens).parse_document() } struct Parser { diff --git a/tests/compiler.rs b/tests/compiler.rs index 585dd11..daf93fd 100644 --- a/tests/compiler.rs +++ b/tests/compiler.rs @@ -1,4 +1,6 @@ -use stack_compiler::{compile, compile_bytes, diagnostic::Severity, ir}; +use stack_compiler::{ + compile, compile_bytes, diagnostic::Severity, ir, lossless::TokenKind, parse_lossless, +}; #[test] fn public_api_applies_defaults_to_a_valid_document() { @@ -60,6 +62,32 @@ fn byte_api_reports_encoding_and_bom_errors() { assert!(bom.diagram.is_none()); } +#[test] +fn public_lossless_api_reconstructs_authored_source() { + let source = concat!( + "// leading\r\n", + "stack 1.0\r\n", + "diagram \"\\u0041\" { node api \"API\" } // trailing\r\n", + ); + let output = parse_lossless(source); + assert!(output.diagnostics.is_empty()); + let Some(document) = output.document else { + return; + }; + + assert_eq!(document.reconstruct(), source); + assert!( + document + .tokens() + .iter() + .any(|token| token.kind == TokenKind::LineComment) + ); + assert!(document.tokens().iter().any( + |token| matches!(&token.kind, TokenKind::String(value) if value == "A") + && token.text == "\"\\u0041\"" + )); +} + #[test] fn complexity_errors_and_degree_warnings_have_distinct_outcomes() { let no_nodes = compile("stack 1.0 diagram \"Empty\" {}"); diff --git a/tests/specification-revision b/tests/specification-revision index 58a0166..0211bb6 100644 --- a/tests/specification-revision +++ b/tests/specification-revision @@ -1 +1 @@ -8a4cf2ec97f5fba702ba3f27388fcab31f039ef5 +e40ad5dc230ab58f8a211130af96ae09c0e523bc