diff --git a/README.md b/README.md index e88d015..48c1860 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,10 @@ JSON support is development-only and does not add a runtime dependency to the co Lossless parsing succeeds for syntactically valid source even when semantic validation would later reject it. Lexical and syntax errors return diagnostics without a partial document. +## Source-Map Sidecar + +`compile_with_source_map` and `compile_bytes_with_source_map` return normalized IR together with a Rust-only `SourceMap`. The sidecar resolves the diagram theme, every node icon, and every diagram or group order hint from semantic identity to either an authored source span or `SourceOrigin::Omitted`. It is deterministic, has no portable JSON representation, and is absent whenever compiler-stage errors prevent normalized IR. + ## Architecture - [`docs/decisions/0001-build-a-portable-rust-compiler-core.md`](./docs/decisions/0001-build-a-portable-rust-compiler-core.md) @@ -59,6 +63,7 @@ Lossless parsing succeeds for syntactically valid source even when semantic vali - [`docs/decisions/0003-use-a-handwritten-parser.md`](./docs/decisions/0003-use-a-handwritten-parser.md) - [`docs/decisions/0004-consume-a-pinned-conformance-suite.md`](./docs/decisions/0004-consume-a-pinned-conformance-suite.md) - [`docs/decisions/0005-add-a-lossless-source-model.md`](./docs/decisions/0005-add-a-lossless-source-model.md) +- [`docs/decisions/0006-add-a-source-map-sidecar.md`](./docs/decisions/0006-add-a-source-map-sidecar.md) - [`docs/specs/compiler-frontend.md`](./docs/specs/compiler-frontend.md) ## License diff --git a/docs/decisions/0006-add-a-source-map-sidecar.md b/docs/decisions/0006-add-a-source-map-sidecar.md new file mode 100644 index 0000000..7cfea83 --- /dev/null +++ b/docs/decisions/0006-add-a-source-map-sidecar.md @@ -0,0 +1,58 @@ +# ADR-0006: Add a Source-Map Sidecar + +## Status + +Accepted + +## Date + +2026-09-03 + +## Context + +The layout, icon, and theme stages run after compiler normalization. Their portable diagnostics must point back to authored Stack source, including an unsatisfied `order` hint, an unavailable icon, or an unavailable theme. Normalized IR deliberately excludes source locations so it remains portable, deterministic semantic data rather than a representation of one source spelling. + +Defaults introduce another distinction: downstream code must know whether `default` or a missing icon came from omitted syntax or from an authored value. Returning only optional spans would conflate an unknown semantic identity with a known value that was omitted. + +## Decision + +Add `compile_with_source_map` and `compile_bytes_with_source_map`. A successful result pairs the unchanged normalized diagram with a Rust-only `SourceMap`. A compiler-stage error returns neither diagram nor source map. + +The sidecar provides: + +- the authored theme identifier span or `SourceOrigin::Omitted`; +- one node-icon origin for every node, addressed by globally unique node identifier; +- one layout-order origin for the diagram and every group, addressed by diagram or group identity. + +An authored node icon points to its string token. An authored theme points to its identifier. An authored order hint covers the complete statement from the `order` keyword through the closing list bracket, including intervening trivia. Omitted values use an explicit `SourceOrigin::Omitted`; an unknown node or group identity returns `None`. + +Node entries follow depth-first declaration order. Layout entries place the diagram first and groups in depth-first declaration order. Lookups use those stable vectors rather than randomized maps because Stack 1.0 limits the collections to small sizes. + +The sidecar reuses lossless tokens to recover the complete order-statement span. It has no JSON schema, is not part of portable interchange, and adds no filesystem, network, clock, or runtime dependency. + +## Alternatives Considered + +### Add spans to normalized IR + +- Pros: One object would contain semantic data and diagnostic locations. +- Cons: Portable equality would depend on source spelling and every downstream schema would inherit compiler implementation details. +- Rejected: Normalized IR must remain source-independent. + +### Return optional spans directly + +- Pros: Smaller API surface. +- Cons: `None` cannot distinguish an omitted default from an unknown semantic identity. +- Rejected: Downstream fallback diagnostics need the distinction explicitly. + +### Re-scan source in the engine + +- Pros: Keeps the compiler API smaller. +- Cons: Duplicates lexical and scope interpretation and can associate diagnostics with the wrong declaration. +- Rejected: The compiler already owns source parsing and semantic identity. + +## Consequences + +- Engine stages can emit diagnostics at stable authored ranges without changing portable IR. +- Omitted defaults are explicit and do not masquerade as missing map entries. +- The source map is available only from the additive mapped compile APIs. +- Invalid programs do not expose potentially ambiguous semantic mappings. diff --git a/docs/specs/compiler-frontend.md b/docs/specs/compiler-frontend.md index d708c2d..847365c 100644 --- a/docs/specs/compiler-frontend.md +++ b/docs/specs/compiler-frontend.md @@ -31,6 +31,7 @@ src/diagnostic.rs Portable diagnostic types and codes src/lexer.rs UTF-8 text tokenization src/lossless.rs Exact tokens, trivia, authored text, and spans src/parser.rs Recursive-descent grammar implementation +src/source_map.rs Engine-facing source origins by semantic identity src/validation.rs Semantic validation and normalization src/validation/ Focused validation unit tests src/ir.rs Renderer-independent normalized model @@ -50,12 +51,16 @@ pub fn parse_lossless(source: &str) -> LosslessParseOutput; pub fn parse_lossless_bytes(source: &[u8]) -> LosslessParseOutput; pub fn compile(source: &str) -> CompileOutput; pub fn compile_bytes(source: &[u8]) -> CompileOutput; +pub fn compile_with_source_map(source: &str) -> SourceMappedCompileOutput; +pub fn compile_bytes_with_source_map(source: &[u8]) -> SourceMappedCompileOutput; ``` `ParseOutput` contains an AST only when decoding, lexing, and parsing succeed. `CompileOutput` contains normalized IR only when all compiler-stage errors are absent. Both outputs contain diagnostics. Semantic warnings may accompany successful IR. `LosslessParseOutput` contains an exact token-and-trivia document only when decoding, lexing, and syntax parsing succeed. Its tokens retain original text and spans, while string tokens additionally expose decoded values. Reconstructing the document concatenates every token and trivia segment without normalization. Semantic validation remains a separate stage. +`SourceMappedCompileOutput` pairs successful normalized IR with a Rust-only source map. The sidecar identifies authored theme and icon values and complete layout order statements, while explicitly distinguishing omitted defaults. Compiler-stage errors suppress both IR and the source map. The normalized IR types and canonical JSON schema contain no source-map fields. + ## Code Style Prefer explicit domain types and exhaustive matches: diff --git a/src/lib.rs b/src/lib.rs index e7de7f9..7d1cac8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod ast; pub mod diagnostic; pub mod ir; pub mod lossless; +pub mod source_map; mod lexer; mod parser; @@ -44,6 +45,17 @@ pub struct LosslessParseOutput { pub diagnostics: Vec, } +/// Output of compilation with the Rust-only source-map sidecar. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceMappedCompileOutput { + /// Normalized diagram, present only when no compiler-stage error occurred. + pub diagram: Option, + /// Source map corresponding to `diagram`, absent whenever `diagram` is absent. + pub source_map: Option, + /// Lexical, syntax, semantic, and complexity diagnostics. + pub diagnostics: Vec, +} + /// Parses UTF-8 Stack source into a source-oriented AST. pub fn parse(source: &str) -> ParseOutput { match parser::parse(source) { @@ -133,6 +145,53 @@ pub fn compile_bytes(source: &[u8]) -> CompileOutput { } } +/// Parses, validates, and normalizes source with an engine-facing source map. +pub fn compile_with_source_map(source: &str) -> SourceMappedCompileOutput { + let tokens = match lexer::tokenize(source) { + Ok(tokens) => tokens, + Err(diagnostic) => { + return SourceMappedCompileOutput { + diagram: None, + source_map: None, + diagnostics: vec![*diagnostic], + }; + } + }; + let document = match parser::parse_tokens(tokens.clone()) { + Ok(document) => document, + Err(diagnostic) => { + return SourceMappedCompileOutput { + diagram: None, + source_map: None, + diagnostics: vec![*diagnostic], + }; + } + }; + let compiled = validate(&document); + let source_map = compiled.diagram.as_ref().map(|_| { + let lossless = lossless::Document::from_lexer_tokens(source, tokens); + source_map::SourceMap::from_document(&document, &lossless) + }); + + SourceMappedCompileOutput { + diagram: compiled.diagram, + source_map, + diagnostics: compiled.diagnostics, + } +} + +/// Decodes and compiles source bytes with an engine-facing source map. +pub fn compile_bytes_with_source_map(source: &[u8]) -> SourceMappedCompileOutput { + match std::str::from_utf8(source) { + Ok(source) => compile_with_source_map(source), + Err(error) => SourceMappedCompileOutput { + diagram: None, + source_map: None, + diagnostics: vec![invalid_utf8_diagnostic(source, error)], + }, + } +} + fn position_after_valid_prefix(prefix: &[u8]) -> diagnostic::SourcePosition { let source = match std::str::from_utf8(prefix) { Ok(source) => source, @@ -172,10 +231,11 @@ fn invalid_utf8_diagnostic(source: &[u8], error: std::str::Utf8Error) -> Diagnos #[cfg(test)] mod tests { use crate::lossless::TokenKind; + use crate::source_map::{LayoutScope, SourceOrigin}; use super::{ - compile, compile_bytes, parse, parse_bytes, parse_lossless, parse_lossless_bytes, - position_after_valid_prefix, validate, + compile, compile_bytes, compile_bytes_with_source_map, compile_with_source_map, parse, + parse_bytes, parse_lossless, parse_lossless_bytes, position_after_valid_prefix, validate, }; #[test] @@ -280,4 +340,123 @@ mod tests { assert!(parse_lossless(source).document.is_some()); assert!(compile(source).diagram.is_none()); } + + #[test] + fn source_map_resolves_authored_values_by_semantic_identity() { + let source = concat!( + "stack 1.0\n", + "diagram \"Mapped\" {\n", + " node root \"Root\"\n", + " group services \"Services\" {\n", + " node api \"API\" { icon \"service\" }\n", + " node worker \"Worker\"\n", + " layout {\n", + " order // Group order\n", + " [api, worker]\n", + " }\n", + " }\n", + " theme dark\n", + " layout { order [root, services] }\n", + "}\n", + ); + + let mapped = compile_with_source_map(source); + let plain = compile(source); + assert_eq!(mapped.diagram, plain.diagram); + assert_eq!(mapped.diagnostics, plain.diagnostics); + let Some(source_map) = mapped.source_map else { + return; + }; + + assert_eq!(authored_text(source, source_map.theme()), Some("dark")); + assert_eq!(source_map.node_icon("root"), Some(SourceOrigin::Omitted)); + assert_eq!(source_map.node_icon("worker"), Some(SourceOrigin::Omitted)); + assert_eq!(source_map.node_icon("missing"), None); + let Some(api_icon) = source_map.node_icon("api") else { + return; + }; + assert_eq!(authored_text(source, api_icon), Some("\"service\"")); + assert_eq!( + source_map + .node_icons() + .iter() + .map(|entry| entry.node_id.as_str()) + .collect::>(), + vec!["root", "api", "worker"] + ); + + assert_eq!( + authored_text(source, source_map.diagram_order()), + Some("order [root, services]") + ); + let Some(group_order) = source_map.group_order("services") else { + return; + }; + assert_eq!( + authored_text(source, group_order), + Some("order // Group order\n [api, worker]") + ); + assert_eq!(source_map.group_order("missing"), None); + assert!(matches!( + source_map.layout_orders()[0].scope, + LayoutScope::Diagram + )); + assert!(matches!( + &source_map.layout_orders()[1].scope, + LayoutScope::Group(identifier) if identifier == "services" + )); + + assert_eq!( + compile_with_source_map(source).source_map, + Some(source_map.clone()) + ); + assert_eq!( + compile_bytes_with_source_map(source.as_bytes()).source_map, + Some(source_map) + ); + } + + #[test] + fn source_map_distinguishes_defaults_and_rejects_error_results() { + let source = concat!( + "stack 1.0 ", + "diagram \"Default\" { ", + "group services \"Services\" { node api \"API\" } ", + "}", + ); + let output = compile_with_source_map(source); + let Some(source_map) = output.source_map else { + return; + }; + assert_eq!(source_map.theme(), SourceOrigin::Omitted); + assert_eq!(source_map.node_icon("api"), Some(SourceOrigin::Omitted)); + assert_eq!(source_map.diagram_order(), SourceOrigin::Omitted); + assert_eq!( + source_map.group_order("services"), + Some(SourceOrigin::Omitted) + ); + assert_eq!(source_map.layout_orders().len(), 2); + + for invalid in [ + "\u{feff}stack 1.0", + "stack 1.0 diagram \"Incomplete\" {", + "stack 1.0 diagram \"Duplicate\" { node api \"A\" node api \"B\" }", + ] { + let output = compile_with_source_map(invalid); + assert!(output.diagram.is_none()); + assert!(output.source_map.is_none()); + assert!(!output.diagnostics.is_empty()); + } + + let encoding = compile_bytes_with_source_map(b"stack 1.0\n\xff"); + assert!(encoding.diagram.is_none()); + assert!(encoding.source_map.is_none()); + assert_eq!(encoding.diagnostics[0].code, "STK1001"); + } + + fn authored_text(source: &str, origin: SourceOrigin) -> Option<&str> { + origin + .span() + .map(|span| &source[span.start.byte_offset..span.end.byte_offset]) + } } diff --git a/src/source_map.rs b/src/source_map.rs new file mode 100644 index 0000000..696e5af --- /dev/null +++ b/src/source_map.rs @@ -0,0 +1,235 @@ +//! Rust source-map sidecar for post-compiler diagnostics. + +use crate::ast::{self, DiagramMember, GroupMember, LayoutStatement, NodeProperty}; +use crate::diagnostic::Span; +use crate::lossless::{self, TokenKind}; + +/// Whether a semantic value was authored or supplied by a language default. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceOrigin { + /// The value was written in source at this end-exclusive span. + Authored(Span), + /// The value was omitted from source. + Omitted, +} + +impl SourceOrigin { + /// Returns the authored span, or `None` for an omitted value. + pub const fn span(self) -> Option { + match self { + Self::Authored(span) => Some(span), + Self::Omitted => None, + } + } +} + +/// Source origin for one node's semantic icon value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NodeIconSource { + /// Globally unique node identifier. + pub node_id: String, + /// Authored icon-string span or omitted default. + pub origin: SourceOrigin, +} + +/// Semantic scope of a layout block. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LayoutScope { + /// The root diagram layout. + Diagram, + /// The layout of the group with this globally unique identifier. + Group(String), +} + +/// Source origin for one scope's semantic order hint. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LayoutOrderSource { + /// Diagram or group identity. + pub scope: LayoutScope, + /// Complete authored `order` statement span or omitted value. + pub origin: SourceOrigin, +} + +/// Deterministic source locations associated with normalized semantic identities. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceMap { + theme: SourceOrigin, + node_icons: Vec, + layout_orders: Vec, +} + +impl SourceMap { + pub(crate) fn from_document(document: &ast::Document, lossless: &lossless::Document) -> Self { + let theme = document + .diagram + .members + .iter() + .find_map(|member| match member { + DiagramMember::Theme(theme) => Some(SourceOrigin::Authored(theme.identifier.span)), + _ => None, + }) + .unwrap_or(SourceOrigin::Omitted); + + let mut node_icons = Vec::new(); + let mut layout_orders = vec![LayoutOrderSource { + scope: LayoutScope::Diagram, + origin: layout_order_origin( + document + .diagram + .members + .iter() + .find_map(diagram_member_layout), + lossless, + ), + }]; + + for member in &document.diagram.members { + match member { + DiagramMember::Node(node) => node_icons.push(node_icon_source(node)), + DiagramMember::Group(group) => { + append_group_sources(group, lossless, &mut node_icons, &mut layout_orders); + } + DiagramMember::Edge(_) | DiagramMember::Theme(_) | DiagramMember::Layout(_) => {} + } + } + + Self { + theme, + node_icons, + layout_orders, + } + } + + /// Returns the authored diagram theme identifier or omitted default. + pub const fn theme(&self) -> SourceOrigin { + self.theme + } + + /// Returns node icon entries in depth-first declaration order. + pub fn node_icons(&self) -> &[NodeIconSource] { + &self.node_icons + } + + /// Finds a node's authored icon string or omitted default by node identifier. + pub fn node_icon(&self, node_id: &str) -> Option { + self.node_icons + .iter() + .find(|entry| entry.node_id == node_id) + .map(|entry| entry.origin) + } + + /// Returns layout order entries with the diagram first, then groups in depth-first order. + pub fn layout_orders(&self) -> &[LayoutOrderSource] { + &self.layout_orders + } + + /// Returns the diagram's authored order statement or omitted value. + pub fn diagram_order(&self) -> SourceOrigin { + self.layout_orders + .first() + .map_or(SourceOrigin::Omitted, |entry| entry.origin) + } + + /// Finds a group's authored order statement or omitted value by group identifier. + pub fn group_order(&self, group_id: &str) -> Option { + self.layout_orders + .iter() + .find_map(|entry| match &entry.scope { + LayoutScope::Group(identifier) if identifier == group_id => Some(entry.origin), + LayoutScope::Diagram | LayoutScope::Group(_) => None, + }) + } +} + +fn diagram_member_layout(member: &DiagramMember) -> Option<&ast::Layout> { + match member { + DiagramMember::Layout(layout) => Some(layout), + _ => None, + } +} + +fn group_member_layout(member: &GroupMember) -> Option<&ast::Layout> { + match member { + GroupMember::Layout(layout) => Some(layout), + _ => None, + } +} + +fn node_icon_source(node: &ast::Node) -> NodeIconSource { + let origin = node + .properties + .iter() + .find_map(|property| match property { + NodeProperty::Icon(icon) => Some(SourceOrigin::Authored(icon.span)), + NodeProperty::Kind(_) | NodeProperty::Detail(_) => None, + }) + .unwrap_or(SourceOrigin::Omitted); + + NodeIconSource { + node_id: node.identifier.value.clone(), + origin, + } +} + +fn append_group_sources( + group: &ast::Group, + lossless: &lossless::Document, + node_icons: &mut Vec, + layout_orders: &mut Vec, +) { + layout_orders.push(LayoutOrderSource { + scope: LayoutScope::Group(group.identifier.value.clone()), + origin: layout_order_origin(group.members.iter().find_map(group_member_layout), lossless), + }); + + for member in &group.members { + match member { + GroupMember::Node(node) => node_icons.push(node_icon_source(node)), + GroupMember::Group(child) => { + append_group_sources(child, lossless, node_icons, layout_orders); + } + GroupMember::Layout(_) => {} + } + } +} + +fn layout_order_origin( + layout: Option<&ast::Layout>, + lossless: &lossless::Document, +) -> SourceOrigin { + let Some(list) = layout.and_then(|layout| { + layout + .statements + .iter() + .find_map(|statement| match statement { + LayoutStatement::Order(list) => Some(list), + LayoutStatement::Direction(_) | LayoutStatement::RankSame(_) => None, + }) + }) else { + return SourceOrigin::Omitted; + }; + + SourceOrigin::Authored(order_statement_span(list.span, lossless.tokens())) +} + +fn order_statement_span(list_span: Span, tokens: &[lossless::Token]) -> Span { + let keyword = tokens + .iter() + .take_while(|token| token.span.end.byte_offset <= list_span.start.byte_offset) + .filter(|token| !matches!(token.kind, TokenKind::Whitespace | TokenKind::LineComment)) + .last(); + + keyword.map_or(list_span, |keyword| Span::covering(keyword.span, list_span)) +} + +#[cfg(test)] +mod tests { + use super::SourceOrigin; + + #[test] + fn source_origin_returns_only_authored_spans() { + let span = crate::diagnostic::Span::point(crate::diagnostic::SourcePosition::start()); + assert_eq!(SourceOrigin::Authored(span).span(), Some(span)); + assert_eq!(SourceOrigin::Omitted.span(), None); + } +} diff --git a/tests/compiler.rs b/tests/compiler.rs index daf93fd..055ae70 100644 --- a/tests/compiler.rs +++ b/tests/compiler.rs @@ -1,5 +1,6 @@ use stack_compiler::{ - compile, compile_bytes, diagnostic::Severity, ir, lossless::TokenKind, parse_lossless, + compile, compile_bytes, compile_with_source_map, diagnostic::Severity, ir, lossless::TokenKind, + parse_lossless, source_map::SourceOrigin, }; #[test] @@ -88,6 +89,33 @@ fn public_lossless_api_reconstructs_authored_source() { )); } +#[test] +fn public_source_map_api_distinguishes_authored_and_default_values() { + let source = concat!( + "stack 1.0\n", + "diagram \"Mapped\" {\n", + " node api \"API\" { icon \"service\" }\n", + " node worker \"Worker\"\n", + " layout { order [api, worker] }\n", + "}\n", + ); + let output = compile_with_source_map(source); + assert!(output.diagnostics.is_empty()); + let Some(source_map) = output.source_map else { + return; + }; + + assert!(matches!( + source_map.node_icon("api"), + Some(SourceOrigin::Authored(_)) + )); + assert_eq!(source_map.node_icon("worker"), Some(SourceOrigin::Omitted)); + assert!(matches!( + source_map.diagram_order(), + SourceOrigin::Authored(_) + )); +} + #[test] fn complexity_errors_and_degree_warnings_have_distinct_outcomes() { let no_nodes = compile("stack 1.0 diagram \"Empty\" {}");