From f8becf2948b310eedd127a0e14a03c51a645a6a0 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:28:01 +0000 Subject: [PATCH] Preserve portable evidence paths and Unicode operator boundaries --- crates/pons-core/src/evidence.rs | 55 +++++++++++++++---- crates/pons-core/src/lang.rs | 2 +- crates/pons-core/src/report/validate.rs | 13 +++-- crates/pons-core/tests/parser_adversarial.rs | 58 ++++++++++++++++++++ docs/PLAN.adoc | 2 +- docs/echo-362-verification.adoc | 11 +++- vendor/tree-sitter-javascript/PATCHES.adoc | 7 ++- vendor/tree-sitter-javascript/src/scanner.c | 5 +- 8 files changed, 131 insertions(+), 22 deletions(-) diff --git a/crates/pons-core/src/evidence.rs b/crates/pons-core/src/evidence.rs index d72c7eb..ff1186b 100644 --- a/crates/pons-core/src/evidence.rs +++ b/crates/pons-core/src/evidence.rs @@ -9,21 +9,40 @@ use serde::{Deserialize, Serialize}; pub(crate) const ASSUMPTION: &str = "Only registered rule semantics and declared parser/model coverage; no runtime safety certification"; pub(crate) fn relative_path(file: &str, root: &str) -> String { - let file = file.replace('\\', "/"); - let root = root.replace('\\', "/"); - let path = std::path::Path::new(&file); - let root = std::path::Path::new(&root); + relative_path_details(file, root).0 +} + +pub(crate) fn relative_path_details(file: &str, root: &str) -> (String, bool) { + let windows = root.as_bytes().get(1) == Some(&b':') + || root.starts_with("\\\\") + || (cfg!(windows) && !root.starts_with('/')); + let file = if windows { + file.replace('\\', "/") + } else { + file.into() + }; + let root = if windows { + root.replace('\\', "/") + } else { + root.into() + }; // A file scan has root == file. Do not consult the validator's filesystem: // the original checkout may no longer exist where evidence is consumed. - let base = if path == root { - root.parent().unwrap_or(root) + let base = if file == root { + root.rsplit_once('/').map_or("", |(parent, _)| parent) } else { - root + root.trim_end_matches('/') }; - path.strip_prefix(base) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/") + let prefix = format!("{base}/"); + let relative = if base.is_empty() && !file.starts_with('/') { + Some(file.as_str()) + } else { + file.strip_prefix(&prefix) + }; + ( + relative.unwrap_or(&file).replace('\\', "/"), + relative.is_some(), + ) } #[cfg(test)] @@ -43,6 +62,20 @@ mod portable_paths { "C:/outside/sample.php" ); } + #[test] + fn posix_filename_boundaries_survive_single_file_scans() { + for file in [r"/tmp/a\b.py", "/tmp/a:b.py"] { + assert_eq!( + super::relative_path(file, file), + super::relative_path(file, "/tmp") + ); + assert!(super::relative_path_details(file, "/tmp").1); + } + assert_eq!( + super::relative_path(r"/tmp/a\b.py", r"/tmp/a\b.py"), + "a/b.py" + ); + } } #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/pons-core/src/lang.rs b/crates/pons-core/src/lang.rs index 89dd696..a7f2a84 100644 --- a/crates/pons-core/src/lang.rs +++ b/crates/pons-core/src/lang.rs @@ -46,7 +46,7 @@ impl Lang { pub fn parser_version(self) -> &'static str { match self { Lang::Python => "0.25.0", - Lang::JavaScript => "0.25.0+jsx-keywords.2", + Lang::JavaScript => "0.25.0+jsx-keywords.3", Lang::TypeScript | Lang::Tsx => "0.23.2", Lang::Rust => "0.24.2", Lang::Php => "0.25.1", diff --git a/crates/pons-core/src/report/validate.rs b/crates/pons-core/src/report/validate.rs index b1ee0ae..8055676 100644 --- a/crates/pons-core/src/report/validate.rs +++ b/crates/pons-core/src/report/validate.rs @@ -240,9 +240,9 @@ fn validate_value(v: Value) -> Result { // is right for rendering and wrong for a join key, so it is refused. if has_corroboration { let file = location["file"].as_str().context("missing finding file")?; - let relative = crate::evidence::relative_path(file, root); + let (relative, stripped) = crate::evidence::relative_path_details(file, root); ensure!( - is_root_relative(&relative), + is_root_relative(&relative, stripped), "finding path is not inside the scan root" ); } @@ -375,9 +375,9 @@ mod tests { // Report paths are portable data, not paths on the reader's filesystem. // Reject drive prefixes on every host. Permit lexical parents only when // they cannot escape the root; retain spelling so frozen fingerprints agree. -fn is_root_relative(path: &str) -> bool { +fn is_root_relative(path: &str, stripped: bool) -> bool { let path = path.replace('\\', "/"); - if path.starts_with('/') || path.as_bytes().get(1) == Some(&b':') { + if path.starts_with('/') || (!stripped && path.as_bytes().get(1) == Some(&b':')) { return false; } let mut depth = 0usize; @@ -410,10 +410,11 @@ mod portable_path_tests { "sub/../../a.php", "//server/share/a.php", ] { - assert!(!is_root_relative(path), "accepted {path}"); + assert!(!is_root_relative(path, false), "accepted {path}"); } for path in ["a.php", "sub/../a.php", "sub/./a.php"] { - assert!(is_root_relative(path), "rejected {path}"); + assert!(is_root_relative(path, false), "rejected {path}"); } + assert!(is_root_relative("a:b.py", true)); } } diff --git a/crates/pons-core/tests/parser_adversarial.rs b/crates/pons-core/tests/parser_adversarial.rs index 99e604a..6aa14ea 100644 --- a/crates/pons-core/tests/parser_adversarial.rs +++ b/crates/pons-core/tests/parser_adversarial.rs @@ -151,3 +151,61 @@ fn reviewed_javascript_boundaries_preserve_valid_syntax() { ); } } + +#[test] +fn comments_and_unicode_operator_boundaries_preserve_expression_structure() { + for newline in ["\r", "\n", "\u{2028}", "\u{2029}"] { + for (suffix, kind) in [ + ("(x)", "call_expression"), + (".x", "member_expression"), + ("+ x", "binary_expression"), + ] { + let src = SourceFile { + path: "comment.js".into(), + lang: Lang::JavaScript, + text: format!("f /* comment{newline} */ {suffix};"), + transcoded: false, + }; + let tree = parse::parse(&src).unwrap(); + let root = tree.root_node(); + assert_eq!( + root.named_child_count(), + 1, + "split expression: {}", + src.text + ); + let statement = root.named_child(0).unwrap(); + assert_eq!(statement.kind(), "expression_statement"); + assert_eq!( + statement.named_child(0).unwrap().kind(), + kind, + "{}", + src.text + ); + } + } + for space in [ + "\u{a0}", "\u{1680}", "\u{2003}", "\u{2028}", "\u{2029}", "\u{202f}", "\u{205f}", + "\u{3000}", "\u{feff}", + ] { + for operator in ["in", "instanceof"] { + let src = SourceFile { + path: "operator.js".into(), + lang: Lang::JavaScript, + text: format!("value\n{operator}{space}Object;"), + transcoded: false, + }; + let tree = parse::parse(&src).unwrap_or_else(|error| panic!("{}: {error}", src.text)); + assert_eq!(tree.root_node().named_child_count(), 1, "{}", src.text); + assert_eq!( + tree.root_node() + .named_child(0) + .unwrap() + .named_child(0) + .unwrap() + .kind(), + "binary_expression" + ); + } + } +} diff --git a/docs/PLAN.adoc b/docs/PLAN.adoc index 807e702..cd5345c 100644 --- a/docs/PLAN.adoc +++ b/docs/PLAN.adoc @@ -385,7 +385,7 @@ declarations, and patch JSX identifiers to admit the existing reserved-word terminals in that context. The pinned upstream source and generated parser ship in `vendor/tree-sitter-javascript`; `PATCHES.adoc` records provenance and regeneration. There is no source rewriting or TSX fallback, and recovered -error trees remain rejected. Parser receipts name `0.25.0+jsx-keywords.1`. +error trees remain rejected. Parser receipts name `0.25.0+jsx-keywords.3`. Parser and CLI regressions cover keyword names, malformed JSX, modern resource declarations, analysis, coverage and report validation. Supported parse failures give partial coverage even when no file was analysed. Frozen 0.4.0 reports diff --git a/docs/echo-362-verification.adoc b/docs/echo-362-verification.adoc index 0185d94..49effb1 100644 --- a/docs/echo-362-verification.adoc +++ b/docs/echo-362-verification.adoc @@ -67,7 +67,7 @@ The next patch extends JSX keywords to member names, removes the invalid `export` expression alternative, and repairs CR-only and identifier-boundary automatic semicolon handling. It includes the upstream parameter-highlight query referenced by the grammar manifest. The receipt is -`0.25.0+jsx-keywords.2`. +`0.25.0+jsx-keywords.3`. `pons-core` directly depends on the vendored grammar so a downstream Git or path consumer receives the same parser. It is explicitly non-publishable @@ -81,3 +81,12 @@ reject drive prefixes and escaping parents on every host, while permitting non-escaping parent components. Default parse-only CLI coverage is exercised. The generator download requires HTTPS for the original URL and redirects, in addition to its pinned digest. Hosted PR checks record final verification. + +The subsequent portability review preserves POSIX single-file basename +boundaries before display normalization. A successfully stripped scan-root +prefix distinguishes in-root colon filenames from drive-qualified paths that +were never inside the root. Both are checked independently of the reader host. +The updated Windows workspace suite passed 258 test executions; all-target +Clippy, eight parser regressions and the 116-parse upstream corpus passed. +The generator provenance is unchanged; the external scanner revision advances +with the explicit Unicode boundary repair. diff --git a/vendor/tree-sitter-javascript/PATCHES.adoc b/vendor/tree-sitter-javascript/PATCHES.adoc index b41c76b..feb6964 100644 --- a/vendor/tree-sitter-javascript/PATCHES.adoc +++ b/vendor/tree-sitter-javascript/PATCHES.adoc @@ -39,7 +39,7 @@ The exact Echo warrant-debugger source is a separate executed reproducer. This locally patched source is a direct path dependency of `pons-core`. Registry publication of `pons-core` is disabled until the patched grammar has a distributable package; downstream Git/path consumers retain it. -The parser receipt identifies it as `0.25.0+jsx-keywords.2`. The generated +The parser receipt identifies it as `0.25.0+jsx-keywords.3`. The generated source, patch, upstream revision and generator identity travel together; neither a crates.io checksum nor an unmodified-upstream claim is invented. @@ -47,3 +47,8 @@ neither a crates.io checksum nor an unmodified-upstream claim is invented. regenerates all committed generated files in Linux CI, comparing their bytes with the shipped files. The three standard JSON metadata files are explicitly recorded as non-JCS in `ci/json-ratchet.kyaml`; I-JSON validity remains enforced. + +Revision 3 makes the identifier-continuation whitespace set explicit and +independent of the C locale. AST controls check that multiline comments before +calls, members and addition preserve one expression, including U+2028/U+2029. +The upstream 116-parse corpus remains unchanged and passes. diff --git a/vendor/tree-sitter-javascript/src/scanner.c b/vendor/tree-sitter-javascript/src/scanner.c index 1971cfe..c8c6b63 100644 --- a/vendor/tree-sitter-javascript/src/scanner.c +++ b/vendor/tree-sitter-javascript/src/scanner.c @@ -111,7 +111,10 @@ static WhitespaceResult scan_whitespace_and_comments(TSLexer *lexer, bool *scann // Match the grammar's identifier-continuation boundary, including escaped and // non-ASCII continuations, instead of the locale-dependent alphabetic test. static bool is_identifier_continue(int32_t c) { - if (c <= 0x20 || iswspace(c) || c == 0xFEFF || c == 0x2060 || c == 0x200B) return false; + // ECMAScript WhiteSpace and LineTerminator are independent of C locale. + if (c <= 0x20 || c == 0x00A0 || c == 0x1680 || (c >= 0x2000 && c <= 0x200A) || + c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F || c == 0x3000 || + c == 0xFEFF || c == 0x2060 || c == 0x200B) return false; switch (c) { case ':': case ';': case '`': case '"': case '\'': case '@': case '#': case '.': case ',': case '|': case '^': case '&': case '<': case '>':