Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 44 additions & 11 deletions crates/pons-core/src/evidence.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,40 @@ use serde::{Deserialize, Serialize};
pub(crate) const ASSUMPTION: &str = "Only registered rule semantics and declared parser/model coverage; no runtime safety certification";

pub(crate) fn relative_path(file: &str, root: &str) -> String {
let file = file.replace('\\', "/");
let root = root.replace('\\', "/");
let path = std::path::Path::new(&file);
let root = std::path::Path::new(&root);
relative_path_details(file, root).0
}

pub(crate) fn relative_path_details(file: &str, root: &str) -> (String, bool) {
let windows = root.as_bytes().get(1) == Some(&b':')
|| root.starts_with("\\\\")
|| (cfg!(windows) && !root.starts_with('/'));
Comment thread
hyperpolymath marked this conversation as resolved.
Resolved
Comment thread
hyperpolymath marked this conversation as resolved.
Resolved
let file = if windows {
file.replace('\\', "/")
} else {
file.into()
};
let root = if windows {
root.replace('\\', "/")
} else {
root.into()
};
// A file scan has root == file. Do not consult the validator's filesystem:
// the original checkout may no longer exist where evidence is consumed.
let base = if path == root {
root.parent().unwrap_or(root)
let base = if file == root {
root.rsplit_once('/').map_or("", |(parent, _)| parent)
Comment thread
hyperpolymath marked this conversation as resolved.
Resolved
} else {
root
root.trim_end_matches('/')
};
path.strip_prefix(base)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/")
let prefix = format!("{base}/");
let relative = if base.is_empty() && !file.starts_with('/') {
Some(file.as_str())
} else {
file.strip_prefix(&prefix)
};
(
relative.unwrap_or(&file).replace('\\', "/"),
relative.is_some(),
)
}

#[cfg(test)]
Expand All @@ -43,6 +62,20 @@ mod portable_paths {
"C:/outside/sample.php"
);
}
#[test]
fn posix_filename_boundaries_survive_single_file_scans() {
for file in [r"/tmp/a\b.py", "/tmp/a:b.py"] {
assert_eq!(
super::relative_path(file, file),
super::relative_path(file, "/tmp")
);
assert!(super::relative_path_details(file, "/tmp").1);
}
assert_eq!(
super::relative_path(r"/tmp/a\b.py", r"/tmp/a\b.py"),
"a/b.py"
);
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
Expand Down
2 changes: 1 addition & 1 deletion crates/pons-core/src/lang.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ impl Lang {
pub fn parser_version(self) -> &'static str {
match self {
Lang::Python => "0.25.0",
Lang::JavaScript => "0.25.0+jsx-keywords.2",
Lang::JavaScript => "0.25.0+jsx-keywords.3",
Lang::TypeScript | Lang::Tsx => "0.23.2",
Lang::Rust => "0.24.2",
Lang::Php => "0.25.1",
Expand Down
13 changes: 7 additions & 6 deletions crates/pons-core/src/report/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -240,9 +240,9 @@ fn validate_value(v: Value) -> Result<String> {
// is right for rendering and wrong for a join key, so it is refused.
if has_corroboration {
let file = location["file"].as_str().context("missing finding file")?;
let relative = crate::evidence::relative_path(file, root);
let (relative, stripped) = crate::evidence::relative_path_details(file, root);
ensure!(
is_root_relative(&relative),
is_root_relative(&relative, stripped),
"finding path is not inside the scan root"
);
}
Expand Down Expand Up @@ -375,9 +375,9 @@ mod tests {
// Report paths are portable data, not paths on the reader's filesystem.
// Reject drive prefixes on every host. Permit lexical parents only when
// they cannot escape the root; retain spelling so frozen fingerprints agree.
fn is_root_relative(path: &str) -> bool {
fn is_root_relative(path: &str, stripped: bool) -> bool {
let path = path.replace('\\', "/");
if path.starts_with('/') || path.as_bytes().get(1) == Some(&b':') {
if path.starts_with('/') || (!stripped && path.as_bytes().get(1) == Some(&b':')) {
return false;
}
let mut depth = 0usize;
Expand Down Expand Up @@ -410,10 +410,11 @@ mod portable_path_tests {
"sub/../../a.php",
"//server/share/a.php",
] {
assert!(!is_root_relative(path), "accepted {path}");
assert!(!is_root_relative(path, false), "accepted {path}");
}
for path in ["a.php", "sub/../a.php", "sub/./a.php"] {
assert!(is_root_relative(path), "rejected {path}");
assert!(is_root_relative(path, false), "rejected {path}");
}
assert!(is_root_relative("a:b.py", true));
}
}
58 changes: 58 additions & 0 deletions crates/pons-core/tests/parser_adversarial.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,3 +151,61 @@ fn reviewed_javascript_boundaries_preserve_valid_syntax() {
);
}
}

#[test]
fn comments_and_unicode_operator_boundaries_preserve_expression_structure() {
for newline in ["\r", "\n", "\u{2028}", "\u{2029}"] {
for (suffix, kind) in [
("(x)", "call_expression"),
(".x", "member_expression"),
("+ x", "binary_expression"),
] {
let src = SourceFile {
path: "comment.js".into(),
lang: Lang::JavaScript,
text: format!("f /* comment{newline} */ {suffix};"),
transcoded: false,
};
let tree = parse::parse(&src).unwrap();
let root = tree.root_node();
assert_eq!(
root.named_child_count(),
1,
"split expression: {}",
src.text
);
let statement = root.named_child(0).unwrap();
assert_eq!(statement.kind(), "expression_statement");
assert_eq!(
statement.named_child(0).unwrap().kind(),
kind,
"{}",
src.text
);
}
}
for space in [
"\u{a0}", "\u{1680}", "\u{2003}", "\u{2028}", "\u{2029}", "\u{202f}", "\u{205f}",
"\u{3000}", "\u{feff}",
] {
for operator in ["in", "instanceof"] {
let src = SourceFile {
path: "operator.js".into(),
lang: Lang::JavaScript,
text: format!("value\n{operator}{space}Object;"),
transcoded: false,
};
let tree = parse::parse(&src).unwrap_or_else(|error| panic!("{}: {error}", src.text));
assert_eq!(tree.root_node().named_child_count(), 1, "{}", src.text);
assert_eq!(
tree.root_node()
.named_child(0)
.unwrap()
.named_child(0)
.unwrap()
.kind(),
"binary_expression"
);
}
}
}
2 changes: 1 addition & 1 deletion docs/PLAN.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ declarations, and patch JSX identifiers to admit the existing reserved-word
terminals in that context. The pinned upstream source and generated parser
ship in `vendor/tree-sitter-javascript`; `PATCHES.adoc` records provenance
and regeneration. There is no source rewriting or TSX fallback, and recovered
error trees remain rejected. Parser receipts name `0.25.0+jsx-keywords.1`.
error trees remain rejected. Parser receipts name `0.25.0+jsx-keywords.3`.
Parser and CLI regressions cover keyword names, malformed JSX, modern resource
declarations, analysis, coverage and report validation. Supported parse failures
give partial coverage even when no file was analysed. Frozen 0.4.0 reports
Expand Down
11 changes: 10 additions & 1 deletion docs/echo-362-verification.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ The next patch extends JSX keywords to member names, removes the invalid
`export` expression alternative, and repairs CR-only and identifier-boundary
automatic semicolon handling. It includes the upstream parameter-highlight
query referenced by the grammar manifest. The receipt is
`0.25.0+jsx-keywords.2`.
`0.25.0+jsx-keywords.3`.

`pons-core` directly depends on the vendored grammar so a downstream Git or
path consumer receives the same parser. It is explicitly non-publishable
Expand All @@ -81,3 +81,12 @@ reject drive prefixes and escaping parents on every host, while permitting
non-escaping parent components. Default parse-only CLI coverage is exercised.
The generator download requires HTTPS for the original URL and redirects,
in addition to its pinned digest. Hosted PR checks record final verification.

The subsequent portability review preserves POSIX single-file basename
boundaries before display normalization. A successfully stripped scan-root
prefix distinguishes in-root colon filenames from drive-qualified paths that
were never inside the root. Both are checked independently of the reader host.
The updated Windows workspace suite passed 258 test executions; all-target
Clippy, eight parser regressions and the 116-parse upstream corpus passed.
The generator provenance is unchanged; the external scanner revision advances
with the explicit Unicode boundary repair.
7 changes: 6 additions & 1 deletion vendor/tree-sitter-javascript/PATCHES.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,16 @@ The exact Echo warrant-debugger source is a separate executed reproducer.
This locally patched source is a direct path dependency of `pons-core`.
Registry publication of `pons-core` is disabled until the patched grammar
has a distributable package; downstream Git/path consumers retain it.
The parser receipt identifies it as `0.25.0+jsx-keywords.2`. The generated
The parser receipt identifies it as `0.25.0+jsx-keywords.3`. The generated
source, patch, upstream revision and generator identity travel together;
neither a crates.io checksum nor an unmodified-upstream claim is invented.

`scripts/check-grammar-generation.sh` verifies the generator archive and
regenerates all committed generated files in Linux CI, comparing their bytes
with the shipped files. The three standard JSON metadata files are explicitly
recorded as non-JCS in `ci/json-ratchet.kyaml`; I-JSON validity remains enforced.

Revision 3 makes the identifier-continuation whitespace set explicit and
independent of the C locale. AST controls check that multiline comments before
calls, members and addition preserve one expression, including U+2028/U+2029.
The upstream 116-parse corpus remains unchanged and passes.
5 changes: 4 additions & 1 deletion vendor/tree-sitter-javascript/src/scanner.c
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,10 @@ static WhitespaceResult scan_whitespace_and_comments(TSLexer *lexer, bool *scann
// Match the grammar's identifier-continuation boundary, including escaped and
// non-ASCII continuations, instead of the locale-dependent alphabetic test.
static bool is_identifier_continue(int32_t c) {
if (c <= 0x20 || iswspace(c) || c == 0xFEFF || c == 0x2060 || c == 0x200B) return false;
// ECMAScript WhiteSpace and LineTerminator are independent of C locale.
if (c <= 0x20 || c == 0x00A0 || c == 0x1680 || (c >= 0x2000 && c <= 0x200A) ||
c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F || c == 0x3000 ||
c == 0xFEFF || c == 0x2060 || c == 0x200B) return false;
switch (c) {
case ':': case ';': case '`': case '"': case '\'': case '@': case '#':
case '.': case ',': case '|': case '^': case '&': case '<': case '>':
Expand Down
Loading