Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@ members = [
]
exclude = ["vendor/tree-sitter-javascript"]

[patch.crates-io]
tree-sitter-javascript = { path = "vendor/tree-sitter-javascript" }

[workspace.package]
edition = "2021"
license = "MPL-2.0"
Expand Down
12 changes: 12 additions & 0 deletions crates/pons-cli/tests/coverage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,18 @@ fn only_parse_errors_are_partial_coverage_not_no_applicable_input() {
validated.is_ok(),
"frozen parse-only report rejected: {validated:?}"
);
frozen["scanned"]["coverage_status"] = "partial".into();
frozen["evidence_contract"]["execution_outcome"] = "partial".into();
frozen["scanned"]["coverage_failures"] =
serde_json::json!(["required JavaScript coverage failed"]);
assert!(pons_core::report::validate::validate(&frozen.to_string()).is_ok());
frozen["evidence_contract"]["execution_outcome"] = "no-applicable-input".into();
assert!(pons_core::report::validate::validate(&frozen.to_string()).is_err());
let default = t.scan(&["--format", "json"]);
assert_eq!(default.status.code(), Some(2));
let default_report: Value = serde_json::from_slice(&default.stdout).unwrap();
assert_eq!(default_report["scanned"]["coverage_status"], "partial");
assert!(pons_core::report::validate::validate(&default_report.to_string()).is_ok());
report["scanned"]["coverage_status"] = "no-applicable-input".into();
report["evidence_contract"]["execution_outcome"] = "no-applicable-input".into();
assert!(pons_core::report::validate::validate(&report.to_string()).is_err());
Expand Down
8 changes: 5 additions & 3 deletions crates/pons-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
[package]
name = "pons-core"
version = "0.1.0"
publish = false
edition.workspace = true
license.workspace = true
authors.workspace = true
Expand All @@ -14,9 +15,10 @@ toml.workspace = true
serde = { workspace = true }
serde_json = { workspace = true }
tree-sitter = "=0.27.0"
# The root patch retains 0.25.0 syntax (including `using`) and fixes JSX
# keyword attributes. Provenance: vendor/tree-sitter-javascript/PATCHES.adoc.
tree-sitter-javascript = "=0.25.0"
# A direct path travels with Git dependencies; root-only patches do not.
# Registry publication requires distributing this patched grammar first.
# Provenance: vendor/tree-sitter-javascript/PATCHES.adoc.
tree-sitter-javascript = { path = "../../vendor/tree-sitter-javascript" }
tree-sitter-python = "=0.25.0"
tree-sitter-rust = "=0.24.2"
tree-sitter-typescript = "=0.23.2"
Expand Down
25 changes: 23 additions & 2 deletions crates/pons-core/src/evidence.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@
pub(crate) const ASSUMPTION: &str = "Only registered rule semantics and declared parser/model coverage; no runtime safety certification";

pub(crate) fn relative_path(file: &str, root: &str) -> String {
let path = std::path::Path::new(file);
let root = std::path::Path::new(root);
let file = file.replace('\\', "/");
let root = root.replace('\\', "/");
Comment thread
hyperpolymath marked this conversation as resolved.
Resolved
let path = std::path::Path::new(&file);
let root = std::path::Path::new(&root);
Comment thread
hyperpolymath marked this conversation as resolved.
Resolved
// A file scan has root == file. Do not consult the validator's filesystem:
// the original checkout may no longer exist where evidence is consumed.
let base = if path == root {
Expand All @@ -23,6 +25,25 @@
.to_string_lossy()
.replace('\\', "/")
}

#[cfg(test)]
mod portable_paths {

Check warning on line 30 in crates/pons-core/src/evidence.rs

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Move this test module after all other items.

See more on https://sonarcloud.io/project/issues?id=hyperpolymath_pons&issues=AaG2tmjnRixY_zePLJ_6&open=AaG2tmjnRixY_zePLJ_6&pullRequest=71
#[test]
fn windows_reports_have_the_same_keys_on_every_reader_host() {
assert_eq!(
super::relative_path(r"C:\repo\sample.php", r"C:\repo"),
"sample.php"
);
assert_eq!(
super::relative_path(r"C:\repo\sample.php", r"C:\repo\sample.php"),
"sample.php"
);
assert_eq!(
super::relative_path(r"C:\outside\sample.php", r"C:\repo"),
"C:/outside/sample.php"
);
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Target {
Expand Down
2 changes: 1 addition & 1 deletion crates/pons-core/src/lang.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ impl Lang {
pub fn parser_version(self) -> &'static str {
match self {
Lang::Python => "0.25.0",
Lang::JavaScript => "0.25.0+jsx-keywords.1",
Lang::JavaScript => "0.25.0+jsx-keywords.2",
Lang::TypeScript | Lang::Tsx => "0.23.2",
Lang::Rust => "0.24.2",
Lang::Php => "0.25.1",
Expand Down
68 changes: 56 additions & 12 deletions crates/pons-core/src/report/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,10 +180,18 @@ fn validate_value(v: Value) -> Result<String> {
} else {
"complete-within-policy"
};
// Earlier 0.4.0 readers accepted both frozen writer output and partial
// zero-file reports with explicit failures. The envelope below still
// requires execution_outcome to agree with the declared coverage status.
let frozen_partial = version == "0.4.0"
&& analysed == 0
&& !failures.is_empty()
&& s["coverage_status"] == "partial";
ensure!(
s["coverage_status"] == status,
s["coverage_status"] == status || frozen_partial,
"coverage status contradicts inventory"
);
let status = if frozen_partial { "partial" } else { status };
if analysed == 0 || !failures.is_empty() {
ensure!(
code.is_null() || code.as_u64() == Some(2),
Expand Down Expand Up @@ -234,17 +242,7 @@ fn validate_value(v: Value) -> Result<String> {
let file = location["file"].as_str().context("missing finding file")?;
let relative = crate::evidence::relative_path(file, root);
ensure!(
!std::path::Path::new(&relative).has_root()
&& !std::path::Path::new(&relative)
.components()
.any(|component| {
matches!(
component,
std::path::Component::Prefix(_) | std::path::Component::ParentDir
)
})
&& !relative.starts_with("../")
&& relative != "..",
is_root_relative(&relative),
"finding path is not inside the scan root"
);
}
Expand Down Expand Up @@ -373,3 +371,49 @@ mod tests {
assert!(!validator.is_valid(&value));
}
}

// Report paths are portable data, not paths on the reader's filesystem.
// Reject drive prefixes on every host. Permit lexical parents only when
// they cannot escape the root; retain spelling so frozen fingerprints agree.
fn is_root_relative(path: &str) -> bool {
let path = path.replace('\\', "/");
if path.starts_with('/') || path.as_bytes().get(1) == Some(&b':') {
Comment thread
hyperpolymath marked this conversation as resolved.
Resolved
return false;
}
let mut depth = 0usize;
for component in path.split('/') {
match component {
"" | "." => (),
".." => {
if depth == 0 {
return false;
}
depth -= 1;
}
_ => depth += 1,
}
}
depth > 0
}

#[cfg(test)]
mod portable_path_tests {
use super::is_root_relative;
#[test]
fn portable_paths_cannot_escape_but_may_cancel_components() {
for path in [
"C:/outside/a.php",
"C:a.php",
"C:\\outside\\a.php",
"/a.php",
"../a.php",
"sub/../../a.php",
"//server/share/a.php",
] {
assert!(!is_root_relative(path), "accepted {path}");
}
for path in ["a.php", "sub/../a.php", "sub/./a.php"] {
assert!(is_root_relative(path), "rejected {path}");
}
}
}
41 changes: 41 additions & 0 deletions crates/pons-core/tests/parser_adversarial.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,3 +110,44 @@ fn deep_ast_is_rejected_before_recursive_rules() {
.to_string()
.contains("budget exhausted"));
}

#[test]
fn reviewed_javascript_boundaries_preserve_valid_syntax() {
for text in [
"const view = <Icons.default />;",
"const view = <default.Icons.class />;",
"const value = {export: 1}; value.export;",
"export const value = 1;",
"const x = 1\rconst y = 2",
"const x = 1 // comment\rconst y = 2",
"const x = 1 /* comment\r */ const y = 2",
"const x = 1\nin_value()",
"const x = 1\ninstanceof_value()",
"const x = 1\nin$()",
"const x = 1\nin0()",
"const x = 1\ninstanceof0()",
"const x = 1\niné()",
"const flag = key\nin object;",
"const flag = value\ninstanceof Object;",
] {
let src = SourceFile {
path: "review.jsx".into(),
lang: Lang::JavaScript,
text: text.into(),
transcoded: false,
};
assert!(parse::parse(&src).is_ok(), "rejected valid syntax: {text}");
}
for text in ["const view = <C value={export} />;", "const x = export;"] {
let src = SourceFile {
path: "invalid.jsx".into(),
lang: Lang::JavaScript,
text: text.into(),
transcoded: false,
};
assert!(
parse::parse(&src).is_err(),
"accepted invalid syntax: {text}"
);
}
}
Comment thread
hyperpolymath marked this conversation as resolved.
Resolved
29 changes: 26 additions & 3 deletions docs/echo-362-verification.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,29 @@ not formal proofs of parser correctness or the coverage classification.
Pearlite is Creusot's specification language, not a third verifier.

The third-party grammar's JavaScript/JSX semantics remain a dependency
assumption (`ASSUMED`), with its exact version and lockfile checksum pinned.
No third-party code or licence was rewritten. A future grammar update must
pass these regressions and the complete existing suite before adoption.
assumption (`ASSUMED`), with upstream commit, local patch and generator digest recorded in
`vendor/tree-sitter-javascript/PATCHES.adoc`. This is modified vendored
source; its path dependency has no registry checksum. The MIT licence is
unchanged. A future grammar update must pass the regressions, regeneration
check and complete existing suite before adoption.

== Review follow-up

The next patch extends JSX keywords to member names, removes the invalid
`export` expression alternative, and repairs CR-only and identifier-boundary
automatic semicolon handling. It includes the upstream parameter-highlight
query referenced by the grammar manifest. The receipt is
`0.25.0+jsx-keywords.2`.

`pons-core` directly depends on the vendored grammar so a downstream Git or
path consumer receives the same parser. It is explicitly non-publishable
until a distributable grammar package is provided; a root-only Cargo patch
cannot guarantee registry consumers receive the repaired code.

Report validation retains both historically accepted 0.4.0 zero-file
classifications when explicit coverage failures justify partial coverage.
Coverage and execution outcomes must still agree. Portable path checks
reject drive prefixes and escaping parents on every host, while permitting
non-escaping parent components. Default parse-only CLI coverage is exercised.
The generator download requires HTTPS for the original URL and redirects,
in addition to its pinned digest. Hosted PR checks record final verification.
26 changes: 25 additions & 1 deletion scripts/check-grammar-generation.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,31 @@ jq -e --arg manifest "$vendor/Cargo.toml" '
[.packages[] | select(.name == "tree-sitter-javascript")] |
length == 1 and .[0].version == "0.25.0" and .[0].source == null and .[0].manifest_path == $manifest
' "$work/metadata.json" > /dev/null
curl --fail --silent --show-error --location --retry 3 \
# An independent workspace must also receive the modified parser, without
# repeating a [patch] section. Exercise the public library through that edge.
mkdir -p "$work/consumer/src"
cat > "$work/consumer/Cargo.toml" <<EOF
[package]
name = "pons-downstream-control"
version = "0.0.0"
edition = "2021"
[workspace]
[dependencies]
pons-core = { path = "$root/crates/pons-core" }
EOF
cat > "$work/consumer/src/main.rs" <<'EOF'
use pons_core::{lang::Lang, parse, source::SourceFile};
fn main() {
let source = SourceFile {
path: "downstream.jsx".into(), lang: Lang::JavaScript,
text: "const v = <Icons.default in='SourceGraphic' />;".into(),
transcoded: false,
};
assert!(parse::parse(&source).is_ok());
}
EOF
cargo run --offline --manifest-path "$work/consumer/Cargo.toml" --target-dir "$root/target"
curl --proto '=https' --proto-redir '=https' --fail --silent --show-error --location --retry 3 \
https://github.com/tree-sitter/tree-sitter/releases/download/v0.27.0/tree-sitter-linux-x64.gz \
-o "$work/generator.gz"
printf '%s %s\n' '20a1f39ec1c45f2211492dcb8881c802b643b554bb196869a29ac3778277fa77' "$work/generator.gz" | sha256sum --check --strict
Expand Down
20 changes: 15 additions & 5 deletions vendor/tree-sitter-javascript/PATCHES.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,24 @@

Upstream: tree-sitter/tree-sitter-javascript v0.25.0,
commit `44c892e0be055ac465d5eeddae6d3e194424e7de`.
The upstream MIT licence, Rust bindings, scanner and queries are preserved.
The upstream MIT licence and Rust bindings are preserved. Queries are copied
from that revision, including the parameter-highlight query required by the manifest.

One grammar correction: admit the existing reserved keyword terminals as
identifier aliases inside `_jsx_identifier`. JSX names do not carry
The JSX grammar correction: admit the existing reserved keyword terminals as
identifier aliases inside `_jsx_identifier`, including nested member names. JSX names do not carry
JavaScript's reserved-word exclusion. This admits SVG's `in`, keyword-named
attributes and namespaces without relaxing keywords in JavaScript statements
or attribute-value expressions. The current grammar, including modern
`using` declarations, remains in use; there is no grammar downgrade or TSX
fallback and no source rewriting.

The second patch also removes `export` from the upstream contextual-identifier
alternative, which incorrectly admitted it in JavaScript expressions. Property
names and export declarations remain valid. In the external scanner, CR joins
LF, U+2028 and U+2029 as a line terminator, including comments; `in` and
`instanceof` distinguish identifiers using the grammar's continuation boundary
instead of a locale-dependent alphabetic-only test.

Generated with the official tree-sitter CLI 0.27.0, native QuickJS runtime,
ABI 15: `tree-sitter generate --js-runtime native --abi 15`.
Append one final LF to `src/grammar.json` and `src/node-types.json`, which
Expand All @@ -28,8 +36,10 @@ reported 116 parses and zero failures. PONS adds keyword-attribute, modern
using-declaration, malformed-JSX and reserved-keyword rejection controls.
The exact Echo warrant-debugger source is a separate executed reproducer.

This locally patched source is declared through the root crates.io patch.
The parser receipt identifies it as `0.25.0+jsx-keywords.1`. The generated
This locally patched source is a direct path dependency of `pons-core`.
Registry publication of `pons-core` is disabled until the patched grammar
has a distributable package; downstream Git/path consumers retain it.
The parser receipt identifies it as `0.25.0+jsx-keywords.2`. The generated
source, patch, upstream revision and generator identity travel together;
neither a crates.io checksum nor an unmodified-upstream claim is invented.

Expand Down
5 changes: 2 additions & 3 deletions vendor/tree-sitter-javascript/grammar.js
Original file line number Diff line number Diff line change
Expand Up @@ -712,9 +712,9 @@ module.exports = grammar({
),

nested_identifier: $ => prec('member', seq(
field('object', choice($.identifier, alias($.nested_identifier, $.member_expression))),
field('object', choice($._jsx_identifier, alias($.nested_identifier, $.member_expression))),
'.',
field('property', alias($.identifier, $.property_identifier)),
field('property', alias($._jsx_identifier, $.property_identifier)),
)),

jsx_namespace_name: $ => seq($._jsx_identifier, ':', $._jsx_identifier),
Expand Down Expand Up @@ -1300,7 +1300,6 @@ module.exports = grammar({
'async',
'await',
'static',
'export',
'let',
),

Expand Down
12 changes: 12 additions & 0 deletions vendor/tree-sitter-javascript/queries/highlights-params.scm
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
(formal_parameters
[
(identifier) @variable.parameter
(array_pattern
(identifier) @variable.parameter)
(object_pattern
[
(pair_pattern value: (identifier) @variable.parameter)
(shorthand_property_identifier_pattern) @variable.parameter
])
]
)
Loading
Loading