diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..c91c3f3 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[net] +git-fetch-with-cli = true diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..fdf23d8 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,39 @@ +{ + "permissions": { + "allow": [ + "Bash(cargo check *)", + "Bash(cargo test *)", + "Bash(cargo build *)", + "Bash(cargo run *)", + "Bash(echo \"exit=$?\")", + "Read(//home/ry/src/tree-sitter-htcl/**)", + "Read(//home/ry/src/tree-sitter-htcl/bindings/**)", + "Bash(tree-sitter generate *)", + "Bash(tree-sitter parse *)", + "Bash(tree-sitter test *)", + "Read(//home/ry/src/redhawk/host/vivado/metro/ip/**)", + "Read(//home/ry/src/amd-htcl/**)", + "Bash(cargo install *)", + "Read(//home/ry/src/htcl/amd/vivado-cmd/**)", + "Read(//home/ry/src/htcl/amd/vivado-cmd/cmd/**)", + "Bash(vw ip *)", + "Read(//tmp/**)", + "Bash(vw check *)", + "Bash(grep -nA20 \"fn diagnostics\\\\|validate\\(\\\\|fn publish_diagnostics\" vw-analyzer/src/htcl_backend.rs)", + "Bash(awk '{sum += $4} END {print \"passed:\", sum}')", + "Bash(vw --help)", + "Bash(awk '{ ok+=$4; failed+=$6 } END { print ok \" passed, \" failed \" failed\" }')", + "Bash(cargo clippy *)", + "Bash(awk *)", + "Bash(/home/ry/src/vw/target/debug/vw run *)", + "Bash(python3 -c ' *)", + "Bash(pdftotext ~/doc-puddle/pg331.pdf -)", + "Bash(pdftotext ~/doc-puddle/pg442.pdf -)", + "Bash(grep *)", + "WebSearch", + "WebFetch(domain:docs.amd.com)", + "WebFetch(domain:www.xilinx.com)", + "WebFetch(domain:ecen220wiki.groups.et.byu.net)" + ] + } +} diff --git a/.github/buildomat/jobs/build-linux.sh b/.github/buildomat/jobs/build-linux.sh index 4347a75..e5490dd 100755 --- a/.github/buildomat/jobs/build-linux.sh +++ b/.github/buildomat/jobs/build-linux.sh @@ -2,11 +2,14 @@ #: #: name = "build-linux" #: variety = "basic" -#: target = "ubuntu-22.04" +#: target = "ubuntu-24.04" #: rust_toolchain = "stable" #: output_rules = [ #: "/work/release/*", #: ] +#: access_repos = [ +#: "oxidecomputer/ipe" +#: ] #: #: [[publish]] #: series = "linux" diff --git a/.github/buildomat/jobs/build.sh b/.github/buildomat/jobs/build.sh index 74e6862..f58e0d1 100755 --- a/.github/buildomat/jobs/build.sh +++ b/.github/buildomat/jobs/build.sh @@ -2,11 +2,14 @@ #: #: name = "build" #: variety = "basic" -#: target = "helios-2.0" +#: target = "helios-3.0" #: rust_toolchain = "stable" #: output_rules = [ #: "/work/release/*", #: ] +#: access_repos = [ +#: "oxidecomputer/ipe" +#: ] #: #: [[publish]] #: series = "illumos" diff --git a/.gitignore b/.gitignore index ea8c4bf..de95887 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ /target +*.jou +*.log +.srcs diff --git a/CLAUDE.md b/CLAUDE.md index c0bf84b..9016f84 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,3 +92,24 @@ project/ ├── vw.toml └── vhdl_ls.toml ``` + +## htcl language stack + +`vw-htcl` (winnow parser, AST, analysis) is the source of truth for +htcl semantics. It feeds `vw run` (interpreter front-end driving an +EDA backend), `vw analyzer` (LSP), and eventually `vw repl`. + +There is a sibling tree-sitter grammar at +`~/src/tree-sitter-htcl` (repo: +https://github.com/oxidecomputer/tree-sitter-htcl) used by editors +for syntax highlighting. **Both must stay in sync.** When changing +the htcl grammar in `vw-htcl/src/parser.rs` or `ast.rs`: + +- Update `~/src/tree-sitter-htcl/grammar.js` to match. +- Update or add corpus tests in `~/src/tree-sitter-htcl/test/corpus/`. +- Update `~/src/tree-sitter-htcl/queries/highlights.scm` if new node + types deserve distinct highlighting. +- Run `tree-sitter generate && tree-sitter test` in that repo. + +The sync contract (vw-htcl leads, divergences are documented) is +written up in `~/src/tree-sitter-htcl/README.md`. diff --git a/Cargo.lock b/Cargo.lock index 16a7be5..632b32a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -82,12 +88,60 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arboard" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" +dependencies = [ + "clipboard-win", + "image 0.25.10", + "log", + "objc2", + "objc2-app-kit", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation", + "parking_lot", + "percent-encoding", + "windows-sys 0.59.0", + "x11rb", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "auto_impl" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -118,6 +172,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.11.1" @@ -130,6 +190,21 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.57" @@ -148,6 +223,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" version = "0.4.44" @@ -201,6 +282,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "clipboard-win" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +dependencies = [ + "error-code", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -223,6 +313,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -309,6 +426,85 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.11.0", + "crossterm_winapi", + "futures-core", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "dirs" version = "5.0.1" @@ -351,6 +547,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.11.0", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -371,6 +577,12 @@ dependencies = [ "libloading", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dunce" version = "1.0.5" @@ -395,6 +607,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "enum-map" version = "2.7.3" @@ -431,12 +649,24 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "error-code" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + [[package]] name = "fdeflate" version = "0.3.7" @@ -446,6 +676,17 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -558,6 +799,104 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.4", + "windows-link", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -625,12 +964,31 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] @@ -646,6 +1004,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -757,6 +1121,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -789,7 +1159,21 @@ dependencies = [ "color_quant", "jpeg-decoder", "num-traits", - "png", + "png 0.17.16", +] + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png 0.18.1", + "tiff", ] [[package]] @@ -805,34 +1189,87 @@ dependencies = [ ] [[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "itertools" -version = "0.14.0" +name = "indicatif" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" dependencies = [ - "either", + "console", + "number_prefix", + "portable-atomic", + "unicode-width 0.2.0", + "web-time", ] [[package]] -name = "itoa" -version = "1.0.18" +name = "indoc" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] [[package]] -name = "jobserver" -version = "0.1.34" +name = "instability" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" dependencies = [ - "getrandom 0.3.4", - "libc", + "darling", + "indoc", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ipxact" +version = "0.1.0" +dependencies = [ + "quick-xml", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", ] [[package]] @@ -928,6 +1365,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -955,6 +1398,37 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "lsp-types" +version = "0.94.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66bfd44a06ae10647fe3f8214762e9369fd4248df1350924b4ef9e770a85ea1" +dependencies = [ + "bitflags 1.3.2", + "serde", + "serde_json", + "serde_repr", + "url", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "memchr" version = "2.8.0" @@ -978,16 +1452,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "netrc" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9a91b326434fca226707ed8ec1fd22d4e1c96801abdf10c412afdc7d97116e0" +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "nucleo-matcher" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf33f538733d1a5a3494b836ba913207f14d9d4a1d3cd67030c5061bdd2cac85" +dependencies = [ + "memchr", + "unicode-segmentation", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -997,6 +1513,85 @@ dependencies = [ "autocfg", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-core-graphics", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.0", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "bitflags 2.11.0", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-io-surface", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1039,7 +1634,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" dependencies = [ - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -1065,6 +1660,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pathfinder_geometry" version = "0.5.1" @@ -1102,6 +1703,26 @@ dependencies = [ "serde", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1128,7 +1749,7 @@ checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "chrono", "font-kit", - "image", + "image 0.24.9", "lazy_static", "num-traits", "pathfinder_geometry", @@ -1153,7 +1774,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ce181e3f6bf82d6c1dc569103ca7b1bd964c60ba03d7e6cdfbb3e3eb7f7405" dependencies = [ "gif", - "image", + "image 0.24.9", "plotters-backend", ] @@ -1179,6 +1800,46 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.11.0", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-pty" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a596a2b3d2752d94f51fac2d4a96737b8705dddd311a32b9af47211f08671e" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "downcast-rs", + "filedescriptor", + "lazy_static", + "libc", + "log", + "nix", + "serial2", + "shared_library", + "shell-words", + "winapi", + "winreg", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -1207,6 +1868,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.45" @@ -1228,6 +1911,27 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.11.0", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum 0.26.3", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + [[package]] name = "rayon" version = "1.11.0" @@ -1317,6 +2021,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1326,7 +2043,7 @@ dependencies = [ "bitflags 2.11.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -1336,6 +2053,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -1400,6 +2123,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -1409,12 +2143,69 @@ dependencies = [ "serde", ] +[[package]] +name = "serial2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb6ea5562eeaed6936b8b54e086aa0f88b9e5b1bef45beb038e2519fa1185b1" +dependencies = [ + "cfg-if", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shared_library" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9e7e0f2bfae24d8a5b5a66c5b257a83c7412304311512a0c054cd5e619da11" +dependencies = [ + "lazy_static", + "libc", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -1431,6 +2222,18 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" @@ -1453,19 +2256,47 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] + [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", ] [[package]] @@ -1487,7 +2318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a9a86e5144f63c2d18334698269a8bfae6eece345c70b64821ea5b35054ec99" dependencies = [ "memchr", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -1521,7 +2352,7 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -1565,6 +2396,29 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -1603,6 +2457,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.23" @@ -1635,7 +2502,7 @@ dependencies = [ "serde_spanned", "toml_datetime", "toml_write", - "winnow", + "winnow 0.7.15", ] [[package]] @@ -1644,24 +2511,179 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-lsp" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ba052b54a6627628d9b3c34c176e7eda8359b7da9acd497b9f20998d118508" +dependencies = [ + "async-trait", + "auto_impl", + "bytes", + "dashmap", + "futures", + "httparse", + "lsp-types", + "memchr", + "serde", + "serde_json", + "tokio", + "tokio-util", + "tower", + "tower-lsp-macros", + "tracing", +] + +[[package]] +name = "tower-lsp-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84fd902d4e0b9a4b27f2f440108dc034e1758628a9b702f8ec61ad66355422fa" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + [[package]] name = "ttf-parser" version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" +[[package]] +name = "tui-textarea" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5318dd619ed73c52a9417ad19046724effc1287fb75cdcc4eca1d6ac1acbae" +dependencies = [ + "crossterm", + "ratatui", + "unicode-width 0.2.0", +] + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + [[package]] name = "unicode-width" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -1678,6 +2700,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -1692,6 +2715,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1710,12 +2739,12 @@ dependencies = [ "enum-map", "fnv", "glob", - "itertools", + "itertools 0.14.0", "pad", "parking_lot", "pinned_vec", "rayon", - "strum", + "strum 0.27.2", "subst", "toml", "vhdl_lang_macros", @@ -1738,10 +2767,89 @@ dependencies = [ "camino", "clap", "colored", + "futures", + "indicatif", + "petgraph", + "serde_json", + "tokio", + "tracing-subscriber", + "vw-analyzer", + "vw-eda", + "vw-htcl", + "vw-htcl-cmd", + "vw-ip", + "vw-lib", + "vw-repl", + "vw-vivado", +] + +[[package]] +name = "vw-analyzer" +version = "0.1.0" +dependencies = [ + "async-trait", + "camino", + "serde", + "serde_json", + "tempfile", "tokio", + "tower-lsp", + "tracing", + "tracing-subscriber", + "vw-htcl", "vw-lib", ] +[[package]] +name = "vw-eda" +version = "0.1.0" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "vw-htcl" +version = "0.1.0" +dependencies = [ + "camino", + "serde", + "tempfile", + "thiserror 1.0.69", + "vw-quote", + "winnow 0.6.26", +] + +[[package]] +name = "vw-htcl-cmd" +version = "0.1.0" +dependencies = [ + "serde", + "tempfile", + "thiserror 1.0.69", + "toml", + "vw-htcl", + "winnow 0.6.26", +] + +[[package]] +name = "vw-ip" +version = "0.1.0" +dependencies = [ + "ipxact", + "quick-xml", + "regex", + "serde", + "tempfile", + "thiserror 1.0.69", + "toml", + "vw-htcl", + "vw-quote", +] + [[package]] name = "vw-lib" version = "0.1.0" @@ -1768,6 +2876,65 @@ dependencies = [ "vhdl_lang", ] +[[package]] +name = "vw-quote" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "vw-htcl", +] + +[[package]] +name = "vw-repl" +version = "0.1.0" +dependencies = [ + "arboard", + "base64", + "camino", + "crossterm", + "dirs 5.0.1", + "futures", + "libc", + "nucleo-matcher", + "ratatui", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "toml", + "tracing", + "tui-textarea", + "vw-eda", + "vw-htcl", + "vw-lib", + "vw-vivado", + "winnow 0.6.26", +] + +[[package]] +name = "vw-vivado" +version = "0.1.0" +dependencies = [ + "async-trait", + "camino", + "colored", + "libc", + "portable-pty", + "serde", + "serde_json", + "similar", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tracing", + "vw-eda", + "vw-htcl", + "vw-lib", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -1891,6 +3058,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "weezl" version = "0.1.12" @@ -2135,6 +3312,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.15" @@ -2144,6 +3330,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "wio" version = "0.2.2" @@ -2247,6 +3442,23 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix 1.1.4", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + [[package]] name = "yeslogic-fontconfig-sys" version = "6.0.0" @@ -2281,6 +3493,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" @@ -2340,3 +3572,18 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index fcfd4f0..b623931 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,23 @@ [workspace] resolver = "2" -members = ["vw-lib", "vw-cli"] +members = [ + "vw-lib", + "vw-cli", + "vw-htcl", + "vw-eda", + "vw-vivado", + "vw-analyzer", + "vw-quote", + "vw-ip", + "vw-htcl-cmd", + "vw-repl", +] [workspace.package] version = "0.1.0" edition = "2021" license = "MPL-2.0" -repository = "https://github.com/your-username/vw" +repository = "https://github.com/oxidecomputer/vw" [workspace.dependencies] # Shared dependencies @@ -24,3 +35,18 @@ url = "2.5" glob = "0.3" petgraph = "0.8.3" plotters = "0.3" +winnow = "0.6" +async-trait = "0.1" +tower-lsp = "0.20" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +futures = "0.3" +portable-pty = "0.9" +ratatui = { version = "0.29", features = ["crossterm"] } +crossterm = { version = "0.28", features = ["event-stream"] } +tui-textarea = { version = "0.7", default-features = false, features = ["crossterm", "ratatui"] } +nucleo-matcher = "0.3" +indicatif = "0.17" +similar = { version = "2.6", features = ["inline"] } +ipxact = { path = "/home/ry/src/ipe/crates/ipxact" } +#ipxact = { git = "https://github.com/oxidecomputer/ipe", branch = "ry/init" } diff --git a/docs/authoring-htcl-libraries.md b/docs/authoring-htcl-libraries.md new file mode 100644 index 0000000..3514484 --- /dev/null +++ b/docs/authoring-htcl-libraries.md @@ -0,0 +1,704 @@ +# Authoring htcl libraries + +This document is a reference for engineers writing **htcl modules +that wrap underlying EDA IP, commands, or workflows**. The intended +output of such a module is one or more `proc` declarations that +downstream code can call from a workflow script to configure +hardware, drive a build, or otherwise script an EDA tool. + +The document covers: + +1. What htcl is and what it is for. +2. The full surface of the language — syntax, semantics, attributes. +3. Where htcl behaves the same as Tcl and where it differs. +4. How to validate an htcl program with `vw`. + +## 1. What htcl is + +htcl — "**h**ardware Tcl" — is a small structured dialect of Tcl +for HDL workflow scripting. It is the language `vw` uses to drive +EDA backends (today, Vivado over a pipe) and to give engineers a +source-controlled, reviewable, tool-checkable surface for everything +that would otherwise live as ad-hoc Tcl: IP configuration, block +design construction, project setup, simulation harnessing, and +custom commands that wrap a vendor tool. + +htcl is **not** specific to any single source of those wrappers. An +htcl library may be: + +- **Hand-written** by an engineer, wrapping a Vivado/Quartus + built-in or an in-house Tcl helper with a typed, doc-commented + interface. +- **Generated** from a vendor IP-XACT description (e.g. `vw ip + generate` emits an htcl wrapper for a Xilinx IP). The result is + ordinary htcl — there is nothing IP-XACT-shaped about it once + generated. +- **Generated from anything else** that has no IP-XACT — a custom + IP repo, a curated set of Tcl recipes, a board-bring-up script. + +What unifies them is the structured `proc` surface: every wrapper +is a proc with documented keyword arguments, default values, and +constraints the analyzer can check. + +At analysis time, the syntax tree drives the LSP (`vw analyzer`) — +completion, hover, signature help, error reporting. At run time, +htcl is lowered to plain Tcl and shipped to the backend. + +### Module shape + +An htcl library is one or more `.htcl` files. Most libraries place +their entry point at a conventional path (`src/.htcl`) and +may `src`-import additional files. A `proc` declared in any +imported file becomes callable in the consumer's scope. Proc names +are flat unless wrapped in a `namespace eval` block, which groups +related helpers under a `::` prefix — see §2.10 below. + +## 2. The language + +### 2.1 File overview + +An htcl file is a sequence of statements separated by newlines or +semicolons. Each statement is one of: + +| Statement | Purpose | +|---|---| +| Comment `# ...` | Free-form comment; ignored. | +| Doc comment `## ...` | Attached to the next `proc` or proc-arg; surfaces in hover. | +| Command `name word word ...` | A call to any command (Tcl builtin, EDA builtin, or htcl proc). | +| `set ` | Variable assignment (same as Tcl). | +| `proc { args } { body }` | Structured proc declaration (the main authoring construct). | +| `src ` | Import another htcl module (htcl-specific; no Tcl analogue). | + +Whitespace is significant only as a word separator. Indentation is +free-form. + +### 2.2 Word forms + +Every command word can be written in one of three forms — the same +three Tcl supports: + +| Form | Example | Semantics | +|---|---|---| +| Bare | `foo` | A literal word. May contain `$var` and `[cmd]`. | +| Quoted | `"hello $world"` | Variable and command substitution still happen; whitespace is preserved. | +| Braced | `{a b c}` | Literal text; no substitution. | + +Inside `[ … ]` command substitution, newlines are treated as +whitespace and do not need backslash continuations. **This is the +canonical form for call sites that don't fit on one line** — wrap +the call in brackets, bind the result with `set`, and let each +keyword argument live on its own line without backslash noise: + +```htcl +set cpm5_pcie1 [ + create_cpm5_cpm_pcie1 + -cell cpm5 + -max_link_speed 32.0_GT/s + -modes PCIE +] +``` + +A bare word ending in `\` does continue onto the next line (the +classic Tcl form), but bracket-bound `set` is the preferred style. + +### 2.3 Comments and doc comments + +htcl has two distinct comment forms with different purposes: + +- **Regular comments — `# ...`.** Free-form. Use these to record + rationale at call sites, to label sections of a workflow script, + to leave notes — anything that's "for the reader." The analyzer + ignores them. +- **Doc comments — `## ...`.** Semantically significant + documentation that attaches to the next `proc` declaration or + the next proc-arg. Tooling — the LSP for hover and signature + help, and documentation generators — consumes them. Use them + only inside a library, on definitions; they serve no purpose at + call sites. + +```htcl +## Configure an AXIS register slice. ;# doc-comment on the proc +proc create_axis_register_slice { + ## Block-design cell name. ;# doc-comment on this arg + cell_name + ... +} + +# Internal streaming bus between DMA and classifier. ;# call-site +# 128-bit because the classifier hits 100 Gb/s. ;# rationale — +set dma_to_classifier [ ;# use #, not ## + create_axis_register_slice + -cell_name dma_to_classifier + -tdata_width 128 +] +``` + +Multiple `##` lines stack into one block of doc-comment text on +the item they precede. + +### 2.4 The `proc` declaration + +This is the central authoring construct. The args list inside the +first `{ … }` is *structured*: each arg is a single identifier +preceded by optional doc comments and optional `@attribute(...)` +annotations. The args grammar is: + +``` +args := arg_item* +arg_item := doc_comment* attribute* IDENT +attribute := '@' IDENT ( '(' value ( ',' value )* ')' )? +value := integer | string | ident +``` + +Example: + +```htcl +## Configure a Versal CIPS instance. +## +## Sets the requested CONFIG.* properties on the supplied block-design cell. +proc create_versal_cips { + ## Block-design cell handle to set the property on. + cell + + ## Boot the secondary PCIe controller as well as the primary. + @enum(0, 1) @default(0) boot_secondary_pcie_enable + + ## Inner dict for the PMC subsystem. + @default("") ps_pmc_config +} { + set_property -dict [list \ + CONFIG.BOOT_SECONDARY_PCIE_ENABLE $boot_secondary_pcie_enable \ + CONFIG.PS_PMC_CONFIG $ps_pmc_config \ + ] $cell +} +``` + +Notes: + +- **htcl is keyword-only.** Every proc you declare accepts its + arguments as `-flag value` pairs at the call site. There is no + positional call syntax — even an arg with no `@default` (an + implicitly-required arg) must be passed as `-arg value`. The + args list above is the **declaration order**, used by the + validator for documentation and stable diagnostics, not by Tcl + for dispatch. +- Args with no `@default` are **required**. Omitting them at a + call site is a compile-time error from the validator. +- The proc body is plain Tcl text. The body refers to each arg by + its bare name (`$cell`, `$boot_secondary_pcie_enable`, …) — the + same way it would for a standard Tcl proc with named parameters. + The lowerer wires up these locals from the caller's `-flag + value` pairs at runtime via a generated `::vw::kwargs` prelude. +- Because the keyword parse happens at runtime (inside the + wrapper), call sites work uniformly at **any** nesting: at the + top level of a file, inside another proc's body, inside a + `namespace eval`, inside a `[ ... ]` command substitution, or + through an `eval`/`uplevel`. The lowerer doesn't need to see + the call site to translate it. +- `proc` itself may be declared at any depth, but only **top-level** + proc declarations are visible to the call-site validator. Procs + defined inside another proc's body ship as raw text and miss + the kwargs-prelude treatment — avoid nested proc declarations + in htcl, or write them in raw Tcl form (`proc inner args { ... + }`). + +### 2.5 Argument attributes + +Attributes go on the line(s) before the arg's identifier. They are +parsed positionally and may stack. Values are written with strings +quoted, integers bare, identifiers bare. + +| Attribute | Meaning | Example | +|---|---|---| +| `@default()` | Default value used when caller omits this arg. Presence makes the arg optional. | `@default(0) boot_secondary_pcie_enable` | +| `@enum(, , ...)` | Caller's value must match one of the listed literals. Validated when the value is a literal. | `@enum(0, 1) enable` | +| `@range(, )` | Caller's integer value must satisfy `lo <= n <= hi`. | `@range(1, 16) num_lanes` | +| `@requires()` | This arg, if set, requires `-` to also be set. | `@requires(has_tuser) tuser_width` | +| `@conflicts()` | This arg cannot coexist with `-`. | `@conflicts(slave_mode) master_mode` | +| `@deprecated[(msg)]` | Warning at call sites; optional human message. | `@deprecated("use -mode instead") legacy_mode` | + +`@enum` and `@range` only check **literal** call-site values. A +value that is itself an interpolation (`$var`, `[cmd]`) is not +statically checkable and silently passes — the runtime sees +whatever the interpolation produces. + +### 2.5.0a Argument types + +An argument may carry a type annotation in a `: TYPE` suffix on +the arg name: + +``` +proc plumb_pin { + ## What to name the external port. + name: string + + ## Identity of the pin to make external. + pin: bd_pin +} unit { + … +} +``` + +The annotation uses the same type vocabulary as the return-type +slot (§2.5.1) — primitives (`string`, `int`, `bool`, `unit`), +newtypes (`bd_cell`, `bd_pin`, …), and generics (`list`, +`dict`, with arbitrary nesting). Compatible with the +existing attribute grammar (`@default(0) count: int`). + +Annotations are optional — untyped args still parse. The +analyzer shows annotated args as `-name: TYPE` in hover and +signature help; the validator uses them to shape-check newtype +`::repr` / `from` / `to` triplets. + +See [htcl-return-types.md](htcl-return-types.md) for the full +type vocabulary, newtype declaration syntax, and worked +examples. For values that can take one of several shapes +(e.g. heterogeneous EDA return values), see +[htcl-enums.md](htcl-enums.md) for tagged sum types with +auto-generated constructors, repr, and overload dispatch. + +### 2.5.1 Return types + +A proc may carry a return-type annotation in a 4th-word slot +between the args block and the body: + +``` +proc make_widget { @arg(name) ... } widget { + …body… +} +``` + +The annotation drives the REPL printer (the result is formatted +through the type's `repr` proc) and the analyzer's hover / +signature-help (`proc NAME → TYPE`). Procs without an annotation +parse and behave identically to today — adoption is gradual. + +Available shapes: + +- Primitives: `string`, `int`, `bool`, `unit`. Built into the + compiler; no declaration needed. +- Generics: `list`, `dict`, with arbitrary nesting. +- User newtypes: any identifier introduced via `type NAME = + UNDERLYING`, accompanied by `::repr` / `from` / `to`. + +`unit` is the type for side-effecting procs that don't return a +meaningful value (logging, configuring, connecting). The REPL +suppresses the empty Result entry on `unit`-typed expressions. + +See [htcl-return-types.md](htcl-return-types.md) for the full +type vocabulary, newtype declaration syntax, and worked examples. + +### 2.6 Call sites + +The canonical call-site shape is `set [ ]` — +bind the call's return value to a name, and let the brackets handle +multi-line wrapping. Each keyword argument goes on its own line, no +backslash continuations needed: + +```htcl +set cips [ + create_versal_cips + -name cips + -cpm_config cpm5 +] + +# 250 MHz / 195 MHz aren't in the preset list but the clock +# generator will synthesize them — chosen for the eth core. +set ps_pmc_config [ + create_versal_cips_ps_pmc_config + -cell cips + -clock_mode Custom + -design_mode 1 + -pcie_apertures_dual_enable 0 + -pcie_apertures_single_enable 0 + -pmc_crp_pl0_ref_ctrl_freqmhz 250 + -pmc_crp_pl1_ref_ctrl_freqmhz 195 + -ps_board_interface Custom + -ps_pcie1_peripheral_enable 0 + -ps_pcie2_peripheral_enable 1 + -ps_pcie_reset {ENABLE 1} + -ps_use_pmcpl_clk0 1 + -ps_use_pmcpl_clk1 1 + -ps_use_pmcpl_iro_clk 0 + -smon_alarms Set_Alarms_On + -smon_enable_temp_averaging 0 + -smon_temp_averaging_samples 0 +] +``` + +A one-line call works the same way — just don't break across lines: + +```htcl +set cpm5 [create_cpm5 -name cpm5] +``` + +Rules the validator enforces on every call to a known proc: + +- Each `-flag` must be one of the declared args. +- Each `-flag` is given exactly one value (the next word). +- Each `-flag` appears at most once (a duplicate is a warning). +- Every required arg (no `@default`) must be present. +- `@requires` / `@conflicts` relationships are checked across + present args. +- Literal values are checked against `@enum` / `@range`. +- `@deprecated` flags produce warnings. + +Calls to commands that aren't declared `proc`s in the loaded +documents are **not** validated — they're assumed to be EDA or +Tcl builtins and are passed through verbatim. + +### 2.7 Variables and substitution + +Variables work as in Tcl: + +```htcl +set ref_clk 100 +puts "ref clock is $ref_clk MHz" + +# Braces suppress substitution. +puts {$ref_clk is literal here} +``` + +`$name` references resolve against the nearest enclosing scope — +local `set`s first, then the enclosing proc's parameter list. +There is no static type checking on variable values. + +### 2.8 Imports — `src` + +```htcl +src @amd-htcl/cpm5 ;# named workspace dependency +src @amd-htcl/cips +src "lib/utils.htcl" ;# relative to the importing file +src "/abs/path/to/file.htcl" ;# absolute filesystem path +``` + +The `src` statement loads and inlines another htcl module. Path +forms: + +- `@/` — resolved via `vw.toml`'s workspace + dependencies (the same dependency resolver `vw` uses for VHDL + deps). +- Anything starting with `/` — filesystem-absolute. +- Anything else — relative to the directory of the importing file. + +The path word must be a literal (bare or quoted text with no +`$var` or `[cmd]` parts). The loader is idempotent on canonical +paths: a file imported twice loads once. + +By the time an htcl program reaches the backend, all `src` imports +have been flattened into a single Tcl stream. + +### 2.9 A complete library example + +Hand-written wrapper around an IP, with no IP-XACT involved: + +```htcl +## A minimal AXIS interface configurator. +## +## Wraps the underlying create_bd_cell and set_property calls so that +## a consumer can request a configured AXIS slice with a few keyword +## arguments. +proc create_axis_register_slice { + ## Block-design cell name to instantiate at. + cell_name + + ## Width of the data bus in bits. + @enum(8, 16, 32, 64, 128, 256, 512) @default(64) tdata_width + + ## Include byte-strobe sideband. + @enum(0, 1) @default(0) has_tkeep + + ## Width of the optional user sideband; required when -has_tuser is on. + @range(1, 32) @requires(has_tuser) tuser_width + + ## Set when a user sideband is desired. + @enum(0, 1) @default(0) has_tuser + + ## Newer designs should use -has_tuser instead. + @deprecated("use -has_tuser") legacy_tuser_mode +} { + create_bd_cell -type ip -vlnv xilinx.com:ip:axis_register_slice:1.1 $cell_name + set_property -dict [list \ + CONFIG.TDATA_NUM_BYTES [expr {$tdata_width / 8}] \ + CONFIG.HAS_TKEEP $has_tkeep \ + CONFIG.HAS_TUSER $has_tuser \ + CONFIG.TUSER_WIDTH $tuser_width \ + ] [get_bd_cells $cell_name] +} +``` + +A call site, with rationale captured in plain comments: + +```htcl +src @oxide-ip/axis + +# Internal streaming bus between the DMA and the packet classifier. +# 128-bit because the classifier hits 100 Gb/s line rate; tuser carries +# the classification verdict (5 bits today, room for one more flag). +set dma_to_classifier [ + create_axis_register_slice + -cell_name dma_to_classifier + -tdata_width 128 + -has_tkeep 1 + -has_tuser 1 + -tuser_width 6 +] +``` + +### 2.10 Namespaces — `namespace eval` + +When several procs share a logical prefix (`project::set_*`, +`ip::*`, `log::*`), wrapping them in a `namespace eval` block lets +each member be defined with a short bare name while still being +*called* under the qualified `::` form: + +```htcl +namespace eval project { + ## Set the target HDL language for new sources in a project. + proc set_target_language { + proj + @enum(VHDL, Verilog) language + } { + set_property -name TARGET_LANGUAGE -value $language -objects $proj + } + + ## Set the default library new sources land in. + proc set_default_library { + proj + @default(xil_defaultlib) library + } { + set_property -name DEFAULT_LIB -value $library -objects $proj + } +} + +# At a call site: +project::set_target_language -proj $proj -language VHDL +project::set_default_library -proj $proj +``` + +The analyzer treats each inner `proc` exactly as if it had been +written `proc project::set_target_language { ... } { ... }` at the +top level — same `@enum` / `@default` / `@requires` validation, +same hover, same signature help, same completion. The only +difference is source organization. + +Mechanics: + +- The `name` word can be a multi-segment Tcl namespace + (`namespace eval foo::bar { ... }`); the analyzer uses the + entire name as the prefix. +- `namespace eval` blocks nest. An inner `proc baz` inside + `namespace eval outer { namespace eval inner { ... } }` + registers as `outer::inner::baz`. +- A call from *inside* a namespace body to a sibling member must + still use the qualified name (no automatic same-namespace + resolution in v1). Write `project::helper $x`, not bare + `helper $x`. +- Lowering walks namespace bodies recursively, so inner procs get + their attributes stripped and the same `::vw::kwargs` runtime + prelude that top-level procs get. + +## 3. How htcl differs from Tcl + +htcl is a strict superset of the Tcl subset most engineers actually +write — anything you'd type in a Vivado console as a one-off +command parses as htcl. The structural differences come from htcl +adding new constructs and tightening the rules around `proc` +declarations. + +### 3.1 What htcl adds + +| Construct | htcl | Tcl | +|---|---|---| +| Doc comments | `## ...` carry to the next `proc` / proc-arg and feed hover. | Plain `#`; no first-class doc concept. | +| Structured `proc` args | Each arg is a doc-commented, attribute-tagged identifier. | Args are flat names or `{name default}` pairs. | +| Keyword call sites | `create_x -foo a -bar b` | Positional `create_x a b`. | +| Static validation | `@enum`, `@range`, `@requires`, `@conflicts`, etc. checked at parse-time. | None — errors only at runtime. | +| Module imports | `src @dep/file` or `src "rel/path.htcl"`. | `source ./foo.tcl`, with no dependency resolution. | +| Bracket-body line continuation | Newlines inside `[ … ]` are whitespace. | Newline terminates a command unless `\`-escaped. | + +### 3.2 What htcl restricts or interprets differently + +- **`proc` args are structured.** A v1 htcl `proc` cannot declare + its args as `{name default}` pairs or as `args` for varargs the + way pure Tcl can. Every arg is a single bare identifier, + optionally preceded by attributes. Defaults live in + `@default(...)`. +- **Required args come from the absence of `@default`.** Any arg + without `@default` is required. There is no `args` catch-all. +- **Call sites must use keyword form.** When the validator sees a + call to a known proc, positional words other than `-flag value` + pairs are reported as errors. Calls to *unknown* commands + (presumed EDA/Tcl builtins) pass through verbatim with no shape + check. +- **Doc comments are semantically significant.** `##` on a `proc` + or proc-arg is consumed by tooling — the LSP for hover, + documentation generators for output — so removing or relocating + one changes observable behavior. Regular `#` comments behave + exactly like Tcl comments. +- **`src` is parsed structurally.** The path word must be a + literal; `src $name` is rejected because the analyzer needs to + follow imports statically. +- **Top-level only for declarations and validated calls.** The + validator builds its signature table from top-level `proc` + declarations. A proc declared inside another proc's body still + parses, but its signature is not used to check call sites. + Likewise, the lowering pass rewrites top-level call sites to + known procs; calls *inside* a proc body are shipped verbatim. + Write your library entry points at the top level. + +### 3.3 What is unchanged + +Everything else is plain Tcl: + +- `$var`, `[cmd]`, `"..."`, `{...}`, backslash escapes. +- `set`, `expr`, `if`, `foreach`, `puts`, `list`, `dict`, … +- The proc body is just Tcl text. Anything you can do in Tcl + works inside a proc body; htcl makes no attempt to constrain it. + +If in doubt, write Tcl. htcl only diverges in service of the +structured proc surface; the body of every command is shipped +through to the backend as written. + +## 4. Checking an htcl program with `vw` + +`vw check` parses, validates, and reports errors and warnings for +one or more `.htcl` files without executing anything. It uses the +same analyzer pipeline the LSP uses, so a clean `vw check` means +the LSP will also be quiet. + +### 4.1 Basic invocation + +```bash +vw check src/cips.htcl +vw check src/lib/*.htcl +``` + +Output on a clean file: + +``` + Checking cips +``` + +Output with errors: + +``` +error: src/cips.htcl:42:23: value 3 for -boot_secondary_pcie_enable is not in @enum. Possible values are 0, 1 +error: src/cips.htcl:58:1: missing required argument -cell +src/cips.htcl: 2 error(s), 0 warning(s) +``` + +Each line carries an absolute file path, line, and column, in +`path:line:col: message` format. Spans inside `src`-imported files +are mapped back to their originating file, so an error in an +imported module reports the imported file's path — not the entry +point's. + +### 4.2 What `vw check` enforces + +The validator runs the rules described above: + +- **Parse errors.** Anything that doesn't lex/parse cleanly: + unterminated brace groups, missing values for `-flag` words, + malformed attributes. +- **Proc shape.** Duplicate proc declarations are an error (the + later one wins, matching Tcl's redefine semantics). +- **Call sites against known procs.** + - Unknown `-flag`: `undefined argument -. Possible values + are `. + - Missing value: `argument - is missing a value`. + - Missing required: `missing required argument -`. + - Duplicate flag (warning): `duplicate argument -`. + - `@enum` violation: `value for - is not in @enum. ...`. + - `@range` violation: `value for - is out of @range(...)`. + - `@range` on a non-integer literal: `argument - expects + an integer, found `. + - `@requires` unmet: `argument - requires - to also be + set`. + - `@conflicts` triggered: `argument - conflicts with -`. + - `@deprecated` (warning): `argument - is deprecated[: msg]`. +- **`src` imports.** Unknown dependency, missing file, non-literal + path, and parse errors inside imported files. + +### 4.3 What `vw check` does *not* enforce + +- Variable type, range, or existence inside a proc body — the + body is opaque to the analyzer in v1. +- EDA- or Tcl-builtin call shapes. A call to `set_property` or + `create_bd_cell` passes through unchecked. +- Values that go through `$var` or `[cmd]` substitution. `@enum` + and `@range` only see literal call-site words. +- Module-level public/private. Every top-level proc in every + loaded file is in scope. + +### 4.4 Related `vw` commands + +- `vw run ` — parses, validates, and executes through + the EDA backend. With `--check` it stops after the parse and + reports errors. Useful when you want to confirm a file is + shippable without spinning up the backend. +- `vw analyzer` — the LSP server (stdio). Editors point at it for + completion, hover, signature help, goto, and live error + reporting. The errors are exactly what `vw check` reports. +- `vw ip generate ` — generates an htcl wrapper + from an IP-XACT component. The generated file is itself a fully + valid htcl library; reading one is a fast way to see a real + wrapper's shape. + +### 4.5 Suggested authoring loop + +1. Sketch the proc signature: name the args, attach doc comments, + set `@default` for everything optional, mark `@enum` / + `@range` where the underlying domain is known and exhaustive. +2. Write the body in plain Tcl — `set_property`, `create_bd_cell`, + whatever the backend needs. +3. Run `vw check` to confirm the proc parses cleanly. +4. Add a call site in a separate `.htcl` file and run `vw check` + on that to confirm the validator agrees with the signature. +5. Open the file in an editor with `vw analyzer` configured for + `.htcl` and verify hover and completion behave as expected — + the doc comments you wrote are what consumers will read. +6. Once the surface looks right, run the call site through + `vw run` to see the lowered Tcl actually do something on the + backend. + +## Reference summary + +```text +File := Statement* +Statement := Command | Comment | DocComment | Proc | Src +Command := Word Word* +Word := Bare | Quoted | Braced +Comment := '#' .* NEWLINE +DocComment := '##' .* NEWLINE ; attaches to next proc / proc-arg +Proc := 'proc' Name '{' ArgList '}' '{' Body '}' +ArgList := ArgItem* +ArgItem := DocComment* Attribute* Ident +Attribute := '@' Ident ( '(' Value (',' Value)* ')' )? +Value := Integer | String | Ident +Src := 'src' PathWord +PathWord := '@''/' | '/' | '' +``` + +Canonical call-site shape: + +```text +set [ <-flag value>...] +``` + +Attributes recognized by the validator: + +```text +@default() ; default value, makes arg optional +@enum(, , ...) ; allowed literal values +@range(, ) ; integer range, inclusive +@requires() ; presence implies - present +@conflicts() ; presence forbids - +@deprecated[()] ; warns at call sites +``` + +Errors and warnings surface through: + +- `vw check ` — one-shot CLI. +- `vw run --check` — same checks, no execution. +- `vw analyzer` — same checks, live in the editor. diff --git a/docs/better-stack-trace-coverage.md b/docs/better-stack-trace-coverage.md new file mode 100644 index 0000000..37c85b5 --- /dev/null +++ b/docs/better-stack-trace-coverage.md @@ -0,0 +1,228 @@ +# Better stack-trace coverage via lowered proc-body instrumentation + +## Problem + +Some Vivado warnings and errors arrive with no stack trace because they +bypass every Tcl-level emission path our shim hooks. The canonical case +is `[IP_Flow 19-7090] Invalid parameter '…' provided, Ignoring`, emitted +from inside `set_property`'s C++ property validator. By the time those +bytes reach the worker's PTY reader, Tcl has already returned control to +the C++ caller and the Tcl call stack that produced them is gone — +neither the `puts` override nor the `send_msg_id` override fired. + +Today we handle this by **per-command instrumentation**: a wrap around +`::set_property` in `vw-vivado/shim/vivado-shim.tcl::install_set_property_context` +captures the Tcl call stack via `info frame` just before delegating into +the underlying C++ command, then ferries the captured frames to the +worker through PTY-side `__VW_CTX_*` markers. The worker tags any +Warning/Error chunks arriving while a context is active. + +This works for `set_property`. It doesn't work for any other Vivado +command that emits async warnings the same way — `create_bd_cell`, +`connect_bd_intf_net`, `validate_bd_design`, etc. Each one needs its own +wrap. + +## Proposal + +Replace per-command wraps with **universal coverage** by instrumenting +every lowered htcl proc body to maintain an explicit htcl-coordinate +stack in Tcl globals. When a warning arrives via PTY, the worker reads +the current top of that stack as the context. + +This is the moral equivalent of "what if we just fed every statement +through Rust one at a time" — it gives us full visibility into the +runtime call chain without rewriting Tcl's proc dispatch. The data +lives in Tcl globals because that's where execution actually happens, +but we control what goes in and when, and the data is the same htcl +coordinates we'd track if we were stepping each statement from Rust. + +### What the lowerer emits + +`vw-htcl/src/lower.rs::lower_proc_decl` currently emits: + +```tcl +proc configure_cips {args} { ::vw::kwargs $args {…} +…body statements at their source line numbers… +} +``` + +We'd extend it to bracket each body statement with push/pop calls: + +```tcl +proc configure_cips {args} { ::vw::kwargs $args {…} + ::vw::stack push "ip/cips.htcl:14 in ::configure_cips" + + ::vw::stack swap "ip/cips.htcl:17 in ::configure_cips" + + ::vw::stack swap "ip/cips.htcl:23 in ::configure_cips" + + … + ::vw::stack pop +} +``` + +`push` adds a new frame, `swap` replaces the current top in-place (so +we don't grow the stack one entry per statement), `pop` removes it at +proc exit. + +Top-level (non-proc) statements get the same treatment in +`dispatch_eval`'s shipped script. + +### Shim helpers + +```tcl +namespace eval ::vw::stack { + variable frames {} + proc push {frame} { + variable frames + lappend frames $frame + } + proc swap {frame} { + variable frames + if {[llength $frames] > 0} { + lset frames end $frame + } else { + lappend frames $frame + } + } + proc pop {} { + variable frames + set frames [lrange $frames 0 end-1] + } + proc snapshot {} { + variable frames + return $frames + } +} +``` + +The shim's existing `attach_stack_if_message` (puts override path) keeps +using `info frame` — it works fine. For the PTY-bypass path, we replace +the per-command marker wraps with a single hook that emits the snapshot +whenever Vivado is about to do something async. The cleanest version: +emit the snapshot **on every statement boundary**, so the worker always +has the latest context without needing per-command opt-in. + +Concretely, every `stack swap` call also writes the new frame to the +PTY as a marker: + +```tcl +proc swap {frame} { + variable frames + if {[llength $frames] > 0} { + lset frames end $frame + } else { + lappend frames $frame + } + ::vw::emit_pty_ctx_replace $frames +} +``` + +`emit_pty_ctx_replace` writes one `__VW_CTX_BEGIN__` / frames / +`__VW_CTX_READY__` group, replacing whatever the worker currently has +active. No `__VW_CTX_END__` is sent — the context is always "the most +recent statement we entered." It gets replaced on the next statement +and reset when an eval completes (worker clears on `EvalDone`). + +### Worker + +The worker already handles `__VW_CTX_BEGIN__` / `__VW_CTX_FRAME__:` / +`__VW_CTX_READY__` / `__VW_CTX_END__` markers via +`worker.rs::consume_ctx_marker` and tags warnings/errors via +`emit_pty_chunk`. No changes needed there beyond: + +- Treat absence of `__VW_CTX_END__` as "always active until eval ends." + The worker should clear `active_pty_context` on `EvalDone` to avoid a + context from one user submission contaminating the next. +- Drop `install_set_property_context` from the shim — universal + coverage subsumes it. + +### Rust-side resolver + +`vw-repl/src/app.rs::resolve_stack_frames` already rewrites +`:N in ::procname` to absolute `(file, line)` via the session's +proc table. No changes needed — the marker frames are already in that +shape. + +## Tradeoffs + +### Pros + +- **Universal coverage.** Any Vivado command emitting async warnings + gets a stack trace, not just `set_property`. We stop playing whack- + a-mole every time a new IP throws a different warning class. +- **Removes per-command wraps.** `install_set_property_context` and + any future siblings (`install_validate_bd_design_context`, etc.) + all go away. +- **Statement-precise.** Currently the tagged frame for an + IP_Flow warning points at the `set_property` call site. Under + this scheme it points at the actual `create_versal_cips` call in + the user's `configure_cips` body, because the most-recent `swap` + captured *that* statement before Tcl dispatched into the wrapper + that eventually called `set_property`. Closer to "what line of my + code is responsible." + +### Cons + +- **Codegen overhead.** Every lowered proc body grows by ~one + `::vw::stack swap` call per statement. For a file like + `vivado-cmd/module.htcl` that's significant but not catastrophic; + the strings are short and Tcl's bytecode compiler handles them + cheaply. For `cpm5/module.htcl` (~880 procs, each with ~5–200 + statements) the lowered text grows by a similar factor — measure + before / after on the `--load` cold-start time to know if it + matters. +- **PTY marker volume.** Each `swap` writes a marker group + (~3 lines) to the PTY. For an eval that fires 1000 statements, + that's 3000 marker lines to filter out on the worker side. + Cheap per line but adds up. Mitigate by only emitting markers when + *about to* call into a typed external — but that gets us back to + per-command opt-in. +- **Coupling to Tcl's eval order.** The htcl-coordinate stack only + stays accurate if every statement reaches its `swap` call. If a + Tcl `error`/`break`/`continue` jumps out of a body mid-statement, + the `pop` at proc exit cleans up, but a partially-walked body + could leave a stale frame as "current" until the next `swap` or + `EvalDone` clears it. In practice this only affects the window + between the error and the next event — same scope as the current + per-command wrap. +- **Visible inside `if` / `foreach` bodies?** Control-flow constructs + in htcl are braced Tcl scripts that Tcl evaluates internally — + the lowerer doesn't walk into braced sub-scripts today, so an + `if { … } { call X }` would only emit `swap` for the outer `if`, + not for the inner `call X`. Whether that resolution matters + depends on how often the user wants line-precise info for code + inside an `if`. Could be added later by lowering braced bodies as + scripts too. + +## When to do it + +Open question. The current per-command wrap covers `set_property`, +which empirically catches ~all the IP_Flow validation warnings the +user has hit so far. Universal coverage becomes worth the codegen +overhead when: + +- A second Vivado command starts emitting async warnings we want + traced (an obvious sign: someone adds an + `install__context` proc and we realize it's the third one). +- The `set_property` wrap starts missing cases (e.g. Vivado adds a + property-setter path that bypasses `::set_property`). +- We want statement-precise warning attribution rather than + "warning happened during a `set_property` call in proc X" — i.e. + the difference between `at ip/cips.htcl:69` (the `create_versal_cips` + call) vs `at vivado-cmd/cmd/set_property.htcl:80` (inside the + wrapper that eventually invoked `set_property`). + +Until one of those bites, the per-command wrap is the cheaper bet. +This file is the breadcrumb for when it doesn't. + +## References + +- `vw-vivado/shim/vivado-shim.tcl::install_set_property_context` — the + current per-command implementation +- `vw-vivado/src/worker.rs::consume_ctx_marker`, + `vw-vivado/src/worker.rs::emit_pty_chunk` — the marker-consumer side +- `vw-htcl/src/lower.rs::lower_proc_decl` — where the + push/swap/pop emission would go +- `vw-repl/src/app.rs::resolve_stack_frames` — the htcl-coordinate + resolver, unchanged by this proposal diff --git a/docs/htcl-enums.md b/docs/htcl-enums.md new file mode 100644 index 0000000..3a1c7d0 --- /dev/null +++ b/docs/htcl-enums.md @@ -0,0 +1,350 @@ +# Enums (tagged sum types) in htcl + +Enums are htcl's way to model values that can be one of several +distinct shapes. They're tagged unions: every value carries a +variant tag at runtime, the compiler auto-generates the +boilerplate (constructors, repr, accessors), and overloaded +handlers dispatch on the tag with no runtime string introspection +on the compiler side. + +This is the principled answer to "I have a value that's +*sometimes* a scalar and *sometimes* a nested dict" — the +canonical case being Vivado property values, where +`get_property NAME $obj` returns a string but +`get_property CONFIG.PS_PMC_CONFIG $obj` returns an embedded +property dict. + +See also: [htcl-return-types.md](htcl-return-types.md) for the +broader type system, [authoring-htcl-libraries.md](authoring-htcl-libraries.md) +for the surrounding arg/return-type annotation grammar. + +## Syntax + +Declaration: + +``` +enum Property = { + Scalar: string + Nested: dict +} + +type Properties = dict +``` + +The variants block is **brace-wrapped and newline-separated** — +the same shape as `proc {arg1; arg2}`. Each variant is +`IDENT (':' TYPE)?`; the payload type is optional, so +empty-payload variants are first-class: + +``` +enum Direction = { + North + South + East + West +} +``` + +Qualified variant types for use in arg annotations: + +``` +proc handle_prop {v: Property::Scalar} string { return "scalar: $v" } +proc handle_prop {v: Property::Nested} string { return "nested children: [llength $v]" } +``` + +The compiler sees that two `handle_prop` procs share a name, that +each first arg is a different variant of `Property`, and +synthesizes a public `handle_prop` dispatcher — no user-written +`proc handle_prop {v: Property} ...` boilerplate. + +## Runtime representation + +Every variant value is a Tcl list: + +- **With payload**: `[list ]` — two elements. + `Property::Scalar "foo"` → `[list Scalar foo]`. +- **Without payload**: `[list ]` — single element. + `Direction::North` → `[list North]`. + +The variant short-name (`Scalar`, not `Property::Scalar`) is +enough for dispatch because the dispatcher already knows which +enum it's switching on. + +## Auto-emitted machinery + +For an enum declaration the compiler emits a single +`namespace eval { … }` block. For +`enum Direction = { North; South: int; East; West }` it looks +roughly like: + +```tcl +namespace eval Direction { + # Constructors — one per variant. + proc North {} { return [list North] } + proc South {v} { return [list South $v] } + proc East {} { return [list East] } + proc West {} { return [list West] } + + # Accessors — explicit unwrap entry points wrappers use when + # bridging to extern:: calls. + proc tag {v} { return [lindex $v 0] } + proc payload {v} { return [lindex $v 1] } + + # repr — switches on tag, calls payload type's repr. Renders + # as `Variant()` for payload variants, bare `Variant` + # for empty ones. + proc repr {v} { … } + + # from / to — identity (exist so generics over enums type-check + # uniformly with newtypes). + proc from {v} { return $v } + proc to {v} { return $v } +} +``` + +**No user-written triplet is required for an enum** (newtypes +require `repr`/`from`/`to`; enums get them auto-generated). If +the user wants custom rendering, they can override the proc +post-hoc — same as any other htcl proc. + +## Bridging to extern (lowering) + +EDA builtins (`extern::create_bd_cell`, `extern::get_property`, +etc.) don't understand tagged tuples — they expect bare Tcl +primitives. Any time an enum value flows into an `extern::` call, +it has to be unwrapped first. + +**v1 policy: explicit unwrap, no auto-lowering.** Wrappers (the +procs that own the `extern::` boundary) explicitly extract the +payload via `::payload`: + +``` +proc vivado_cmd::set_string_prop {obj: bd_cell; name: string; val: Property} unit { + # Property::payload extracts the inner value; if val is + # Scalar("foo"), this yields "foo". The wrapper is responsible + # for knowing this is the right shape — the compiler doesn't + # auto-coerce. + extern::set_property -dict [list $name [Property::payload $val]] -objects $obj +} +``` + +Newtypes don't need unwrap — `bd_cell` IS a string at runtime, +so `extern::foo $cell` already passes the right thing. + +Compiler-side **auto-lowering** (walk the expression tree, find +every enum-typed value being passed to an extern, insert the +unwrap automatically) requires full type inference across +expressions and is out of scope for v1. The explicit +`::payload` form is principled (no magic at call sites) +and gives wrapper authors visibility into where lowering +happens. + +## Lifting from extern + +The other direction — taking an EDA function's raw Tcl return +value and tagging it into a typed enum — is per-function +business. `extern::get_property NAME $obj` returns a scalar; +`extern::get_property CONFIG.PS_PMC_CONFIG $obj` returns an +embedded dict. Whether a given property is one or the other is +**metadata the wrapper queries from the EDA tool** (e.g. +`extern::report_property -type`), not a shape-of-string +heuristic. + +**v1 policy: lifting lives in the wrapper, not the compiler.** +Each wrapper that returns an enum decides which variant to +construct. The compiler doesn't try to be smart — there's no +shape-guessing path in compiler-emitted code. + +To avoid every wrapper reinventing the wheel, the +`~/src/htcl/amd/vivado-cmd/lift.htcl` library provides a small +set of reusable helpers: + +``` +# Structural check: is the string a well-formed Tcl list with +# an even length and bare-ident keys? Used by wrappers that +# already have other evidence the value MIGHT be a paired dict +# and need a sanity check — NOT as a primary classifier. +proc lift::looks_like_paired_dict {raw: string} bool { … } + +# Vivado-specific: lift a property value to Property using +# `extern::report_property -type` metadata. The classifier IS +# the heuristic — but it's named, scoped, and called from one +# place instead of being baked into the compiler. +proc lift::vivado_property {obj: bd_cell; name: string; raw: string} Property { … } +``` + +Wrappers compose these. Custom cases write their own lifters — +the helper library is convenience, not a requirement. + +**Future direction**: F# data providers as inspiration for +`vw ip generate`. Given an IP-XACT schema, the generator could +emit not just wrappers but also the per-component tagging logic +— declarative schema in, typed lifting out. Worth investigating +once the v1 enum machinery is in user hands and we see which +lifting patterns actually recur. + +## Overload dispatch + +When two or more procs share a name AND each one's first arg is +declared as a different variant of the same enum, the compiler +treats them as **a single overloaded function** rather than a +duplicate-definition warning. + +``` +proc handle_prop {v: Property::Scalar} string { return $v } +proc handle_prop {v: Property::Nested} string { + set parts [list] + foreach {k val} $v { lappend parts "$k=[handle_prop $val]" } + return [join $parts ", "] +} +``` + +The compiler: + +1. **Verifies exhaustiveness** — every variant of `Property` + must have a handler. Missing variants are a hard error + pointing at the first overload, listing the gaps. +2. **Verifies tail-arg agreement** — every overload must + declare identical args after the dispatched first one + (same names, attributes, type annotations). +3. **Verifies return-type agreement** — every annotated + return must be identical. Mixing annotated and unannotated + is an error. +4. **Renames specializations** to `__handle_prop__Scalar` + and `__handle_prop__Nested` internally. User procs whose + names start with `__` are forbidden — that prefix is + reserved for compiler-emitted names. +5. **Synthesizes a public dispatcher**: + ```tcl + proc handle_prop {v args} { + switch -- [lindex $v 0] { + Scalar { return [__handle_prop__Scalar [lindex $v 1] {*}$args] } + Nested { return [__handle_prop__Nested [lindex $v 1] {*}$args] } + } + } + ``` + The payload is unwrapped before the specialization runs, so + the body of `proc handle_prop {v: Property::Scalar}` sees + `$v` as the bare string — matches Haskell `case` semantics. +6. **Registers a synthetic public signature** in the proc table + under the public name. Specializations register under their + mangled names so analyzer drill-down still finds them. + +### What's NOT allowed + +Two procs sharing a name where the first args aren't both +variants of one enum is a **hard error** ("ad-hoc overloading +not supported"). Examples: + +- `proc foo {x: int}` + `proc foo {x: string}` — different + primitives, no enum to dispatch on. +- `proc foo {x: Property::Scalar}` + `proc foo {x: Color::Red}` + — different enums. +- `proc foo {x: Property::Scalar}` + `proc foo {x: Property::Scalar}` + — duplicate variants. + +If you legitimately want a single function that handles +unrelated types, rename one of them or wrap the union in an +enum. + +## Recursive types + +Enums and the types they reference can be mutually recursive: + +``` +enum Property = { + Scalar: string + Nested: Properties +} +type Properties = dict +``` + +`Property` references `Properties`, which references `Property`. +Codegen handles this fine — Tcl resolves proc references at call +time, not parse time, so the order in which the namespaces are +emitted doesn't matter. The validator's type-decl-table +collection runs to completion before per-type checks fire, so +forward references work. + +## Worked example: `util::props` + +The motivating case. Vivado property values are heterogeneous — +some scalars (`NAME cips`), some embedded dicts +(`CONFIG.PS_PMC_CONFIG CLOCK_MODE Custom DESIGN_MODE 1 …`). + +Pre-enum (today): `util::props` returns `dict`. +Embedded dict values render as long single lines that wrap at +the terminal — visually confusing. + +With enums: + +``` +# types.htcl +enum Property = { + Scalar: string + Nested: Properties +} +type Properties = dict + +# lift.htcl — the heuristic lives in a named, scoped place. +proc lift::vivado_property {obj: bd_cell; name: string; raw: string} Property { + set kind [extern::report_property -type $obj $name] + if {$kind eq "bool" || $kind eq "string" || $kind eq "long"} { + return [Property::Scalar $raw] + } + # Composite: recurse through the embedded dict. + set inner [dict create] + foreach {k v} $raw { + dict set inner $k [lift::vivado_property $obj "$name.$k" $v] + } + return [Property::Nested $inner] +} + +# util.htcl — the wrapper just builds the typed result; the +# compiler handles the rendering via the auto-generated +# Property::repr and the monomorphized Properties::repr. +proc util::props {object: bd_cell} Properties { + set result [dict create] + foreach name [extern::list_property $object] { + set raw [extern::get_property $name $object] + dict set result $name [lift::vivado_property $object $name $raw] + } + return $result +} +``` + +In the REPL: + +``` +› util::props -object $cips + CLASS Scalar(bd_cell) + NAME Scalar(cips) + CONFIG.PS_PMC_CONFIG Nested( + CLOCK_MODE Scalar(Custom) + DESIGN_MODE Scalar(1) + PCIE_APERTURES_DUAL_ENABLE Scalar(0) + … + ) + … +``` + +— recursive structure rendered with no string-shape heuristics +in the compiler-emitted code. The `Property::repr` switch +dispatches on tag, `Properties::repr` (auto-monomorphized from +`dict`) iterates pairs and recurses. + +## Out of scope for v1 + +- **Compiler-side auto-lowering at extern:: call sites** — would + need cross-expression type inference. Wrappers explicitly + unwrap via `::payload`. +- **Ad-hoc overloading** (procs sharing a name where args aren't + variants of one enum) — hard error; add as a distinct feature + later if needed. +- **Multi-arg dispatch** (Julia-style) — first-arg dispatch only. +- **Generic enums** (`enum Result = Ok: T | Err: E`) — needs + type-parameter machinery; defer. +- **Pattern guards / nested patterns** — single arm per variant. +- **F#-style data-provider generation** for IP-XACT schemas — + declarative schema → typed lifting code is a multi-week + project of its own. Note as future direction. diff --git a/docs/htcl-return-types.md b/docs/htcl-return-types.md new file mode 100644 index 0000000..2bcfd48 --- /dev/null +++ b/docs/htcl-return-types.md @@ -0,0 +1,246 @@ +# Return-type annotations in htcl + +htcl procs may declare a return type in a 4th-word slot between the +args block and the body: + +``` +proc make_widget { @arg(name) ... } widget { + …body… +} +``` + +The annotation is purely additive — procs without it parse and run +identically to today. With it, the REPL printer and the analyzer +(hover, signature help) start treating the proc's result as +typed. + +## Syntax + +Three pieces: + +``` +proc NAME { ARGS } TYPE { BODY } +``` + +The args list and body keep their existing shapes (see +[authoring-htcl-libraries.md](authoring-htcl-libraries.md) for +the arg attribute grammar). The new `TYPE` slot is a single htcl +word: + +- A bare identifier: `string`, `int`, `bool`, `unit`, `bd_cell`, + `widget`, … +- A generic with no whitespace: `list`, + `dict`, `list>`. +- A brace-wrapped type when the expression contains spaces: + `{dict}` — the parser strips the outer braces + before type-parsing. + +### Grammar + +``` +Type ::= IDENT ('<' Type (',' Type)* '>')? +``` + +Nested generics work to arbitrary depth. + +For heterogeneous values that can take one of several distinct +shapes — e.g. Vivado property values that are sometimes scalars +and sometimes embedded dicts — use **enums** (tagged sum types). +See [htcl-enums.md](htcl-enums.md) for the full design. + +## Type vocabulary + +### Primitives (built into the compiler) + +| Type | Repr | +| --------- | ------------------------------------------------------- | +| `string` | identity | +| `int` | `[format %d $v]` | +| `bool` | `true` / `false` | +| `unit` | empty string; the REPL suppresses the Result entry | + +`unit` is the "I don't return a meaningful value" type — use it on +side-effecting procs (logging, connecting, configuring) so the +REPL stops trying to render their empty return as output. + +### Newtypes (user-declared) + +Any other type is a newtype, introduced by: + +``` +type NAME = UNDERLYING +``` + +Every newtype declaration **must** be accompanied by three procs +in a namespace matching the type name — the validator rejects the +program otherwise: + +| Proc | Signature | Purpose | +| -------------- | ---------------------------------- | -------------------------------------------- | +| `::repr` | `proc ::repr { v } string { … }` | Render an instance for display. | +| `::from` | `proc ::from { v } { … }` | Validate + lift an underlying value into T. | +| `::to` | `proc ::to { v } { … }` | Extract the underlying value back out of T. | + +The `from` proc is the one place to validate — e.g. reject +strings that don't match the Vivado-path shape `^/[\w/]+$` before +they get treated as a `bd_cell`. + +### Example: Vivado's typed handles + +The whole `bd_*` family lives in +`~/src/htcl/amd/vivado-cmd/types.htcl`: + +``` +type bd_cell = string + +proc bd_cell::repr {v} string { return $v } +proc bd_cell::from {v} bd_cell { + if {![regexp {^/[\w/]+$} $v]} { + error "bd_cell::from: '$v' is not a valid block-design path" + } + return $v +} +proc bd_cell::to {v} string { return $v } +``` + +All `bd_pin`, `bd_intf_pin`, etc. follow the same template. + +### Example: a domain newtype + +A user library can introduce its own types the same way: + +``` +type pcie_lane_count = int + +proc pcie_lane_count::repr {v} string { + return "x$v" ;# render as "x1", "x2", "x4", "x8", "x16" +} + +proc pcie_lane_count::from {v} pcie_lane_count { + if {$v ni {1 2 4 8 16}} { + error "pcie_lane_count must be one of {1 2 4 8 16}, got $v" + } + return $v +} + +proc pcie_lane_count::to {v} int { return $v } +``` + +Now a proc annotated `} pcie_lane_count {` will render `x4` in +the REPL instead of `4`. + +### Generics + +`list` and `dict` work over any composition of primitives +and newtypes — the compiler monomorphizes a `repr` proc per unique +instantiation, dispatching to the user's per-type `::repr` at +element boundaries. + +``` +proc list_of_cells {} list { return [list /a /b /c] } +``` + +The REPL invokes the compiler-generated +`list_bd_cell::repr` on the result, which iterates the list and +joins each element's `bd_cell::repr` rendering with newlines: + +``` +› list_of_cells + /a + /b + /c +``` + +For dicts the rendering is `KEY VAL` pairs, one per line: + +``` +proc props {} dict { … } + +› props -object $cips + CLASS bd_cell + NAME cips + … +``` + +## What the type drives + +| Subsystem | Behavior | +| ---------------------- | ---------------------------------------------------------------------------------------- | +| REPL result printer | Wraps the expression with the type's `repr` proc; `unit` suppresses the Result entry. | +| Analyzer hover | Shows `proc NAME → TYPE` in the hover popup. | +| Analyzer signature help | Appends ` → TYPE` to the signature label. | + +Unannotated procs keep the legacy heuristic formatter as a fallback, +so adopting annotations is gradual — annotate as you go. + +## Argument types + +Arguments use the same vocabulary as return types, declared with +a `: TYPE` suffix on the arg name: + +``` +proc plumb_pin { + ## What to name the external port. + name: string + + ## Identity of the pin to make external. + pin: bd_pin +} unit { + … +} +``` + +Rules: + +- Same grammar as return types — primitives, newtypes, and + generics with arbitrary nesting. +- Compatible with existing attributes (`@default(0) count: int`, + `@enum(Master, Slave) mode: string`). +- Optional. Untyped args still parse — adoption is gradual. + +What the annotation drives: + +- **Validator shape checks.** Newtype `::repr/from/to` procs + get a full shape check: `repr` must take `v: T` and return + `string`; `from` must take `v: ` and return `T`; + `to` must take `v: T` and return ``. Annotations + are *optional* on these procs — unannotated args/returns pass + as "trust the user". Annotated mismatches are a hard error. +- **Analyzer display.** Hover and signature help render the arg + as `-name: TYPE` instead of the bare `-name` form. + +Out of scope for v1 (future work): call-site validation +("you're passing a `string` where a `bd_cell` is expected"), +unions, and inference for unannotated args. + +## Authoring conventions + +- **Annotate as you write.** Same effort as documenting an arg, + same payoff as a TypeScript return-type hint. +- **Prefer specific named newtypes over `string`** for values + that have a well-defined shape (paths, IDs, port names). The + `from`/`to` triplet documents the invariant and the `from` + validator catches typos at the boundary. +- **Use `unit` for side-effecting procs.** Anything that calls + `set_property`, `connect_*`, `puts`, or `log::*` is almost + certainly `unit`. The REPL won't bother trying to display + whatever Tcl-internal value falls out. +- **Generics nest freely.** `dict>` is fine. + Don't be afraid to be specific. + +## Limitations (v1) + +- **No arg-type annotations yet.** Args still use only the + attribute grammar (`@default`, `@enum`, etc.). The + `::repr/from/to` validator only checks the procs EXIST, + not that their signatures match shape — that arrives with arg + types. +- **No inference.** Unannotated procs are simply untyped; we + don't walk the body to derive a return type from `return`. +- **No union or function types.** Start small; extend the grammar + as need shows up. + +For the implementation, see `vw-htcl/src/repr.rs` (codegen), +`vw-htcl/src/type_parse.rs` (mini-parser), and the design notes +in [the original plan](../docs/plans/return-types.md) if it +survives the cleanup. diff --git a/docs/new-structure.md b/docs/new-structure.md new file mode 100644 index 0000000..ff87745 --- /dev/null +++ b/docs/new-structure.md @@ -0,0 +1,136 @@ +# New Structure + +We're going to be putting together a new structure for `vw` workspaces. A vw +workspace contains + +- A VHDL design +- A Rust-based operating system driver for the VHDL hardware design +- Test suites at multiple levels + - Mixed mode analog/digital testbenches for the hardware/wire interface + - Pure digital testbenches for the VHDL design + - Pure unit and rust integration tests for the Rust driver + - Codesign tests that integrate combinations of Rust, digital and mixed mode + digital/analog tests. + +The filesystem structure that captures this is the following + +workspace-root +|- `design.htcl`: entry point for HTCL-based configuration and automation +|- `ip/**/*.htcl`: IP configuration files used by design.htcl +|- `hdl/**/*.vhd`: VHDL design sources +|- `bench/**/*.rs`: Rust-based hardware testbenches, pure digital and mixed mode +|- `driver/**/*.rs`: Rust-based kernel driver for the hardware design + +All of this machinery exists in ~/src/redhawk. But the organization is quite +haphazard. Within ~/src/redhawk most of the design and testbenches live within +host/hdl/n1 under design and bench respectively. This is basically the simulated +world. But then when we need to enter the synthesized world, we have to go to +vivado/vpk120-evb where a set of completely unmaintainable TCL scripts smash +a vivado project onto host/hdl/n1/design that's locally symlinked into +vivado/vpk120-evb. Within vivado/ there is also the metro folder which is a +vivado project for our custom metro motherboard with a versal on it, as opposed +to the vpk120-evb which is a vivado project for an AMD evaluation board. + +The bottom line is redhawk has become a hot mess and we're going to start +incrementally pulling redhawk sources here into metroid using the organization +I described above. + +Some other things to be aware of are: + +## Anodizer + +Much of our test infra depends on anodizer (at ~/src/anodizer) +which is machinery for generating Rust structures that correspond to VHDL +records. + +## Rust Co-sim + +We have developed Rust-based cosimulation machinery (at ~/src/rust-cosim). This +allows us to define our test benches in Rust instead of VHDL. Recently, this +machinery grew support for testing VHDL entities directly in rust, without +having to build a VHDL test bench wrapper. This is the path forward, but we +still have many test benches with explicit VHDL testbench harnesses that are +driven by Rust. + +## RSF + +Eventually we want to generate RSF specifications (~/src/rsf) for registe +interfaces. Right now these are manually maintained between our VHDL register +interfaces and our Rust kernel drivers. + +## Builds from the bottom up + +One of the goals here is to build form the bottom up. + +1. Configure IP, generate wrappers and synthesize IP. +2. Elaborate/Synthesize/implement VHDL which depends on IP synth and wrappers + from (1) +4. Generate RSF specs from VHDL (this has yet to be done, maybe with anodizer + later?) +5. Build rust testbenches using anodizer +6. Build driver code using cargo with a build.rs that pulls in generated RSF + and anodizer artifacts (if/when needed) + +A lot of the machinery here has yet to be built, but this is the overall flow +we are looking for, and for it to be completely automatic. Data structures +should not be manually maintaind across the hardware/software boundary. + +## Vhdl-ls + +There is currently a strange relationship between vhdl-ls and vw, in that vw +uses vhdl-ls config to find VHDL files. I want to stop that. I'd like vhdl-ls +to get it's config from vw and not the other way around. I would even like to +explore embedding vhdl-ls as a library and have the vw LSP be the entry point +for HTCL, VHDL and Rust sources combined (also need to figure out the Rust +side of that). I've put the vhdl-ls sources at ~/src/rust_hdl. + +To further drive home the point, the structure described at the beginning of +this document means that vw knows where ALL files are according to that +structure, so we don't need to glean it from other configs. + +## Where to start + +### 1. Pull in design code and synthesize it + +I would like to start by pulling in our VHDL design code and then synthesize it. +This will involve + +1. Copying redhawk hdl design code into the hdl directory +2. Adding a `vhdl_dependency_sources` function to the vw htcl module. This will + be rust under the hood that we call via our RPC mechanism that returns all + the VHDL sources VW has pulled in as dependencies. +3. Adding a `vhdl_design_sources` function to the vw htcl module. This will be + rust under the hood that we call via our RPC mechanism that finds all the + design sources in the vw workspace and returns them as a list. +4. Extending design.htcl to synthesize our VHDL design sources together with + our VHDL dependency sources. + +### 2. Sort out the vw/vhdl-ls relationship + +Now that (1) is done and we can synthesize our VHDL design sources, we need +to get vhdl-ls working. Let's explore having our existing vw LSP use vhdl-ls +as a library to see if that is a reasonable approach. Something else to +consider is using vhdl-ls standalone, but figure out a way for vw to provide +it with a dynamic configuration source. + +### 3. Start to bring over test benches + +We've got quite a few test benches to bring over. This may be one of the +more compex tasks. I'd like them to be runnable with `vw bench` and have +the same nice interface we worked to develop for `vw test` for HTCL tests. +This will take a fair amount of iteration to get right and to fully integrate +anodizer, rust-cosim and mixed-mode simulation support that uses Xyce under +the hood. There are some heavyweight C/C++ dependencies lurking beneath here +that we want to integrate in a way that can capture all the build and runtime +complexity there in a reasonable way that does not make using the testbenches +a constant pain. + +### 4. Bring the driver over + +We can start by just bringing the rhdrv cargo workspace over from redhawk. The +harder part is going to be thinking about how we integate this with everything +else. e.g. generating RSF specs from VHDL code, propagating those up to a place +where build.rs can capture them and having all that be automatic. Automatic +means robust change and rebuild detection as well. + + diff --git a/docs/whypoints.md b/docs/whypoints.md new file mode 100644 index 0000000..c763de9 --- /dev/null +++ b/docs/whypoints.md @@ -0,0 +1,74 @@ +# Why + +Why HTCL + +1. Structured, documented discoverable interface. +2. Catch common programming errors at analysis time. + +The H stands for holomorphic, meaning this type of TCL captures the whole +shape of a tool's interface. + +## Notes + +1. Documentation versus engineering reality + - There is always a detla between Xilinix product guides and the configuation + surface reality. + - DCMAC `tx_data_out_*` is an example of this. It's not mentioned anywhere in + PG 369. Other examples are abundant. + - Tightening up PDF documentation is not the answer. Decoupled PDF as a means + of documenting an engineering interface is a failure by design. + - The interface that engineers actually use must itself be documented, it's + only way to do this. + +2. Structured interfaces + - Configuring IP is complex, both in the number of parameters available and + how those paramters are structured as an overall configuration. + - Structured interfaces make it clear what configuration opions there are + and how they can be composed. + - Structured interfaces enforce structurally correct configurations by + construction, e.g. they make it impossible to compose structurually invalid + configurations. + - Building structure into interfaces empowers analyzers to catch many clases + of bugs. + - Incorrect assignment of values can be caught by the type system through + enums and composite types. + - Subtle issues like multiple assignment of the same paramter can be caught + at compile/analysis time rather than runtime (actually *running* an IP + configuration and synthesis script can take hours) + - Configuration key typos manifest as compile/analysis time errors. + - Structured interfaces enable discoverability, through tools like LSPs as + well as documentation generators. When configuration takes place through + functions with well documented arguments, the design surface of interest + is readily discoverable by the engineer. And critically, there are no more + guessing games on what the right paramter actually is. + - The analyzer can catch and warn about unused variables. Forgetting to + actually use a variable can lead to subtle bugs that can take hours if not + days to catch for complex designs. + - Configuration options as typed enumerations is extremely powerful + - If a config option just takes a string, it's + 1. Not at all clear what the valid options are + 2. Even if you think you know the valid options, it's easy to be wrong or + make a type + - Typed enumerations take away both problems entirely. + - They structually define what values are valid, making them *discoverable* + - This means analyzers can catch invalid issues, not the runtime after an + hour of running. + - It provides a natural surface for documenting the input alternatives. + +3. Catch common programming errors at analysis time + - Design build scripts can take an enormous amount of time to run. + - There are many things that can be caught by basic analysis and not having + to run them at all. + - Functions that declare a return type but: + - Return nothing + - Return the wrong type + - Don't _always_ return the right type + - Passing incorrect arguments types to procs + - Passing the wrong arguments to procs + - Calling undefined procs + - Refrencing undefined variables + - Sourcing dependencies that don't exist + - Supplying enumeration values as strings with typos + - Catching these hours into a built is a punch in the face that is completely + unnecessary and avoidable. + - HTCL catches all of these at analysis time in seconds. diff --git a/projects/htcl-project-plan.md b/projects/htcl-project-plan.md new file mode 100644 index 0000000..463d73e --- /dev/null +++ b/projects/htcl-project-plan.md @@ -0,0 +1,1736 @@ +# vw: extending for HDL workflow scripting + +## Audience and intent + +This document is a working plan for extending [`vw`](https://github.com/oxidecomputer/vw) +with first-class support for HDL workflow scripting: a structured TCL dialect +("htcl"), a workflow-aware analyzer (LSP), an interactive REPL, and a +Vivado-driving executor. The audience is Claude Code working with the +author. Treat it as a living spec — open questions are called out +explicitly; close them with the author before locking in design decisions. + +## Goal + +The underlying purpose of this work is **complexity management for HDL +designs**: making IP configuration and workflow scripting first-class +source-controlled artifacts that engineers can read, write, review, and +evolve over years. See "Strategic context" below for the full framing. +The concrete capabilities the project adds to `vw`: + +1. **Provide a best-in-class interactive experience for HDL workflow + code, in both the editor and a REPL.** This is the primary goal. + Completion, hover, diagnostics, and navigation should match what a + Rust or TypeScript developer expects from their IDE — and the same + capabilities should be available in an interactive shell that + replaces Vivado's TCL console. Both surfaces consume the same + analysis backend (`vw-htcl`), so a feature built for one is + available to the other for free. Every other language-design + decision in this document is partly in service of this goal — the + parser, the proc grammar, the module system, and the reuse of + `vw`'s dependency resolver all exist in forms designed to be + statically analyzable. +2. **Establish a unified multi-language LSP for the HDL workflow.** + `vw analyzer` is designed from day one as a multi-language language + server. htcl is the first language wired up (and the focus of v1); + VHDL is the planned second, initially via a `vhdl_ls` proxy and + eventually via direct integration with Oxide's developing VHDL + frontend. The architecture (a `LanguageBackend` abstraction and + per-file dispatch) is in place from the initial analyzer phase even + while only htcl is wired up. See "LSP design" for the full + treatment. +3. Provide an ergonomic dialect of TCL ("htcl") for HDL workflow + scripting, with first-class support for structured proc + declarations, modules, and dependencies resolved via `vw`. This + dialect is not typed in v1; the structural improvements + (per-argument doc comments, attributes like `@default` / `@enum` / + `@required`, real imports) deliver most of the value without + committing to a type system before we know what shape it should + take. +4. Execute htcl by talking to Vivado's built-in TCL interpreter over a + pipe, with a thin TCL shim on the Vivado side. +5. Stay vendor-aware: Vivado first, but the architecture should + accommodate Quartus and other backends later. + +This is an alternative to `set_property -dict {...}` bag-of-strings IP +config, ad-hoc `source [file join $::ROOT ...]` module loading, and +Vivado's generally unpleasant interpreter as a development environment. + +## Strategic context: complexity management + +The underlying problem this project addresses is that today's IP +integration workflow is intrinsically lossy with respect to source +control. Integrating IP into a design currently means reading user +guides and architecture manuals, then mapping what's learned into +GUI-based configuration in Vivado. The connection between official +documentation and GUI configuration is ambiguous and — critically — +not reproducible from an engineering-process perspective. The +artifacts that end up in source control (TCL block design exports, +generated wrappers, project files) don't capture how those +configurations were created or why specific parameterizations were +chosen. There's nowhere in the workflow to record rationale, no way to +review configuration changes the way code is reviewed, and no +mechanism to evolve a design over years and remember why it looks the +way it does. + +htcl is a complexity-management tool first, and a TCL replacement +second. Configuration becomes textual source code: reviewable, +diffable, doc-commentable, version-controlled, and analyzed by +tooling. Rationale lives next to the configuration it explains. The +artifact in source control is the authoritative record of *what was +chosen and why*, not a lossy projection of decisions made in a GUI. + +### Conceptual layering: specification, interface, instantiation + +These are three distinct things. Confusing them leads to bad design +decisions, so they're named explicitly here: + +**IP specification (IP-XACT).** Describes what an IP *is* — its full +parameter space, the parameterized port set, the parameterized memory +map, and the relationships between configuration choices and the +resulting structure. The specification is static; it covers all valid +configurations. IP-XACT is the format vendors use for this. + +**Configuration interface (htcl).** A means to invoke an IP at a +specific configuration, with human ergonomics. An htcl wrapper is a +proc whose arguments are the IP's parameters and whose body emits the +underlying `create_ip` / `set_property` calls. The htcl proc *is not* +a description of the IP — it's a means to pick a configuration and +record the rationale for that pick. Different layer entirely from +IP-XACT. + +**Instantiation (RTL + memory map).** What you get when you run a +configuration interface at chosen parameter values: a specific VHDL +entity (the wrapper Vivado generates) and a specific memory map (RSF, +in our case). These are the concrete artifacts at one configuration. + +The lifecycle is: + +1. *Specification* (IP-XACT, authored by the IP vendor): all valid + configurations and their resulting structure. +2. *Configuration interface* (htcl wrapper, generated from IP-XACT by a + sideband tool): the ergonomic surface for picking a configuration. +3. *Configuration choice* (htcl call site, hand-written by the + engineer): specific parameter values, with doc comments capturing + rationale, source-controlled and reviewed. +4. *Instantiation* (generated VHDL + RSF, produced by running htcl + through Vivado): the actual artifacts at the chosen configuration. + +htcl sits squarely at layer 2, with call sites at layer 3. It does +*not* attempt to subsume IP-XACT (layer 1) or replace generated RTL +and RSF (layer 4). The value of htcl is in giving layer 3 a +first-class, source-controlled, tool-analyzable form. + +### The Vivado-team pitch + +There is an active conversation with the Vivado team about Xilinx +publishing htcl configuration interfaces alongside the IP-XACT they +already publish. The pitch: + +> IP-XACT is your specification format and stays the source of truth. +> What's missing is a published *configuration interface* — a layer +> where engineers can record what configuration they chose and why, in +> a form that is source-controllable, reviewable, and analyzable by +> tooling. Today engineers do this in GUIs, and the resulting TCL +> dumps don't capture intent. htcl wrappers, generated from your +> IP-XACT, fill that gap. You keep the specification; we get a +> rationale-preserving configuration layer that bridges to your +> existing pipelines unchanged. + +This is a smaller, more defensible pitch than "replace IP-XACT with +htcl." htcl complements IP-XACT; it doesn't compete with it. The +showcase that earns the conversation is a set of generated htcl +wrappers that a Vivado engineer would be happy to publish — ergonomic, +documented, idiomatic. + +### IP as distributed packages + +IP configuration interfaces live in packages that vw resolves as +ordinary dependencies. A Xilinx-published `xilinx-ip` package +(generated from Xilinx's IP-XACT) contains `.htcl` wrappers for each +IP. A custom-IP repository at Oxide ships its own htcl wrappers, +generated from its own IP-XACT. A third party publishes wrappers for +their IP. Consumers add the relevant package to `vw.toml` and `src +@xilinx-ip/axis_register_slice` works the same as any other import. + +There is no IP database, no central registry, no special-case +infrastructure. Repositories of htcl distributed through vw +dependencies *are* the catalog of available configuration interfaces, +decentralized by construction. This matches how the Rust crate +ecosystem works and how Oxide's existing vw-managed VHDL dependencies +work. + +### Scope of htcl as a configuration interface + +htcl describes how to *invoke* an IP. It does not describe the IP's +ports, the IP's memory map, or the IP's parameterized structure — +those are IP-XACT's job, and the artifacts at any specific +configuration come out the other side as generated RTL and RSF. + +What an htcl wrapper proc declares: + +- Parameter names, types, defaults, constraints, and inter-parameter + dependencies — the configuration interface itself. +- Doc comments on each parameter (sourced from IP-XACT descriptions + when generated). +- Doc comments on the wrapper as a whole. + +What an htcl wrapper proc *emits* (in its body): + +- `create_ip` and `set_property` calls that hand the configuration + choice to Vivado. +- Optionally, directives that influence Vivado's wrapper generation + (see "Wrapper documentation" below). + +What an htcl wrapper proc does *not* contain: + +- Port lists. The ports of any specific instantiation come from the + generated VHDL/Verilog wrapper. The space of possible ports across + all configurations is in the IP-XACT specification. +- Memory maps. The register interface of any specific instantiation + comes from RSF generated by an IP-XACT-aware pipeline (see "RSF + generation" below). The space of possible memory maps is in the + IP-XACT specification. + +This scope is the point: htcl is small, focused on the human-ergonomic +configuration layer, and stays out of the description and +instantiation layers where IP-XACT and generated artifacts already +serve well. + +### RSF generation + +Software needs to know the register map of any IP it talks to. RSF is +Oxide's register-spec format and the natural target for this +information. + +The RSF for a specific IP instantiation is a function of two inputs: +the IP-XACT specification (which describes the parameterized memory +map) and the chosen configuration values (which pin the parameters). +The pipeline is: + +``` +IP-XACT spec + chosen parameter values --> RSF for this instance +``` + +This pipeline is not part of htcl, vw, the analyzer, or the REPL. It +is its own tool (call it `ipxact2rsf` or whatever the team names it) +that reads the IP-XACT spec, takes the chosen configuration values +(extracted from the htcl call site, or passed in directly, or queried +from Vivado post-instantiation), and emits RSF. + +What htcl contributes to this pipeline: it is the place where the +configuration values are pinned down in a source-controlled form. The +RSF generator can extract those values by reading the htcl call site, +by running the htcl through to Vivado and querying the instantiated +IP, or by both. The mechanics are for that tool to decide; htcl just +needs to ensure the configuration is recoverable. + +### Wrapper documentation + +Vivado's generated VHDL/Verilog wrappers around configured IP are +notoriously undocumented. Port semantics, parameter effects, and +intended usage patterns are absent from the wrapper file. This is a +real obstacle to using IP correctly and reviewing changes to its +configuration. + +The information flow we want: + +``` +htcl wrapper proc (doc comments on parameters, derived from IP-XACT) + --> create_ip with directives capturing those docs + --> Vivado wrapper generator (would need to honor the directives) + --> generated VHDL with carry-through documentation +``` + +The first step we control (htcl carries the docs). The last step we +want (Vivado emits documented wrappers). The middle step is the open +question: what mechanism gets the doc strings from `create_ip` +arguments into Vivado's wrapper-generation output? Possibilities +include TCL directives Vivado already supports (likely insufficient), +TCL directives Xilinx would need to add (this is what we'd pitch), or +post-processing of generated wrappers by a separate tool (works but +fragile). + +Worth pitching to Xilinx alongside htcl adoption: a documented +mechanism for `create_ip` to accept doc strings that flow into the +generated wrapper. Independently useful even for users who never adopt +htcl. + +## Why this lives in `vw` + +`vw` is already structured library-first: `vw-lib` is the core, the `vw` +CLI is a thin clap-based shim, and `vw-lib` is already consumed by +external tools (e.g., the remote build service for Vivado projects). This +is the same architectural pattern as Cargo's relationship to +rust-analyzer: one library underneath, thin CLIs on top, all tools sharing +the manifest, lockfile, and resolved dependency graph. + +Adding htcl-language support, an analyzer, and a REPL as additional +subcommands of `vw` (or as sibling binaries that share `vw-lib`) gives us: + +- **One manifest, one lockfile, one cache.** `vw.toml` and `vw.lock` + cover VHDL and htcl dependencies uniformly. Packages can ship both + languages from the same git source, versioned together (as + discussed in earlier design conversations: a package version is a + single value across all languages it contains). +- **One LSP serving both languages.** `vw analyzer` is designed from + day one as a multi-language language server. htcl is wired up + natively from the initial analyzer phase; VHDL arrives in a + subsequent phase via a `vhdl_ls` proxy, with eventual replacement + by Oxide's developing VHDL frontend. The user-facing surface (one + LSP per workspace) is stable across that transition. See "LSP + design" for the architecture. +- **Cross-language analysis.** An htcl file that wraps a user VHDL + entity and the VHDL file defining that entity live in the same + workspace, analyzed by the same tool through a shared backend + abstraction. Go-to-definition can cross language boundaries. +- **Shared dependency resolution.** No new resolver, no new fetch + logic, no new cache layout. We add an htcl file-selection layer on + top of `vw-lib`'s existing dependency model, but the resolution + mechanism is unchanged. +- **One mental model for users.** They learn `vw` once and get + everything. + +The model: `vw analyzer` is the LSP (modeled on `rust-analyzer`'s +relationship to Cargo). `vw repl` is the interactive htcl shell. `vw run` +(or similar) executes an htcl script against a Vivado worker. All of +these are thin subcommands over `vw-lib` plus a new `vw-htcl` crate that +holds the htcl-specific analysis. + +## Architecture overview + +``` + +-----------------------------+ + | vw-lib (existing) | + | - manifest / lockfile | + | - dependency resolver | + | - cache management | + | - VHDL file selection | + +--------------+--------------+ + | + +---------------------------+---------------------------+ + | | | + v v v + +----------+-----------+ +-----------+----------+ +-----------+----------+ + | vw CLI (existing) | | vw-htcl (new) | | other vw-lib users | + | vw add / update / | | - htcl parser | | (remote build svc, | + | test / etc. | | - module resolver | | future tools) | + +----------------------+ | - signature check | +----------------------+ + | - htcl -> Vivado | + | TCL emission | + +-----------+----------+ + | + +--------------------------------+--------------------------------+ + | | | + v v v + +---------+----------+ +------------+---------+ +------------+---------+ + | vw analyzer (new) | | vw repl (new) | | vw run (new) | + | LSP server, serves | | interactive htcl | | execute htcl script | + | VHDL + htcl from | | against Vivado | | against Vivado | + | one process | | worker | | worker | + +--------------------+ +----------+-----------+ +----------+-----------+ + | | + +--------------+-----------------+ + | + v wire protocol (newline- + delimited commands + + structured responses) + | + +-------------+--------------+ + | Vivado process | + | (long-lived) | + | - vivado-shim.tcl | + | (small TCL layer that | + | wraps commands and | + | emits JSON for | + | structured results) | + +----------------------------+ +``` + +Key invariants: + +- `vw-lib` is unchanged in spirit; we add new crates beside it + (`vw-htcl`, `vw-analyzer`, `vw-repl`, etc.) rather than restructuring + what exists. +- All language semantics live in Rust. The Vivado shim is a dispatcher + and serializer, nothing more. +- The wire protocol is newline-delimited commands in, structured (mixed + text/JSON) responses out. Hand-written; not RPC-framework-heavy. +- Vivado runs as a long-lived worker process. Cold-start is too expensive + for the hot path. +- The Vivado-driving binaries are self-contained; no Vivado-specific + linking. Distribution is one (or a few) Rust binaries plus the shim TCL + file. + +## Language design + +### Naming + +Working name for the dialect: `htcl`. This is the language name, used as +the file extension (`.htcl`) and as the LSP language identifier. It is not +a separate tool — htcl is a thing `vw` supports, alongside VHDL. + +Open question: final name. Avoid `tcl` in the name to reduce confusion +about it being a TCL implementation; it isn't. + +### Relationship to TCL + +htcl is **not** a TCL superset in the way TypeScript is a JS superset. +Existing Vivado TCL files are not valid htcl. The reasons: + +- We want a structured proc-argument grammar that vanilla TCL can't parse. +- We want static module imports (`src` / `use`) that aren't `source`. +- We want to reject TCL features that defeat static analysis (`upvar`, + `uplevel`, `trace`, dynamic command rewriting). + +However, htcl emits TCL when talking to Vivado, and users can drop down to raw +TCL via an escape hatch for things htcl doesn't model. Existing TCL scripts can +be `source`d through the shim if needed. + +### Proc grammar + +Procs declare structured arguments with per-argument doc comments and attributes: + +```htcl +proc axis_interface { + ## Add a TKEEP sideband signal. Indicates valid bytes in the beat. + @default(0) + has_tkeep + + ## Add a TLAST sideband signal. Indicates the last beat in a packet. + @default(1) + has_tlast + + ## Width of TDATA in bytes. + @default(8) + @enum(1, 2, 4, 8, 16, 32, 64, 128) + tdata_num_bytes + + ## Width of TUSER in bits. Only meaningful if has_tuser is set. + @default(0) + @requires(has_tuser) + tuser_width +} { + # body: emits CONFIG.* set_property calls +} +``` + +Call site uses keyword arguments: + +```htcl +set lrq_request [axis_interface -has_tkeep 1 -tdata_num_bytes 128] +``` + +Attribute set for v1 (extensible): + +- `@default(value)` — value if omitted +- `@required` — error if omitted +- `@enum(a, b, c)` — value must be one of these +- `@range(min, max)` — numeric bounds +- `@requires(other_arg)` — dependency between args +- `@conflicts(other_arg)` — mutual exclusion +- `@deprecated("message")` — soft warning + +Open question: should attributes go before or after the doc comment, or be +order-insensitive? Recommend: doc comments first, then attributes in any order, +then the argument name. Matches Rust/TypeScript conventions. + +### Module system + +Replace `source` with `src` (or `use` — see open questions): + +```htcl +src common/project # relative to current file's directory +src /opt/xilinx/lib/foo # filesystem-absolute (use sparingly, non-portable) +src @quartz/ip/bacd # named dependency, resolved via vw.toml + vw.lock +``` + +Resolution rules: + +- Leading identifier: relative to the directory of the importing file. + Subdirectory traversal allowed (`src ip/cips` is fine). +- Leading `/`: filesystem-absolute. Permitted but discouraged; lint warning + in v1, since these break across machines. +- Leading `@name/`: resolved via `vw.toml`'s `[dependencies.name]` entry. + The cached path comes from `vw-lib`'s resolution (which reads `vw.lock` + and the cache layout under `~/.vw/deps/`). The `@name` prefix means the + same thing as it does to VHDL consumers of vw — same dependency entry, + same commit, same cache directory. +- No upward traversal (`../`) in v1. Force cross-tree references to be + absolute (filesystem or named). + +Extension is implicit: `src foo/bar` resolves to `foo/bar.htcl` (or +whatever extension we settle on). Exactly one extension is recognized; +ambiguity is an error. + +Idempotence: a module is loaded at most once per interpreter run, keyed by +canonical (realpath'd) file path. Repeated `src` calls are no-ops. + +The "project root" — the base for filesystem-absolute imports' meaning of +the manifest, and the directory the analyzer walks for workspace symbols — +is the directory containing `vw.toml`. Same convention as for VHDL. + +Open question: namespace semantics. Does `src foo/bar` populate +`foo::bar::*`, or do top-level definitions land in the global namespace as +they would with `source`? Recommend: top-level definitions are scoped to +the module's namespace by default, with an explicit `export` list +controlling what's visible. Importers use bare names after `src` (the +imports of the module are pulled into the importer's namespace). This is +the bigger semantic change; if too invasive for v1, fall back to +global-namespace semantics and add scoping later. + +### Types — defer until we have usage experience + +Types are out of scope for v1, and we should be in no hurry to add them. The +proc grammar, module system, and dependency manager are the highest-value +features and don't require a type system to be useful. We need real usage +experience with htcl-without-types before we can design types that pay for +themselves. + +The risk of adding types prematurely is well-attested across other +ecosystems: type systems designed before the language has settled tend to +encode the wrong abstractions, become hard to evolve, and impose annotation +overhead that the underlying use cases don't justify. Better to live without +them, collect concrete cases where they would have caught real bugs or +documented intent better, and design to those cases later. + +If types do land eventually, they will be: + +- Optional and gradual. Unannotated code keeps working. +- Focused on HDL-specific concerns where the leverage is clear (likely + candidates: IP handle types so CONFIG.* completion works without flow + analysis, units like Hz/MHz/ns, bit widths). General-purpose typing of + TCL values is not the goal. +- Driven by accumulated evidence, not speculative design. + +The LSP's type-directed CONFIG.* completion (see LSP design section) is +achieved in v1 via flow tracking of IP handles, not a user-facing type +system. The user writes `set lrq [create_ip -name axis_register_slice ...]` +with no annotations; the analyzer infers that `$lrq` holds an +`axis_register_slice` handle. This is a narrow, internal analysis — not a +type system users interact with — and it covers the headline IP-completion +case without committing to broader type design. + +## Dependency management + +Dependency management is `vw-lib`. We do not design a new manifest, a new +lockfile, a new resolver, or a new cache. We extend the existing model in +the minimal ways htcl needs. + +### Manifest and lockfile + +`vw.toml` and `vw.lock`, as they exist today. Reference: +[vw README](https://github.com/oxidecomputer/vw/blob/main/README.md). + +Each dependency entry already specifies `repo` + `branch`/`commit`/`tag` +and a `src` selector for VHDL files. For htcl consumption, we add an +optional `htcl` selector alongside `src`: + +```toml +[dependencies.quartz] +repo = "https://github.com/oxidecomputer/quartz" +branch = "main" +src = "hdl/ip/vhd" # VHDL files (existing; consumed by VHDL flow) +htcl = "hdl/ip/htcl" # htcl files (new; consumed by vw-htcl) +recursive = true +``` + +Either or both selectors may be present. A package that ships only htcl +omits `src`; a package that ships only VHDL omits `htcl`; a package that +ships both (the common case for shared IP wrappers) has both. + +Open question: whether to keep them as separate keys (`src` / `htcl`) or +generalize to a polymorphic selector keyed by file type. The separate-keys +version is the smallest change to vw-lib and the most explicit; keep it +unless there's a reason to generalize. If we later add other languages +(SystemVerilog, Quartus TCL), we add more keys. + +### Versioning model (unchanged from vw) + +- A package version is a single value across all languages it contains. + VHDL and htcl contents of `quartz` ship together at the same commit; no + per-language versions. +- `vw.lock` pins exact commit SHAs, not tags. + +This was previously called out as a design decision; it is, but it's +already the model vw uses, so there's nothing to design. + +### Resolution and fetch (unchanged from vw) + +- `vw update` fetches and locks. Same command, same semantics, now also + resolves htcl dependencies if `htcl` selectors are present. +- Cache at `~/.vw/deps/-/`, unchanged. +- Authentication defers to git, unchanged. + +### What `vw-htcl` does on top of `vw-lib` + +A thin file-selection and module-resolution layer: + +- Given a resolved dependency from `vw-lib`, find the htcl files within + it using the `htcl` selector. Same selector semantics as the existing + `src` field (directory / single file / glob). +- Build an index from `@name` to the resolved dependency's htcl root + directory. +- The htcl module resolver consults this index when it sees `src + @name/path`. + +This is the only dependency-related code htcl needs to write. Everything +upstream of "I have a cache directory for `@quartz`" is already done. + +### Path dependencies + +`vw` currently supports git-sourced dependencies. For monorepo/sibling- +checkout development loops, htcl's plan called for `path = "../foo"` +dependencies. Confirm whether `vw-lib` currently supports this; if not, +adding it is a small extension that benefits both VHDL and htcl +consumers. Likely worth doing in `vw-lib` itself rather than as +htcl-specific behavior. + +### Coordination questions for `vw-lib` + +Things to confirm before phase 1 implementation, and possibly upstream +changes to schedule: + +1. **Does `vw-lib` expose a stable Rust API for "give me the resolved + path for dependency X"?** Yes (per the existing remote-build-service + consumer), but confirm the exact shape — a `Resolved { name, commit, + path, src_files }` struct, or similar — so the htcl resolver can + consume it cleanly. +2. **Is the `src`-selector logic factored well enough to reuse for `htcl` + selectors?** Ideally yes; the directory / single-file / glob handling + is generic and shouldn't be reimplemented. +3. **Path dependencies:** present, absent, or in progress? +4. **Workspace concept:** if `vw.toml` ever grows multi-package workspace + support (cargo-style), how does that interact with htcl? Likely fine, + but worth a thought. + +## Wire protocol + +### Transport + +Vivado is spawned once per workflow run (or per LSP session) and stays alive. +htcl talks to it over stdin/stdout pipes. No sockets, no daemon, no +multi-user complexity in v1. + +### Request format + +Newline-delimited commands. Each command is a JSON object: + +```json +{"id": 42, "op": "eval", "tcl": "set_property -dict {CONFIG.HAS_TKEEP 1} $lrq"} +{"id": 43, "op": "eval_structured", "tcl": "report_property -all $cell"} +``` + +`id` is a monotonic request ID for matching responses. + +Two ops in v1: + +- `eval`: run the TCL, return the result as a string (or error info). +- `eval_structured`: run the TCL through a wrapper that emits JSON for known- + structured commands. The shim has a dispatch table from command name to + wrapper function. + +### Response format + +```json +{"id": 42, "ok": true, "result": ""} +{"id": 43, "ok": true, "result": {"CONFIG.HAS_TKEEP": "1", ...}} +{"id": 44, "ok": false, "error": {"message": "...", "code": "...", "info": "..."}} +``` + +For `eval_structured`, `result` is the JSON shape produced by the per-command +wrapper. For `eval`, it's a string. + +### Vivado-side shim + +A small TCL file (`vivado-shim.tcl`) loaded into Vivado at worker startup. It: + +- Reads newline-delimited JSON from stdin. +- Dispatches to a handler per `op`. +- For `eval`, calls `uplevel #0 $tcl`, captures result or error, emits + response. +- For `eval_structured`, parses the command name from the TCL, looks up a + wrapper, runs the wrapper to produce JSON. +- Wrappers are hand-written per command family. Start with: `report_property`, + `report_timing` (summary), `get_cells` / `get_pins` / `get_nets` / + `get_clocks` lists, `list_property`. Grow as needed. + +Most commands don't need a structured wrapper. Default to passthrough as a +string; opt into structure for the commands where text parsing on the Rust +side would be painful. + +### Batching and fencing + +v1: one request, one response. No batching. If round-trip latency becomes a +measured bottleneck, add a `batch` op that runs a list of commands with +fence semantics (stop on first error, or run-all-collect-errors). + +## LSP design + +LSP is a first-class concern, not a feature bolted on late. This +section describes the design in detail because LSP quality is the primary +goal of the project, and because the rest of the language design either +serves the LSP or constrains it. + +### Scope: a multi-language LSP, htcl first, VHDL next + +`vw analyzer` is designed from the start as a multi-language LSP for +the entire HDL workflow, not as an htcl-only LSP that might grow VHDL +later. The bulk of the initial implementation focus is htcl — that's +where the new language design and the headline complexity-management +features live — but the LSP server architecture, the workspace model, +and the configuration shape all assume a multi-language future and +must accommodate it from day one. + +The motivating reality: + +- Oxide's HDL work spans both htcl (this project) and VHDL (extensive + existing codebase, the larger of the two by volume). +- Today, VHDL editor support comes from `vhdl_ls`, the open-source + VHDL language server, configured via `vhdl_ls.toml` that `vw` + generates. This works well and isn't going anywhere short-term. +- Long-term, Oxide is developing its own VHDL frontend and synthesizer + as part of a complete VHDL stack. The eventual goal is for `vw + analyzer` to integrate directly with that frontend, replacing + `vhdl_ls`. + +The path: `vw analyzer` serves htcl natively from day one, and +provides VHDL support initially by proxying to `vhdl_ls`, eventually +by integrating with the Oxide VHDL frontend. The user-facing surface +(one LSP serving both languages from a unified `vw.toml` workspace) +is stable across that transition; the implementation under the hood +changes. + +Two consequences for the htcl-focused work in this plan: + +1. The LSP server architecture is multi-language from phase 3, even + while only htcl is wired up. The language-backend abstraction + exists; it has exactly one implementation initially. +2. The htcl-side analysis must surface cross-language queries through + that abstraction rather than directly to a specific VHDL + implementation. This keeps the abstraction honest from day one and + means the eventual swap from `vhdl_ls` to the Oxide frontend + doesn't ripple into htcl-side code. + +### Architectural principle: one source of truth per language + +The LSP is **not** a separate analyzer with its own parser and signature +checker. It is the same code as the CLI, exposed over the LSP protocol. +For htcl, the parser, name resolver, signature checker, and +(eventually) any type analysis are written once and used by both `vw +run` / `vw check` and `vw analyzer`. For VHDL (later), the same +discipline applies via whichever backend serves VHDL at the time. + +This matters because the dominant failure mode of language tooling is +divergence between "what the compiler does" and "what the IDE shows." +The moment they're separate implementations, they drift, and users +learn not to trust the IDE. Sharing the implementation is the only +durable fix. + +Concretely: + +- All htcl semantic analysis lives in the `vw-htcl` crate, consumed by + every subcommand that needs it (`vw run`, `vw check`, `vw analyzer`, + `vw repl`). +- VHDL analysis lives behind a language-backend abstraction (see + below). Initially: `vhdl_ls` proxy. Later: direct integration with + the Oxide VHDL frontend. htcl-side code talks to this abstraction + for cross-language queries, never to a specific VHDL implementation. +- `vw analyzer` is the LSP server process. It does protocol plumbing + (LSP request/response, text-sync, capability negotiation), dispatch + by file type, and cross-language coordination. It contains minimal + language logic of its own. +- `vw check` runs the same analyses the LSP runs on save, with the + same diagnostics. CI uses `vw check`; editors use `vw analyzer`. + Same diagnostics either way. + +### Language backend abstraction + +The LSP server dispatches per-file based on extension and routes +requests to a language backend. Each backend implements a common +trait (working name `LanguageBackend`): + +```rust +trait LanguageBackend { + fn diagnostics(&self, file: FileId) -> Vec; + fn hover(&self, file: FileId, pos: Position) -> Option; + fn completion(&self, file: FileId, pos: Position) -> Vec; + fn definition(&self, file: FileId, pos: Position) -> Vec; + fn document_symbols(&self, file: FileId) -> Vec; + // ... cross-language query surface, see below ... + fn find_symbol(&self, query: SymbolQuery) -> Vec; +} +``` + +(Exact shape is for implementation to decide; the point is the +abstraction exists from day one.) + +Initial implementations: + +- `HtclBackend` — uses `vw-htcl` directly. Native, in-process. +- `VhdlBackend` (initial) — `vhdl_ls` proxy. Spawns `vhdl_ls` as a + subprocess and forwards file-scoped requests over the standard LSP + protocol. The proxy generates `vhdl_ls.toml` from `vw.toml` (which + `vw` already does for standalone editor support) and points the + subprocess at it. + +Later, the `VhdlBackend` is replaced by a direct integration with the +Oxide VHDL frontend — same trait, different implementation. Call +sites in `vw analyzer` and in `HtclBackend` don't change. + +The cross-language query surface (`find_symbol` above, plus whatever +else accumulates as cross-language features grow) is the contract +that lets htcl ask "is there a VHDL entity named X?" without caring +which backend answers. For the `vhdl_ls` proxy, `find_symbol` is +implemented by querying `vhdl_ls` with `workspace/symbol` and +translating the results. For the Oxide frontend, `find_symbol` is a +direct API call. Same shape from the htcl side. + +### Cross-language analysis (htcl ↔ VHDL) + +A different case from the IP-XACT-generated wrapper flow described in +Strategic Context: hand-written htcl that wraps user VHDL entities. A +team's own VHDL design has entities with generics, and an htcl proc +gives those entities ergonomic instantiation interfaces with doc +comments, defaults, and validation. Because `vw analyzer` sees both +languages (through the backend abstraction), it can offer +cross-language navigation between the htcl wrapper and the underlying +VHDL entity: + +- **Go-to-definition from htcl into VHDL.** An htcl proc that wraps a + VHDL entity — e.g., `instantiate_uart` taking parameters that map + to the `uart` entity's generics — can declare its target entity via + an attribute (likely `@vhdl_entity(uart)`). When the user invokes + go-to-definition on the proc call, the LSP server asks the VHDL + backend for the location of entity `uart` and returns it. The + htcl-side code that issues this query doesn't know whether + `vhdl_ls` or the Oxide frontend answered. +- **Find references across languages.** "Find references" on a VHDL + entity surfaces both VHDL instantiations (from the VHDL backend's + references query) and htcl wrapper procs that target it (from the + htcl backend's index of `@vhdl_entity` attributes). +- **Generic-to-argument mapping.** If an htcl wrapper declares which + of its proc arguments map to which VHDL generics, the htcl backend + queries the VHDL backend for the entity's generic list and checks + for missing or extra mappings. Warns on drift when the entity's + generic list changes. + +Note: this is distinct from IP-XACT-generated wrappers. Those target +Vivado IP via `create_ip` and don't have a VHDL entity in the +workspace to navigate to — the entity comes out the other side as +generated RTL. Cross-language analysis applies to user-authored +htcl-over-VHDL, not to vendor-IP wrappers. + +Open question: how aggressively to pursue cross-language features in +v1. The minimum is "go-to-definition from htcl into VHDL"; the rest +is nice to have. I'd recommend the minimum lands in v1 (it's the +headline demo for the multi-language model) and the more +sophisticated checks come after. + +### Incremental analysis + +A useful LSP must re-analyze on every keystroke. The analysis layer is +designed for this from the start, not retrofitted. + +Approach: + +- The unit of caching is the file (module). Parsing a file produces a syntax + tree; resolving its imports produces a module-level binding. Files cache + their parsed and resolved state, keyed by content hash. +- Cross-module analysis (resolving an import, looking up an external proc + signature) reads from the cache. A change to a file invalidates that file + and any file that depends on it, transitively. +- Consider `salsa` (the framework rust-analyzer uses) for memoization and + invalidation. It's heavy machinery for a small project, but it solves + exactly this problem and the alternative is hand-rolling the same thing + badly. Decide after phase 0 whether to adopt it; for the smallest possible + v1, a hand-rolled cache keyed on file mtimes is fine. +- Parsing should be tolerant of incomplete input. The user is in the middle + of typing; the parser must produce a usable AST with error nodes rather + than bailing on the first syntax error. This shapes the parser choice + (see below). + +### Parser + +The parser is the foundation of every LSP feature. Built with +[`winnow`](https://docs.rs/winnow), the parser library used pervasively +across Oxide. Familiarity, code-review consistency, and shared idioms with +the rest of the codebase outweigh case-by-case evaluation of alternatives. + +Requirements the implementation must meet within winnow: + +- **Error-tolerant.** Recover from syntax errors and continue parsing. A + half-typed proc declaration should still produce a tree where the rest + of the file is analyzable. Winnow's `cut_err` and combinator-level + recovery are the building blocks; design recovery points around + statement boundaries (newline-terminated top-level forms, proc bodies, + `src` statements). +- **Position-preserving.** Every node knows its source span. Winnow's + `Located` adapter or equivalent span-tracking is used throughout. No + AST node without a span. +- **CST-shaped, trivia-preserving.** The output is a concrete syntax tree + that retains whitespace and comments, not a stripped abstract syntax + tree. We need comments for doc-comment extraction and trivia for + accurate formatting (`vw fmt` is a likely future feature). The AST + layer used by name resolution and signature checking is derived from + the CST. +- **Reusable across editor and CLI.** Same parser code runs in `vw run`, + `vw check`, `vw analyzer`, `vw repl`. No editor-only or CLI-only + variants. + +Incremental reparse is deferred. Most htcl files will be small enough +that full reparse per edit is fast; revisit if measurement shows +otherwise. If incremental parsing becomes necessary later, the CST +boundary makes it tractable to swap in a different strategy for hot +paths without rewriting downstream analysis. + +Open question: how to structure the CST → AST lowering. Two reasonable +shapes: (a) a single AST with optional trivia attached to nodes, or (b) +a separate AST that holds references back into the CST for source +positions. Pick after the first non-trivial grammar pass. + +### Feature inventory + +Each feature has acceptance criteria specific enough to be implementable. + +#### Completion + +What completion offers depends on cursor context. The completion system +needs a notion of "what kind of position is this," determined by the +surrounding syntax tree. + +Positions and their completion sets: + +- **Top-level statement.** Suggest: keywords (`proc`, `src`, `set`, control + flow), in-scope procs, in-scope variables. +- **After `src `.** Suggest: relative module names (subdirectories and + `.htcl` files reachable from the current file), `/` to start a filesystem + path, `@name/` for declared dependencies. After `@name/`, suggest paths + within that dependency. +- **Command position (start of a statement).** Suggest in-scope procs and + Vivado builtins. For Vivado builtins, the suggestion source is a + generated table from UG835 (see "Vivado builtins" below). +- **Argument position of a known proc call.** If the cursor is after + `axis_interface ` and the next token is `-`, suggest the proc's keyword + arguments. If the cursor is after `-has_tkeep `, suggest values + appropriate to that argument's type / `@enum` set. +- **Inside a `$variable` reference.** Suggest in-scope variable names. +- **Inside an attribute (`@`).** Suggest known attribute names + (`@default`, `@required`, `@enum`, etc.) and, where appropriate, + their arguments. + +Note: parameter completion on IP instantiation sites (the +highest-value HDL use case) is the same code path as proc-argument +completion above. An IP wrapper is an htcl proc; its parameters are +proc arguments with attributes; completion works the same way as for +any other proc. There is no special-case "IP property" completion. + +Acceptance criteria: +- Completion responds in <50ms for files under 1000 lines. +- Completion items include `detail` (short type/signature info) and + `documentation` (full doc comment) fields. +- Snippets supported for procs with required arguments — completing a proc + call inserts the proc name plus placeholders for required keyword args. + +#### Hover + +Acceptance criteria: +- Hovering on a proc name shows the proc's doc comment, signature + (arguments with their attributes), and source location. +- Hovering on a proc argument at a call site shows that argument's doc + comment, default value, and any attributes (`@enum`, `@range`, etc.). +- Hovering on a `src` import shows the resolved file path and, if + available, the module's top-level doc comment. +- Hovering on a Vivado builtin shows UG835-derived documentation. +- Hovering on an IP wrapper proc (imported from a vw package) shows + its doc comment and per-parameter docs, same as any other proc. If + the package was generated from IP-XACT, that documentation flows + through unchanged. + +#### Diagnostics + +Diagnostics are produced by the same analyzer the CLI uses. The LSP just +ships them over the wire. + +Categories: + +- **Syntax errors.** Parse failures, recovered to the best position the + parser can manage. +- **Unresolved imports.** `src foo/bar` where `foo/bar.htcl` doesn't exist. +- **Unknown procs.** Call to a name that isn't defined or imported. +- **Argument errors.** Unknown keyword argument, missing required argument, + value outside `@enum` or `@range`, `@requires` / `@conflicts` violation. +- **Unused declarations.** Unused imports, unused local variables. Warning + level, suppressible. +- **Deprecation warnings.** Call sites of procs marked `@deprecated`. + +Note: there is no separate "IP property error" diagnostic category. +Unknown arguments to an IP wrapper proc are caught by the standard +"unknown keyword argument" check; out-of-range values are caught by +the standard `@enum` / `@range` check. The diagnostics machinery +doesn't distinguish IP wrappers from other procs. + +Each diagnostic has: source range, severity, message, optional related +information (e.g., "this is the proc declaration whose required argument +you're missing"), optional code action (e.g., "add missing argument"). + +Acceptance criteria: +- Diagnostics update within 200ms of an edit. +- Every diagnostic has a precise source range, not just a line number. +- Diagnostics are stable: editing an unrelated part of a file doesn't + cause diagnostics elsewhere to flicker. + +#### Go-to-definition + +- **Proc reference → proc declaration.** Across files, following imports. +- **Variable reference → assignment.** Within a scope; "definition" for a + variable is its first assignment in the current scope or a containing + scope. +- **`src` target → the imported file.** Open the imported `.htcl` file. +- **Vivado builtin → UG835 entry.** Either open a generated stub file with + the documentation, or open the UG835 URL. Implementation-defined; the + point is the user can find the docs. + +#### Find references + +- For procs, find all call sites and any explicit references (passing as a + value, etc.). +- For variables, find all reads and writes in scope. +- For modules, find all `src` statements that import them. + +Acceptance criteria: +- Find references on a proc returns results across the whole project, + searching all `.htcl` files transitively reachable from the project root. +- Results include the source range and one line of context. + +#### Rename + +Lowest priority but high value when it works. Rename a proc, variable, or +module and update all references atomically. + +Caveats: +- Renaming across the project boundary (into dependencies) is forbidden. +- Renaming requires the LSP to be confident about every reference; if any + reference is ambiguous (e.g., dynamically constructed), abort with an + error rather than rename incorrectly. + +#### Document symbols and workspace symbols + +- Document symbols: every proc, top-level variable, and module-level + declaration in the current file. Used for the editor's outline view. +- Workspace symbols: same, across the project. Used for "go to symbol in + project" pickers. + +#### Code actions + +A small set in v1, expanded over time: + +- "Add missing required argument." +- "Remove unused import." +- "Convert raw `set_property -dict` to a structured IP configuration call." + (Big one for migration off existing Vivado TCL.) +- "Extract selection to proc." + +#### Formatting + +`htcl fmt` is a separate CLI command; the LSP exposes it via the +`textDocument/formatting` request. Formatter implementation is a phase past +v1, but the architectural slot for it should exist from the start (the CST +must preserve enough information to reformat). + +### Configuration completion on IP instances + +The highest-impact LSP feature for HDL work is parameter completion at +IP instantiation sites. The user imports an IP from a vw dependency: + +```htcl +src @xilinx-ip/axis_register_slice +# ... +set lrq [axis_register_slice -has_tkeep 1 -tdata_num_bytes | + ^cursor here +``` + +The analyzer offers completion for the IP's parameters (`tuser_width`, +`tdest_width`, etc.), with hover documentation, defaults, and `@enum` +constraint values pulled from the proc's declared signature. + +Crucially: **this is the same code path as completion on any other +htcl proc.** The IP wrapper is a proc; the proc has structured +arguments with attributes (per the proc grammar); completion of those +arguments works the same as completion of arguments on a hand-written +proc. There is no special-case "IP property" subsystem in the +analyzer. + +This is the architectural payoff of treating IP as ordinary htcl +packages distributed through vw dependencies: the LSP doesn't need to +know anything about IP-XACT, IP catalogs, or Vivado-specific +introspection. It just analyzes htcl. + +Flow tracking of IP handles is still useful for downstream features — +"this variable is an instance of `axis_register_slice`, so when it's +passed to `connect_axis`, here's what we can validate" — but it's a +narrow extension of proc return-type tracking, not a separate +mechanism for IP. Defer until the cross-IP wiring story matures. + +### Vivado builtins + +htcl needs knowledge of Vivado's built-in TCL commands (`get_cells`, +`report_timing`, `current_design`, etc.) to provide completion and +hover for them. These aren't IP — they're the underlying Vivado +language surface htcl sits on top of. + +Sources: + +- UG835 (the Tcl Command Reference) parsed into a structured form. The + doc has consistent enough structure to be machine-readable, though + it's not trivial. +- `help ` output from a live Vivado, scraped at + builtin-data-generation time. +- Hand-written annotations layered on top for things UG835 gets wrong + or doesn't explain. + +The result ships with vw (in a `vw-vivado-data` crate, or similar) as +a generated data file consumed by the analyzer. Regenerated per Vivado +release; the version targeted in `vw.toml` selects which data file is +used. + +### LSP server implementation + +- Crate: `vw-analyzer`, a binary crate. Invoked as `vw analyzer` from + the `vw` CLI dispatcher, or directly via `vw-analyzer`. +- Framework: `tower-lsp` is the standard Rust LSP framework, + well-maintained and used by rust-analyzer-adjacent projects. Use it + unless there's a specific reason not to. +- Transport: stdio. The editor spawns `vw analyzer` and talks to it. +- Concurrency: file analysis runs on a worker thread pool; the protocol + handler thread stays responsive to cancellation requests. + Long-running analyses (full project re-resolution) are cancellable. +- Language scope: htcl and VHDL in one server process, dispatched + per-file via the `LanguageBackend` abstraction (see "Language + backend abstraction" above). htcl is wired up natively from phase 3; + VHDL is wired up via the `vhdl_ls` proxy in a subsequent phase, with + eventual replacement by the Oxide VHDL frontend. Cross-language + queries are first-class through the backend trait. + +### Editor integration + +VS Code is the primary target. A minimal extension: + +- Activates on `.htcl` and `.vhd`/`.vhdl` files, and on workspaces + containing `vw.toml`. +- Spawns `vw analyzer` as the language server. +- Ships a TextMate grammar for htcl syntax highlighting (VS Code's + native highlighting format). A tree-sitter grammar can come later if + we want to support editors that consume those directly (Zed, Neovim + with `nvim-treesitter`). The editor-side highlighting grammar is + separate from the LSP parser; the LSP uses winnow, while highlighting + is whatever the editor consumes. +- Provides commands: "vw: restart analyzer," "vw: update dependencies," + "vw: show IP property reference." + +Open question: relationship to any existing vw VS Code extension. If one +exists, extend it; if not, this is a new package. Either way, one +extension per project, not separate VHDL and htcl extensions. + +Other editors (Neovim, Emacs, Helix, Zed) get LSP support for free via +their generic LSP clients; we don't ship extensions for them initially, +but configuration snippets in the README are a low-cost way to support +them. + +### LSP testing strategy + +LSP regressions are easy to ship and hard to notice. Test infrastructure +from the start: + +- Snapshot tests for analysis output. Each test is a small `.htcl` fixture; + the expected output (diagnostics, symbol tables, completions at marked + positions) is a checked-in snapshot. Mismatches fail the test. +- End-to-end LSP tests using a test client that speaks the protocol. Verify + that a `textDocument/completion` request at a given position returns the + expected set of items. +- Don't test through VS Code. Test the LSP server directly; the VS Code + extension is a thin enough wrapper that manual smoke-testing is fine for + it. + +### Phasing within the LSP work + +The analyzer is built incrementally alongside the rest of the language, +with `vw analyzer` introduced as a real subcommand at phase 3 and +growing features as later phases land: + +- **Phase 3 (analyzer initial, htcl only):** `vw analyzer` binary + exists. `LanguageBackend` abstraction in place with `HtclBackend` as + the sole implementation. Provides diagnostics, document symbols, + go-to-definition for `src` targets and proc references, hover for + proc docs, completion for proc arguments. Crucially, this includes + parameter completion on IP wrapper procs imported from vw packages + — the headline IP-completion case falls out of the proc-argument + completion path. This is the point where the analyzer is genuinely + useful for htcl. +- **Phase 4 (structured wire responses):** No direct analyzer impact, + but enables typed result handling that the REPL builds on. +- **Phase 5 (VHDL via vhdl_ls proxy):** `VhdlBackend` lands as a + proxy to `vhdl_ls`. `vw analyzer` now serves both languages from a + single process; the user-facing multi-language LSP surface is in + place. +- **Phase 6 (cross-language):** htcl ↔ VHDL go-to-definition and find + references, building on the backends from phase 5. +- **Phase 8 (polish):** Find references across the workspace, rename, + workspace symbols, code actions, performance tuning, editor + extension packaging. + +Note: the analyzer benefits from being built alongside language +features rather than after them. Each language feature (modules, proc +grammar, cross-language) lands its analyzer support in the same phase +that introduces the feature. The "phase 8 polish" pass is for +analyzer-only features that don't have an underlying-language +counterpart. + +The eventual replacement of the `vhdl_ls` proxy with direct Oxide VHDL +frontend integration is a "Later" item (see implementation plan). +Because the swap stays within the `LanguageBackend` abstraction, +no htcl-side code needs to change when it happens. + +### Non-goals for the LSP + +- Debugging protocol (DAP). Out of scope; debugging happens inside Vivado. +- Semantic tokens for syntax highlighting. Editor-side grammars are cheaper + and good enough. +- Inlay hints. Possibly later; not needed for v1. +- Refactorings beyond rename and the small code-action set listed above. + +## REPL design + +The REPL is, architecturally, the same product as the analyzer with a +different presentation layer. The LSP serves an editor; the REPL serves +a TUI. Both query the same `vw-htcl` analysis. This isn't a coincidence +to exploit — it's the design. + +The traditional captive-CLI help model (Vivado's `help foo`, Cisco IOS's +`?`) was a 1990s answer to "I don't have a graphics-capable terminal but +I have screen-clearing escape codes." It conflates discovery (what +exists?), reference (what does this do?), and navigation (where am I?) +into a single text-dump idiom that clutters scrollback and answers none +of those questions well. With a modern TUI and the analyzer's data +already on hand, we can do substantially better without reimplementing +anything. + +Built with [`ratatui`](https://ratatui.rs/), the TUI library used +pervasively across Oxide. Line editing uses +[`reedline`](https://docs.rs/reedline) — Nushell's modern readline +replacement, well-suited to hint and menu rendering. Both choices are +Oxide-conventional rather than case-evaluated; same reasoning as winnow. + +### Architectural principle: history hygiene + +The defining UX commitment: anything the user explicitly ran (commands +and their results) belongs in scrollback. Anything that was a navigation +aid (completion menus, signature help, hover dialogs, help overlays) +does not. Navigation aids appear in transient overlays or inline +ghost-text and disappear when the user moves on. The scrollback is what +the user chose to do, not how they figured out what to do. + +This is the failure mode of Vivado's REPL: a `help` invocation dumps +200 lines into history, and the user's actual work is buried. Ours +won't. + +### Virtual document model + +The REPL maintains an in-memory document representing the session: +successful evaluations are appended, and the current input line is +treated as the tail. The analyzer's queries operate on (document + +current input), with the cursor positioned within the current input. + +Consequences: + +- Variables and procs defined earlier in the session are in scope for + completion, hover, and diagnostics on the current input. +- Diagnostics on the current line surface *before* the user submits it + — a typo'd argument name is flagged inline, not after Vivado returns + an error. +- Sourced modules contribute their definitions to the session document, + so completion includes everything reachable from the import graph. + +The same analyzer code that powers `vw analyzer`'s editor support +powers the REPL's interactive features. The analyzer doesn't know +whether it's serving an editor or a TUI. + +### Feature inventory + +#### Tab completion + +The primary discovery mechanism. Triggered on Tab, optionally +auto-suggested as a menu after a short typing pause. + +Completion sources match the LSP's: in-scope procs, proc arguments at +call sites (including arguments on IP wrapper procs imported from vw +packages), `@enum` values, variable names, module imports, and Vivado +builtins. + +Rendered as a popup menu *below* the input line. Arrow keys navigate; +Enter or Tab accepts; Escape dismisses. The menu does not enter +scrollback. + +#### Signature help while typing + +When the user is partway through a proc call, a non-intrusive line +*below* the input shows the proc's signature with the current argument +highlighted. The current value's `@enum` or `@range` constraint is +shown alongside. + +The signature line updates as the user types and disappears as soon as +they move past the call. Never enters scrollback. + +#### Modal help overlay + +Bound to F1 (or `?` if a more keyboard-friendly approach is preferred). +Pops a transient overlay — a centered panel or split-pane — with the +full documentation for the symbol under the cursor: proc signature, +all argument docs, attribute constraints, source location, related +procs. For IP wrappers, this surfaces the same documentation that's in +the proc's doc comments and parameter attributes — which (for +IP-XACT-sourced packages) carries the IP-XACT descriptions through to +the user. + +Dismissed with Escape. Nothing lands in scrollback. + +This is what Vivado's `help` command should have been: a brief takeover +of the screen that returns control unchanged. + +#### Inline ghost-text suggestions + +As the user types, faintly render the most likely completion in dim +text ahead of the cursor (fish-shell / Copilot style). Right-arrow or +Tab accepts. Any other key dismisses. + +For HDL workflows where proc and argument names are long and +repetitive (`tdata_num_bytes`, `axis_register_slice`), this saves real +keystrokes. Reedline supports this natively. + +#### Discoverable command palette + +Bound to Ctrl-P (or similar). Opens a fuzzy-searchable overlay listing +in-scope procs, recent commands, and (optionally) workspace symbols. +Same data the LSP uses for `workspace/symbol`. + +The failure mode of current Vivado REPLs is "I know I want to do X but +I don't remember what it's called." This is the fix. + +#### Per-instance exploration + +Dedicated mode for the common HDL workflow question: "I have an IP +instance; what are all its current properties and values in the live +Vivado design?" Triggered by `:describe ` or by a hotkey on a +variable in scope. + +Renders a sortable, filterable table built from live `report_property` +data on the instance, joined with the IP wrapper proc's parameter +documentation (so each property has its description and constraints +visible). Navigable with arrow keys; Enter on a property opens its +full documentation. Escape closes. + +Substantially better than `report_property` dumped into scrollback as +text. + +#### Pretty-printed structured results + +When `eval_structured` (phase 4) returns a typed result, the REPL +renders it as a navigable structure rather than a flat string. Timing +reports become collapsible trees; property dumps become tables; lists +of cells/pins/nets become selectable lists where each entry can be +hovered for details. + +The text representation is still available — `:plain` or a config +option turns off pretty-printing for screencasts and pipe-friendly +output. + +#### Lightweight text help fallback + +A `:help foo` command (or similar) prints help to scrollback as plain +text. Useful for SSH over slow links, grepping history, screencasts, +and copying into chat/issues. The TUI overlay is the default for +interactive use; the text command exists for cases where the overlay +isn't what's wanted. + +This is the one place we accept scrollback clutter, because it's the +user explicitly asking for it. + +### Implementation notes + +**Debouncing.** Analysis runs on keystrokes; if it takes more than +~20ms the UI feels laggy. Debounce completion and hover queries (fire +~50ms after input stability), and run analysis on a worker thread that +the ratatui frame loop polls. + +**Cancellation.** A new keystroke invalidates the previous analysis +request. The analyzer is already cancellable (LSP requirement); the +REPL inherits that. + +**History.** Persistent across sessions, stored in +`~/.local/state/vw/repl-history` (or platform-equivalent). Reedline +handles this. + +**Multiline input.** htcl procs span multiple lines; the editor must +support multi-line buffers with proper indentation. Reedline supports +this; pair with the parser to detect when a buffer is syntactically +complete vs. needs more lines. + +**Vivado worker lifecycle.** A REPL session corresponds to one +long-lived Vivado worker. Cold start happens at REPL launch (with a +spinner during the multi-second Vivado startup); the worker persists +until exit. `:restart` rebuilds the worker without exiting the REPL. + +**Module hot reload.** If a sourced module changes on disk, the REPL +detects it (file watcher), re-sources, and updates the session +document. The user keeps any session-local definitions made after the +module was first loaded. Conflicts (same name now means something +different) are surfaced as warnings. + +### Phasing within the REPL work + +The REPL doesn't need to wait for every other phase to land. It can +ship with a meaningful subset early and grow: + +- **Initial REPL (phase 7 below):** ratatui shell, reedline line + editor, tab completion, signature help, history, multi-line input, + Vivado worker integration, pretty-printed results (phase 4 has + already landed by this point). Parameter completion on IP wrapper + procs works the same as parameter completion on any other proc — + no separate path. +- **`:describe` for live instances:** lands when wired up; depends on + the structured-wire-response work from phase 4 to read live + properties cleanly. +- **Polish phase (alongside LSP phase 8):** Command palette, modal + help overlay, ghost-text suggestions, file-watcher-based module hot + reload. + +### Non-goals for the REPL + +- Mouse interaction. Keyboard-only TUI. Mouse support is an + accessibility win we can add later; not v1. +- Replacing the Vivado GUI. The REPL is for scripted/exploratory + workflows; users who need waveform viewers and floorplanning still + use Vivado proper. +- Persistent named sessions / tmux-style detach. Run inside tmux if + you want that. +- Custom keybinding configuration in v1. Pick sane defaults; expose + config later if requested. + +## Implementation plan + +The plan is organized around extending `vw` with new crates and +subcommands. The existing `vw-lib` and `vw` CLI are not restructured; we +add alongside. + +New crates introduced over the phases: + +- `vw-htcl` — htcl parser, AST, name resolution, signature checking, + TCL emission. The language layer. +- `vw-vivado` — Vivado worker spawn/connect, wire protocol, embedded + shim TCL. The execution layer. +- `vw-vivado-data` — generated database of UG835 builtin commands. + Regenerated per Vivado release; not user-edited. +- `vw-analyzer` — LSP server. Binary. +- `vw-repl` — interactive shell. Binary. + +Not in this project (separate downstream tooling): + +- IP-XACT → htcl wrapper generation. A sideband tool that reads an + IP's IP-XACT `component.xml` and emits an `.htcl` configuration + interface (a wrapper proc whose parameters match the IP's + parameters). Lives in its own repo, ships its own binary, produces + vw-consumable packages. The output is ordinary htcl that vw doesn't + need to know was generated. See "Strategic context" section. +- IP-XACT + configuration values → RSF. A separate tool that produces + the register-spec file for a specific IP instantiation. Reads the + IP-XACT memory map and the configuration values, emits RSF. Not in + vw; see "RSF generation" in Strategic Context. + +The `vw` CLI grows subcommands `run`, `check`, `repl`, `analyzer` (plus +existing `add`, `update`, `test`, etc.). + +### Phase 0: skeleton + +Goal: smallest end-to-end thing that proves the architecture. + +- New crates `vw-htcl` and `vw-vivado` created in the vw repo. +- `vw run` subcommand added to the CLI dispatcher. +- htcl parser for a minimal subset: literals, variables, `set`, `proc`, + command invocation, comments. No control flow yet. Built with + [`winnow`](https://docs.rs/winnow); see the LSP design section's + "Parser" subsection for the full rationale and requirements. +- Vivado worker spawn-and-connect logic. +- Vivado shim with `eval` op only. +- `vw run file.htcl` reads the file, sends each top-level command to + Vivado, prints results. + +Deliverable: `vw run hello.htcl` where `hello.htcl` is `puts "hello"` +prints `hello`. + +### Phase 1: module system + +Goal: `src` works. + +- Use `vw-lib` to find the project root (location of `vw.toml`). +- Implement `src` with relative and filesystem-absolute resolution. +- `@name/...` resolution: query `vw-lib` for the dependency's resolved + cache path, then index into it via the `htcl` selector. +- Module loading: parse file, execute top-level forms, track loaded set + for idempotence. +- Decide and implement namespace semantics (see open question above). +- Coordinate `vw-lib` extensions: the `htcl` selector key in dependency + entries; path dependencies if not already supported. + +Deliverable: a multi-file project loads and runs, including imports from +a `vw`-managed dependency. + +### Phase 2: proc grammar + +Goal: structured proc declarations with attributes. + +- Extend parser for the proc-arg grammar (doc comments, attributes, + names). +- AST representation for procs with metadata. +- At call time, validate keyword args against the declared signature: + required args present, no unknown args, `@enum` / `@range` / + `@requires` / `@conflicts` checked. +- Generate a TCL-side `proc` that takes positional args in canonical + order; callers pass keyword args, vw-htcl reorders them and emits a + positional call. +- Introduce `vw check` subcommand that runs analysis and reports + diagnostics without executing. + +Deliverable: the `axis_interface` example from the language design +section works end-to-end with validation, and `vw check` flags malformed +calls. + +### Phase 3: analyzer (LSP) — initial version + +Goal: editor support lands as soon as the analysis is meaningful. + +- `vw-analyzer` crate, `vw analyzer` subcommand. +- LSP server using `tower-lsp` over stdio. +- `LanguageBackend` trait introduced; `HtclBackend` is the only + implementation initially. The dispatch by file extension is in + place from the start (it just always routes to `HtclBackend`). +- Wire up the existing `vw-htcl` analysis through `HtclBackend`: + diagnostics, document symbols, hover for proc docs, go-to-definition + for `src` targets and proc references. +- Completion for proc arguments (using the signature data from phase + 2). +- VS Code extension stub: activates on `.htcl` and `vw.toml`, launches + `vw analyzer`. + +Deliverable: opening an htcl project in VS Code gives diagnostics, +hover, and basic completion. The LSP is genuinely useful for htcl +from this point forward; later phases add features and bring VHDL +into the same server. + +### Phase 4: structured wire responses + +Goal: avoid Rust-side TCL parsing for structured outputs. + +- Add `eval_structured` op to wire protocol. +- Write Vivado-shim wrappers for the initial command set + (`report_property`, `get_cells`-family, etc.). +- Rust-side types for the parsed results. + +Deliverable: `report_property` returns a typed Rust value, not a string, +in the executor. + +### Phase 5: VHDL via vhdl_ls proxy + +Goal: bring VHDL into `vw analyzer` so it serves both languages from a +single process; the user-facing surface for a unified LSP is in place. + +- `VhdlBackend` implementation that spawns `vhdl_ls` as a subprocess + and proxies LSP requests for `.vhd` / `.vhdl` files. +- Generate `vhdl_ls.toml` from `vw.toml` (reuse the existing `vw` + logic for this) and point the subprocess at it. Regenerate when + `vw.toml` changes. +- File-type dispatch in `vw-analyzer` now routes htcl files to + `HtclBackend` and VHDL files to `VhdlBackend`. +- Cross-language query surface (`find_symbol` etc.) implemented on + `VhdlBackend` via `workspace/symbol` and related `vhdl_ls` + queries. +- Cancellation, lifecycle, and error handling for the subprocess. +- Performance: confirm the proxy adds acceptable overhead. If + noticeable, profile and optimize. + +Deliverable: a single `vw analyzer` process serves htcl and VHDL. +Editors configured to use it see consistent behavior across both +languages without needing a separate `vhdl_ls` configuration. +Cross-language queries from htcl to VHDL work but aren't yet +user-facing (next phase wires up the htcl-side attributes). + +### Phase 6: cross-language analysis + +Goal: htcl ↔ VHDL navigation (the user-facing cross-language +features, building on phase 5's backend wiring). + +- `@vhdl_entity(name)` attribute on htcl procs declaring the entity + they wrap. +- `HtclBackend` resolves entity references by issuing `find_symbol` + to `VhdlBackend`. Go-to-definition surfaces the resulting location. +- Find-references on a VHDL entity surfaces both VHDL instantiations + (from `VhdlBackend`) and htcl wrappers (from `HtclBackend`'s index + of `@vhdl_entity` attributes). +- Generic-to-argument mapping: optional in this phase, depending on + how much work the `find_symbol` extension to "give me this entity's + generics" turns out to be. + +Deliverable: clicking through an htcl proc into its VHDL entity +works, in both directions. + +### Phase 7: REPL + +Goal: ship the REPL as a meaningful interactive environment. See the +dedicated "REPL design" section above for the full treatment. + +- `vw-repl` crate, `vw repl` subcommand. +- Built with `ratatui` and `reedline`. +- Initial feature set (per the REPL phasing subsection): tab completion, + signature help, persistent history, multi-line input, Vivado worker + lifecycle management, pretty-printed structured results (relies on + phase 4). + +Deliverable: a meaningfully better experience than the Vivado console +for exploring a live design — discoverable commands, inline validation, +overlay-based help that doesn't clutter scrollback. + +### Phase 8: LSP polish + +Goal: bring the analyzer up to "rust-analyzer-quality" expectations for +the features that matter most. + +- Find references across the workspace. +- Rename (cautious; abort on ambiguity). +- Workspace symbols. +- Code actions (add missing required argument, remove unused import, + convert raw `set_property -dict` to a structured call). +- Performance tuning; consider `salsa` if hand-rolled caching shows its + limits. + +Deliverable: an analyzer that meets the acceptance criteria in the LSP +design section. + +### Later (not in initial plan) + +- Type system (typed IP handles, units, phases, constraint scopes). +- Quartus backend. +- **Oxide VHDL frontend integration.** Replace the `vhdl_ls` proxy + `VhdlBackend` with a direct integration with Oxide's developing + VHDL frontend. Same `LanguageBackend` trait, different + implementation. Timing depends on the frontend's maturity; the + `LanguageBackend` abstraction exists from phase 3 specifically to + make this swap possible without rippling into htcl-side code. +- Tracing / profiling of TCL execution. +- Distributed worker pools for parallel synthesis runs. +- `vw fmt` (htcl formatter). +- Mechanism for htcl parameter doc comments to propagate into + Vivado-generated wrappers (requires Xilinx-side support; see + "Wrapper documentation" in Strategic Context). + +Not in this project at all (separate downstream tooling): + +- IP-XACT → htcl wrapper generation (a sideband tool). +- IP-XACT + configuration values → RSF generation (a separate tool; + see "RSF generation" in Strategic Context). + +## Open questions to resolve with the author + +1. **Final name for the htcl dialect.** Working name; pick something + durable before shipping anything publicly. This is just the language + name now, not a tool name. +2. **Module namespace semantics.** Global (TCL-compatible, simple) or + scoped with explicit exports (better complexity management, bigger + change)? Recommend scoped, but flag for discussion. +3. **`src` vs `use` vs `import` vs `mod` keyword.** Recommend `use` for + familiarity (Rust) and to avoid the `src/` directory collision. +4. **Shim distribution.** Ship the shim TCL embedded in the `vw-vivado` + binary, written to a temp file at worker startup? Or expect it on + disk somewhere? Embedded is simpler for users; do that unless there's + a reason not to. +5. **`vw-lib` extensions to confirm or schedule:** + - Stable Rust API for "give me the resolved cache path for dependency + X" — confirm shape. + - Generalization of the dependency selector to per-language keys + (`src` for VHDL, `htcl` for htcl), or staying with `src` plus a + parallel `htcl` field. + - Path dependencies (`path = "..."`) — present, absent, or in + progress? +6. **Cross-language wrapper attribute name.** `@vhdl_entity(name)` is + the working syntax for declaring which VHDL entity an htcl proc + wraps. Confirm or rename. +7. **Showcase IP selection.** For the Vivado-team pitch, which IPs do + we cover in the initial generated `xilinx-ip` package? The + IP-XACT → htcl generator (a separate sideband tool) produces + wrappers mechanically, but the showcase needs to demonstrate + quality at a level that earns the conversation. Pick a small set + where the generated wrappers will look genuinely good, plus one or + two complex IPs (DCMAC, CIPS) where the value of source-controlled + configuration is most visible. +8. **`vhdl_ls` proxy specifics.** Phase 5 wires up VHDL via a + subprocess proxy to `vhdl_ls`. Open: does `vhdl_ls`'s + `workspace/symbol` interface answer the cross-language queries we + need (entity location, generic lists)? If not, what's the + smallest extension to either the proxy or to `vhdl_ls` itself that + closes the gap? Confirm before phase 5 starts. +9. **Evidence-gathering for eventual types.** Not a v1 question, but + worth a habit from day one: when working with htcl, keep a log of + cases where a type system would have caught a real bug or documented + intent meaningfully. Revisit the types decision only when there's a + concrete case file to design against. + +## Non-goals + +- TCL language compatibility. We are not implementing TCL; we are implementing + a different language that happens to share TCL's value model and emits TCL + to Vivado. +- General TCL extension authorship. The Vivado shim is the only TCL we write + intentionally; it stays small. +- Replacing Vivado's interpreter in-process. We talk to it over a pipe. +- Supporting every Vivado command natively. Most commands pass through as + strings; we add structured wrappers only where they pay off. +- A package registry. Git-source dependencies cover the realistic needs; a + registry is a separate company. +- **IP-XACT awareness in vw, the analyzer, or the REPL.** IP-XACT is a + source format for *generating* htcl IP wrapper packages via a + separate sideband tool. The tooling described in this plan consumes + only htcl; it has no IP-XACT-specific code paths, data structures, + or features. See "Strategic context" for the rationale. +- **A replacement for IP-XACT.** htcl is a configuration interface + layer that sits above IP-XACT (specification) and below generated + RTL/RSF (instantiation). It does not describe ports, memory maps, + or any other aspect of an IP's structure — those remain IP-XACT's + responsibility. See "Conceptual layering" in Strategic Context. +- **Port-level analysis of generated RTL.** htcl wrappers don't + describe the ports their instantiation will emit; ports come from + the VHDL/Verilog Vivado generates. Cross-IP wiring analysis is + possible but lives in the VHDL analyzer, not in htcl. +- **Memory-map description.** Not htcl's job. Register interfaces are + generated from IP-XACT plus configuration values by a separate + pipeline targeting RSF; see "RSF generation" in Strategic Context. + +## Reference points + +- `vw`: https://github.com/oxidecomputer/vw — the host project for this + work. `vw-lib` is the existing library that handles dependency + resolution, caching, and manifest/lockfile management. The plan + extends `vw` with htcl-language support and an analyzer/repl modeled + on rust-analyzer's relationship to Cargo. +- rust-analyzer + Cargo: the architectural model. rust-analyzer reads + `Cargo.toml` / `Cargo.lock`, resolves dependencies through Cargo's + data model, and provides editor support without reimplementing the + build tool. `vw analyzer` plays the same role for `vw.toml` / + `vw.lock`. +- UG835: Vivado Design Suite Tcl Command Reference — the authoritative + source for what commands exist and what they return. +- IP-XACT (IEEE 1685): the schema for IP component metadata. *Not used + internally by vw, the analyzer, or the REPL.* IP-XACT is the source + format consumed by a separate sideband tool that generates `.htcl` + IP packages, which vw then resolves like any other dependency. +- TypeScript: model for "additive features over an existing language" + done well. Discipline: existing code keeps working (except htcl + breaks this intentionally for the proc-arg case), new features are + opt-in, output is consumable by tools that don't know about the new + features. diff --git a/vw-analyzer/Cargo.toml b/vw-analyzer/Cargo.toml new file mode 100644 index 0000000..d805726 --- /dev/null +++ b/vw-analyzer/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "vw-analyzer" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Multi-language LSP server for the vw HDL workflow (htcl native; VHDL via vhdl_ls proxy in a later phase)" + +[[bin]] +name = "vw-analyzer" +path = "src/main.rs" + +[dependencies] +vw-htcl = { path = "../vw-htcl" } +vw-lib = { path = "../vw-lib" } +camino.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +async-trait.workspace = true +tower-lsp.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-analyzer/src/backend.rs b/vw-analyzer/src/backend.rs new file mode 100644 index 0000000..f7ce618 --- /dev/null +++ b/vw-analyzer/src/backend.rs @@ -0,0 +1,187 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! [`LanguageBackend`] — per-language analysis surface consumed by the +//! LSP server. +//! +//! Even though only [`HtclBackend`](crate::HtclBackend) exists today, +//! defining the trait from day one is the architectural commitment +//! described in the project plan: VHDL via a `vhdl_ls` proxy (phase 5) +//! and a future direct Oxide-VHDL-frontend integration both slot in as +//! additional implementations without changing the server or +//! cross-language htcl code. + +use async_trait::async_trait; +use tower_lsp::lsp_types::{ + CompletionItem, Diagnostic, DocumentSymbol, Hover, Location, Position, + SignatureHelp, SymbolInformation, Url, WorkspaceEdit, +}; + +#[async_trait] +pub trait LanguageBackend: Send + Sync { + /// Language id (`"htcl"`, `"vhdl"`, ...) — used for tracing and + /// dispatch. + fn language_id(&self) -> &str; + + /// Whether this backend handles the given file. Default: match by + /// extension. + fn handles(&self, uri: &Url) -> bool; + + /// Update the backend's view of `uri`'s contents. Called on + /// `did_open` and every `did_change`. The backend should treat + /// this as the new authoritative source and may eagerly compute + /// (and cache) analysis results. + async fn set_text(&self, uri: Url, text: String); + + /// Trigger an immediate re-index of `uri` — no debounce, no + /// wait. Called from `did_save` so `Ctrl-s` in the editor + /// forces a fresh diagnostic sweep even after a small edit + /// that hadn't yet reached the `set_text` debounce window. + /// Backends without a debounce can leave this as the no-op + /// default: their `set_text` already committed synchronously. + async fn save(&self, _uri: &Url) {} + + /// Block until the next full re-index / re-analysis of `uri` + /// commits. Called by the server AFTER `set_text` from a + /// detached task so it can wrap the wait in an LSP + /// `window/workDoneProgress` notification — that's how the + /// editor's "indexing…" spinner comes back on. + /// + /// Default implementation returns immediately (backends without + /// a background reindex signal don't have anything to wait on; + /// the progress spinner just flashes briefly and disappears). + /// Backends that do have a background rebuild (like + /// [`crate::HtclBackend`]) override this to await their + /// indexer's commit notification. + async fn wait_for_reindex(&self, _uri: &Url) {} + + /// Editor-supplied workspace roots (from LSP `rootUri` / + /// `workspaceFolders`, plus updates via + /// `didChangeWorkspaceFolders`). Backends may use them as + /// fallback dep-lookup sources when a file being analyzed sits + /// outside the nearest `vw.toml` — e.g. a goto-def landed the + /// user in a dep cache dir whose own workspace doesn't declare + /// the same deps as the editor's root. Default impl is a no-op + /// because most backends won't care. + async fn set_workspace_roots(&self, _roots: Vec) {} + + /// Forget any state for `uri`. + async fn close(&self, uri: &Url); + + /// Diagnostics for the current text of `uri`. The server pushes + /// these to the editor via `textDocument/publishDiagnostics` after + /// every text update. + async fn diagnostics(&self, uri: &Url) -> Vec; + + /// Workspace-wide diagnostics, keyed by file URI. The server + /// serves these to the editor via `workspace/diagnostic` (LSP + /// 3.17 pull-based workspace diagnostics — Helix's `space-D` + /// picker consumes them). + /// + /// Backends compute these from the workspace view attached to + /// every open document — a validator diagnostic that landed in + /// an imported file's region gets routed back to that file's + /// URI. Files that no open document transitively `src`s stay + /// silent; that's a soft edge of the model but a good default + /// (an isolated file with no reachable entry is unlikely to be + /// what the user is looking for). + /// + /// Default impl returns empty so backends without workspace- + /// diagnostic support don't have to stub it out. + async fn workspace_diagnostics(&self) -> Vec<(Url, Vec)> { + Vec::new() + } + + /// Document symbols ("outline view") for `uri`. + async fn document_symbols(&self, uri: &Url) -> Vec; + + /// Workspace-wide symbol search for the LSP `workspace/symbol` + /// request. `query` is the user's filter text; the backend should + /// return at minimum every symbol whose name matches `query` (case- + /// insensitive substring is fine — the editor applies its own fuzzy + /// scoring on top). An empty `query` should return every symbol the + /// backend knows about (capped at a sensible upper bound to keep + /// the response small enough to render). + async fn workspace_symbols(&self, query: &str) -> Vec; + + /// Hover content for the construct at `position`. Returns `None` + /// if the cursor isn't on anything the backend has something to + /// say about. + async fn hover(&self, uri: &Url, position: Position) -> Option; + + /// Definition site for the reference at `position`. Returns + /// `None` if the cursor isn't on a known reference. Returns + /// possibly multiple locations because, in general, a name may + /// have several defining sites (overloads, conditional + /// definitions); the Phase 2 htcl backend only returns one. + async fn goto_definition( + &self, + uri: &Url, + position: Position, + ) -> Vec; + + /// Completion items for the cursor at `position`. Empty when the + /// backend has nothing to offer in that context. + async fn completion( + &self, + uri: &Url, + position: Position, + ) -> Vec; + + /// Signature help for the call enclosing `position`. `None` when + /// the cursor isn't inside a call the backend recognizes. + async fn signature_help( + &self, + uri: &Url, + position: Position, + ) -> Option; + + /// Compute the workspace edit that renames the identifier at + /// `position` to `new_name`. `None` when the cursor isn't on a + /// renamable symbol, when `new_name` is invalid for the target + /// language, or when the rename would need to touch symbols the + /// backend can't safely reach (e.g. cross-file references). + /// + /// Default `None` so language backends without rename support + /// (or where rename hasn't landed yet) don't have to stub the + /// method out; the server treats the response as "not + /// supported here" and the editor shows an unobtrusive error. + async fn rename( + &self, + _uri: &Url, + _position: Position, + _new_name: &str, + ) -> Option { + None + } + + /// All locations that reference the symbol at `position`, + /// including `position`'s own location. `include_declaration` + /// mirrors the LSP `ReferenceContext` flag — when `false` the + /// backend should omit the decl span from the response. + /// + /// Returns an empty vec when the cursor isn't on a known + /// symbol. Default impl returns empty so backends without a + /// references implementation don't have to stub it out. + async fn references( + &self, + _uri: &Url, + _position: Position, + _include_declaration: bool, + ) -> Vec { + Vec::new() + } +} + +/// A symbol surfaced from a backend, language-neutral. Backends that +/// need richer fields can build [`DocumentSymbol`] directly; this is +/// a convenience for the common cases that fit a flat name+kind+span. +#[derive(Clone, Debug)] +pub struct SymbolInfo { + pub name: String, + pub kind: tower_lsp::lsp_types::SymbolKind, + pub detail: Option, + pub range: tower_lsp::lsp_types::Range, + pub selection_range: tower_lsp::lsp_types::Range, +} diff --git a/vw-analyzer/src/htcl_backend.rs b/vw-analyzer/src/htcl_backend.rs new file mode 100644 index 0000000..f12a9b5 --- /dev/null +++ b/vw-analyzer/src/htcl_backend.rs @@ -0,0 +1,3383 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! htcl [`LanguageBackend`] — native, in-process, using `vw-htcl`. + +use std::collections::HashMap; +use std::fmt::Write; +use std::sync::Arc; + +use async_trait::async_trait; +use tokio::sync::{watch, RwLock}; +use tower_lsp::lsp_types::{ + CompletionItem, CompletionItemKind, Diagnostic, DiagnosticSeverity, + DocumentSymbol, Documentation, Hover, HoverContents, InsertTextFormat, + Location, MarkupContent, MarkupKind, ParameterInformation, ParameterLabel, + Position, Range, SignatureHelp, SignatureInformation, SymbolInformation, + SymbolKind, TextEdit, Url, WorkspaceEdit, +}; +use tracing::debug; +use vw_htcl::{ + definition_at, find_references_in, hover_at, identify_at, parse, + signature_help_at, validate_with_all_extras_and_vars, Attribute, + AttributeValue, CommandKind, Completion, CompletionKind, HoverTarget, + LineCol, LineIndex, ParseOutput, ProcArg, ProcSignature, ReferenceTarget, + Severity, Span, Stmt, +}; + +use crate::backend::LanguageBackend; + +#[derive(Default)] +pub struct HtclBackend { + docs: Arc>>, + /// Editor-supplied workspace roots (LSP `rootUri` / + /// `workspaceFolders`). Consulted as a fallback when the file + /// currently being analyzed sits outside the enclosing + /// `vw.toml` — e.g. after a goto-def has taken the user into a + /// dep-cache dir. Without this, dep names declared only in the + /// editor-root workspace fail to resolve and every `@name/…` + /// import in the visited file goes dead. + workspace_roots: Arc>>, +} + +/// Cached analysis for one open document. Populated by the +/// background indexer spawned from `set_text`, read by every +/// request handler. Parses + workspace-view + diagnostics all +/// bundled together — the source strings live in +/// `view.view_source` and `local_text`, and every span in +/// `parsed_view`/`parsed_local` indexes into them, so passing the +/// whole `Arc` around keeps span interpretation safe. +pub(crate) struct DocAnalysis { + /// Document text at index time — sources for every span in + /// `parsed_local`, and for line/col translation via + /// `local_line_index`. + pub local_text: String, + /// Concatenated workspace view (local text + every + /// transitively `src`d file). Source for spans in + /// `parsed_view` and for `line_index`. + pub view: crate::workspace::WorkspaceView, + pub parsed_local: ParseOutput, + pub parsed_view: ParseOutput, + pub local_line_index: LineIndex, + /// Diagnostics computed at index time — parse errors from the + /// local doc plus validator errors from the workspace view + /// filtered to local-file spans. Served verbatim by + /// `LanguageBackend::diagnostics`. + pub diagnostics: Vec, + /// Same validator errors, but for spans that land in + /// TRANSITIVELY-imported files. Each entry is + /// `(origin_file_uri, diagnostic_with_file_local_range)`. + /// The `workspace/diagnostic` handler routes these back to the + /// files that actually contain the error, giving the editor a + /// workspace-wide picker (`space-D` in Helix) even for files + /// the user hasn't opened. + pub cross_file_diagnostics: Vec<(Url, Diagnostic)>, +} + +struct DocState { + text: String, + /// Monotonic per-URI counter bumped on every `set_text`. The + /// spawned indexer captures the generation it was created for + /// and only pushes its result to the watch channel if the + /// counter still matches — newer set_text having bumped it + /// means the newer indexer will supersede us. + generation: u64, + /// Watch channel carrying the latest completed analysis. + /// `None` while indexing is in flight; `Some(Arc<..>)` once + /// the indexer commits. `set_text` sends `None` to invalidate. + /// Request handlers subscribe and `.changed().await` until + /// `borrow()` returns `Some`. + tx: watch::Sender>>, + /// Handle to the in-flight indexer. `set_text` aborts the + /// previous handle before spawning a new one — keeps only ONE + /// index running at a time so rapid typing doesn't back up + /// (each keystroke's stale index runs to completion under the + /// generation guard, but the ABORT drops it at the next .await + /// point which shortens the wall-clock for the freshest text). + index_task: Option>, +} + +impl HtclBackend { + pub fn new() -> Self { + Self::default() + } + + /// Test-only convenience: update text and wait for the indexer + /// to commit. Production `set_text` is deliberately fire-and- + /// forget so keystrokes never block; tests want synchronous + /// behavior so their assertions run against a committed + /// analysis. + #[cfg(test)] + pub(crate) async fn set_text_sync(&self, uri: Url, text: String) { + use crate::backend::LanguageBackend as _; + self.set_text(uri.clone(), text).await; + self.wait_for_reindex(&uri).await; + } + + /// Snapshot of the editor-supplied workspace roots. Callers + /// pass this into [`crate::workspace::build_view`] etc. as + /// fallback dep-lookup roots — see `workspace_roots` + /// on the struct for the rationale. Cloned so the lock isn't + /// held across the (potentially I/O-heavy) view build. + async fn workspace_roots_snapshot(&self) -> Vec { + self.workspace_roots.read().await.clone() + } + + /// Snapshot the current in-memory text for `uri`. Used by + /// completion / hover / signature-help handlers that need the + /// CURRENT text (what the user just typed) rather than the + /// analysis snapshot's stale copy. Cheap: one hashmap read + a + /// String clone. + pub(crate) async fn current_text(&self, uri: &Url) -> Option { + let docs = self.docs.read().await; + docs.get(uri).map(|s| s.text.clone()) + } + + /// Resolve a `src` import path from `entry_file`'s directory, + /// honoring the editor-supplied root fallback. + async fn resolve_import( + &self, + entry_file: &std::path::Path, + raw: &str, + ) -> Option { + let roots = self.workspace_roots_snapshot().await; + crate::workspace::resolve_import(entry_file, raw, &roots) + } + + /// Return the cached analysis for `uri`, awaiting the in-flight + /// indexer if one is currently running. Returns `None` when the + /// document isn't tracked (never `set_text`'d or already + /// `close`d). + /// + /// Contract with `set_text`: every time a new set_text fires + /// on this URI, the watch channel receives `Some(...)` only + /// after the indexer for THAT set_text's text commits under + /// the generation guard. So `analysis_for` naturally waits for + /// the LATEST index. Older in-flight indexers that finish + /// under a stale generation are silently discarded — waiters + /// don't get bogus results. + pub(crate) async fn analysis_for( + &self, + uri: &Url, + ) -> Option> { + // Non-blocking snapshot: return whatever's currently in the + // watch channel — `Some(previous_analysis)` while a rebuild + // is in flight (serve-stale), `None` before the very first + // indexer has committed. Callers that CAN work with a stale + // snapshot (completion, hover, goto-def, references) use + // this directly; callers that need a fresh commit + // (diagnostics publish + progress indicator) use + // `wait_for_reindex` explicitly. + // + // We intentionally do NOT `await` a `changed()` here: rapid + // typing plus a 7s workspace-validate means every keystroke + // aborts the in-flight indexer, and if we haven't yet had a + // first commit, waiting would block indefinitely. Returning + // `None` immediately lets handlers degrade gracefully to a + // local-only analysis or empty results. + let docs = self.docs.read().await; + let state = docs.get(uri)?; + let out = state.tx.borrow().clone(); + out + } + + /// Build a fresh `DocAnalysis` for `text` synchronously + /// (parses + workspace view + validate). Called from the + /// spawned indexer task. Extracted so it can also be invoked + /// synchronously from tests without needing the async task + /// plumbing. + pub(crate) fn build_analysis( + uri: &Url, + text: String, + workspace_roots: &[std::path::PathBuf], + ) -> Arc { + let view = crate::workspace::build_view(uri, &text, workspace_roots); + let parsed_local = parse(&text); + let local_line_index = LineIndex::new(&text); + let parsed_view = parse(&view.view_source); + let line_index = LineIndex::new(&view.view_source); + + let mut diagnostics = Vec::new(); + // Target-compatibility check. For LSP diagnostics we anchor + // the message on the `src @` statement in the local + // file — much more informative than a whole-file gutter + // marker. Only fires when the entry workspace declares a + // target-part; libraries (no target) are no-op. + if let Ok(file_path) = uri.to_file_path() { + if let Some(ws) = + file_path.parent().and_then(vw_lib::find_workspace_dir) + { + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + // LSP checks the DEFAULT part only — cheap and + // reflects what `vw run` would boot with. In + // variant-mode workspaces the default variant's + // part is what runs by default; in part-mode + // workspaces we fall back to the default target + // part. The CLI's `vw check --all-parts` / + // `--all-variants` covers the wider matrix on + // demand. + let resolved_part: Option = + if !cfg.workspace.variants.is_empty() { + cfg.workspace + .default_variant() + .ok() + .flatten() + .map(|v| v.part.clone()) + } else { + cfg.workspace + .default_target_part() + .ok() + .flatten() + .map(|p| p.to_string()) + }; + if let Some(target_part) = resolved_part { + let target_part = target_part.as_str(); + let dep_targets = vw_lib::collect_dep_targets(&ws); + let mismatches = vw_lib::check_target_compatibility( + Some(target_part), + &dep_targets, + ); + // Anchor each mismatch on the specific + // `src @` line in the LOCAL file. + for m in &mismatches { + for stmt in &parsed_local.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + let vw_htcl::CommandKind::Src(src) = &cmd.kind + else { + continue; + }; + let Some(raw) = &src.path else { continue }; + // Match `@` or `@/...`. + let dep_name = raw + .strip_prefix('@') + .and_then(|rest| { + rest.split_once('/') + .map(|(n, _)| n) + .or(Some(rest)) + }) + .unwrap_or(""); + if dep_name != m.dep { + continue; + } + let (start, end) = + local_line_index.range(src.path_span); + let hint = target_mismatch_families_hint(m); + let (severity, message) = match m.kind { + vw_lib::TargetMismatchKind::NotSupported => ( + DiagnosticSeverity::ERROR, + format!( + "target-part `{}` matches dep \ + `{}`'s `not-supported` list \ + — Xilinx has attested the IP \ + is not usable on this part \ + ({})", + m.target_part, m.dep, hint, + ), + ), + vw_lib::TargetMismatchKind::Unblessed => ( + DiagnosticSeverity::WARNING, + format!( + "target-part `{}` isn't blessed \ + by dep `{}` ({}); the IP may \ + still work but Xilinx hasn't \ + blessed the combination", + m.target_part, m.dep, hint, + ), + ), + }; + diagnostics.push(Diagnostic { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + severity: Some(severity), + source: Some("vw-htcl".into()), + message, + ..Default::default() + }); + } + } + } + } + } + } + let mut cross_file_diagnostics: Vec<(Url, Diagnostic)> = Vec::new(); + // Local parse errors. + for err in &parsed_local.errors { + let (start, end) = local_line_index.range(err.span); + diagnostics.push(Diagnostic { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + severity: Some(DiagnosticSeverity::ERROR), + source: Some("vw-htcl".into()), + message: err.message.clone(), + ..Default::default() + }); + } + // Precompute a per-imported-file `LineIndex` on demand. + // Kept keyed by region index so multiple diagnostics + // landing in the same file only build the index once. + let mut import_line_indexes: HashMap = HashMap::new(); + // Workspace-view validator. Diagnostics whose span sits + // in the local prefix land in `diagnostics`; those that + // land in an imported region get retranslated into that + // file's own line/col and stashed for the workspace- + // diagnostic path. + for d in validate_with_all_extras_and_vars( + &parsed_view.document, + &view.view_source, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + &std::collections::HashSet::new(), + &view.dep_names, + ) { + let severity = match d.severity { + Severity::Error => DiagnosticSeverity::ERROR, + Severity::Warning => DiagnosticSeverity::WARNING, + }; + if d.span.start < view.local_len { + let (start, end) = line_index.range(d.span); + diagnostics.push(Diagnostic { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + severity: Some(severity), + source: Some("vw-htcl".into()), + message: d.message, + ..Default::default() + }); + continue; + } + // Locate which imported file the span belongs to and + // translate its offset into that file's coordinates. + // Skip diagnostics that don't land in any tracked + // region — shouldn't happen with the current view + // builder, but the model allows for view-source + // regions unmapped to files (empty for now). + let Some((region_idx, region, file_offset_start)) = + view.imports.iter().enumerate().find_map(|(i, r)| { + (d.span.start >= r.start && d.span.start < r.end) + .then(|| (i, r, d.span.start - r.start)) + }) + else { + continue; + }; + let file_offset_end = d.span.end.saturating_sub(region.start); + let file_text_range = + &view.view_source[region.start as usize..region.end as usize]; + let li = import_line_indexes + .entry(region_idx) + .or_insert_with(|| LineIndex::new(file_text_range)); + let (start, end) = li.range(vw_htcl::Span { + start: file_offset_start, + end: file_offset_end.min(region.end - region.start), + }); + cross_file_diagnostics.push(( + region.file_uri.clone(), + Diagnostic { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + severity: Some(severity), + source: Some("vw-htcl".into()), + message: d.message, + ..Default::default() + }, + )); + } + + Arc::new(DocAnalysis { + local_text: text, + view, + parsed_local, + parsed_view, + local_line_index, + diagnostics, + cross_file_diagnostics, + }) + } + + /// Spawn a fresh indexer task for `uri` + `text`. Returns the + /// `JoinHandle` so the caller can install it into `DocState` + /// (and abort it on a subsequent update). `debounce` gates how + /// long the task waits before starting the ~7s build — used + /// by `set_text` (250ms, to coalesce rapid typing) and by + /// `save` (0ms, so `Ctrl-s` re-checks immediately). + /// + /// The `spawn_blocking` inner run to completion even if the + /// outer task is aborted; the generation guard at commit time + /// discards any superseded result. + fn spawn_indexer( + &self, + uri: Url, + text: String, + generation: u64, + debounce: std::time::Duration, + ) -> tokio::task::JoinHandle<()> { + let docs_arc = self.docs.clone(); + let roots_arc = self.workspace_roots.clone(); + tokio::spawn(async move { + if !debounce.is_zero() { + tokio::time::sleep(debounce).await; + } + let roots = roots_arc.read().await.clone(); + let uri_inner = uri.clone(); + let analysis = match tokio::task::spawn_blocking(move || { + HtclBackend::build_analysis(&uri_inner, text, &roots) + }) + .await + { + Ok(a) => a, + Err(_) => return, + }; + let docs = docs_arc.read().await; + if let Some(state) = docs.get(&uri) { + if state.generation == generation { + debug!(uri = %uri, generation, "index committed"); + let _ = state.tx.send_replace(Some(analysis)); + } else { + debug!( + uri = %uri, + generation, + current = state.generation, + "index superseded, discarded", + ); + } + } + }) + } +} + +#[async_trait] +impl LanguageBackend for HtclBackend { + fn language_id(&self) -> &str { + "htcl" + } + + fn handles(&self, uri: &Url) -> bool { + uri.path().ends_with(".htcl") + } + + async fn set_text(&self, uri: Url, text: String) { + debug!(%uri, bytes = text.len(), "set_text"); + // Capture the previous indexer task (if any) so we can abort + // it AFTER releasing the write lock — abort() itself is + // cheap but keeping the lock held for it stalls other + // handlers wanting to read the docs map. + let (tx, generation, prev_task) = { + let mut docs = self.docs.write().await; + match docs.get_mut(&uri) { + Some(state) => { + state.generation += 1; + state.text = text.clone(); + // Serve-stale-while-rebuild: do NOT clear the + // previous analysis. Reads (`analysis_for`, + // via completion/hover/goto-def/references) + // will see the pre-keystroke snapshot + // immediately instead of waiting the ~7s a + // fresh `build_analysis` takes on a large + // workspace (the metroid tree hits ~7s in + // the validator alone). Diagnostics update + // one commit behind — an acceptable tradeoff + // for interactive latency. When the freshly + // spawned indexer commits, `send_replace(Some + // (new))` swaps the snapshot in-place with + // no observable gap. + let prev = state.index_task.take(); + (state.tx.clone(), state.generation, prev) + } + None => { + let (tx, _rx) = watch::channel(None); + docs.insert( + uri.clone(), + DocState { + text: text.clone(), + generation: 1, + tx: tx.clone(), + index_task: None, + }, + ); + (tx, 1, None) + } + } + }; + if let Some(handle) = prev_task { + handle.abort(); + } + + let handle = self.spawn_indexer( + uri.clone(), + text, + generation, + std::time::Duration::from_millis(250), + ); + + // Store the handle so a subsequent set_text can abort us. + let mut docs = self.docs.write().await; + if let Some(state) = docs.get_mut(&uri) { + if state.generation == generation { + state.index_task = Some(handle); + } else { + // A newer set_text landed between our two lock + // acquisitions. Kill our task, the newer set_text + // has already installed its own. + handle.abort(); + } + } else { + // Doc was closed while we were spawning. Kill our task. + handle.abort(); + } + let _ = tx; // silence unused warning when watching sends aren't used further + } + + async fn set_workspace_roots(&self, roots: Vec) { + *self.workspace_roots.write().await = roots; + } + + async fn save(&self, uri: &Url) { + // Save is the user's explicit "I'm done for now" signal — + // skip the debounce entirely so their edit is checked + // immediately. Bump the generation so any in-flight + // debounced indexer from the last `set_text` gets + // superseded when this one commits. Text is whatever's + // currently stored; Helix sends `did_change` before + // `did_save` on save operations so the buffer's already + // in sync. + let (text, generation, prev_task) = { + let mut docs = self.docs.write().await; + let Some(state) = docs.get_mut(uri) else { + return; + }; + state.generation += 1; + let prev = state.index_task.take(); + (state.text.clone(), state.generation, prev) + }; + if let Some(handle) = prev_task { + handle.abort(); + } + let handle = self.spawn_indexer( + uri.clone(), + text, + generation, + std::time::Duration::ZERO, + ); + let mut docs = self.docs.write().await; + if let Some(state) = docs.get_mut(uri) { + if state.generation == generation { + state.index_task = Some(handle); + } else { + handle.abort(); + } + } else { + handle.abort(); + } + } + + async fn wait_for_reindex(&self, uri: &Url) { + // Subscribe to the doc's analysis-watch channel, then wait + // for the NEXT commit — i.e. the indexer task's + // `send_replace(Some(new))` at the end of `set_text`'s + // spawned future. Used by the server to wrap the wait in + // an LSP `workDoneProgress` notification so the editor's + // "indexing…" spinner reflects the actual rebuild + // duration, not just a fire-and-forget millisecond. + // + // `mark_unchanged` is the key call: `subscribe()` seeds the + // receiver at the current sender value (which may be the + // stale-serve analysis we've KEPT alive across `set_text` + // — see the serve-stale comment there). Without + // `mark_unchanged` the immediate `changed().await` would + // return instantly on that already-seen value and the + // spinner would flash for a millisecond instead of + // spanning the whole rebuild. + // + // If the sender is dropped (doc closed), or the current + // watch has never held a value (uri unknown), we return + // immediately — no rebuild to wait on. + let mut rx = { + let docs = self.docs.read().await; + match docs.get(uri) { + Some(state) => state.tx.subscribe(), + None => return, + } + }; + rx.mark_unchanged(); + let _ = rx.changed().await; + } + + async fn close(&self, uri: &Url) { + let prev_task = { + let mut docs = self.docs.write().await; + docs.remove(uri).and_then(|s| s.index_task) + }; + if let Some(handle) = prev_task { + handle.abort(); + } + } + + async fn diagnostics(&self, uri: &Url) -> Vec { + // All diagnostics precomputed at index time. No work here. + let Some(analysis) = self.analysis_for(uri).await else { + return Vec::new(); + }; + analysis.diagnostics.clone() + } + + async fn workspace_diagnostics(&self) -> Vec<(Url, Vec)> { + // Walk every open document's committed analysis. Each + // carries its own local diagnostics AND the retranslated + // diagnostics from every file it transitively `src`s. + // + // Snapshot the URI list first so we can drop the docs + // read lock before calling `analysis_for` on each (which + // takes its own read). + let uris: Vec = self.docs.read().await.keys().cloned().collect(); + let roots = self.workspace_roots_snapshot().await; + // Group by origin URI so a file `src`d by multiple open + // docs doesn't get its diagnostics duplicated; when the + // same file surfaces via more than one analysis, we keep + // just the first non-empty set. In practice the analyses + // agree because the validator is deterministic per input. + let mut by_uri: HashMap> = HashMap::new(); + for uri in &uris { + let Some(analysis) = self.analysis_for(uri).await else { + continue; + }; + // Every open document contributes an entry — even an + // EMPTY one — so the editor clears stale errors when + // the user fixes them. Without this, a file that + // *used* to have errors keeps showing them until it's + // reopened. + by_uri + .entry(uri.clone()) + .or_insert_with(|| analysis.diagnostics.clone()); + // Seed an empty entry for every imported file that + // sits inside the workspace. This is what makes fixed + // errors CLEAR: after the fix, `cross_file_diagnostics` + // has no entry for that file, but the fan-out still + // sees the URI (from `view.imports`) and publishes an + // empty payload — the editor's cached "there were + // errors here" state gets overwritten with "no + // errors." Without this seed, cleared files wouldn't + // reappear in the output map at all. + for import in &analysis.view.imports { + if !roots.is_empty() + && !uri_under_roots(&import.file_uri, &roots) + { + continue; + } + by_uri.entry(import.file_uri.clone()).or_default(); + } + for (u, d) in &analysis.cross_file_diagnostics { + // Only surface diagnostics from files that sit + // inside the editor's own workspace roots. Deps + // (`~/.vw/deps`, the amd/ trees, etc.) get walked + // by `build_view` for symbol resolution, but the + // user isn't editing them from this workspace — + // reporting a dep-side error in `space-D` is + // noise. When no roots are set (e.g. the file was + // opened standalone), fall through: nothing to + // filter against. + if !roots.is_empty() && !uri_under_roots(u, &roots) { + continue; + } + by_uri.entry(u.clone()).or_default().push(d.clone()); + } + } + by_uri.into_iter().collect() + } + + async fn document_symbols(&self, uri: &Url) -> Vec { + let Some(analysis) = self.analysis_for(uri).await else { + return Vec::new(); + }; + let parsed = &analysis.parsed_local; + let line_index = &analysis.local_line_index; + let mut symbols = Vec::new(); + for stmt in &parsed.document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Proc(proc) = &cmd.kind else { + continue; + }; + let name = proc.name.clone().unwrap_or_else(|| "".into()); + let (cmd_start, cmd_end) = line_index.range(cmd.span); + let (name_start, name_end) = line_index.range(proc.name_span); + let detail = if cmd.doc_comments.is_empty() { + None + } else { + Some(cmd.doc_comments.join("\n")) + }; + #[allow(deprecated)] + symbols.push(DocumentSymbol { + name, + detail, + kind: SymbolKind::FUNCTION, + tags: None, + deprecated: None, + range: Range { + start: lc_to_pos(cmd_start), + end: lc_to_pos(cmd_end), + }, + selection_range: Range { + start: lc_to_pos(name_start), + end: lc_to_pos(name_end), + }, + children: None, + }); + } + symbols + } + + async fn workspace_symbols(&self, query: &str) -> Vec { + // Cap the response so a wide picker scroll doesn't pay for + // thousands of entries when the user hasn't narrowed yet. The + // editor applies its own scoring on top, so any reasonable + // ceiling keeps the UX responsive. + const MAX_RESULTS: usize = 500; + + let needle = query.to_ascii_lowercase(); + // Snapshot the list of open URIs — we release the docs + // lock before calling `analysis_for` on each so an + // in-flight indexer's write-lock acquisition doesn't + // deadlock against our read lock. + let uris: Vec = self.docs.read().await.keys().cloned().collect(); + // Files we've already harvested — dedupe so a header imported + // by multiple open docs doesn't double up. Keyed on the URI as + // a string for hashability. + let mut seen_files: HashMap = HashMap::new(); + let mut out: Vec = Vec::new(); + + for uri in &uris { + let Some(analysis) = self.analysis_for(uri).await else { + continue; + }; + // Visit the open doc itself first, then everything it + // transitively `src`s. `build_view` already canonicalizes + // paths during the walk, so the import file_uris are + // stable across docs. + if seen_files.insert(uri.to_string(), ()).is_none() { + collect_workspace_symbols( + uri, + &analysis.local_text, + &needle, + &mut out, + MAX_RESULTS, + ); + if out.len() >= MAX_RESULTS { + return out; + } + } + + for import in &analysis.view.imports { + let key = import.file_uri.to_string(); + if seen_files.insert(key, ()).is_some() { + continue; + } + let text = &analysis.view.view_source + [import.start as usize..import.end as usize]; + collect_workspace_symbols( + &import.file_uri, + text, + &needle, + &mut out, + MAX_RESULTS, + ); + if out.len() >= MAX_RESULTS { + return out; + } + } + } + out + } + + async fn goto_definition( + &self, + uri: &Url, + position: Position, + ) -> Vec { + let Some(analysis) = self.analysis_for(uri).await else { + return Vec::new(); + }; + let line_index = &analysis.local_line_index; + let offset = line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + + // Special case: cursor on a `src @dep/foo` path → jump to the + // imported file. Resolved through the same `vw-lib` machinery + // the CLI uses, so editor and CLI agree on the same target. + let parsed_local = &analysis.parsed_local; + if let Some(import) = src_import_at(&parsed_local.document, offset) { + if let Some(raw) = import.path.as_deref() { + let Ok(file_path) = uri.to_file_path() else { + return Vec::new(); + }; + if let Some(resolved) = + self.resolve_import(&file_path, raw).await + { + if let Ok(target_uri) = Url::from_file_path(resolved) { + return vec![Location { + uri: target_uri, + range: Range::default(), + }]; + } + } + } + return Vec::new(); + } + + // General case: resolve against the workspace view so calls to + // imported procs jump to the right file. + let view = &analysis.view; + let parsed_view = &analysis.parsed_view; + let Some(target_span) = + definition_at(&parsed_view.document, &view.view_source, offset) + else { + return Vec::new(); + }; + + // Translate the target span back to its source file: local + // file when in the original region, otherwise the imported + // file whose appended region contains it. + match view.locate(target_span.start) { + None => { + // Local hit — line_index is over analysis.local_text. + let (start, end) = analysis.local_line_index.range(target_span); + vec![Location { + uri: uri.clone(), + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + }] + } + Some((region, _)) => { + // Read the imported file's text so we can build a + // file-local line index. (Already on disk; cheap.) + let Ok(import_path) = region.file_uri.to_file_path() else { + return Vec::new(); + }; + let Ok(import_text) = std::fs::read_to_string(&import_path) + else { + return Vec::new(); + }; + let import_index = LineIndex::new(&import_text); + let local_start = target_span.start - region.start; + let local_end = target_span.end - region.start; + let (s, e) = import_index + .range(vw_htcl::Span::new(local_start, local_end)); + vec![Location { + uri: region.file_uri.clone(), + range: Range { + start: lc_to_pos(s), + end: lc_to_pos(e), + }, + }] + } + } + } + + async fn hover(&self, uri: &Url, position: Position) -> Option { + // Same strategy as completion: serve hover off the CURRENT + // in-memory text so what the user's cursor is on maps to + // real content. When a workspace analysis is available + // (usually — 99% of hover requests fire between typing + // bursts, when there IS a committed snapshot), we consult + // it for cross-file lookups; when there isn't (fresh file, + // still-building initial index), we degrade to local-only + // — still useful for hovering over locally-defined procs. + let current_text = self.current_text(uri).await.unwrap_or_default(); + let current_line_index = LineIndex::new(¤t_text); + let offset = current_line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + let current_parsed = vw_htcl::parse(¤t_text); + + // Prefer the workspace snapshot's parsed_view (it contains + // the imported proc definitions we want to hover through). + // Fall back to the CURRENT local parse when nothing's + // committed yet. + let stale = self.analysis_for(uri).await; + let (hover_doc, hover_source, hover_offset, doc_for_comments) = + if let Some(a) = stale.as_ref() { + // The offset was computed against CURRENT text. It + // maps 1:1 into the workspace view AS LONG AS the + // stale view's local prefix is a prefix of the + // current text (typical: adds/dels midway through + // the line only shift bytes past that point, but + // the workspace view is only ~correct in the local + // prefix anyway). For hover, the miscarriage is + // harmless — worst case we hover on the wrong + // token and return None. + ( + &a.parsed_view.document, + a.view.view_source.as_str(), + offset, + &a.parsed_view.document, + ) + } else { + ( + ¤t_parsed.document, + current_text.as_str(), + offset, + ¤t_parsed.document, + ) + }; + let target = hover_at(hover_doc, hover_source, hover_offset)?; + // Prefer LOCAL line index for translating spans → line/col: + // that's what Helix expects. + let (start, end) = current_line_index.range(target.span()); + // The proc's own doc comments live on the surrounding Command, + // not on its `Proc` payload — fetch them up here so the + // formatters can stay focused on shape, not lookup plumbing. + let proc_doc_comments = match &target { + HoverTarget::ProcDef { proc, .. } => { + proc_doc_comments_for(doc_for_comments, proc) + } + HoverTarget::CallSite { proc_name, .. } => { + proc_doc_comments_by_name(doc_for_comments, proc_name) + } + _ => Vec::new(), + }; + let markdown = format_hover(&target, &proc_doc_comments); + Some(Hover { + contents: HoverContents::Markup(MarkupContent { + kind: MarkupKind::Markdown, + value: markdown, + }), + range: Some(Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }), + }) + } + + async fn completion( + &self, + uri: &Url, + position: Position, + ) -> Vec { + // Serve completion off the CURRENT in-memory text (what the + // user just typed), not the stale-cached workspace analysis's + // local_text. Otherwise cmdline::analyze scans backward from + // an offset in TEXT THAT DOESN'T CONTAIN WHAT WAS JUST TYPED + // — the `partial` comes out blank or wrong, and completion + // silently returns nothing. This is the "typed `-preset` and + // got no enum values" symptom. + // + // Cross-file proc lookups (`gtwiz_versal::configure`, etc.) + // still come from the stale workspace analysis via + // `parsed_view` — those signatures don't change while the + // user types locally, so stale is fine. + let current_text = self.current_text(uri).await.unwrap_or_default(); + let current_line_index = LineIndex::new(¤t_text); + let offset = current_line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + let current_parsed = vw_htcl::parse(¤t_text); + + // `src ` is filesystem-aware, so it takes its own + // path before we fall back to the htcl-level analyzer. + let line = vw_htcl::cmdline::analyze(¤t_text, offset); + if crate::src_complete::is_src_path_context(&line) { + if let Ok(entry_file) = uri.to_file_path() { + let resolver = crate::workspace::build_resolver(&entry_file); + return crate::src_complete::src_path_completions( + &entry_file, + &line, + ¤t_line_index, + &resolver, + ); + } + } + + // Grab the workspace analysis if it exists (stale or fresh). + // If we've never had one commit, we complete against just + // the local file — better than blocking indefinitely. + let analysis = self.analysis_for(uri).await; + let workspace_docs: Vec<&vw_htcl::Document> = analysis + .as_ref() + .map(|a| vec![&a.parsed_view.document]) + .unwrap_or_default(); + + vw_htcl::complete_at_with_extras( + ¤t_parsed.document, + ¤t_text, + offset, + &workspace_docs, + ) + .into_iter() + .map(|c| completion_item(c, ¤t_line_index)) + .collect() + } + + async fn signature_help( + &self, + uri: &Url, + position: Position, + ) -> Option { + let analysis = self.analysis_for(uri).await?; + let offset = analysis.local_line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + // Workspace view so signatures of imported procs surface, and + // so the cmdline scan can step into a `[ … ]` substitution + // (the parser now carries a `body` inside `CmdSubst` and the + // scan already treats `[` as a command boundary). + let view = &analysis.view; + let parsed = &analysis.parsed_view; + let help = + signature_help_at(&parsed.document, &view.view_source, offset)?; + Some(signature_help_response(&help)) + } + + async fn rename( + &self, + uri: &Url, + position: Position, + new_name: &str, + ) -> Option { + let (target, _decl_uri) = + self.identify_target_at(uri, position).await?; + // Build a per-URI edit list for every file the target + // reaches. For file-local kinds (Local, ProcArg) this + // resolves to just the current file; for cross-file + // kinds it walks every `.htcl` file under the workspace + // root. + let per_file = self.collect_reference_spans(uri, &target).await; + if per_file.is_empty() { + return None; + } + let replacement = rename_replacement_for(&target, new_name)?; + let mut changes: HashMap> = HashMap::new(); + for (file_uri, text, spans) in per_file { + let line_index = LineIndex::new(&text); + let edits: Vec = spans + .into_iter() + .map(|span| span_to_text_edit(span, &line_index, &replacement)) + .collect(); + if !edits.is_empty() { + changes.insert(file_uri, edits); + } + } + if changes.is_empty() { + return None; + } + Some(WorkspaceEdit { + changes: Some(changes), + document_changes: None, + change_annotations: None, + }) + } + + async fn references( + &self, + uri: &Url, + position: Position, + include_declaration: bool, + ) -> Vec { + let Some((target, _)) = self.identify_target_at(uri, position).await + else { + return Vec::new(); + }; + let per_file = self.collect_reference_spans(uri, &target).await; + let mut locations = Vec::new(); + for (file_uri, text, mut spans) in per_file { + let line_index = LineIndex::new(&text); + if !include_declaration { + // Best-effort decl filter: the target's own decl + // is contained inside the ref set (procs' name- + // span, types' name-span, enum-variant name- + // spans). The reference finder returns them all; + // remove those that lie inside the current + // target's OWN declaration span when the target + // came from this file. For cross-file callers + // there's no ambiguity — the decl is only in the + // decl file. + spans.retain(|s| { + !span_looks_like_decl(&target, *s, &file_uri, uri) + }); + } + for span in spans { + let (start, end) = line_index.range(span); + locations.push(Location { + uri: file_uri.clone(), + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + }); + } + } + locations + } +} + +impl HtclBackend { + /// Identify the reference target at `position` in `uri`. Also + /// returns the URI that owned the identification so the + /// per-file collector knows which document to treat as the + /// origin (matters for the file-local Local/ProcArg kinds). + async fn identify_target_at( + &self, + uri: &Url, + position: Position, + ) -> Option<(ReferenceTarget, Url)> { + let analysis = self.analysis_for(uri).await?; + let line_index = &analysis.local_line_index; + let offset = line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + let parsed = &analysis.parsed_local; + let target = + identify_at(&parsed.document, &analysis.local_text, offset)?; + Some((target, uri.clone())) + } + + /// For each file the target reaches, return `(uri, text, + /// spans)`. File-local targets stay in the origin file; cross- + /// file targets get every `.htcl` file under the workspace + /// root read and scanned. + async fn collect_reference_spans( + &self, + origin: &Url, + target: &ReferenceTarget, + ) -> Vec<(Url, String, Vec)> { + match target { + ReferenceTarget::Local { .. } | ReferenceTarget::ProcArg { .. } => { + let Some(analysis) = self.analysis_for(origin).await else { + return Vec::new(); + }; + let spans = find_references_in( + &analysis.parsed_local.document, + &analysis.local_text, + target, + ); + if spans.is_empty() { + Vec::new() + } else { + vec![(origin.clone(), analysis.local_text.clone(), spans)] + } + } + ReferenceTarget::Proc { .. } + | ReferenceTarget::Type { .. } + | ReferenceTarget::EnumVariant { .. } => { + let mut files = self.workspace_htcl_files(origin).await; + // Fallback: no workspace root (test URIs, files + // opened outside a `vw.toml` tree, etc.) → operate + // on the origin file only. The rename still + // works locally; users adopting the LSP outside a + // workspace get local-only semantics until they + // set up a `vw.toml`. + if files.is_empty() { + if let Some(analysis) = self.analysis_for(origin).await { + files.push(( + origin.clone(), + analysis.local_text.clone(), + )); + } + } + let mut out = Vec::new(); + for (file_uri, text) in files { + let parsed = parse(&text); + let spans = + find_references_in(&parsed.document, &text, target); + if !spans.is_empty() { + out.push((file_uri, text, spans)); + } + } + out + } + } + } + + /// Enumerate every `.htcl` file under the workspace root that + /// contains `origin`. Reads their current on-disk content — + /// for files also open in the editor this may be one tick + /// behind, but that's the cost of not requiring the editor to + /// pre-open every workspace file. Skips typical non-workspace + /// directories (`target/`, `.git/`, `.vw/`). + /// + /// The origin file itself is served from the in-memory + /// analysis so unsaved edits round-trip through the rename. + async fn workspace_htcl_files(&self, origin: &Url) -> Vec<(Url, String)> { + let Ok(origin_path) = origin.to_file_path() else { + return Vec::new(); + }; + let Some(origin_utf8) = camino::Utf8Path::from_path(&origin_path) + else { + return Vec::new(); + }; + let Some(root) = crate::workspace::workspace_root(origin_utf8) else { + return Vec::new(); + }; + let mut paths: Vec = Vec::new(); + walk_htcl_files(root.as_std_path(), &mut paths); + let mut visited: std::collections::HashSet = + std::collections::HashSet::new(); + let mut out: Vec<(Url, String)> = Vec::new(); + for path in paths { + let canonical = + path.canonicalize().unwrap_or_else(|_| path.clone()); + if !visited.insert(canonical.clone()) { + continue; + } + let file_uri = match Url::from_file_path(&canonical) { + Ok(u) => u, + Err(_) => continue, + }; + // For the origin file, prefer the in-memory analysis + // text so unsaved edits round-trip. + let text = if file_uri == *origin { + if let Some(analysis) = self.analysis_for(&file_uri).await { + analysis.local_text.clone() + } else { + std::fs::read_to_string(&canonical).unwrap_or_default() + } + } else { + std::fs::read_to_string(&canonical).unwrap_or_default() + }; + if text.is_empty() { + continue; + } + out.push((file_uri, text)); + } + out + } +} + +/// Recursively walk `dir` collecting `.htcl` files. Skips `target/`, +/// `.git/`, `.vw/`, `node_modules/` at any depth. Silently swallows +/// I/O errors on individual directories — a permission-denied +/// subtree just contributes nothing to the results. +/// True when `uri`'s filesystem path lies under any of `roots`. +/// Non-file URIs and paths that don't resolve into any root fall +/// through as `false` — the caller's default behavior is "not in +/// the workspace, don't fan out." Both sides get canonicalized so +/// a symlinked workspace root matches a real-path URI (`Path:: +/// starts_with` is purely lexical). Canonicalization failures +/// (missing files, permission errors) fall back to the lexical +/// compare, which still catches the common case. +/// Same helper as vw-cli's — split blessed vs. banned family lists +/// so a dep whose `[targets]` only carries a `not-supported` list +/// doesn't get misreported as "declared families: versal" when +/// the versal families there are BANNED, not blessed. +fn target_mismatch_families_hint(m: &vw_lib::TargetMismatch) -> String { + match ( + m.supported_families.is_empty(), + m.not_supported_families.is_empty(), + ) { + (true, true) => { + "no `[targets]` families declared — the dep has patterns \ + but none carry family names" + .to_string() + } + (false, true) => { + format!("blessed families: {}", m.supported_families.join(", ")) + } + (true, false) => { + format!( + "no blessed families — only `not-supported` entries for {}", + m.not_supported_families.join(", "), + ) + } + (false, false) => { + format!( + "blessed families: {}; also `not-supported` entries for {}", + m.supported_families.join(", "), + m.not_supported_families.join(", "), + ) + } + } +} + +fn uri_under_roots(uri: &Url, roots: &[std::path::PathBuf]) -> bool { + let Ok(path) = uri.to_file_path() else { + return false; + }; + let canonical_path = path.canonicalize().unwrap_or(path); + roots.iter().any(|r| { + let canonical_root = r.canonicalize().unwrap_or_else(|_| r.clone()); + canonical_path.starts_with(&canonical_root) + }) +} + +fn walk_htcl_files(dir: &std::path::Path, out: &mut Vec) { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + Err(_) => return, + }; + for entry in entries.flatten() { + let path = entry.path(); + let file_type = match entry.file_type() { + Ok(t) => t, + Err(_) => continue, + }; + if file_type.is_dir() { + let name = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); + if matches!(name, "target" | ".git" | ".vw" | "node_modules") { + continue; + } + walk_htcl_files(&path, out); + } else if file_type.is_file() + && path.extension().and_then(|s| s.to_str()) == Some("htcl") + { + out.push(path); + } + } +} + +/// Pick the exact text to substitute at each rename span for a +/// given target. Preserves namespace prefixes when the user typed +/// a bare replacement name. +fn rename_replacement_for( + target: &ReferenceTarget, + new_name: &str, +) -> Option { + if new_name.is_empty() { + return None; + } + // Validate: bare identifier or `ns::segment(::segment)*`. + for seg in new_name.split("::") { + if seg.is_empty() { + return None; + } + let mut bytes = seg.bytes(); + let first = bytes.next().unwrap(); + if !(first.is_ascii_alphabetic() || first == b'_') { + return None; + } + if !bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return None; + } + } + Some(match target { + ReferenceTarget::Proc { name } + if name.contains("::") && !new_name.contains("::") => + { + let ns = name.rsplit_once("::").map(|(n, _)| n).unwrap_or(""); + format!("{ns}::{new_name}") + } + ReferenceTarget::EnumVariant { enum_name, .. } + if !new_name.contains("::") => + { + format!("{enum_name}::{new_name}") + } + _ => new_name.to_string(), + }) +} + +/// Map a source `Span` + replacement text to an LSP `TextEdit`. +fn span_to_text_edit( + span: Span, + line_index: &LineIndex, + new_text: &str, +) -> TextEdit { + let (start, end) = line_index.range(span); + TextEdit { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + new_text: new_text.to_string(), + } +} + +/// Best-effort filter for the `!include_declaration` case. Skips +/// spans that plausibly correspond to a decl site by name-matching +/// the target's shape. This is imperfect (a proc named `X` and a +/// call `X` are indistinguishable at the span level), but the LSP +/// clients that pass `include_declaration=false` usually just want +/// to hide the decl in the results — an occasional inclusion is +/// benign. +fn span_looks_like_decl( + _target: &ReferenceTarget, + _span: Span, + _file_uri: &Url, + _origin_uri: &Url, +) -> bool { + // Placeholder — the LSP protocol says clients CAN filter locally, + // and most do. Returning false means we always include; safer + // than accidentally dropping too much. + false +} + +// (`rename_edit_to_lsp` removed — the rename handler now emits +// `TextEdit`s directly via `span_to_text_edit` on the raw +// reference spans, so the intermediate `RenameEdit` type isn't +// crossed over anymore.) + +// --- completion / signature-help formatters ------------------------------- + +fn completion_item(c: Completion, line_index: &LineIndex) -> CompletionItem { + let kind = match c.kind { + CompletionKind::Proc => CompletionItemKind::FUNCTION, + CompletionKind::Flag => CompletionItemKind::FIELD, + CompletionKind::EnumValue => CompletionItemKind::ENUM_MEMBER, + CompletionKind::Constructor => CompletionItemKind::CONSTRUCTOR, + }; + let (start, end) = line_index.range(c.replace); + let insert = c.insert_text.clone().unwrap_or_else(|| c.label.clone()); + let text_edit = TextEdit { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + new_text: insert, + }; + let insert_text_format = if c.snippet { + InsertTextFormat::SNIPPET + } else { + InsertTextFormat::PLAIN_TEXT + }; + CompletionItem { + label: c.label, + kind: Some(kind), + detail: c.detail, + documentation: c.documentation.map(|value| { + Documentation::MarkupContent(MarkupContent { + kind: MarkupKind::Markdown, + value, + }) + }), + insert_text_format: Some(insert_text_format), + text_edit: Some(tower_lsp::lsp_types::CompletionTextEdit::Edit( + text_edit, + )), + ..Default::default() + } +} + +fn signature_help_response(help: &vw_htcl::SignatureHelp<'_>) -> SignatureHelp { + // Build the rendered signature label and, in lockstep, the + // [start, end) offsets each parameter occupies within it so the + // editor highlights the active one. Names are identifiers, so + // UTF-16 and char counts coincide. + let mut label = help.proc_name.clone(); + let mut parameters = Vec::with_capacity(help.signature.args.len()); + for arg in &help.signature.args { + label.push(' '); + let start = label.chars().count() as u32; + label.push('-'); + label.push_str(&arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + label.push_str(": "); + label.push_str(&render_type(ty)); + } + let end = label.chars().count() as u32; + parameters.push(ParameterInformation { + label: ParameterLabel::LabelOffsets([start, end]), + documentation: vw_htcl::doc::brief(&arg.doc_comments) + .map(Documentation::String), + }); + } + // Append the return type to the signature label when present. + // Renders as `proc-name -arg1 -arg2 → bd_cell`. + if let Some(ty) = help.signature.return_type.as_ref() { + label.push_str(" → "); + label.push_str(&render_type(ty)); + } + + let reflowed = vw_htcl::doc::reflow_doc_comments(help.doc_comments); + let documentation = (!reflowed.is_empty()).then_some({ + Documentation::MarkupContent(MarkupContent { + kind: MarkupKind::Markdown, + value: reflowed, + }) + }); + + #[allow(deprecated)] // `active_parameter` field on SignatureInformation + let info = SignatureInformation { + label, + documentation, + parameters: Some(parameters), + active_parameter: help.active_parameter, + }; + + SignatureHelp { + signatures: vec![info], + active_signature: Some(0), + active_parameter: help.active_parameter, + } +} + +// --- src import lookup ---------------------------------------------------- + +/// If the cursor at `offset` is on the path word of a `src ` +/// statement, return that import. Used by `goto_definition` to jump +/// to the imported module. +fn src_import_at( + document: &vw_htcl::Document, + offset: u32, +) -> Option<&vw_htcl::SrcImport> { + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(import) = &cmd.kind else { + continue; + }; + if import.path_span.contains(offset) { + return Some(import); + } + } + None +} + +// --- doc-comment lookup --------------------------------------------------- + +fn proc_doc_comments_for( + document: &vw_htcl::Document, + proc: &vw_htcl::Proc, +) -> Vec { + proc_doc_comments_for_in(&document.stmts, proc).unwrap_or_default() +} + +fn proc_doc_comments_for_in( + stmts: &[Stmt], + proc: &vw_htcl::Proc, +) -> Option> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(p) + // Pointer-identity match: `proc` was looked up out + // of this same parse, so its address inside the AST + // is unique. + if std::ptr::eq(p, proc) => { + return Some(cmd.doc_comments.clone()); + } + CommandKind::NamespaceEval(ns) => { + if let Some(found) = proc_doc_comments_for_in(&ns.body, proc) { + return Some(found); + } + } + _ => {} + } + } + None +} + +fn proc_doc_comments_by_name( + document: &vw_htcl::Document, + name: &str, +) -> Vec { + proc_doc_comments_by_name_in(&document.stmts, "", name).unwrap_or_default() +} + +fn proc_doc_comments_by_name_in( + stmts: &[Stmt], + prefix: &str, + name: &str, +) -> Option> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(p) => { + let Some(decl_name) = p.name.as_deref() else { + continue; + }; + let qualified = if prefix.is_empty() { + decl_name.to_string() + } else { + format!("{prefix}::{decl_name}") + }; + if qualified == name { + return Some(cmd.doc_comments.clone()); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(ns_name) = ns.name.as_deref() else { + continue; + }; + let nested = if prefix.is_empty() { + ns_name.to_string() + } else { + format!("{prefix}::{ns_name}") + }; + if let Some(found) = + proc_doc_comments_by_name_in(&ns.body, &nested, name) + { + return Some(found); + } + } + _ => {} + } + } + None +} + +/// Render a type expression in the canonical user-facing form — +/// `dict`, `list`, etc. Used by hover and +/// signature-help so the displayed type matches what the user +/// would write in source. +fn render_type(ty: &vw_htcl::TypeExpr) -> String { + match ty { + vw_htcl::TypeExpr::Named { name, .. } => name.clone(), + vw_htcl::TypeExpr::Generic { name, args, .. } => { + let inner: Vec = args.iter().map(render_type).collect(); + format!("{name}<{}>", inner.join(",")) + } + vw_htcl::TypeExpr::Qualified { + namespace, variant, .. + } => { + format!("{namespace}::{variant}") + } + } +} + +// --- markdown formatters -------------------------------------------------- + +fn format_hover(target: &HoverTarget, proc_doc_comments: &[String]) -> String { + match target { + HoverTarget::ProcDef { proc, .. } => format_proc( + proc.name.as_deref().unwrap_or(""), + proc.signature.as_ref(), + proc_doc_comments, + ), + HoverTarget::CallSite { + proc_name, + signature, + .. + } => format_proc(proc_name, Some(signature), proc_doc_comments), + HoverTarget::ProcArgDef { arg, .. } + | HoverTarget::CallArg { arg, .. } => format_arg(arg), + HoverTarget::LocalVar { name, ty, .. } => { + format_local_var(name, ty.as_ref()) + } + HoverTarget::EnumDef { decl, .. } => format_enum(decl), + HoverTarget::TypeDef { decl, .. } => format_type_def(decl), + } +} + +fn format_type_def(decl: &vw_htcl::TypeDecl) -> String { + let name = decl.name.as_deref().unwrap_or(""); + let mut out = String::new(); + writeln!(out, "```htcl").unwrap(); + match decl.underlying.as_ref() { + Some(ty) => writeln!(out, "type {name} = {}", render_type(ty)).unwrap(), + None => writeln!(out, "type {name} = ").unwrap(), + } + writeln!(out, "```").unwrap(); + out +} + +fn format_enum(decl: &vw_htcl::EnumDecl) -> String { + let mut out = String::new(); + let name = decl.name.as_deref().unwrap_or(""); + writeln!(out, "```htcl").unwrap(); + writeln!(out, "enum {name} = {{").unwrap(); + for v in &decl.variants { + match v.payload.as_ref() { + Some(p) => { + writeln!(out, " {}: {}", v.name, render_type(p)).unwrap() + } + None => writeln!(out, " {}", v.name).unwrap(), + } + } + writeln!(out, "}}").unwrap(); + writeln!(out, "```").unwrap(); + out.push_str("\nTagged sum type. The compiler auto-generates "); + out.push_str("constructors (`::`), repr, and "); + out.push_str("`tag`/`payload` accessors. See "); + out.push_str("docs/htcl-enums.md for the full semantics.\n"); + out +} + +fn format_local_var(name: &str, ty: Option<&vw_htcl::TypeExpr>) -> String { + let mut out = String::new(); + writeln!(out, "```htcl").unwrap(); + match ty { + Some(t) => writeln!(out, "${name}: {}", render_type(t)).unwrap(), + None => writeln!(out, "${name}").unwrap(), + } + writeln!(out, "```").unwrap(); + out.push_str("\nLocal variable.\n"); + out +} + +fn format_proc( + name: &str, + signature: Option<&ProcSignature>, + proc_doc_comments: &[String], +) -> String { + let mut out = String::new(); + writeln!(out, "```htcl").unwrap(); + // Include the return type in the proc header when annotated: + // proc foo → string + // Unannotated procs render unchanged (`proc foo`). + let return_ty = signature.and_then(|s| s.return_type.as_ref()); + match return_ty { + Some(ty) => { + writeln!(out, "proc {name} → {}", render_type(ty)).unwrap(); + } + None => { + writeln!(out, "proc {name}").unwrap(); + } + } + writeln!(out, "```").unwrap(); + let reflowed = vw_htcl::doc::reflow_doc_comments(proc_doc_comments); + if !reflowed.is_empty() { + out.push('\n'); + out.push_str(&reflowed); + out.push('\n'); + } + if let Some(sig) = signature { + if !sig.args.is_empty() { + out.push_str("\n### Parameters\n\n"); + for arg in &sig.args { + match arg.type_annotation.as_ref() { + Some(ty) => { + write!(out, "- `-{}: {}`", arg.name, render_type(ty)) + .unwrap(); + } + None => { + write!(out, "- `-{}`", arg.name).unwrap(); + } + } + let reflowed = + vw_htcl::doc::reflow_doc_comments(&arg.doc_comments); + let mut paragraphs = reflowed.split("\n\n"); + if let Some(brief) = paragraphs.next().filter(|s| !s.is_empty()) + { + write!(out, " — {brief}").unwrap(); + } + out.push('\n'); + for extra in paragraphs.filter(|s| !s.is_empty()) { + writeln!(out, " {extra}").unwrap(); + } + for attr in &arg.attributes { + writeln!(out, " - `{}`", format_attribute(attr)).unwrap(); + } + } + } + } + out +} + +fn format_arg(arg: &ProcArg) -> String { + let mut out = String::new(); + writeln!(out, "```htcl").unwrap(); + match arg.type_annotation.as_ref() { + Some(ty) => { + writeln!(out, "-{}: {}", arg.name, render_type(ty)).unwrap() + } + None => writeln!(out, "-{}", arg.name).unwrap(), + } + writeln!(out, "```").unwrap(); + let reflowed = vw_htcl::doc::reflow_doc_comments(&arg.doc_comments); + if !reflowed.is_empty() { + out.push('\n'); + out.push_str(&reflowed); + out.push('\n'); + } + if !arg.attributes.is_empty() { + out.push('\n'); + for attr in &arg.attributes { + writeln!(out, "- `{}`", format_attribute(attr)).unwrap(); + } + } + out +} + +fn format_attribute(attr: &Attribute) -> String { + if attr.values.is_empty() { + format!("@{}", attr.name) + } else { + let values: Vec = + attr.values.iter().map(format_attribute_value).collect(); + format!("@{}({})", attr.name, values.join(", ")) + } +} + +fn format_attribute_value(v: &AttributeValue) -> String { + v.to_tcl_literal() +} + +fn lc_to_pos(lc: LineCol) -> Position { + Position { + line: lc.line, + character: lc.character, + } +} + +/// Parse one htcl `text` and push every `proc` / `type` / `enum` +/// declaration whose name contains `needle` (case-insensitive, empty +/// `needle` matches all) into `out`. Stops as soon as `out` reaches +/// `cap` entries so a `workspace/symbol` request never assembles an +/// unbounded response. Variants of an enum are emitted as siblings +/// with `container_name` set to the enum, matching how +/// rust-analyzer surfaces variants in the workspace picker. +fn collect_workspace_symbols( + uri: &Url, + text: &str, + needle: &str, + out: &mut Vec, + cap: usize, +) { + let parsed = parse(text); + let line_index = LineIndex::new(text); + for stmt in &parsed.document.stmts { + if out.len() >= cap { + return; + } + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(name) = proc.name.as_deref() { + push_symbol( + uri, + &line_index, + name, + proc.name_span, + SymbolKind::FUNCTION, + None, + needle, + out, + ); + } + } + CommandKind::TypeDecl(td) => { + if let Some(name) = td.name.as_deref() { + push_symbol( + uri, + &line_index, + name, + td.name_span, + SymbolKind::STRUCT, + None, + needle, + out, + ); + } + } + CommandKind::EnumDecl(ed) => { + let enum_name = ed.name.as_deref(); + if let Some(name) = enum_name { + push_symbol( + uri, + &line_index, + name, + ed.name_span, + SymbolKind::ENUM, + None, + needle, + out, + ); + } + for v in &ed.variants { + if out.len() >= cap { + return; + } + push_symbol( + uri, + &line_index, + &v.name, + v.name_span, + SymbolKind::ENUM_MEMBER, + enum_name.map(str::to_string), + needle, + out, + ); + } + } + _ => {} + } + } +} + +#[allow(clippy::too_many_arguments)] +fn push_symbol( + uri: &Url, + line_index: &LineIndex, + name: &str, + span: vw_htcl::Span, + kind: SymbolKind, + container_name: Option, + needle: &str, + out: &mut Vec, +) { + if !needle.is_empty() && !name.to_ascii_lowercase().contains(needle) { + return; + } + let (start, end) = line_index.range(span); + #[allow(deprecated)] + out.push(SymbolInformation { + name: name.to_string(), + kind, + tags: None, + deprecated: None, + location: Location { + uri: uri.clone(), + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + }, + container_name, + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn uri() -> Url { + Url::parse("file:///tmp/x.htcl").unwrap() + } + + #[tokio::test] + async fn handles_htcl_extension() { + let backend = HtclBackend::new(); + assert!(backend.handles(&uri())); + assert!(!backend.handles(&Url::parse("file:///tmp/x.vhd").unwrap())); + } + + #[tokio::test] + async fn diagnostics_for_unterminated_string() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "puts \"oops\nputs ok\n".into()) + .await; + let diags = backend.diagnostics(&uri()).await; + assert!(!diags.is_empty(), "expected at least one diagnostic"); + assert_eq!(diags[0].severity, Some(DiagnosticSeverity::ERROR)); + assert!(diags[0].message.contains("unterminated string")); + } + + #[tokio::test] + async fn document_symbols_include_proc() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "## greet someone\nproc greet {name} { puts hi }\n".into(), + ) + .await; + let symbols = backend.document_symbols(&uri()).await; + assert_eq!(symbols.len(), 1); + assert_eq!(symbols[0].name, "greet"); + assert_eq!(symbols[0].kind, SymbolKind::FUNCTION); + assert_eq!(symbols[0].detail.as_deref(), Some("greet someone")); + } + + #[tokio::test] + async fn workspace_symbols_surface_procs_types_and_enum_variants() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "proc greet {name} { puts hi }\n\ + type Foo = int\n\ + enum Color = {\n Red\n Green\n Blue\n}\n" + .into(), + ) + .await; + let all = backend.workspace_symbols("").await; + let names: Vec<&str> = all.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"greet"), "{names:?}"); + assert!(names.contains(&"Foo"), "{names:?}"); + assert!(names.contains(&"Color"), "{names:?}"); + assert!(names.contains(&"Red"), "{names:?}"); + let red = all.iter().find(|s| s.name == "Red").unwrap(); + assert_eq!(red.kind, SymbolKind::ENUM_MEMBER); + assert_eq!(red.container_name.as_deref(), Some("Color")); + + // Substring filter, case-insensitive. + let filtered = backend.workspace_symbols("gre").await; + assert!(filtered.iter().any(|s| s.name == "greet")); + assert!(filtered.iter().any(|s| s.name == "Green")); + assert!(!filtered.iter().any(|s| s.name == "Foo")); + } + + #[tokio::test] + async fn validator_diagnostics_surface_in_lsp() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "proc axis {\n @enum(1, 2, 4) width\n} { puts $width }\n\ + axis -width 3\n" + .into(), + ) + .await; + let diags = backend.diagnostics(&uri()).await; + assert!( + diags.iter().any(|d| d.message.contains("@enum")), + "{:?}", + diags + ); + } + + /// Unused-variable warnings from the `vw-htcl::unused` pass + /// reach LSP clients with `DiagnosticSeverity::WARNING` and + /// point at the offending decl. Underscore-prefixed names are + /// exempt. + #[tokio::test] + async fn unused_var_warning_surfaces_in_lsp() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "proc f {unused_arg} { return 1 }\n".into()) + .await; + let diags = backend.diagnostics(&uri()).await; + let warnings: Vec<_> = diags + .iter() + .filter(|d| d.severity == Some(DiagnosticSeverity::WARNING)) + .filter(|d| d.message.contains("unused proc arg")) + .collect(); + assert_eq!(warnings.len(), 1, "{:?}", diags); + assert!( + warnings[0].message.contains("unused_arg"), + "{:?}", + warnings[0] + ); + } + + #[tokio::test] + async fn unused_var_underscore_prefix_suppresses_lsp_warning() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "proc f {_ignored} { return 1 }\n".into()) + .await; + let diags = backend.diagnostics(&uri()).await; + assert!( + !diags.iter().any(|d| d.message.contains("unused")), + "{:?}", + diags + ); + } + + /// Rename produces a WorkspaceEdit whose TextEdits, when + /// applied in reverse order, transform the source correctly. + /// Covers the end-to-end LSP path: cursor → offset → rename_at → + /// edits → LSP `WorkspaceEdit`. + #[tokio::test] + async fn rename_local_via_lsp() { + let backend = HtclBackend::new(); + // `mode` is a local; renaming it should update the decl and + // the two `$mode` refs. + let src = "\ +proc f {} { + set mode fast + puts $mode + return $mode +} +"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on the `m` of `set mode` (line 1, column 6). 0-indexed. + let workspace_edit = backend + .rename( + &uri(), + Position { + line: 1, + character: 6, + }, + "kind", + ) + .await + .expect("rename should succeed"); + let changes = workspace_edit.changes.expect("expected changes"); + let text_edits = changes.get(&uri()).expect("edits for this file"); + assert_eq!(text_edits.len(), 3, "{text_edits:?}"); + // Apply edits from tail to head to preserve earlier offsets. + let mut renamed = src.to_string(); + let mut edits = text_edits.clone(); + edits.sort_by_key(|e| (e.range.start.line, e.range.start.character)); + for edit in edits.iter().rev() { + let start = position_to_offset(&renamed, edit.range.start); + let end = position_to_offset(&renamed, edit.range.end); + renamed.replace_range(start..end, &edit.new_text); + } + assert!(renamed.contains("set kind fast"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + assert!(renamed.contains("return $kind"), "{renamed}"); + assert!(!renamed.contains("mode"), "{renamed}"); + } + + /// Proc-name rename now works within the local file — the + /// declaration span and every call site in the same document + /// get rewritten. Cross-file callers (in other `.htcl` files + /// the LSP hasn't yet been asked about) still need the + /// workspace-scan variant. + #[tokio::test] + async fn rename_proc_name_via_lsp_covers_decl_and_call() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "proc greet {} { puts hi }\ngreet\n".into()) + .await; + // Cursor on the `g` of the proc's own name. + let result = backend + .rename( + &uri(), + Position { + line: 0, + character: 5, + }, + "hello", + ) + .await; + let ws = result.expect("expected an edit set"); + let changes = ws.changes.expect("expected changes map"); + let edits = changes.get(&uri()).expect("edits for the local uri"); + assert_eq!(edits.len(), 2, "decl + one call site"); + } + + #[tokio::test] + async fn references_returns_all_local_call_sites() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "proc greet {} { puts hi }\ngreet\nproc other {} { greet }\n" + .into(), + ) + .await; + let locs = backend + .references( + &uri(), + Position { + line: 0, + character: 5, + }, + true, + ) + .await; + // 3 hits: decl name + top-level call + nested call in `other`. + assert_eq!(locs.len(), 3, "{locs:?}"); + for loc in &locs { + assert_eq!(loc.uri, uri()); + } + } + + #[tokio::test] + async fn references_on_type_covers_annotations() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "type MyThing = string\nproc a {v: MyThing} MyThing { }\nproc b {v: MyThing} { }\n" + .into(), + ) + .await; + // Cursor on `MyThing` at the type decl (char 5..12 = "MyThing"). + let locs = backend + .references( + &uri(), + Position { + line: 0, + character: 5, + }, + true, + ) + .await; + // decl + a's arg-type + a's return-type + b's arg-type = 4. + assert_eq!(locs.len(), 4, "{locs:?}"); + } + + #[tokio::test] + async fn rename_type_covers_all_annotations() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "type MyThing = string\nproc a {v: MyThing} MyThing { }\n" + .into(), + ) + .await; + let ws = backend + .rename( + &uri(), + Position { + line: 0, + character: 5, + }, + "YourThing", + ) + .await + .expect("edit set"); + let changes = ws.changes.expect("changes"); + let edits = changes.get(&uri()).expect("local edits"); + // Same 3 hits: decl + arg-type + return-type. + assert_eq!(edits.len(), 3, "{edits:?}"); + for e in edits { + assert_eq!(e.new_text, "YourThing"); + } + } + + /// Utility: convert an LSP `Position` (line + UTF-16 char offset, + /// but at ASCII we treat as byte offset) into a byte index in the + /// given source. Used to apply text edits in tests. + fn position_to_offset(source: &str, pos: Position) -> usize { + let mut cur_line = 0u32; + let mut cur_col = 0u32; + for (idx, byte) in source.bytes().enumerate() { + if cur_line == pos.line && cur_col == pos.character { + return idx; + } + if byte == b'\n' { + cur_line += 1; + cur_col = 0; + } else { + cur_col += 1; + } + } + source.len() + } + + #[tokio::test] + async fn hover_on_call_site_shows_signature() { + let backend = HtclBackend::new(); + let src = "\ +## Greet someone by name.\n\ +proc greet {\n\ + ## Who to greet.\n\ + @default(\"world\") name\n\ +} { puts \"hi $name\" }\n\ +greet -name there\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on the `g` of the call-site `greet`. Line indices + // are 0-based. + let hover = backend + .hover( + &uri(), + Position { + line: 5, + character: 0, + }, + ) + .await + .expect("hover should return content"); + let body = match hover.contents { + HoverContents::Markup(m) => m.value, + _ => panic!("expected markup"), + }; + assert!(body.contains("proc greet"), "{body}"); + assert!(body.contains("Greet someone by name."), "{body}"); + assert!(body.contains("### Parameters"), "{body}"); + assert!(body.contains("-name"), "{body}"); + assert!(body.contains("Who to greet."), "{body}"); + assert!(body.contains("@default"), "{body}"); + } + + #[tokio::test] + async fn hover_on_call_arg_shows_arg_doc() { + let backend = HtclBackend::new(); + let src = "\ +proc greet {\n\ + ## Who to greet.\n\ + @default(\"world\") name\n\ +} { puts hi }\n\ +greet -name there\n"; + backend.set_text_sync(uri(), src.into()).await; + // Position cursor on `-name` of the call site (line 4 in the + // 0-indexed scheme). + let hover = backend + .hover( + &uri(), + Position { + line: 4, + character: 7, + }, + ) + .await + .expect("hover should return content"); + let body = match hover.contents { + HoverContents::Markup(m) => m.value, + _ => panic!("expected markup"), + }; + assert!(body.contains("-name"), "{body}"); + assert!(body.contains("Who to greet."), "{body}"); + assert!(body.contains("@default"), "{body}"); + // Shouldn't include the proc-level header. + assert!(!body.contains("### Parameters"), "{body}"); + } + + #[tokio::test] + async fn hover_outside_known_construct_returns_none() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "puts hello world\n".into()) + .await; + let hover = backend + .hover( + &uri(), + Position { + line: 0, + character: 0, + }, + ) + .await; + assert!(hover.is_none()); + } + + #[tokio::test] + async fn goto_definition_jumps_call_to_proc_decl() { + let backend = HtclBackend::new(); + let src = "\ +proc greet {\n name\n} { puts hi }\n\ +greet -name there\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on the `g` of the call-site `greet` (line 3). + let locs = backend + .goto_definition( + &uri(), + Position { + line: 3, + character: 0, + }, + ) + .await; + assert_eq!(locs.len(), 1); + // Decl name `greet` is on line 0 at character 5. + assert_eq!(locs[0].range.start.line, 0); + assert_eq!(locs[0].range.start.character, 5); + } + + #[tokio::test] + async fn goto_definition_resolves_attribute_ident() { + let backend = HtclBackend::new(); + let src = "\ +proc f {\n has_a\n @requires(has_a) has_b\n} { }\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on `has_a` inside `@requires(has_a)`. + let locs = backend + .goto_definition( + &uri(), + Position { + line: 2, + character: 13, + }, + ) + .await; + assert_eq!(locs.len(), 1); + // Decl `has_a` is on line 1 at character 2. + assert_eq!(locs[0].range.start.line, 1); + assert_eq!(locs[0].range.start.character, 2); + } + + #[tokio::test] + async fn completion_offers_proc_names_in_command_position() { + let backend = HtclBackend::new(); + let src = "\ +proc greet {} { }\n\ +proc grumble {} { }\n\ +gr\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor at end of `gr` on line 2. + let items = backend + .completion( + &uri(), + Position { + line: 2, + character: 2, + }, + ) + .await; + let mut labels: Vec = + items.iter().map(|i| i.label.clone()).collect(); + labels.sort(); + assert_eq!(labels, vec!["greet", "grumble"]); + assert_eq!(items[0].kind, Some(CompletionItemKind::FUNCTION)); + } + + #[tokio::test] + async fn completion_offers_flags_in_argument_position() { + let backend = HtclBackend::new(); + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg \n"; + backend.set_text_sync(uri(), src.into()).await; + // Line 4, just after `cfg ` (character 4). + let items = backend + .completion( + &uri(), + Position { + line: 4, + character: 4, + }, + ) + .await; + let mut labels: Vec = + items.iter().map(|i| i.label.clone()).collect(); + labels.sort(); + assert_eq!(labels, vec!["-depth", "-width"]); + assert_eq!(items[0].kind, Some(CompletionItemKind::FIELD)); + } + + #[tokio::test] + async fn signature_help_highlights_active_parameter() { + let backend = HtclBackend::new(); + let src = "\ +## Configure the bus.\n\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -depth \n"; + backend.set_text_sync(uri(), src.into()).await; + // Line 5, after `cfg -depth ` (character 11). + let help = backend + .signature_help( + &uri(), + Position { + line: 5, + character: 11, + }, + ) + .await + .expect("signature help expected"); + assert_eq!(help.active_parameter, Some(1)); + let info = &help.signatures[0]; + assert!(info.label.starts_with("cfg "), "{}", info.label); + assert_eq!(info.parameters.as_ref().unwrap().len(), 2); + match &info.documentation { + Some(Documentation::MarkupContent(m)) => { + assert!(m.value.contains("Configure the bus."), "{}", m.value); + } + other => panic!("expected markup documentation, got {other:?}"), + } + } + + #[tokio::test] + async fn signature_help_includes_return_type_arrow() { + let backend = HtclBackend::new(); + let src = "\ +proc make_widget {} bd_cell { return foo }\n\ +make_widget \n"; + backend.set_text_sync(uri(), src.into()).await; + let help = backend + .signature_help( + &uri(), + Position { + line: 1, + character: 12, + }, + ) + .await + .expect("signature help expected"); + let info = &help.signatures[0]; + // Label should carry the `→ bd_cell` suffix. + assert!(info.label.contains("→ bd_cell"), "{}", info.label); + } + + #[tokio::test] + async fn hover_on_enum_decl_shows_variants() { + let backend = HtclBackend::new(); + let src = "\ +enum Property = {\n Scalar: string\n Nested: int\n}\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on the enum name (line 0, col 5: 'Property'). + let hover = backend + .hover( + &uri(), + Position { + line: 0, + character: 7, + }, + ) + .await + .expect("hover on enum decl name"); + if let HoverContents::Markup(MarkupContent { value, .. }) = + hover.contents + { + assert!(value.contains("enum Property"), "{value}"); + assert!(value.contains("Scalar: string"), "{value}"); + assert!(value.contains("Nested: int"), "{value}"); + } else { + panic!("expected Markup hover"); + } + } + + #[tokio::test] + async fn hover_proc_def_includes_return_type() { + let backend = HtclBackend::new(); + let src = "\ +## Builds a widget.\n\ +proc make_widget {} dict { return {} }\n"; + backend.set_text_sync(uri(), src.into()).await; + // Hover on the proc name `make_widget` at line 1. + let hover = backend + .hover( + &uri(), + Position { + line: 1, + character: 8, + }, + ) + .await + .expect("hover expected on proc def"); + if let HoverContents::Markup(MarkupContent { value, .. }) = + hover.contents + { + assert!( + value.contains("→ dict"), + "expected return type in hover: {value}" + ); + } else { + panic!("expected Markup hover contents"); + } + } + + #[tokio::test] + async fn signature_help_none_outside_call() { + let backend = HtclBackend::new(); + backend.set_text_sync(uri(), "puts hi\n".into()).await; + let help = backend + .signature_help( + &uri(), + Position { + line: 0, + character: 0, + }, + ) + .await; + assert!(help.is_none()); + } + + #[tokio::test] + async fn goto_definition_unknown_returns_empty() { + let backend = HtclBackend::new(); + backend.set_text_sync(uri(), "puts hello\n".into()).await; + let locs = backend + .goto_definition( + &uri(), + Position { + line: 0, + character: 0, + }, + ) + .await; + assert!(locs.is_empty()); + } + + // --- cross-file (workspace view) tests -------------------------------- + + /// Build a temp workspace with a `lib.htcl` defining `greet` and + /// a `main.htcl` that imports it. Returns the backend with both + /// files already opened and the URIs. + async fn temp_workspace_with_import() -> ( + tempfile::TempDir, + HtclBackend, + Url, // main.htcl + Url, // lib.htcl + ) { + let dir = tempfile::tempdir().unwrap(); + let lib_path = dir.path().join("lib.htcl"); + std::fs::write( + &lib_path, + "## Greet someone.\n\ +proc greet {\n ## Who to greet.\n who\n} { puts \"hi $who\" }\n", + ) + .unwrap(); + let main_path = dir.path().join("main.htcl"); + let main_src = "src lib\ngreet -who world\n"; + std::fs::write(&main_path, main_src).unwrap(); + + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let lib_uri = Url::from_file_path(&lib_path).unwrap(); + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + (dir, backend, main_uri, lib_uri) + } + + #[tokio::test] + async fn goto_on_src_import_jumps_to_imported_file() { + let (_dir, backend, main_uri, lib_uri) = + temp_workspace_with_import().await; + // Cursor on the `l` of `src lib` (line 0, col 4). + let locs = backend + .goto_definition( + &main_uri, + Position { + line: 0, + character: 4, + }, + ) + .await; + assert_eq!(locs.len(), 1); + assert_eq!(locs[0].uri, lib_uri); + } + + #[tokio::test] + async fn goto_on_call_to_imported_proc_jumps_to_lib() { + let (_dir, backend, main_uri, lib_uri) = + temp_workspace_with_import().await; + // Cursor on `greet` at line 1. + let locs = backend + .goto_definition( + &main_uri, + Position { + line: 1, + character: 0, + }, + ) + .await; + assert_eq!(locs.len(), 1, "{locs:?}"); + assert_eq!(locs[0].uri, lib_uri); + // The declaration of `greet` is on lib.htcl line 1 col 5. + assert_eq!(locs[0].range.start.line, 1); + assert_eq!(locs[0].range.start.character, 5); + } + + /// Regression: a call from inside an `if { … }` body should + /// still find its proc's declaration. The parser leaves the + /// brace-body as an opaque word, so without an explicit + /// reparse pass in [`vw_htcl::goto`] the search never reaches + /// the nested call. + #[tokio::test] + async fn goto_from_inside_if_body() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("m.htcl"); + let src = "proc target { x } { }\n\ + proc caller { } {\n \ + if {1} {\n \ + target -x 1\n \ + }\n\ + }\n"; + std::fs::write(&path, src).unwrap(); + let backend = HtclBackend::new(); + let uri = Url::from_file_path(&path).unwrap(); + backend.set_text_sync(uri.clone(), src.into()).await; + let locs = backend + .goto_definition( + &uri, + Position { + line: 3, + character: 4, + }, + ) + .await; + assert!( + !locs.is_empty(), + "goto-def from inside `if {{…}}` body failed" + ); + } + + /// Regression: a call from inside `[…]` command substitution + /// inside `if {…} { … }` — the double-nested shape the IP + /// wrapper's `if {$bd} { set cell [create_bd_cell …] } + /// else { set cell [create_ip …] }` scaffold produces. The + /// reparse pass has to also run `populate_procs` so the + /// inner CmdSubst.body gets filled in. + #[tokio::test] + async fn goto_from_cmdsubst_inside_if_body() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("m.htcl"); + let src = "proc target { x } { }\n\ + proc caller { } {\n \ + if {1} {\n \ + set cell [target -x 1]\n \ + }\n\ + }\n"; + std::fs::write(&path, src).unwrap(); + let backend = HtclBackend::new(); + let uri = Url::from_file_path(&path).unwrap(); + backend.set_text_sync(uri.clone(), src.into()).await; + // Cursor on `target` inside `[target -x 1]` on line 3. + // Line 3 is ` set cell [target -x 1]`; `target` starts + // at col 17. + let locs = backend + .goto_definition( + &uri, + Position { + line: 3, + character: 17, + }, + ) + .await; + assert!( + !locs.is_empty(), + "goto-def from inside `[[…]]`-inside-`if` failed" + ); + } + + /// Companion to [`goto_from_cmdsubst_inside_if_body`] for hover. + #[tokio::test] + async fn hover_from_cmdsubst_inside_if_body() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("m.htcl"); + let src = "## Target proc doc.\n\ + proc target { x } { }\n\ + proc caller { } {\n \ + if {1} {\n \ + set cell [target -x 1]\n \ + }\n\ + }\n"; + std::fs::write(&path, src).unwrap(); + let backend = HtclBackend::new(); + let uri = Url::from_file_path(&path).unwrap(); + backend.set_text_sync(uri.clone(), src.into()).await; + // Cursor on `target` inside `[target -x 1]` on line 4. + let hover = backend + .hover( + &uri, + Position { + line: 4, + character: 17, + }, + ) + .await; + assert!( + hover.is_some(), + "hover from inside `[[…]]`-inside-`if` returned None" + ); + } + + /// Same regression as [`goto_from_inside_if_body`], but for + /// hover — the two share the "reparse brace-body" fix in + /// [`vw_htcl::goto`] / [`vw_htcl::hover`]. + #[tokio::test] + async fn hover_from_inside_if_body() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("m.htcl"); + let src = "## Target proc doc.\n\ + proc target { x } { }\n\ + proc caller { } {\n \ + if {1} {\n \ + target -x 1\n \ + }\n\ + }\n"; + std::fs::write(&path, src).unwrap(); + let backend = HtclBackend::new(); + let uri = Url::from_file_path(&path).unwrap(); + backend.set_text_sync(uri.clone(), src.into()).await; + let hover = backend + .hover( + &uri, + Position { + line: 4, + character: 4, + }, + ) + .await; + assert!( + hover.is_some(), + "hover from inside `if {{…}}` body returned None" + ); + } + + /// Reproduces the exact user scenario against the on-disk + /// `~/src/htcl/amd/` tree. Only runs when that path exists, so + /// the test is a no-op in CI / fresh checkouts. + #[tokio::test] + async fn goto_finds_sibling_workspace_dep_real_htcl_tree() { + let cpm5_module = + std::path::PathBuf::from("/home/ry/src/htcl/amd/cpm5/module.htcl"); + if !cpm5_module.exists() { + eprintln!("skipping — {} not present", cpm5_module.display()); + return; + } + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + let text = std::fs::read_to_string(&cpm5_module).unwrap(); + backend.set_text_sync(cpm5_uri.clone(), text.clone()).await; + + // Find the line + column of `vivado_cmd::set_property` — + // avoids hard-coding a line number that will drift as the + // wrapper regenerates. + let mut target_line = None; + for (i, line) in text.lines().enumerate() { + if let Some(col) = line.find("vivado_cmd::set_property") { + // Cursor on the `set_property` word, past the + // `vivado_cmd::` prefix (12 chars). + target_line = Some((i as u32, (col + 12) as u32)); + break; + } + } + let Some((line, character)) = target_line else { + panic!("no `vivado_cmd::set_property` in cpm5/module.htcl"); + }; + let locs = backend + .goto_definition(&cpm5_uri, Position { line, character }) + .await; + assert!( + !locs.is_empty(), + "goto-def against real htcl tree returned no location \ + for cpm5/module.htcl:{line}:{character}" + ); + let hit = &locs[0]; + let path = hit.uri.to_file_path().unwrap(); + assert!( + path.to_string_lossy().contains("vivado-cmd"), + "expected to land in the vivado-cmd tree, got {:?}", + hit + ); + } + + /// Regression against the on-disk cpm5 tree for goto and hover + /// on `vivado_cmd::create_bd_cell` — the sole cell-creation + /// call at the top of `create_cpm5`. (Previously covered + /// `vivado_cmd::create_ip` too, but the IP generator's + /// `-bd 0` path is now rejected up front with an `error`, so + /// the generated wrapper only calls `create_bd_cell`.) + #[tokio::test] + async fn goto_and_hover_on_create_bd_cell_and_create_ip_in_cpm5() { + let cpm5_module = + std::path::PathBuf::from("/home/ry/src/htcl/amd/cpm5/module.htcl"); + if !cpm5_module.exists() { + return; + } + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + let text = std::fs::read_to_string(&cpm5_module).unwrap(); + backend.set_text_sync(cpm5_uri.clone(), text.clone()).await; + for needle in &["vivado_cmd::create_bd_cell"] { + let (line, character) = text + .lines() + .enumerate() + .find_map(|(i, l)| { + l.find(needle).map(|c| (i as u32, (c + 12) as u32)) + }) + .unwrap_or_else(|| panic!("no {needle} in cpm5/module.htcl")); + let locs = backend + .goto_definition(&cpm5_uri, Position { line, character }) + .await; + assert!( + !locs.is_empty(), + "goto-def on {needle} at {line}:{character} returned nothing" + ); + let hover = + backend.hover(&cpm5_uri, Position { line, character }).await; + assert!( + hover.is_some(), + "hover on {needle} at {line}:{character} returned None" + ); + } + } + + /// Companion to [`goto_finds_sibling_workspace_dep_real_htcl_tree`] + /// for hover — same file, same cursor position, same expected + /// outcome: the imported proc's signature resolves and hover + /// returns something rather than `None`. + #[tokio::test] + async fn hover_finds_imported_proc_real_htcl_tree() { + let cpm5_module = + std::path::PathBuf::from("/home/ry/src/htcl/amd/cpm5/module.htcl"); + if !cpm5_module.exists() { + return; + } + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + let text = std::fs::read_to_string(&cpm5_module).unwrap(); + backend.set_text_sync(cpm5_uri.clone(), text.clone()).await; + let target = text.lines().enumerate().find_map(|(i, line)| { + line.find("vivado_cmd::set_property") + .map(|col| (i as u32, (col + 12) as u32)) + }); + let Some((line, character)) = target else { + panic!("no `vivado_cmd::set_property` in cpm5/module.htcl"); + }; + let hover = + backend.hover(&cpm5_uri, Position { line, character }).await; + assert!( + hover.is_some(), + "hover against real htcl tree returned None \ + for cpm5/module.htcl:{line}:{character}" + ); + } + + /// Hover + goto on a namespaced newtype (`dcmac::MacPortProps`) + /// used as a return-type annotation resolve to the type + /// declaration. Guards the analyzer's type-annotation path + /// against regressions and validates end-to-end with a real + /// generated wrapper. + #[tokio::test] + async fn hover_goto_on_namespaced_newtype_return_type() { + let dcmac_module = + std::path::PathBuf::from("/home/ry/src/htcl/amd/dcmac/module.htcl"); + if !dcmac_module.exists() { + return; + } + let backend = HtclBackend::new(); + let dcmac_uri = Url::from_file_path(&dcmac_module).unwrap(); + let text = std::fs::read_to_string(&dcmac_module).unwrap(); + backend.set_text_sync(dcmac_uri.clone(), text.clone()).await; + // Find a real type-annotation site (arg-type slot on + // `MacPortProps::from` etc.) — not the `namespace eval + // dcmac::MacPortProps {}` word, which passes the string as + // a namespace name rather than a type annotation. + let target = text.lines().enumerate().find_map(|(i, line)| { + line.find(": dcmac::MacPortProps") + .map(|col| (i as u32, (col + 9) as u32)) + }); + let Some((line, character)) = target else { + panic!("no `: dcmac::MacPortProps` in dcmac/module.htcl"); + }; + let hover = backend + .hover(&dcmac_uri, Position { line, character }) + .await; + assert!( + hover.is_some(), + "hover on `dcmac::MacPortProps` at line {line}:{character} \ + returned None — type-annotation path not wired" + ); + let locs = backend + .goto_definition(&dcmac_uri, Position { line, character }) + .await; + assert!( + !locs.is_empty(), + "goto on `dcmac::MacPortProps` at line {line}:{character} \ + returned no locations" + ); + } + + /// Sibling-workspace fallback with a NESTED src chain — mirrors + /// the real vivado-cmd layout where `module.htcl` re-sources + /// per-command files under `cmd/`. `set_property` doesn't live + /// in the module.htcl entry directly; it's reached through + /// `src "cmd/set_property.htcl"` inside the dep module. This + /// caught the actual reproduction case where a shallower test + /// (proc in the dep's module.htcl) passed but goto-def against + /// the real vivado-cmd tree still returned nothing. + #[tokio::test] + async fn goto_finds_sibling_workspace_dep_via_nested_src() { + let dir = tempfile::tempdir().unwrap(); + let amd = dir.path().join("amd"); + let cpm5 = amd.join("cpm5"); + let vivado_cmd = amd.join("vivado-cmd"); + let vivado_cmd_cmd = vivado_cmd.join("cmd"); + std::fs::create_dir_all(&cpm5).unwrap(); + std::fs::create_dir_all(&vivado_cmd_cmd).unwrap(); + std::fs::write( + cpm5.join("vw.toml"), + "[workspace]\nname=\"cpm5\"\nversion=\"0.1.0\"\n\n[dependencies]\n", + ) + .unwrap(); + std::fs::write( + vivado_cmd.join("vw.toml"), + "[workspace]\nname=\"vivado-cmd\"\nversion=\"0.1.0\"\n\n\ + [dependencies]\n", + ) + .unwrap(); + // vivado-cmd/module.htcl re-sources set_property.htcl — + // matching the real layout. + std::fs::write( + vivado_cmd.join("module.htcl"), + "src \"cmd/set_property.htcl\"\n", + ) + .unwrap(); + // vivado-cmd/cmd/set_property.htcl defines the proc. + let set_property_path = vivado_cmd_cmd.join("set_property.htcl"); + std::fs::write( + &set_property_path, + "namespace eval vivado_cmd {\n \ + proc set_property { args } { }\n}\n", + ) + .unwrap(); + let cpm5_module = cpm5.join("module.htcl"); + std::fs::write( + &cpm5_module, + "src @vivado-cmd\nvivado_cmd::set_property -dict {} -objects x\n", + ) + .unwrap(); + + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + backend + .set_text_sync( + cpm5_uri.clone(), + std::fs::read_to_string(&cpm5_module).unwrap(), + ) + .await; + // Cursor on `set_property` — the call is + // `vivado_cmd::set_property ...` (col 0). `vivado_cmd::` + // is 12 chars; `set_property` starts at col 12. + let locs = backend + .goto_definition( + &cpm5_uri, + Position { + line: 1, + character: 12, + }, + ) + .await; + assert!(!locs.is_empty(), "goto-def returned no location"); + let set_property_uri = Url::from_file_path(&set_property_path).unwrap(); + assert_eq!( + locs[0].uri, set_property_uri, + "expected jump to {set_property_uri}, got {:?}", + locs[0] + ); + } + + /// Sibling-workspace fallback: when a file's own workspace + /// doesn't declare a `@dep/…` import but a sibling directory + /// under a shared parent DOES have its own `vw.toml` with a + /// matching basename, the resolver should still find it. + /// + /// Layout (mirrors `~/src/htcl/amd/{cpm5,vivado-cmd}` as the + /// user's actual reproduction): + /// + /// /amd/cpm5/vw.toml # empty deps + /// /amd/cpm5/module.htcl # calls vivado_cmd::foo + /// /amd/vivado-cmd/vw.toml + /// /amd/vivado-cmd/module.htcl # namespace eval vivado_cmd { proc foo … } + /// + /// Regression for "goto-def returns 'No definition found' once + /// I'm in a vw-tracked dependency." + #[tokio::test] + async fn goto_finds_sibling_workspace_dep() { + let dir = tempfile::tempdir().unwrap(); + let amd = dir.path().join("amd"); + let cpm5 = amd.join("cpm5"); + let vivado_cmd = amd.join("vivado-cmd"); + std::fs::create_dir_all(&cpm5).unwrap(); + std::fs::create_dir_all(&vivado_cmd).unwrap(); + std::fs::write( + cpm5.join("vw.toml"), + "[workspace]\nname=\"cpm5\"\nversion=\"0.1.0\"\n\n[dependencies]\n", + ) + .unwrap(); + std::fs::write( + vivado_cmd.join("vw.toml"), + "[workspace]\nname=\"vivado-cmd\"\nversion=\"0.1.0\"\n\n\ + [dependencies]\n", + ) + .unwrap(); + // The vivado-cmd module: define namespace `vivado_cmd` with + // a `foo` proc so a call to `vivado_cmd::foo` from cpm5 has + // somewhere to land. + let vivado_module = vivado_cmd.join("module.htcl"); + std::fs::write( + &vivado_module, + "namespace eval vivado_cmd {\n proc foo { x } { }\n}\n", + ) + .unwrap(); + let cpm5_module = cpm5.join("module.htcl"); + std::fs::write(&cpm5_module, "src @vivado-cmd\nvivado_cmd::foo -x 1\n") + .unwrap(); + + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + backend + .set_text_sync( + cpm5_uri.clone(), + std::fs::read_to_string(&cpm5_module).unwrap(), + ) + .await; + + // Cursor on `foo` — line 1, at the start of the call word + // (`vivado_cmd::foo` starts at column 0, `foo` starts after + // `vivado_cmd::` which is 12 chars). + let locs = backend + .goto_definition( + &cpm5_uri, + Position { + line: 1, + character: 12, + }, + ) + .await; + assert!(!locs.is_empty(), "goto-def returned no location"); + let vivado_uri = Url::from_file_path(&vivado_module).unwrap(); + assert_eq!(locs[0].uri, vivado_uri, "landed on wrong file"); + } + + #[tokio::test] + async fn completion_in_command_position_lists_imported_procs() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + // Append a partial proc name at end of file so cursor lands in + // command position. + let new_text = "src lib\ngreet -who world\ngre\n"; + backend + .set_text_sync(main_uri.clone(), new_text.into()) + .await; + let items = backend + .completion( + &main_uri, + Position { + line: 2, + character: 3, + }, + ) + .await; + let labels: Vec<_> = items.iter().map(|i| i.label.as_str()).collect(); + assert!(labels.contains(&"greet"), "labels = {labels:?}"); + } + + #[tokio::test] + async fn hover_on_imported_call_shows_signature() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + // Hover on `greet` on line 1. + let hover = backend + .hover( + &main_uri, + Position { + line: 1, + character: 0, + }, + ) + .await + .expect("hover"); + let body = match hover.contents { + HoverContents::Markup(m) => m.value, + _ => panic!(), + }; + assert!(body.contains("proc greet"), "{body}"); + assert!(body.contains("Greet someone."), "{body}"); + assert!(body.contains("-who"), "{body}"); + } + + #[tokio::test] + async fn diagnostics_accept_calls_to_imported_procs() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + // No errors when the call matches the imported signature. + let diags = backend.diagnostics(&main_uri).await; + let errs: Vec<_> = diags + .iter() + .filter(|d| d.severity == Some(DiagnosticSeverity::ERROR)) + .collect(); + assert!(errs.is_empty(), "{errs:?}"); + } + + #[tokio::test] + async fn hover_works_on_call_inside_command_substitution() { + // Mirrors the user's cips.htcl shape: + // src lib + // set cell [greet -who x] + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + let new_text = "src lib\nset cell [greet -who x]\n"; + backend + .set_text_sync(main_uri.clone(), new_text.into()) + .await; + // Cursor on `greet` inside the `[ … ]` on line 1. + let hover = backend + .hover( + &main_uri, + Position { + line: 1, + character: 11, + }, + ) + .await + .expect("hover should resolve calls inside `[…]`"); + let body = match hover.contents { + HoverContents::Markup(m) => m.value, + _ => panic!(), + }; + assert!(body.contains("proc greet"), "{body}"); + } + + #[tokio::test] + async fn signature_help_works_on_call_inside_command_substitution() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + // Cursor right after `greet ` inside `[ … ]`. + let new_text = "src lib\nset cell [greet ]\n"; + backend + .set_text_sync(main_uri.clone(), new_text.into()) + .await; + let help = backend + .signature_help( + &main_uri, + Position { + line: 1, + character: 16, + }, + ) + .await + .expect("signature help inside `[…]`"); + assert!( + help.signatures[0].label.starts_with("greet"), + "{:?}", + help.signatures[0].label + ); + } + + #[tokio::test] + async fn diagnostics_still_flag_wrong_flag_on_imported_call() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + backend + .set_text_sync( + main_uri.clone(), + "src lib\ngreet -whoz world\n".into(), + ) + .await; + let diags = backend.diagnostics(&main_uri).await; + assert!( + diags + .iter() + .any(|d| d.message.contains("undefined argument -whoz")), + "{diags:?}" + ); + } + + #[tokio::test] + async fn workspace_diagnostics_surface_errors_in_imported_files() { + // Break the imported lib (return with a value in an + // unannotated proc — one of the new checks) and open the + // main file that `src`s it. The main file itself is + // error-free. workspace_diagnostics must report the lib's + // diagnostic against the LIB's URI so the editor's + // workspace picker points to the right file. + let dir = tempfile::tempdir().unwrap(); + let lib_path = dir.path().join("broken.htcl"); + std::fs::write(&lib_path, "proc broken {} { return 42 }\n").unwrap(); + let main_path = dir.path().join("main.htcl"); + let main_src = "src broken\n"; + std::fs::write(&main_path, main_src).unwrap(); + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let lib_uri = Url::from_file_path(&lib_path).unwrap(); + // Set the editor's workspace root to the temp dir so the + // filter accepts the lib file (which lives inside it). + backend + .set_workspace_roots(vec![dir.path().to_path_buf()]) + .await; + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + // Main entry gets an entry (possibly empty), so the editor + // can clear stale state. + assert!(ws.contains_key(&main_uri), "main uri missing: {ws:?}"); + let lib_diags = ws.get(&lib_uri).unwrap_or_else(|| { + panic!("no diagnostics routed to {lib_uri}: {ws:?}") + }); + assert!( + lib_diags + .iter() + .any(|d| d.message.contains("no declared return type")), + "expected the return-in-unannotated-proc error in lib: {lib_diags:?}", + ); + } + + #[tokio::test] + async fn workspace_diagnostics_clear_when_import_is_fixed() { + // When the user fixes an error in an imported file, the + // next workspace_diagnostics call must include an + // entry for that file — with an EMPTY diagnostic list. + // That empty payload is what the editor overwrites its + // cached "had errors" state with; without it, the + // stale errors linger in `space-D` even after the fix. + let dir = tempfile::tempdir().unwrap(); + let lib_path = dir.path().join("lib.htcl"); + std::fs::write(&lib_path, "proc broken {} { return 42 }\n").unwrap(); + let main_path = dir.path().join("main.htcl"); + let main_src = "src lib\n"; + std::fs::write(&main_path, main_src).unwrap(); + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let lib_uri = Url::from_file_path(&lib_path).unwrap(); + backend + .set_workspace_roots(vec![dir.path().to_path_buf()]) + .await; + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + // Sanity: broken lib produces a workspace diagnostic. + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + assert!( + !ws.get(&lib_uri).map(|v| v.is_empty()).unwrap_or(true), + "expected non-empty lib diagnostics before fix: {ws:?}", + ); + // Fix the lib on disk. Since main.htcl is what's open, + // resetting main's text re-triggers the workspace build + // and reloads lib from disk. + std::fs::write(&lib_path, "proc fixed {} { puts hi }\n").unwrap(); + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + // The lib URI must appear with an empty list so the + // editor clears its cached errors. + let lib_after = ws.get(&lib_uri).unwrap_or_else(|| { + panic!("lib uri missing from post-fix workspace diags: {ws:?}") + }); + assert!( + lib_after.is_empty(), + "expected empty lib diagnostics after fix, got {lib_after:?}", + ); + } + + #[tokio::test] + async fn workspace_diagnostics_skip_out_of_workspace_deps() { + // The workspace is `main_dir`; the imported `dep.htcl` + // lives OUTSIDE it. Errors in the dep should NOT show up + // in workspace diagnostics — that's just noise for a file + // the user isn't editing from this workspace. + let dep_dir = tempfile::tempdir().unwrap(); + let dep_path = dep_dir.path().join("dep.htcl"); + std::fs::write(&dep_path, "proc broken {} { return 42 }\n").unwrap(); + let main_dir = tempfile::tempdir().unwrap(); + let main_path = main_dir.path().join("main.htcl"); + // Use an absolute `src` pointing at the dep tempfile. + let dep_str = dep_path.to_string_lossy().into_owned(); + // Strip the .htcl since `src` re-adds it. + let dep_no_ext = dep_str.trim_end_matches(".htcl"); + let main_src = format!("src {dep_no_ext}\n"); + std::fs::write(&main_path, &main_src).unwrap(); + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let dep_uri = Url::from_file_path(&dep_path).unwrap(); + backend + .set_workspace_roots(vec![main_dir.path().to_path_buf()]) + .await; + backend + .set_text_sync(main_uri.clone(), main_src.clone()) + .await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + assert!( + !ws.contains_key(&dep_uri), + "dep diagnostics should be filtered out: {ws:?}", + ); + } + + #[tokio::test] + async fn save_skips_debounce_and_commits_immediately() { + // Simulates the "small edit + Ctrl-s" flow: the user made + // a tiny change (so set_text just fired a 250ms-debounced + // indexer that hasn't started yet), then saved. `save` + // must abort the pending debounced task and commit its + // OWN indexer without waiting. + // + // We verify by racing `save` against a bounded timeout: + // if `save` still went through the debounce, this would + // time out because the sleep would still be pending. + let backend = HtclBackend::new(); + let uri = Url::parse("file:///tmp/save-test.htcl").unwrap(); + // `set_text` puts a debounced indexer in flight — it + // won't commit for 250ms even though the analysis itself + // is fast for this trivial input. + backend + .set_text(uri.clone(), "proc f {} unit { puts hi }\n".into()) + .await; + // `save` bumps the generation and spawns a fresh + // zero-debounce indexer, which should commit essentially + // right away. If we're wrong and save honors the + // debounce, the `changed()` await would block ~250ms; + // set a 100ms timeout to catch that regression. + let saved = backend.save(&uri); + let waited = tokio::time::timeout( + std::time::Duration::from_millis(500), + async { + saved.await; + backend.wait_for_reindex(&uri).await; + }, + ) + .await; + assert!(waited.is_ok(), "save didn't commit within timeout"); + // And the committed analysis must actually exist. + let a = backend.analysis_for(&uri).await; + assert!(a.is_some(), "save didn't leave a committed analysis"); + } +} diff --git a/vw-analyzer/src/lib.rs b/vw-analyzer/src/lib.rs new file mode 100644 index 0000000..0d989e7 --- /dev/null +++ b/vw-analyzer/src/lib.rs @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw analyzer` — multi-language LSP for the vw HDL workflow. +//! +//! The server is built around a [`LanguageBackend`] abstraction even +//! while only [`HtclBackend`] is wired up. This keeps the architectural +//! slot for VHDL (initially a `vhdl_ls` proxy, later a direct Oxide +//! VHDL frontend integration) open from day one — see the project +//! plan's "LSP design" section. + +mod backend; +mod htcl_backend; +mod server; +mod src_complete; +mod workspace; + +pub use backend::{LanguageBackend, SymbolInfo}; +pub use htcl_backend::HtclBackend; +pub use server::Analyzer; + +use tower_lsp::{LspService, Server}; + +/// Run the LSP server on stdio. Returns when the editor disconnects. +/// +/// Both the standalone `vw-analyzer` binary and the `vw analyzer` +/// subcommand call this so the editor sees identical behavior either +/// way. +pub async fn run_stdio() { + let stdin = tokio::io::stdin(); + let stdout = tokio::io::stdout(); + let (service, socket) = LspService::new(Analyzer::new); + Server::new(stdin, stdout, socket).serve(service).await; +} diff --git a/vw-analyzer/src/main.rs b/vw-analyzer/src/main.rs new file mode 100644 index 0000000..c349930 --- /dev/null +++ b/vw-analyzer/src/main.rs @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw-analyzer` binary entry point. +//! +//! Spawns the LSP server on stdio. The editor (or `vw analyzer` +//! subcommand) exec's this binary directly. + +#[tokio::main] +async fn main() { + // Silent by default — Helix and most LSP clients flag any stderr + // output from a language server as an error. Opt in with + // `VW_ANALYZER_LOG=info` (or `debug`/`trace`) for development. + // ANSI off so colors don't show up as escape codes in the + // client's log viewer. + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_ansi(false) + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_env("VW_ANALYZER_LOG") + .unwrap_or_else(|_| "vw_analyzer=off".into()), + ) + .init(); + + vw_analyzer::run_stdio().await; +} diff --git a/vw-analyzer/src/server.rs b/vw-analyzer/src/server.rs new file mode 100644 index 0000000..de95c87 --- /dev/null +++ b/vw-analyzer/src/server.rs @@ -0,0 +1,564 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! LSP server entry point. Owns the per-language backends and +//! dispatches `textDocument/*` requests by URI. + +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use tower_lsp::jsonrpc::Result; +use tower_lsp::lsp_types::notification::Progress; +use tower_lsp::lsp_types::request::WorkDoneProgressCreate; +use tower_lsp::lsp_types::*; +use tower_lsp::{Client, LanguageServer}; +use tracing::{debug, info}; + +use crate::backend::LanguageBackend; +use crate::htcl_backend::HtclBackend; + +pub struct Analyzer { + client: Client, + backends: Vec>, + /// Monotonic counter for `$/progress` tokens. Every user-facing + /// slow operation (diagnostics, goto-def, hover, completion) + /// generates a fresh token and reports begin/end so Helix and + /// other LSP clients render a pulsating "indexing" indicator + /// while the request is in flight. Wrapped in Arc so the + /// background diagnostic-publish task fired from did_change + /// can share the counter with the foreground handlers. + progress_seq: Arc, +} + +impl Analyzer { + pub fn new(client: Client) -> Self { + let backends: Vec> = + vec![Arc::new(HtclBackend::new())]; + Self { + client, + backends, + progress_seq: Arc::new(AtomicU64::new(0)), + } + } + + fn backend_for(&self, uri: &Url) -> Option> { + self.backends.iter().find(|b| b.handles(uri)).cloned() + } + + /// Fire a background diagnostics publish for `uri`. Returns + /// immediately; the spawned task awaits the current indexer + /// via the backend's `analysis_for` and publishes when it + /// completes. Used from `did_open`/`did_change` so the LSP's + /// notification queue isn't stalled by the ~1s indexer + /// wall-clock — rapid typing no longer serializes into a + /// queue of stale re-indexes. + /// + /// The progress token creation is fire-and-forget from a + /// separate task, wrapping only the `analysis_for` await — + /// the diagnostics publish is NOT gated on the client's + /// progress-create response. If we wrapped publish itself in + /// `with_progress`, a client that never responds to + /// `window/workDoneProgress/create` would hang the entire + /// pipeline. Progress is UX polish; diagnostics are the + /// contract, so diagnostics win. + fn spawn_publish_diagnostics(&self, uri: Url, version: Option) { + let Some(backend) = self.backend_for(&uri) else { + return; + }; + let client = self.client.clone(); + let progress_seq = self.progress_seq.clone(); + let uri_task = uri.clone(); + // Detached task: wait for the next indexer commit while an + // LSP progress spinner is active, then publish the diagnostics + // from THAT fresh analysis. Wrapping the wait in + // `with_progress` is what makes Helix's pulsing "indexing…" + // indicator show up during the rebuild — the previous version + // wrapped an empty `async {}` future so Begin+End fired in + // the same millisecond, effectively no-op. + // + // Reads (completion, hover, goto-def) DO NOT go through this + // task — they use `backend.diagnostics` / `analysis_for` + // directly and are served instantly from the stale-cache. So + // typing latency stays great; only the diagnostics-refresh + + // progress spinner are gated on the actual rebuild. + tokio::spawn(async move { + with_progress( + &client, + &progress_seq, + "Indexing", + uri_task.as_ref(), + backend.wait_for_reindex(&uri_task), + ) + .await; + let diags = backend.diagnostics(&uri_task).await; + debug!( + uri = %uri_task, + count = diags.len(), + "publishing diagnostics" + ); + client + .publish_diagnostics(uri_task.clone(), diags, version) + .await; + // Fan out cross-file diagnostics for EVERY file the + // just-completed analysis touched. Helix's `space-D` + // workspace-diagnostic view reads its cache of pushed + // `publishDiagnostics` — the LSP 3.17 pull path we + // implement in `workspace_diagnostic` isn't wired in + // there yet, so without this fan-out the picker stays + // empty until the user actually opens each broken + // file. We publish empty diagnostics for files with + // no findings too, so a fixed error clears from the + // picker as soon as the change commits. + for (uri, diagnostics) in backend.workspace_diagnostics().await { + if uri == uri_task { + continue; + } + client.publish_diagnostics(uri, diagnostics, None).await; + } + }); + } +} + +/// Free-function `with_progress` that takes just the components +/// needed to negotiate a workDoneProgress token. Sharing the impl +/// this way lets the background diagnostic-publish task fire +/// progress notifications without cloning the whole Analyzer. +async fn with_progress( + client: &Client, + progress_seq: &AtomicU64, + title: &str, + message: &str, + fut: impl Future, +) -> T { + let seq = progress_seq.fetch_add(1, Ordering::Relaxed); + let token = NumberOrString::String(format!("vw-analyzer-{seq}")); + // Server-initiated progress: create the token first. If the + // client refuses, fall through to the future without + // reporting — the request still runs, just without the + // spinner. + let created = client + .send_request::(WorkDoneProgressCreateParams { + token: token.clone(), + }) + .await + .is_ok(); + if created { + let begin = ProgressParams { + token: token.clone(), + value: ProgressParamsValue::WorkDone(WorkDoneProgress::Begin( + WorkDoneProgressBegin { + title: title.to_string(), + cancellable: Some(false), + message: Some(message.to_string()), + percentage: None, + }, + )), + }; + client.send_notification::(begin).await; + } + let result = fut.await; + if created { + let end = ProgressParams { + token, + value: ProgressParamsValue::WorkDone(WorkDoneProgress::End( + WorkDoneProgressEnd { message: None }, + )), + }; + client.send_notification::(end).await; + } + result +} + +#[tower_lsp::async_trait] +impl LanguageServer for Analyzer { + async fn initialize( + &self, + params: InitializeParams, + ) -> Result { + info!("vw-analyzer initializing"); + // Capture the editor's workspace roots so backends can use + // them as fallback dep-lookup dirs when analyzing files + // opened outside the nearest `vw.toml`. Newer LSP clients + // send `workspaceFolders`; older ones use `rootUri` — we + // accept whichever is present. Missing → empty (each + // file's own workspace still resolves in isolation, which + // was the pre-fallback behavior). + let roots = collect_workspace_roots(¶ms); + for backend in &self.backends { + backend.set_workspace_roots(roots.clone()).await; + } + Ok(InitializeResult { + server_info: Some(ServerInfo { + name: "vw-analyzer".into(), + version: Some(env!("CARGO_PKG_VERSION").into()), + }), + capabilities: ServerCapabilities { + // FULL sync — the client sends the whole buffer + // on every change. Do NOT switch this to + // `TextDocumentSyncCapability::Options { ... }` + // to opt into `didSave`: Helix's LSP client + // stopped sending `didChange` altogether when we + // tried that (verified 2026-07 — no + // notifications reached the server after the + // switch, and diagnostics froze until reload). + // Keep this `Kind(FULL)` until we find a Helix- + // safe way to also receive save events (e.g. + // dynamic registration via + // `client/registerCapability`). + text_document_sync: Some(TextDocumentSyncCapability::Kind( + TextDocumentSyncKind::FULL, + )), + document_symbol_provider: Some(OneOf::Left(true)), + workspace_symbol_provider: Some(OneOf::Left(true)), + hover_provider: Some(HoverProviderCapability::Simple(true)), + definition_provider: Some(OneOf::Left(true)), + completion_provider: Some(CompletionOptions { + // `-` opens a flag list; a space after a flag pops + // its `@enum(…)` choices (or the next available + // flags when there are no enum constraints), so + // the user doesn't have to start typing blind to + // discover options. + trigger_characters: Some(vec![ + "-".to_string(), + " ".to_string(), + ]), + ..Default::default() + }), + signature_help_provider: Some(SignatureHelpOptions { + trigger_characters: Some(vec![ + " ".to_string(), + "-".to_string(), + ]), + retrigger_characters: Some(vec!["-".to_string()]), + work_done_progress_options: Default::default(), + }), + rename_provider: Some(OneOf::Left(true)), + references_provider: Some(OneOf::Left(true)), + // LSP 3.17 pull-based diagnostics — Helix uses this + // for `space-D`'s workspace-wide diagnostic picker. + // `workspace_diagnostics: true` also opts us into + // the `workspace/diagnostic` request; without it the + // editor only knows about diagnostics we've + // proactively pushed for open files. + diagnostic_provider: Some( + DiagnosticServerCapabilities::Options(DiagnosticOptions { + identifier: Some("vw-htcl".into()), + inter_file_dependencies: true, + workspace_diagnostics: true, + work_done_progress_options: Default::default(), + }), + ), + ..Default::default() + }, + }) + } + + async fn initialized(&self, _: InitializedParams) { + info!("vw-analyzer initialized"); + } + + async fn did_change_workspace_folders( + &self, + params: DidChangeWorkspaceFoldersParams, + ) { + // Rebuild the roots list from scratch on any change. We + // don't retain state between initialize and here, so an + // added-only event still tells us the FULL updated set — + // both `added` and `removed` are already applied by the + // client before this notification per LSP spec. + let added: Vec = params + .event + .added + .iter() + .filter_map(|f| f.uri.to_file_path().ok()) + .collect(); + for backend in &self.backends { + backend.set_workspace_roots(added.clone()).await; + } + } + + async fn shutdown(&self) -> Result<()> { + info!("vw-analyzer shutting down"); + Ok(()) + } + + async fn did_open(&self, params: DidOpenTextDocumentParams) { + let uri = params.text_document.uri.clone(); + let version = Some(params.text_document.version); + debug!(%uri, "did_open"); + if let Some(backend) = self.backend_for(&uri) { + backend + .set_text(uri.clone(), params.text_document.text) + .await; + } + // Fire the diagnostic publish as a background task so the + // ~1s indexer wall-clock doesn't stall tower-lsp's + // notification queue. Rapid typing (each keystroke fires + // did_change → set_text → publish) previously serialized + // into a queue of stale re-indexes; now every did_change + // returns in microseconds and the LATEST index's + // diagnostics arrive whenever it wins the abort race. + self.spawn_publish_diagnostics(uri, version); + } + + async fn did_change(&self, params: DidChangeTextDocumentParams) { + let uri = params.text_document.uri.clone(); + let version = Some(params.text_document.version); + let Some(backend) = self.backend_for(&uri) else { + return; + }; + // FULL sync: each change is the entire new text. + if let Some(change) = params.content_changes.into_iter().last() { + backend.set_text(uri.clone(), change.text).await; + } + // Background publish (see `did_open`). + self.spawn_publish_diagnostics(uri, version); + } + + async fn did_close(&self, params: DidCloseTextDocumentParams) { + let uri = params.text_document.uri; + if let Some(backend) = self.backend_for(&uri) { + backend.close(&uri).await; + } + } + + async fn did_save(&self, params: DidSaveTextDocumentParams) { + let uri = params.text_document.uri.clone(); + debug!(%uri, "did_save"); + if let Some(backend) = self.backend_for(&uri) { + // Zero-debounce reindex — the whole point of + // handling save is that `Ctrl-s` should force a + // fresh check now, not 250ms from now. + backend.save(&uri).await; + } + // Same publish path as did_change so the fresh index's + // diagnostics land in the editor. + self.spawn_publish_diagnostics(uri, None); + } + + async fn document_symbol( + &self, + params: DocumentSymbolParams, + ) -> Result> { + let uri = params.text_document.uri; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + let symbols = backend.document_symbols(&uri).await; + if symbols.is_empty() { + Ok(None) + } else { + Ok(Some(DocumentSymbolResponse::Nested(symbols))) + } + } + + async fn symbol( + &self, + params: WorkspaceSymbolParams, + ) -> Result>> { + let query = params.query; + // Walk every registered backend (today just one) and merge the + // matches — keeps the same dispatch shape as `backend_for` so + // adding a second language later doesn't need a refactor. + let mut symbols = Vec::new(); + for backend in &self.backends { + symbols.extend(backend.workspace_symbols(&query).await); + } + if symbols.is_empty() { + Ok(None) + } else { + Ok(Some(symbols)) + } + } + + async fn hover(&self, params: HoverParams) -> Result> { + // Reads from the cached DocAnalysis — no per-request + // progress wrapping needed since the answer lands in + // microseconds after indexing has completed. + let uri = params.text_document_position_params.text_document.uri; + let position = params.text_document_position_params.position; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + Ok(backend.hover(&uri, position).await) + } + + async fn goto_definition( + &self, + params: GotoDefinitionParams, + ) -> Result> { + // Cached — see the note on `hover`. + let uri = params + .text_document_position_params + .text_document + .uri + .clone(); + let position = params.text_document_position_params.position; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + let locs = backend.goto_definition(&uri, position).await; + if locs.is_empty() { + Ok(None) + } else { + Ok(Some(GotoDefinitionResponse::Array(locs))) + } + } + + async fn completion( + &self, + params: CompletionParams, + ) -> Result> { + // Cached — see the note on `hover`. + let uri = params.text_document_position.text_document.uri; + let position = params.text_document_position.position; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + let items = backend.completion(&uri, position).await; + if items.is_empty() { + Ok(None) + } else { + Ok(Some(CompletionResponse::Array(items))) + } + } + + async fn signature_help( + &self, + params: SignatureHelpParams, + ) -> Result> { + let uri = params + .text_document_position_params + .text_document + .uri + .clone(); + let position = params.text_document_position_params.position; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + Ok(backend.signature_help(&uri, position).await) + } + + async fn rename( + &self, + params: RenameParams, + ) -> Result> { + let uri = params.text_document_position.text_document.uri.clone(); + let position = params.text_document_position.position; + let new_name = params.new_name; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + Ok(backend.rename(&uri, position, &new_name).await) + } + + async fn references( + &self, + params: ReferenceParams, + ) -> Result>> { + let uri = params.text_document_position.text_document.uri.clone(); + let position = params.text_document_position.position; + let include_declaration = params.context.include_declaration; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + let locs = backend + .references(&uri, position, include_declaration) + .await; + if locs.is_empty() { + Ok(None) + } else { + Ok(Some(locs)) + } + } + + async fn diagnostic( + &self, + params: DocumentDiagnosticParams, + ) -> Result { + // Pull-based single-file diagnostics. Same payload the + // push path serves via `publishDiagnostics` — the editor + // may request it explicitly (Helix does when the buffer + // opens, before any push has fired) as a + // no-guess-when-they-arrive alternative. + let uri = params.text_document.uri.clone(); + let items = match self.backend_for(&uri) { + Some(backend) => backend.diagnostics(&uri).await, + None => Vec::new(), + }; + Ok(DocumentDiagnosticReportResult::Report( + DocumentDiagnosticReport::Full( + RelatedFullDocumentDiagnosticReport { + related_documents: None, + full_document_diagnostic_report: + FullDocumentDiagnosticReport { + result_id: None, + items, + }, + }, + ), + )) + } + + async fn workspace_diagnostic( + &self, + _params: WorkspaceDiagnosticParams, + ) -> Result { + // Collect from every backend. Each returns a set of + // (uri, diagnostics) tuples pulled from its open docs' + // workspace-view analyses — files transitively `src`d by + // an open document surface their errors here even if the + // user hasn't opened them, which is what makes Helix's + // `space-D` picker useful for whole-workspace triage. + let mut items = Vec::new(); + for backend in &self.backends { + for (uri, diagnostics) in backend.workspace_diagnostics().await { + items.push(WorkspaceDocumentDiagnosticReport::Full( + WorkspaceFullDocumentDiagnosticReport { + uri, + version: None, + full_document_diagnostic_report: + FullDocumentDiagnosticReport { + result_id: None, + items: diagnostics, + }, + }, + )); + } + } + Ok(WorkspaceDiagnosticReportResult::Report( + WorkspaceDiagnosticReport { items }, + )) + } +} + +/// Extract workspace roots from an `initialize` request as +/// filesystem paths. Prefers `workspaceFolders` (LSP 3.6+, sent +/// by every modern client) and falls back to `rootUri` for older +/// clients. Both may be absent — e.g. when the editor opens a +/// file with no folder context — in which case we return an +/// empty vec and each file's own `vw.toml` is the only source +/// of dep names, matching the pre-fallback behavior. +fn collect_workspace_roots( + params: &InitializeParams, +) -> Vec { + if let Some(folders) = params.workspace_folders.as_ref() { + if !folders.is_empty() { + return folders + .iter() + .filter_map(|f| f.uri.to_file_path().ok()) + .collect(); + } + } + // `root_uri` is deprecated but still what a lot of clients + // (including bare-bones LSP integrations) send. + #[allow(deprecated)] + if let Some(uri) = params.root_uri.as_ref() { + if let Ok(p) = uri.to_file_path() { + return vec![p]; + } + } + Vec::new() +} diff --git a/vw-analyzer/src/src_complete.rs b/vw-analyzer/src/src_complete.rs new file mode 100644 index 0000000..09059dc --- /dev/null +++ b/vw-analyzer/src/src_complete.rs @@ -0,0 +1,367 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Filesystem-aware completion for `src` import paths. +//! +//! The `vw-htcl` crate stays free of filesystem concerns, so this +//! lives in the analyzer alongside the workspace resolver. When the +//! cursor sits in the path-position of a `src` command, we +//! enumerate the directory implied by the partial path and offer: +//! +//! - every `.htcl` file at that level, labelled by basename (no +//! extension), and +//! - every subdirectory at that level that transitively contains at +//! least one `.htcl` file, labelled with a trailing `/`. +//! +//! Three flavors of partial are recognized, matching +//! [`vw_htcl::src_path::classify`]: +//! +//! - `@/...` — resolve against the workspace dependency's cached +//! root. +//! - `/abs/...` — filesystem-absolute. +//! - anything else — relative to the importing file's directory. +//! +//! When the partial is just `@` or `@` (no `/` yet), suggest +//! dependency names from the workspace resolver instead. + +use std::path::{Path, PathBuf}; + +use tower_lsp::lsp_types::{ + CompletionItem, CompletionItemKind, CompletionTextEdit, InsertTextFormat, + Position, Range, TextEdit, +}; +use vw_htcl::cmdline::CmdLine; +use vw_htcl::src_path::{classify, PathKind}; +use vw_htcl::{LineCol, LineIndex, Resolver, Span}; + +/// True when the cursor sits in the path-position of a `src` command +/// (i.e. the first complete word is `src` and we're typing the path). +pub fn is_src_path_context(line: &CmdLine<'_>) -> bool { + line.words.first().copied() == Some("src") && line.words.len() == 1 +} + +/// Generate path completions for `line.partial`, treating it as the +/// `` of `src `. +/// +/// `entry_file` is the open file's path on disk; it anchors relative +/// imports and lets us walk up to the workspace's `vw.toml` for dep +/// resolution. `line_index` maps source offsets to LSP positions. +pub fn src_path_completions( + entry_file: &Path, + line: &CmdLine<'_>, + line_index: &LineIndex, + resolver: &Resolver, +) -> Vec { + let partial = line.partial; + + // `@` with no `/` yet → dep-name completion. Replace the + // whole partial with `@/` so the next completion fires on + // the contents. + if let Some(prefix_after_at) = partial.strip_prefix('@') { + if !prefix_after_at.contains('/') { + return dep_name_completions( + resolver, + prefix_after_at, + line.partial_span, + line_index, + ); + } + } + + // Otherwise: resolve the directory the partial points into, then + // enumerate it. + let Some((dir, segment_start)) = resolve_dir(entry_file, resolver, partial) + else { + return Vec::new(); + }; + let segment = &partial[segment_start..]; + let replace = Span::new( + line.partial_span.start + segment_start as u32, + line.partial_span.end, + ); + enumerate_entries(&dir, segment, replace, line_index) +} + +/// Resolve the *directory* part of `partial` to an on-disk path, plus +/// the byte offset into `partial` where the trailing (still-being- +/// typed) segment begins. Returns `None` when the partial points at a +/// dep that doesn't exist or a path that can't be classified. +fn resolve_dir( + entry_file: &Path, + resolver: &Resolver, + partial: &str, +) -> Option<(PathBuf, usize)> { + let kind = classify(partial).kind; + let (base, body) = match &kind { + PathKind::Relative => { + let dir = entry_file.parent()?.to_path_buf(); + (dir, partial) + } + PathKind::Absolute => { + (PathBuf::from("/"), partial.trim_start_matches('/')) + } + PathKind::Named { name, subpath } => { + let root = resolver.dep_root(name)?.to_path_buf(); + (root, subpath.as_str()) + } + }; + // Split `body` at its last `/`: everything before is the + // sub-directory walk; everything after is the segment being typed + // (used for the replace range and ignored for enumeration). + let (subdir, trailing_segment) = match body.rfind('/') { + Some(i) => (&body[..i], &body[i + 1..]), + None => ("", body), + }; + let mut dir = base; + if !subdir.is_empty() { + dir.push(subdir); + } + let segment_start = partial.len() - trailing_segment.len(); + Some((dir, segment_start)) +} + +fn enumerate_entries( + dir: &Path, + segment: &str, + replace: Span, + line_index: &LineIndex, +) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let _ = segment; // LSP client filters by prefix; we list everything. + + let mut out: Vec<(String, CompletionItemKind)> = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + if name.starts_with('.') { + continue; + } + let ft = entry.file_type().ok(); + if ft.is_some_and(|t| t.is_dir()) { + if dir_has_htcl(&path) { + out.push((format!("{name}/"), CompletionItemKind::FOLDER)); + } + } else if path.extension().and_then(|s| s.to_str()) == Some("htcl") { + let stem = + path.file_stem().and_then(|s| s.to_str()).unwrap_or(name); + // `module.htcl` is the dep's default entry point, already + // reachable as bare `@` — listing it here as + // `@/module` would just be a noisier alias. + if stem == vw_htcl::src_path::DEFAULT_MODULE { + continue; + } + out.push((stem.to_string(), CompletionItemKind::FILE)); + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + let range = lsp_range(replace, line_index); + out.into_iter() + .map(|(label, kind)| build_item(label, kind, range)) + .collect() +} + +fn dep_name_completions( + resolver: &Resolver, + _prefix: &str, + partial_span: Span, + line_index: &LineIndex, +) -> Vec { + let mut deps: Vec<(&str, &Path)> = resolver.deps().collect(); + deps.sort_by_key(|(n, _)| *n); + let range = lsp_range(partial_span, line_index); + // Bare `@` is a complete import on its own (resolves to the + // dep's `module.htcl`), so don't append a trailing `/` — that + // would leave behind invalid syntax for a user who just wanted + // the default module. Users who want to drill in still type `/` + // themselves, which retriggers completion against the dep root. + deps.into_iter() + .map(|(name, _)| { + build_item(format!("@{name}"), CompletionItemKind::MODULE, range) + }) + .collect() +} + +/// True if `dir` contains, or transitively contains, any `.htcl` file. +/// Short-circuits on the first hit. +fn dir_has_htcl(dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(dir) else { + return false; + }; + for entry in entries.flatten() { + let path = entry.path(); + let Ok(ft) = entry.file_type() else { continue }; + if ft.is_file() { + if path.extension().and_then(|s| s.to_str()) == Some("htcl") { + return true; + } + } else if ft.is_dir() { + // Skip dot-dirs to keep `.git`, `.svn`, etc. out of the + // walk. + if path + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with('.')) + { + continue; + } + if dir_has_htcl(&path) { + return true; + } + } + } + false +} + +fn build_item( + label: String, + kind: CompletionItemKind, + range: Range, +) -> CompletionItem { + let new_text = label.clone(); + CompletionItem { + label, + kind: Some(kind), + insert_text_format: Some(InsertTextFormat::PLAIN_TEXT), + text_edit: Some(CompletionTextEdit::Edit(TextEdit { range, new_text })), + ..Default::default() + } +} + +fn lsp_range(span: Span, line_index: &LineIndex) -> Range { + let (start, end) = line_index.range(span); + Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + } +} + +fn lc_to_pos(lc: LineCol) -> Position { + Position { + line: lc.line, + character: lc.character, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use vw_htcl::cmdline; + + fn workspace_fixture() -> (tempfile::TempDir, PathBuf, Resolver) { + // amd-htcl/ + // module.htcl ← default entry, HIDDEN from list + // cmd.htcl + // ip.htcl + // cmd/foo.htcl + // scripts/ ← no .htcl, should NOT appear + // ip/bd/cell.htcl ← nested, ip/ should appear + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("amd-htcl"); + fs::create_dir_all(dep.join("cmd")).unwrap(); + fs::create_dir_all(dep.join("scripts")).unwrap(); + fs::create_dir_all(dep.join("ip/bd")).unwrap(); + fs::write(dep.join("module.htcl"), "# entry").unwrap(); + fs::write(dep.join("cmd.htcl"), "# stub").unwrap(); + fs::write(dep.join("ip.htcl"), "# stub").unwrap(); + fs::write(dep.join("cmd/foo.htcl"), "# stub").unwrap(); + fs::write(dep.join("scripts/notes.txt"), "not htcl").unwrap(); + fs::write(dep.join("ip/bd/cell.htcl"), "# stub").unwrap(); + // entry file + let entry = dir.path().join("prime.htcl"); + fs::write(&entry, "src @amd-htcl/cmd\n").unwrap(); + let resolver = Resolver::new().with_dep("amd-htcl", dep); + // hold dir handle so files persist for the test + (dir, entry, resolver) + } + + fn labels_for(src: &str, entry: &Path, resolver: &Resolver) -> Vec { + let line = cmdline::analyze(src, src.len() as u32); + let idx = LineIndex::new(src); + let items = src_path_completions(entry, &line, &idx, resolver); + let mut labels: Vec = + items.into_iter().map(|c| c.label).collect(); + labels.sort(); + labels + } + + #[test] + fn lists_dep_root_after_trailing_slash() { + let (_dir, entry, resolver) = workspace_fixture(); + let labels = labels_for("src @amd-htcl/", &entry, &resolver); + // .htcl files: cmd, ip. dirs with .htcl: cmd/, ip/. + // scripts/ is omitted (no .htcl inside). + assert_eq!(labels, vec!["cmd", "cmd/", "ip", "ip/"]); + } + + #[test] + fn lists_dep_subdirectory() { + let (_dir, entry, resolver) = workspace_fixture(); + let labels = labels_for("src @amd-htcl/ip/", &entry, &resolver); + // ip/ has bd/ (containing cell.htcl) — bd/ should show; no other entries. + assert_eq!(labels, vec!["bd/"]); + } + + #[test] + fn partial_segment_replaces_only_the_segment() { + // User has typed `src @amd-htcl/c` — replace should cover just + // the `c`, not the whole `@amd-htcl/c`. + let src = "src @amd-htcl/c"; + let (_dir, entry, resolver) = workspace_fixture(); + let line = cmdline::analyze(src, src.len() as u32); + let idx = LineIndex::new(src); + let items = src_path_completions(&entry, &line, &idx, &resolver); + let labels: Vec = + items.iter().map(|c| c.label.clone()).collect(); + // Both `cmd` and `cmd/` start with `c`. + assert!(labels.contains(&"cmd".to_string()), "{labels:?}"); + // The text-edit range should cover only the `c` (single char on line 0). + let edit = match items[0].text_edit.as_ref() { + Some(CompletionTextEdit::Edit(e)) => e, + _ => panic!("expected text edit"), + }; + assert_eq!(edit.range.start.character, 14, "{:?}", edit.range); + assert_eq!(edit.range.end.character, 15); + } + + #[test] + fn dep_name_completion_when_no_slash_yet() { + // Bare `@` is a complete import on its own, so the + // completion shouldn't append `/` — selecting `@amd-htcl` + // alone should leave valid syntax that resolves to + // `/module.htcl`. + let (_dir, entry, resolver) = workspace_fixture(); + let labels = labels_for("src @", &entry, &resolver); + assert_eq!(labels, vec!["@amd-htcl"]); + } + + #[test] + fn dep_root_listing_hides_module_htcl() { + // `module.htcl` is the default entry — already importable as + // bare `@amd-htcl`, so it should not show up as `module` in + // the per-dep file listing. + let (_dir, entry, resolver) = workspace_fixture(); + let labels = labels_for("src @amd-htcl/", &entry, &resolver); + assert!(!labels.contains(&"module".to_string()), "{labels:?}"); + // Sanity: the non-default modules still show. + assert!(labels.contains(&"cmd".to_string())); + assert!(labels.contains(&"ip".to_string())); + } + + #[test] + fn relative_completion_uses_entry_directory() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("ip")).unwrap(); + fs::write(dir.path().join("ip/cips.htcl"), "# stub").unwrap(); + let entry = dir.path().join("prime.htcl"); + fs::write(&entry, "src ip/\n").unwrap(); + let resolver = Resolver::new(); + let labels = labels_for("src ip/", &entry, &resolver); + assert_eq!(labels, vec!["cips"]); + } +} diff --git a/vw-analyzer/src/workspace.rs b/vw-analyzer/src/workspace.rs new file mode 100644 index 0000000..4e106a4 --- /dev/null +++ b/vw-analyzer/src/workspace.rs @@ -0,0 +1,374 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Workspace-aware helpers for the analyzer. +//! +//! The bare LSP backend deals with one file at a time. Cross-file +//! features — goto-definition into an imported module, completion of +//! procs defined in `@dep/foo`, validating a call against a signature +//! that lives elsewhere — need a view that spans the importing file +//! plus everything it pulled in via `src`. +//! +//! This module computes that view on demand. It's deliberately +//! re-computed per query rather than cached: htcl files are tiny next +//! to a Vivado IP wrapper, the LSP's edits-per-second is modest, and a +//! cache would have to deal with invalidation when an `@dep/...` +//! file on disk changes. A targeted cache is a sensible follow-up once +//! the access pattern is settled. + +use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use camino::{Utf8Path, Utf8PathBuf}; +use tower_lsp::lsp_types::Url; + +use vw_htcl::{parse, CommandKind, Resolver, SrcImport, Stmt}; + +/// A flattened source view used for cross-file analysis. +/// +/// `view_source` is the local file's text *first* (so the cursor's +/// byte offset in the open document is the same offset in the view — +/// hover/goto/etc. don't need offset translation for the local file), +/// followed by every transitively imported file's text concatenated. +/// Each appended region is recorded in [`imports`](Self::imports) so +/// spans landing there can be mapped back to the file they came from. +pub struct WorkspaceView { + pub view_source: String, + /// Byte length of the *local* file's contribution. Spans whose + /// `start < local_len` belong to the open file; everything past + /// that lives in some imported file. + pub local_len: u32, + pub imports: Vec, + /// Names of every dep the file's resolver knows about + /// (workspace `vw.toml` + editor extra_roots + sibling + /// scan). Passed to the validator's undefined-src-module + /// check so `src @` where `` isn't in the set + /// gets a spanned Error diagnostic. Empty when no workspace + /// context resolved. + pub dep_names: std::collections::HashSet, +} + +pub struct ImportRegion { + /// Inclusive start offset in `view_source`. + pub start: u32, + /// Exclusive end offset in `view_source`. + pub end: u32, + pub file_uri: Url, +} + +impl WorkspaceView { + /// If `offset` lies inside an imported file's region, return the + /// import region plus the file-local offset of that span; `None` + /// means the offset is in the open file itself. + pub fn locate(&self, offset: u32) -> Option<(&ImportRegion, u32)> { + if offset < self.local_len { + return None; + } + self.imports + .iter() + .find(|r| offset >= r.start && offset < r.end) + .map(|r| (r, offset - r.start)) + } +} + +/// Build a workspace view by reading every file the entry transitively +/// `src`s. Returns a view with `imports` empty when the entry can't be +/// resolved to a filesystem path or has no imports — the analyzer can +/// still use it; it just won't see anything cross-file. +/// +/// `extra_roots` supplies fallback workspace roots (typically the +/// editor's `rootUri` / `workspaceFolders`) so files opened outside +/// the enclosing `vw.toml` — e.g. via goto-def into a dep cache +/// dir — still resolve `@name/…` imports through the outer +/// workspace's dep graph. Pass `&[]` when the caller doesn't have +/// that context. +pub fn build_view( + file_uri: &Url, + local_text: &str, + extra_roots: &[PathBuf], +) -> WorkspaceView { + let mut view = WorkspaceView { + view_source: local_text.to_string(), + local_len: local_text.len() as u32, + imports: Vec::new(), + dep_names: std::collections::HashSet::new(), + }; + + let Ok(file_path) = file_uri.to_file_path() else { + return view; + }; + let parent = file_path + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + let resolver = build_resolver_with(&file_path, extra_roots); + // Snapshot dep names now — the resolver may get moved into + // collect_imports below; the diagnostics pass needs the + // set as a plain HashSet. + view.dep_names = + resolver.deps().map(|(name, _)| name.to_string()).collect(); + + let mut loaded: HashSet = HashSet::new(); + if let Ok(canonical) = file_path.canonicalize() { + loaded.insert(canonical); + } + let mut queue: Vec<(PathBuf, String)> = Vec::new(); + collect_imports(local_text, &parent, &resolver, &mut loaded, &mut queue); + + while let Some((path, text)) = queue.pop() { + view.view_source.push('\n'); + // Record `start` *after* the separator so a span's local + // offset within the imported file is `span.start - start` + // with no off-by-one for the inserted newline. + let start = view.view_source.len() as u32; + view.view_source.push_str(&text); + let end = view.view_source.len() as u32; + if let Ok(import_uri) = Url::from_file_path(&path) { + view.imports.push(ImportRegion { + start, + end, + file_uri: import_uri, + }); + } + // Recurse into this file's own imports. + let import_parent = path + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + collect_imports( + &text, + &import_parent, + &resolver, + &mut loaded, + &mut queue, + ); + } + + view +} + +/// Build a [`Resolver`] for the workspace that owns `entry_file`. +/// Convenience wrapper — see [`build_resolver_with`] for the full +/// variant that also honors editor-supplied fallback workspace +/// roots (LSP `rootUri` / `workspaceFolders`). +pub fn build_resolver(entry_file: &Path) -> Resolver { + build_resolver_with(entry_file, &[]) +} + +/// Build a [`Resolver`] for `entry_file`, merging dep declarations +/// from every source that could plausibly resolve a `@name/…` +/// import when the file's own workspace doesn't declare it: +/// +/// 1. The file's own workspace — walk up to the nearest `vw.toml` +/// and expand its dep graph via +/// [`vw_lib::transitive_dep_cache_paths`]. Highest precedence. +/// 2. Each path in `extra_roots` — treated as a workspace directory +/// and expanded the same way. Used to plumb the LSP's `rootUri` +/// (or `workspaceFolders`) so a file opened via goto-def *out* +/// of the editor's root workspace still inherits its dep names. +/// 3. Sibling-workspace layout scan — for every ancestor directory +/// of `entry_file`, treat each direct subdirectory that itself +/// contains a `vw.toml` as an implicit dep whose name is the +/// subdirectory basename. This lets a +/// `~/src/htcl/amd/cpm5/module.htcl` still see `@vivado-cmd` +/// at `~/src/htcl/amd/vivado-cmd/` even when neither its own +/// workspace nor the editor's root declares the dep — a monorepo +/// layout that's typical of a `foo-htcl` collection of siblings. +/// +/// First-seen wins on name collisions in the order above, so the +/// file's own workspace's choice never gets overridden. +/// True when `entry_file` sits inside the workspace's `test/` +/// directory subtree. Used to decide whether the analyzer's +/// resolver includes `[test-dependencies]` for LSP goto-def / +/// hover / diagnostics inside test files. +fn is_test_file(entry_file: &Path, workspace_dir: &Utf8Path) -> bool { + let ws_std = workspace_dir.as_std_path(); + let Ok(rel) = entry_file.strip_prefix(ws_std) else { + return false; + }; + rel.components() + .next() + .is_some_and(|c| std::path::Component::Normal("test".as_ref()) == c) +} + +pub fn build_resolver_with( + entry_file: &Path, + extra_roots: &[PathBuf], +) -> Resolver { + let mut merged: std::collections::HashMap = + std::collections::HashMap::new(); + if let Some(workspace_dir) = find_workspace_dir(entry_file) { + // A file under `/test/**` is a test file — pull in + // `[test-dependencies]` too so `src @` resolves + // in the analyzer just like it does in `vw test`. Matches + // the CLI's `check_htcl_with_mode(_, include_test)` + // behavior. + let include_test = is_test_file(entry_file, &workspace_dir); + // Transitive: a library that does `src @other-lib/...` + // shouldn't force every consumer to redeclare `other-lib` + // in their own `vw.toml`. The walker pulls in each dep's + // own deps so the resolver sees the whole graph + // (Cargo-style first-seen-wins on name conflicts). + if let Ok(paths) = vw_lib::transitive_dep_cache_paths_with_test( + &workspace_dir, + include_test, + ) { + for (name, path) in paths { + merged.entry(name).or_insert(path); + } + } + // Cargo-parity self-reference: a workspace named `foo` + // resolves `src @foo/bar` to `/bar.htcl`. Uses + // `entry(...).or_insert(...)` so a legitimately-declared + // external `foo` (rare but possible) still wins. + if let Ok(cfg) = vw_lib::load_workspace_config(&workspace_dir) { + merged + .entry(cfg.workspace.name) + .or_insert_with(|| workspace_dir.as_std_path().to_path_buf()); + } + } + for root in extra_roots { + let Ok(root_utf8) = Utf8PathBuf::from_path_buf(root.clone()) else { + continue; + }; + if !root_utf8.join("vw.toml").exists() { + continue; + } + if let Ok(paths) = vw_lib::transitive_dep_cache_paths(&root_utf8) { + for (name, path) in paths { + merged.entry(name).or_insert(path); + } + } + } + collect_sibling_workspaces(entry_file, &mut merged); + let mut resolver = Resolver::new(); + for (name, path) in merged { + resolver = resolver.with_dep(name, path); + } + resolver +} + +/// Walk up from `entry_file`, and at each ancestor directory add +/// every subdirectory that contains its own `vw.toml` as an +/// implicit dep — keyed by the subdirectory basename. +/// +/// This mirrors a monorepo layout that's common for htcl workspaces: +/// `~/src/htcl/amd/{cips,cpm5,clk-wizard,vivado-cmd}/`. From any +/// one of those, the others are visible as siblings even when +/// no `vw.toml` explicitly declares them. Without this heuristic, +/// jumping into a dep-module file from an editor whose LSP has +/// restarted rooted at that dep's own `vw.toml` (helix's +/// `roots = ["vw.toml"]` behavior) would strand the analyzer with +/// no way to resolve `@sibling-name/…` imports. +/// +/// Stops at the filesystem root or after a handful of ancestors — +/// scanning `~` or `/` for candidate workspaces would be both slow +/// and semantically wrong. +fn collect_sibling_workspaces( + entry_file: &Path, + merged: &mut std::collections::HashMap, +) { + // Cap the walk so we don't scan every level up to `/`. Six + // ancestors covers the typical `~/src////` + // + a few extra tolerance for deeper nestings. + const MAX_ANCESTORS: usize = 6; + let mut cursor = match entry_file.parent() { + Some(p) => p.to_path_buf(), + None => return, + }; + for _ in 0..MAX_ANCESTORS { + let read_dir = match std::fs::read_dir(&cursor) { + Ok(r) => r, + Err(_) => break, + }; + for entry in read_dir.flatten() { + let Ok(ft) = entry.file_type() else { continue }; + if !ft.is_dir() { + continue; + } + let sub = entry.path(); + if !sub.join("vw.toml").is_file() { + continue; + } + let Some(name) = sub.file_name().and_then(|n| n.to_str()) else { + continue; + }; + merged.entry(name.to_string()).or_insert(sub); + } + cursor = match cursor.parent() { + Some(p) => p.to_path_buf(), + None => break, + }; + } +} + +/// Walk up from `start`'s parent directory looking for a `vw.toml`. +fn find_workspace_dir(start: &Path) -> Option { + let mut cur = start.parent()?.to_path_buf(); + loop { + if cur.join("vw.toml").exists() { + return Utf8PathBuf::from_path_buf(cur).ok(); + } + cur = cur.parent()?.to_path_buf(); + } +} + +/// Parse `text` and queue each new (not yet seen) `src` resolution as +/// `(canonical_path, file_text)` for the caller to incorporate. +fn collect_imports( + text: &str, + parent_dir: &Path, + resolver: &Resolver, + loaded: &mut HashSet, + queue: &mut Vec<(PathBuf, String)>, +) { + let parsed = parse(text); + for stmt in &parsed.document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(SrcImport { + path: Some(raw), .. + }) = &cmd.kind + else { + continue; + }; + let Ok(resolved) = resolver.resolve(parent_dir, raw) else { + continue; + }; + // Resolver already canonicalizes when possible; defensive + // dedup either way. + if !loaded.insert(resolved.clone()) { + continue; + } + let Ok(content) = fs::read_to_string(&resolved) else { + continue; + }; + queue.push((resolved, content)); + } +} + +/// Public helper: resolve the import at `raw` from `entry_file`'s +/// directory. Used by goto-on-import-path so the analyzer can return +/// a Location pointing at the imported file. +/// +/// `extra_roots` — see [`build_view`] for the same rationale — lets +/// callers plumb through the editor's workspace roots so a +/// `src @dep/file.htcl` in a file outside the enclosing workspace +/// still resolves. +pub fn resolve_import( + entry_file: &Path, + raw: &str, + extra_roots: &[PathBuf], +) -> Option { + let parent = entry_file.parent()?; + build_resolver_with(entry_file, extra_roots) + .resolve(parent, raw) + .ok() +} + +/// Allow `&Utf8Path` callers to canonicalize through us. +#[allow(dead_code)] +pub fn workspace_root(entry_file: &Utf8Path) -> Option { + find_workspace_dir(Path::new(entry_file.as_str())) +} diff --git a/vw-cli/Cargo.toml b/vw-cli/Cargo.toml index c27a1aa..36fdb12 100644 --- a/vw-cli/Cargo.toml +++ b/vw-cli/Cargo.toml @@ -14,7 +14,19 @@ path = "src/main.rs" [dependencies] vw-lib = { path = "../vw-lib" } +vw-htcl = { path = "../vw-htcl" } +vw-eda = { path = "../vw-eda" } +vw-vivado = { path = "../vw-vivado" } +vw-analyzer = { path = "../vw-analyzer" } +vw-ip = { path = "../vw-ip" } +vw-htcl-cmd = { path = "../vw-htcl-cmd" } +vw-repl = { path = "../vw-repl" } clap = { version = "4.0", features = ["derive"] } colored = "2.0" tokio.workspace = true camino.workspace = true +tracing-subscriber.workspace = true +petgraph.workspace = true +futures.workspace = true +indicatif.workspace = true +serde_json.workspace = true diff --git a/vw-cli/src/htcl_test.rs b/vw-cli/src/htcl_test.rs new file mode 100644 index 0000000..108be2a --- /dev/null +++ b/vw-cli/src/htcl_test.rs @@ -0,0 +1,875 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw test` — htcl-level test runner. +//! +//! Discovers `@test`-annotated procs under `/test/**/ +//! *.htcl`, drives them against a Vivado session, and reports +//! results in a nextest-inspired format. +//! +//! Design shape: +//! - Each test file's non-`@test` top-level statements are treated +//! as setup (proc decls, `src` imports). They're shipped to +//! Vivado once per file per session. +//! - Tests without a specific attribute value run in a SHARED +//! Vivado session; `@test(dedicated-eda)` marks tests that need +//! their own Vivado process (spawned per test, capped by +//! `--test-threads`). +//! - Assertion failures throw Tcl errors, which the runner catches +//! as `BackendError::Tcl` and marks the enclosing test FAILED. + +use std::path::PathBuf; +use std::time::Instant; + +use camino::{Utf8Path, Utf8PathBuf}; +use colored::*; + +use vw_eda::EdaBackend; + +use crate::load_htcl_program_for_test; + +/// Entry point wired from `main.rs::Commands::Test`. +#[allow(clippy::too_many_arguments)] +pub async fn run_htcl_tests( + cwd: &Utf8Path, + filter: Option, + list: bool, + test_threads: usize, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, +) -> Result<(), Box> { + let ws = vw_lib::find_workspace_dir(cwd.as_std_path()) + .ok_or("not in a vw workspace (no vw.toml in the parent chain)")?; + let test_files = vw_lib::list_htcl_tests(&ws)?; + if test_files.is_empty() { + eprintln!("no tests found in {}", ws.join("test").as_str().dimmed()); + return Ok(()); + } + + // Discover phase — load each test file's program and enumerate + // its `@test` procs. + let mut all_tests: Vec = Vec::new(); + for path in &test_files { + let path_utf8 = Utf8PathBuf::from_path_buf(path.clone()) + .map_err(|p| format!("non-UTF8 path: {p:?}"))?; + let discovered = + discover_tests_in_file(&path_utf8, &ws, filter.as_deref()).await?; + all_tests.extend(discovered); + } + + if list { + for t in &all_tests { + let tag = if t.dedicated { " [dedicated-eda]" } else { "" }; + println!("{}::{}{}", t.display_path, t.name.cyan(), tag.dimmed()); + } + return Ok(()); + } + + if all_tests.is_empty() { + eprintln!( + "no tests matched the filter {}", + filter.as_deref().unwrap_or("").dimmed() + ); + return Ok(()); + } + + let overall_start = Instant::now(); + let mut summary = RunSummary::default(); + + // Shared bucket — one Vivado for all `@test` (without + // `dedicated-eda`) tests. + let (shared, dedicated): (Vec<_>, Vec<_>) = + all_tests.into_iter().partition(|t| !t.dedicated); + + println!( + "\nrunning {} tests", + (shared.len() + dedicated.len()).to_string().bold() + ); + + if !shared.is_empty() { + run_shared_bucket( + &ws, + shared, + &mut summary, + part, + variant, + log_level, + info_with_stack, + ) + .await?; + } + if !dedicated.is_empty() { + run_dedicated_bucket( + &ws, + dedicated, + test_threads, + &mut summary, + part, + variant, + log_level, + info_with_stack, + ) + .await?; + } + + print_summary(&summary, overall_start.elapsed()); + if summary.failed > 0 { + std::process::exit(1); + } + Ok(()) +} + +/// One `@test`-annotated proc discovered inside a test file. The +/// `program` and `setup_tcl_lines` fields carry everything the +/// runner needs to ship setup to Vivado before invoking the test. +struct TestCase { + /// Test file path, workspace-relative. + display_path: String, + /// Absolute path — used to key "which file's setup has been + /// shipped in this Vivado session already." + file_path: PathBuf, + /// Proc name as it appears at Tcl call level. + name: String, + /// True if `@test(dedicated-eda)` — this test wants its own + /// Vivado process. + dedicated: bool, + /// `@target(part=)` override — swaps the auto-project's + /// `-part` argument for this specific test. Meaningful only + /// for `dedicated-eda` tests (shared-bucket tests all reuse + /// the workspace-default project). + target_part_override: Option, + /// `@test(dedicated-eda variant=)` override — selects a + /// specific `[[workspace.variants]]` entry for this test. + /// The variant's `part` becomes the auto-project part; its + /// name flows through to `vw::vhdl_design_sources` so + /// variant-exclusive files filter correctly. Same + /// dedicated-eda restriction as `target_part_override`. + /// Meaningful only in variant-mode workspaces. + variant_override: Option, + /// Concrete Tcl lines to ship as setup before invoking the + /// test proc. Includes proc declarations, `src`-imported code, + /// and any top-level setup statements. + setup_tcl: Vec, +} + +async fn discover_tests_in_file( + file: &Utf8Path, + ws: &Utf8Path, + filter: Option<&str>, +) -> Result, Box> { + let program = load_htcl_program_for_test(file).await?; + let source = program.source.clone(); + let parsed = vw_htcl::parse(&source); + // Only reject on parse errors — validator warnings shouldn't + // block tests, matching cargo test's behavior. + if !parsed.errors.is_empty() { + let mut msg = format!("parse errors in {}:\n", file); + for e in &parsed.errors { + msg.push_str(&format!(" {}: {}\n", e.span.start, e.message)); + } + return Err(msg.into()); + } + + let putr_map = vw_htcl::putr::rewrite(&source, &parsed.document); + let signature_table = vw_htcl::signature_table(&parsed.document); + let line_index = vw_htcl::LineIndex::new(&source); + + // Pre-ship: primitive prelude + enum preludes + overload + // dispatchers. Same shape `run_htcl` uses (main.rs:1631-1667) + // and required for wrapped user code to install its procs + // correctly under the shim's `install_proc_body_wrap` machinery. + let mut _ignored: Vec = Vec::new(); + let enum_decl_table = + vw_htcl::build_enum_decl_table(&parsed.document, &mut _ignored); + let type_decl_table = + vw_htcl::build_type_decl_table(&parsed.document, &mut _ignored); + let type_decl_names: std::collections::HashSet = + type_decl_table.keys().cloned().collect(); + let (_full_sigs, overload_table) = + vw_htcl::build_signature_table_with_overloads( + &parsed.document, + &type_decl_names, + &mut _ignored, + ); + let mut setup_tcl: Vec = Vec::new(); + for p in vw_htcl::emit_primitive_prelude() { + setup_tcl.push(p); + } + for ed in enum_decl_table.values() { + let prelude = vw_htcl::emit_enum_prelude(ed); + if !prelude.trim().is_empty() { + setup_tcl.push(prelude); + } + } + for info in overload_table.values() { + let dispatcher = vw_htcl::emit_dispatcher(info); + if !dispatcher.trim().is_empty() { + setup_tcl.push(dispatcher); + } + } + + // Partition statements: `@test`-proc decls are collected as + // tests; everything else (including proc decls WITHOUT + // `@test`, `src` imports, and top-level setup) becomes setup. + let mut test_procs: Vec<(String, bool, Option, Option)> = + Vec::new(); + for stmt in &parsed.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + if let vw_htcl::CommandKind::Proc(proc) = &cmd.kind { + if let Some(test_attr) = proc.attribute("test") { + let Some(name) = proc.name.clone() else { + continue; + }; + let dedicated = test_attr.values.iter().any(|v| { + matches!( + v, + vw_htcl::AttributeValue::Ident { value, .. } + if value == "dedicated-eda" + ) + }); + // Per-test target override, e.g. + // `@test(dedicated-eda target="xcvp1202-vsva2785-3HP-e-S")`. + // Swaps the auto-project's `-part` for this test's + // Vivado session. Only meaningful for dedicated-eda + // tests (the validator warns otherwise); shared tests + // silently fall back to the workspace default. + let target_part_override = + extract_target_from_test_attr(test_attr); + let variant_override = + extract_variant_from_test_attr(test_attr); + test_procs.push(( + name, + dedicated, + target_part_override, + variant_override, + )); + // Test proc decls MUST also be shipped as setup — + // Vivado has to know about the proc before we call + // it. Fall through to the setup-emit below. + } + } + let lowered = vw_htcl::lower_command_with_putr_and_index( + cmd, + &source, + &signature_table, + &putr_map, + &line_index, + ); + // `rewrite_externs` — same as `vw run`. + let stripped = vw_htcl::rewrite_externs(&lowered).text; + if !stripped.trim().is_empty() { + setup_tcl.push(stripped); + } + } + + let display_path = file + .strip_prefix(ws) + .map(|p| p.to_string()) + .unwrap_or_else(|_| file.to_string()); + + let mut out = Vec::new(); + for (name, dedicated, target_part_override, variant_override) in test_procs + { + if let Some(filt) = filter { + if !name.contains(filt) { + continue; + } + } + out.push(TestCase { + display_path: display_path.clone(), + file_path: file.as_std_path().to_path_buf(), + name, + dedicated, + target_part_override, + variant_override, + setup_tcl: setup_tcl.clone(), + }); + } + Ok(out) +} + +/// Pull the `variant=` keyed value out of a `@test(...)` +/// attribute. Same shape as [`extract_target_from_test_attr`] +/// but for the variant-mode workspace path. +fn extract_variant_from_test_attr(attr: &vw_htcl::Attribute) -> Option { + for v in &attr.values { + let vw_htcl::AttributeValue::Keyed { key, value, .. } = v else { + continue; + }; + if key != "variant" { + continue; + } + return match value.as_ref() { + vw_htcl::AttributeValue::String { value, .. } => { + Some(value.clone()) + } + vw_htcl::AttributeValue::Ident { value, .. } => Some(value.clone()), + _ => None, + }; + } + None +} + +/// Pull the `target=` keyed value out of a `@test(...)` +/// attribute. Returns `None` if the attribute has no `target` +/// key or the value isn't a string/ident. +fn extract_target_from_test_attr(attr: &vw_htcl::Attribute) -> Option { + for v in &attr.values { + let vw_htcl::AttributeValue::Keyed { key, value, .. } = v else { + continue; + }; + if key != "target" { + continue; + } + return match value.as_ref() { + vw_htcl::AttributeValue::String { value, .. } => { + Some(value.clone()) + } + vw_htcl::AttributeValue::Ident { value, .. } => Some(value.clone()), + _ => None, + }; + } + None +} + +/// Run all shared-bucket tests in ONE Vivado session. First-file +/// setup lands once, then each proc-invocation is a separate eval. +async fn run_shared_bucket( + ws: &Utf8Path, + tests: Vec, + summary: &mut RunSummary, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, +) -> Result<(), Box> { + let verbose = matches!(log_level, vw_vivado::LogLevel::Debug); + // Shared bucket uses one Vivado process for every test in it, + // so per-test `@target(part=…)` / `@target(variant=…)` cannot + // apply — those overrides only make sense in dedicated-eda + // tests where each run gets a fresh session. The validator + // flags this at check time; here we silently fall back to the + // workspace default (or the CLI's `--part` / `--variant` + // selector if the user passed one). + let auto_project = workspace_auto_project(ws, part, variant)?; + // Show the first test's name up front with a "starting vivado" + // status so the run doesn't look hung during the ~10-30s + // Vivado boot. `run_one_test` overwrites this line as soon as + // it fires its own "running" line. + if let Some(first) = tests.first() { + let label = format!("{}::{}", first.display_path, first.name); + print_status_line("starting vivado", &label); + } + let active_variant = resolve_active_variant_name(ws, variant)?; + let mut backend = spawn_backend( + ws, + verbose, + info_with_stack, + auto_project, + active_variant, + ) + .await?; + // Track which file's setup we've already shipped. + let mut shipped: std::collections::HashSet = + std::collections::HashSet::new(); + for test in tests { + run_one_test( + &mut backend, + &test, + summary, + &mut shipped, + /*display_shipped=*/ true, + ) + .await; + } + // Detach shutdown — see `run_dedicated_bucket` for the + // rationale; here it saves the summary block from a 10s + // Vivado-exit wait tacked onto the tail of `vw test`. + tokio::spawn(async move { + let _ = backend.shutdown().await; + }); + Ok(()) +} + +/// Run `@test(dedicated-eda)` tests, each in its own Vivado +/// process. `test_threads` caps parallelism. +#[allow(clippy::too_many_arguments)] +async fn run_dedicated_bucket( + ws: &Utf8Path, + tests: Vec, + test_threads: usize, + summary: &mut RunSummary, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, +) -> Result<(), Box> { + let verbose = matches!(log_level, vw_vivado::LogLevel::Debug); + let threads = test_threads.max(1); + // Sequential for MVP — a proper semaphore-based parallel run + // would use tokio::task::JoinSet. Vivado processes each hold + // ~1-2 GB of RAM so caution around parallelism is warranted; + // parallel exec is a follow-up. + let _ = threads; + let ws_default = workspace_auto_project(ws, part, variant)?; + // Precompute variant → part for cheap per-test lookup below. + // `None` when the workspace has no variants block; a missing + // variant name gets a targeted error rather than a silent + // fallback so a typo in `@test(dedicated-eda variant=vpk20)` + // fails loud. + let ws_cfg = vw_lib::load_workspace_config(ws).ok(); + for test in tests { + // Resolution order per-test: + // 1. `variant=` → variant.part, override name from ws_default + // 2. `target=` → the literal part + // 3. no override → workspace default (whatever `part` selected) + let auto_project = if let Some(vname) = &test.variant_override { + let Some(cfg) = &ws_cfg else { + return Err(format!( + "test `{}` requested variant `{}` but the workspace \ + has no `vw.toml` — declare a `[[workspace.variants]]` \ + block first", + test.name, vname, + ) + .into()); + }; + let variant = cfg + .workspace + .select_variant(Some(vname)) + .map_err(|e| e.to_string())? + .ok_or_else(|| { + format!( + "test `{}` requested variant `{}` but the workspace \ + has no `[[workspace.variants]]` block", + test.name, vname, + ) + })?; + Some(vw_vivado::AutoProject { + name: ws_default + .as_ref() + .map(|p| p.name.clone()) + .unwrap_or_else(|| "vw_test".to_string()), + part: variant.part.clone(), + // `vw test` never persists — parallel `@test(dedicated-eda)` + // workers would trip over a shared on-disk project, and + // test scratch has no business dirtying `/target/`. + persist_dir: None, + }) + } else if let Some(p) = &test.target_part_override { + Some(vw_vivado::AutoProject { + name: ws_default + .as_ref() + .map(|p| p.name.clone()) + .unwrap_or_else(|| "vw_test".to_string()), + part: p.clone(), + persist_dir: None, + }) + } else { + ws_default.clone() + }; + // Show the test up front so the runner announces what's + // pending during Vivado boot (~10-30s per dedicated-eda + // test). Overwritten by run_one_test's "running" once + // spawn returns. + let label = format!("{}::{}", test.display_path, test.name); + print_status_line("starting vivado", &label); + // Per-test variant precedence: `@test(dedicated-eda variant=…)` + // wins over the CLI `--variant`, which in turn wins over the + // workspace default. Only meaningful when the workspace is + // variant-mode; part-mode workspaces get `None` and the + // session-scoped filter stays inert. + let active_variant = if test.variant_override.is_some() { + test.variant_override.clone() + } else { + resolve_active_variant_name(ws, variant)? + }; + let mut backend = spawn_backend( + ws, + verbose, + info_with_stack, + auto_project, + active_variant, + ) + .await?; + let mut shipped: std::collections::HashSet = + std::collections::HashSet::new(); + run_one_test( + &mut backend, + &test, + summary, + &mut shipped, + /*display_shipped=*/ false, + ) + .await; + // Detach shutdown. Vivado's tear-down is a 10s-bounded + // graceful-exit dance (see VivadoBackend::shutdown), and + // for an in-memory test session nothing about that wait is + // load-bearing — the child process exit / kill happens + // either way. Fire-and-forget so the runner reaches the + // summary block immediately after the last test, and so + // one test's shutdown overlaps the next test's Vivado + // boot instead of serializing. If the process exits before + // the shutdown task finishes, VivadoBackend's Drop still + // SIGKILLs the child — no orphan. + tokio::spawn(async move { + let _ = backend.shutdown().await; + }); + } + Ok(()) +} + +async fn run_one_test( + backend: &mut vw_vivado::VivadoBackend, + test: &TestCase, + summary: &mut RunSummary, + shipped: &mut std::collections::HashSet, + _display_shipped: bool, +) { + let started = Instant::now(); + let label = format!("{}::{}", test.display_path, test.name); + // Live "RUN" line — on a TTY we print with `\r` (no newline) + // and overwrite in place when the test finishes; on non-TTY + // we suppress it entirely so log files stay linear. Same + // convention nextest uses. Vivado boot takes ~10-30s per + // shared bucket, so without this the runner looks hung until + // the first PASS/FAIL prints. + print_running_line(&label); + // Ship setup once per file per session. Any error during setup + // is a hard-fail — subsequent tests in the same file would + // observe a broken state, so we bail on the whole file. + if shipped.insert(test.file_path.clone()) { + for line in &test.setup_tcl { + if let Err(e) = backend.eval(line).await { + let elapsed = format_secs(started.elapsed().as_secs_f64()); + clear_running_line(); + println!( + " {} [{}] {} — setup error", + "FAIL".red().bold(), + elapsed.red(), + label, + ); + summary.failed += 1; + summary.failures.push(TestFailure { + display: label, + err: e, + }); + return; + } + } + } + // Actually invoke the test proc. + match backend.eval(&test.name).await { + Ok(_) => { + let elapsed = format_secs(started.elapsed().as_secs_f64()); + clear_running_line(); + println!( + " {} [{}] {}", + "PASS".green().bold(), + elapsed.green(), + label, + ); + summary.passed += 1; + } + Err(e) => { + let elapsed = format_secs(started.elapsed().as_secs_f64()); + clear_running_line(); + println!(" {} [{}] {}", "FAIL".red().bold(), elapsed.red(), label,); + summary.failed += 1; + summary.failures.push(TestFailure { + display: label, + err: e, + }); + } + } +} + +/// Print the "currently running" line for `label`, overwriting +/// itself on the next `clear_running_line`. No-op on non-TTY — +/// the label would just fill the log with cursor-return sequences +/// that get rendered as literal `\r`. +fn print_running_line(label: &str) { + print_status_line("running", label); +} + +/// Same shape as [`print_running_line`] but with a caller-chosen +/// status word. Used to show `starting vivado` up front while the +/// backend boots (~10-30s) so the runner doesn't look hung on +/// dedicated-eda tests before their first eval. +fn print_status_line(status: &str, label: &str) { + use std::io::{IsTerminal, Write}; + let mut out = std::io::stdout(); + if !out.is_terminal() { + return; + } + // Pad status to a consistent width so successive re-renders + // ("starting vivado" → "running") overwrite cleanly without + // trailing chars leaking through. + let _ = write!( + out, + "\r {} [ {:16} ] {}", + "RUN ".cyan().bold(), + status.cyan(), + label, + ); + let _ = out.flush(); +} + +fn clear_running_line() { + use std::io::{IsTerminal, Write}; + let mut out = std::io::stdout(); + if !out.is_terminal() { + return; + } + // Overwrite the RUN line with spaces then \r back to column 0, + // so the PASS/FAIL println that follows lands at the start of + // the line. 120 cols is enough for any reasonable test name + + // decoration; longer names truncate the RUN line in place, + // which is fine because it's transient. + let _ = write!(out, "\r{}\r", " ".repeat(120)); + let _ = out.flush(); +} + +async fn spawn_backend( + ws: &Utf8Path, + verbose: bool, + info_with_stack: bool, + auto_project: Option, + active_variant: Option, +) -> Result> { + let rpc_handler = vw_vivado::make_handler_with_variant( + Some(ws.as_std_path().to_path_buf()), + active_variant, + ); + // Raw byte-log per spawn — test runs may spawn multiple Vivados + // in the dedicated bucket; each gets its own timestamped log + // under target/logs/. Silent-on-error: the test runner already + // owns the output surface, and a broken target/ dir shouldn't + // fail the test. + let raw_log = vw_vivado::raw_log_path_for_workspace(ws.as_std_path()).ok(); + let backend = vw_vivado::VivadoBackend::spawn(vw_vivado::VivadoConfig { + verbose, + info_with_stack, + rpc_handler: Some(rpc_handler), + auto_project, + raw_log, + ..Default::default() + }) + .await + .map_err(|e| format!("failed to start Vivado worker: {e}"))?; + Ok(backend) +} + +/// Resolve the active variant name for a workspace given the +/// CLI's `--variant` selector. Returns `None` for workspaces with +/// no `[[workspace.variants]]` block (variant-mode is inactive). +/// Errors mirror `workspace_auto_project`'s selector-mismatch +/// checks — a workspace-mode/selector mismatch is caller error. +fn resolve_active_variant_name( + ws: &Utf8Path, + variant: Option<&str>, +) -> Result, Box> { + let Ok(cfg) = vw_lib::load_workspace_config(ws) else { + return Ok(None); + }; + if cfg.workspace.variants.is_empty() { + return Ok(None); + } + let selected = cfg + .workspace + .select_variant(variant) + .map_err(|e| e.to_string())?; + Ok(selected.map(|v| v.name.clone())) +} + +/// Derive the auto-project from a workspace's `[[target-parts]]` +/// or `[[workspace.variants]]` block — same rule `vw run` uses. +/// - `part` is the CLI's `--part` selector; ignored when the +/// workspace is variant-mode. +/// - `variant` is the CLI's `--variant` selector; ignored when the +/// workspace is part-mode. +/// +/// Returns: +/// - `Ok(None)` for library workspaces (neither block declared) +/// - `Ok(Some(_))` when a part / variant resolves cleanly +/// - `Err(_)` when the CLI selector is bogus, the multi-entry list +/// has no default marked, or the user passed a selector that +/// doesn't match the workspace's declared mode. +fn workspace_auto_project( + ws: &Utf8Path, + part: Option<&str>, + variant: Option<&str>, +) -> Result, Box> { + let Ok(cfg) = vw_lib::load_workspace_config(ws) else { + return Ok(None); + }; + if !cfg.workspace.variants.is_empty() { + if part.is_some() { + return Err(format!( + "workspace `{}` declares `[[workspace.variants]]` — use \ + `--variant ` instead of `--part`", + cfg.workspace.name, + ) + .into()); + } + let selected = cfg + .workspace + .select_variant(variant) + .map_err(|e| e.to_string())?; + return Ok(selected.map(|v| vw_vivado::AutoProject { + name: cfg.workspace.name.clone(), + part: v.part.clone(), + // See sibling site above: `vw test` intentionally never + // persists the Vivado project. + persist_dir: None, + })); + } + if variant.is_some() { + return Err(format!( + "workspace `{}` does not declare `[[workspace.variants]]` — \ + remove `--variant` or add a variants block", + cfg.workspace.name, + ) + .into()); + } + let selected = cfg + .workspace + .select_target_part(part) + .map_err(|e| e.to_string())?; + Ok(selected.map(|p| vw_vivado::AutoProject { + name: cfg.workspace.name.clone(), + part: p.to_string(), + persist_dir: None, + })) +} + +#[derive(Default)] +struct RunSummary { + passed: usize, + failed: usize, + failures: Vec, +} + +struct TestFailure { + display: String, + /// The raw backend error kept intact so `print_summary` can + /// render it with the same colored/split treatment `vw run` + /// and the REPL apply — message in bright red, stdout in + /// default, stack in dimmed gray. + err: vw_eda::BackendError, +} + +fn print_summary(summary: &RunSummary, elapsed: std::time::Duration) { + if !summary.failures.is_empty() { + println!("\n{}\n", "failures:".red().bold()); + for f in &summary.failures { + print_failure_block(f); + } + } + let outcome = if summary.failed == 0 { + "ok".green().bold().to_string() + } else { + "FAILED".red().bold().to_string() + }; + println!( + "\ntest result: {outcome}. {} passed; {} failed; finished in {}", + summary.passed.to_string().green(), + summary.failed.to_string().red(), + format_secs(elapsed.as_secs_f64()), + ); +} + +/// Print one failure block. Layout mirrors nextest's: +/// separator + STDOUT capture + STDERR/error + stack. Colors +/// match `vw run` / REPL — bright red for the failing message, +/// dimmed for the stack, default for captured stdout. +fn print_failure_block(f: &TestFailure) { + let bar = "─".repeat(64); + println!("{}", bar.red()); + println!(" {} {}", "✗".red().bold(), f.display.bold()); + match &f.err { + vw_eda::BackendError::Tcl { + message, + info, + stdout, + .. + } => { + if !stdout.trim().is_empty() { + println!("\n{}", "STDOUT:".bright_black().bold()); + println!("{}", stdout.trim_end()); + } + println!("\n{}", "ERROR:".red().bold()); + for line in message.lines() { + println!(" {}", line.red()); + } + if let Some(info) = info { + // Tcl's `errorInfo` embeds the error message at its + // top and appends the "while executing…" frames + // after it. Rendering the whole thing after the + // ERROR block duplicates every line of the message + // (which for `assert_file_eq` is the entire diff). + // Strip the message prefix off `info` and show + // only the frame tail; skip the section entirely + // if nothing but the message was present. + let frames = strip_message_prefix(info, message); + if !frames.trim().is_empty() { + println!("\n{}", "stack:".bright_black().bold()); + for line in frames.lines() { + println!(" {}", line.bright_black()); + } + } + } + } + other => { + println!(" {}", other.to_string().red()); + } + } + println!("{}\n", bar.red()); +} + +/// Trim Tcl's echo of the error message from the start of +/// `errorInfo`. Tcl formats `errorInfo` as +/// `\n while executing "..."\n ...`, so the frame +/// tail always starts after `message` (whose newlines and +/// whitespace we match verbatim). When the message doesn't appear +/// at the top for some reason, we fall back to returning the raw +/// info unchanged rather than silently swallowing the whole stack. +fn strip_message_prefix<'a>(info: &'a str, message: &str) -> &'a str { + let trimmed = message.trim_end(); + if let Some(rest) = info.strip_prefix(trimmed) { + // Skip any leading whitespace / newlines between the + // message and the first `while executing "..."` frame. + rest.trim_start_matches(['\n', '\r', ' ']) + } else { + info + } +} + +fn format_secs(secs: f64) -> String { + format!("{secs:>6.3}s") +} + +#[cfg(test)] +mod tests { + use super::strip_message_prefix; + + #[test] + fn strip_removes_message_echo_and_leading_whitespace() { + let message = + "assert_file_eq: files differ\n actual: a\n expected: b"; + let info = "assert_file_eq: files differ\n actual: a\n expected: b\n while executing\n\"error \\\"foo\\\"\""; + let out = strip_message_prefix(info, message); + assert!(out.starts_with("while executing"), "got {out:?}"); + } + + #[test] + fn strip_falls_back_when_prefix_missing() { + let info = "totally different\n while executing\n\"foo\""; + let out = strip_message_prefix(info, "assert_file_eq: files differ"); + assert_eq!(out, info); + } +} diff --git a/vw-cli/src/main.rs b/vw-cli/src/main.rs index ce4386b..07aed45 100644 --- a/vw-cli/src/main.rs +++ b/vw-cli/src/main.rs @@ -2,21 +2,24 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -use camino::Utf8PathBuf; +use camino::{Utf8Path, Utf8PathBuf}; use clap::{Parser, Subcommand, ValueEnum}; use colored::*; use std::collections::HashSet; use std::fmt; use std::process; +use vw_eda::EdaBackend; use vw_lib::{ - add_dependency_with_token, clear_cache, extract_hostname_from_repo_url, - generate_deps_tcl, get_access_credentials_from_netrc, init_workspace, - list_dependencies, list_testbenches, load_workspace_config, - remove_dependency, run_testbench, update_workspace_with_token, Credentials, - VersionInfo, VhdlStandard, + add_dependency_with_token, clear_cache, generate_deps_tcl, + get_access_credentials_for_repo, get_access_credentials_for_workspace, + init_workspace, list_dependencies, list_testbenches, remove_dependency, + run_testbench, update_workspace_with_token, VersionInfo, VhdlStandard, }; +mod htcl_test; +mod parallel_load; + #[derive(Clone, Copy, Debug, ValueEnum)] enum CliVhdlStandard { #[value(name = "2008")] @@ -43,6 +46,38 @@ impl From for VhdlStandard { } } +/// Clap ValueEnum mirror of [`vw_vivado::LogLevel`]. Lives here +/// rather than in vw-vivado so the backend crate stays clap-free. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum CliLogLevel { + /// Show every block raw — including Vivado's banners, tables, + /// hierarchy summaries, and other non-diagnostic noise. The + /// same content the raw `vivado.log` receives. + Debug, + /// Show INFO+ diagnostics. NONE-block noise renders dimmed + /// (vw run) or collapsed (repl). Default. + Info, + /// Show WARNING+ diagnostics. NONE + INFO are elided. + Warning, + /// Show CRITICAL WARNING+ diagnostics. NONE + INFO + WARNING + /// are elided. + Critical, + /// Show only ERRORs. Everything else is elided. + Error, +} + +impl From for vw_vivado::LogLevel { + fn from(lvl: CliLogLevel) -> Self { + match lvl { + CliLogLevel::Debug => vw_vivado::LogLevel::Debug, + CliLogLevel::Info => vw_vivado::LogLevel::Info, + CliLogLevel::Warning => vw_vivado::LogLevel::Warning, + CliLogLevel::Critical => vw_vivado::LogLevel::Critical, + CliLogLevel::Error => vw_vivado::LogLevel::Error, + } + } +} + #[derive(Parser)] #[command(name = "vw")] #[command(about = "A VHDL workspace management tool")] @@ -91,8 +126,8 @@ enum Commands { List, #[command(about = "Generate deps.tcl file with all dependency VHDL files")] DepsToTcl, - #[command(about = "Run testbench using NVC")] - Test { + #[command(about = "Run VHDL testbench using NVC")] + Bench { #[arg(help = "Name of the testbench entity to run")] testbench: Option, #[arg(long, help = "VHDL standard", default_value_t = CliVhdlStandard::Vhdl2019)] @@ -130,37 +165,263 @@ enum Commands { )] scaffold: bool, }, + #[command(about = "Run an htcl script against a Vivado worker. \ + With no file, discovers `/design.htcl`.")] + Run { + #[arg(help = "Path to an .htcl source file. Omit to run the \ + workspace's `design.htcl`.")] + file: Option, + #[arg( + long, + help = "Parse and print diagnostics only; don't launch Vivado" + )] + check: bool, + #[arg( + long, + value_name = "ID", + conflicts_with = "variant", + help = "Select a non-default `[[target-parts]]` entry by full \ + part ID or unique substring (e.g. `--part 3HP`). \ + Mutually exclusive with `--variant`." + )] + part: Option, + #[arg( + long, + value_name = "NAME", + conflicts_with = "part", + help = "Select a non-default `[[workspace.variants]]` entry by \ + exact name. Variants own their parts inline." + )] + variant: Option, + #[arg( + long = "log-level", + value_enum, + default_value_t = CliLogLevel::Info, + help = "Minimum severity to show. `debug` shows everything \ + (including Vivado's non-diagnostic noise); INFO+ \ + filters and dims/collapses non-diagnostic output. \ + The full raw stream always lands in \ + `/target/logs/vivado-*.log`." + )] + log_level: CliLogLevel, + #[arg( + long = "info-with-stack", + help = "Attach the Tcl call stack to INFO messages too \ + (WARNING / ERROR / CRITICAL always include the stack)" + )] + info_with_stack: bool, + }, + #[command(about = "Launch the vw analyzer LSP server on stdio")] + Analyzer, + #[command( + about = "Interactive htcl REPL backed by a long-lived Vivado worker" + )] + Repl { + #[arg( + long = "log-level", + value_enum, + default_value_t = CliLogLevel::Info, + help = "Minimum severity shown in scrollback. `debug` shows \ + every block raw; INFO+ collapses non-diagnostic \ + output into a togglable placeholder." + )] + log_level: CliLogLevel, + #[arg( + long = "load", + value_name = "FILE", + help = "Source FILE into the session as soon as Vivado is up" + )] + initial_load: Option, + #[arg( + long, + value_name = "ID", + conflicts_with = "variant", + help = "Select a non-default `[[target-parts]]` entry by full \ + part ID or unique substring. Mutually exclusive with \ + `--variant`." + )] + part: Option, + #[arg( + long, + value_name = "NAME", + conflicts_with = "part", + help = "Select a non-default `[[workspace.variants]]` entry by \ + exact name." + )] + variant: Option, + #[arg( + long = "info-with-stack", + help = "Attach the Tcl call stack to INFO messages too \ + (WARNING / ERROR / CRITICAL always include the stack)" + )] + info_with_stack: bool, + }, + #[command(about = "Parse and analyze htcl. With no args, discovers the \ + workspace's module.htcl AND test/*.htcl and checks both")] + Check { + #[arg(help = "One or more .htcl source files. Empty → discover \ + from the workspace root.")] + files: Vec, + #[arg( + long, + value_name = "ID", + conflicts_with_all = ["all_parts", "variant", "all_variants"], + help = "Check against a specific `[[target-parts]]` entry by \ + full part ID or unique substring. Default: the workspace's \ + default part." + )] + part: Option, + #[arg( + long = "all-parts", + conflicts_with_all = ["part", "variant", "all_variants"], + help = "Check against every declared `[[target-parts]]` entry \ + instead of just the default" + )] + all_parts: bool, + #[arg( + long, + value_name = "NAME", + conflicts_with_all = ["part", "all_parts", "all_variants"], + help = "Check against a specific `[[workspace.variants]]` entry \ + by exact name. Default: the workspace's default variant." + )] + variant: Option, + #[arg( + long = "all-variants", + conflicts_with_all = ["part", "all_parts", "variant"], + help = "Check against every declared `[[workspace.variants]]` \ + entry (each variant's `.part`) instead of just the \ + default" + )] + all_variants: bool, + }, + #[command(about = "Run htcl-level tests (@test procs under test/)")] + Test { + #[arg(help = "Substring filter — run only tests whose name matches")] + filter: Option, + #[arg(long, help = "List discovered tests without running them")] + list: bool, + #[arg( + long, + help = "Max concurrent dedicated-eda Vivado processes", + default_value_t = 2 + )] + test_threads: usize, + #[arg( + long, + value_name = "ID", + conflicts_with = "variant", + help = "Select the workspace-default `[[target-parts]]` entry \ + for tests without their own `@test(target=…)`; matches \ + by full ID or unique substring" + )] + part: Option, + #[arg( + long, + value_name = "NAME", + conflicts_with = "part", + help = "Select a specific `[[workspace.variants]]` entry as the \ + default for tests without their own \ + `@test(variant=…)`" + )] + variant: Option, + #[arg( + long = "log-level", + value_enum, + default_value_t = CliLogLevel::Info, + help = "Minimum severity to show during test evals. See \ + `vw run --help` for the full model." + )] + log_level: CliLogLevel, + #[arg( + long = "info-with-stack", + help = "Attach the Tcl call stack to INFO messages too" + )] + info_with_stack: bool, + }, + #[command(subcommand, about = "IP-XACT tooling")] + Ip(IpCommand), + #[command( + subcommand, + name = "htcl-cmd", + about = "Generate htcl wrappers from Vivado command references" + )] + HtclCmd(HtclCmdCommand), } -/// Helper function to get access credentials for a repository URL from netrc if available -async fn get_access_credentials_for_repo( - repo_url: &str, -) -> Option { - if let Ok(hostname) = extract_hostname_from_repo_url(repo_url) { - if let Ok(Some(creds)) = get_access_credentials_from_netrc(&hostname) { - return Some(creds); - } - } - None +#[derive(Subcommand)] +enum HtclCmdCommand { + #[command( + about = "Generate an htcl wrapper from a Vivado man-page command \ + reference" + )] + Generate { + #[arg(help = "Path to a Vivado man-page file (e.g. \ + /doc/eng/man/add_files)")] + input: Utf8PathBuf, + #[arg(short, long, help = "Output file (defaults to stdout)")] + output: Option, + #[arg( + long, + help = "Command name to wrap (defaults to the input file stem)" + )] + name: Option, + #[arg( + long, + value_name = "FILE", + help = "Per-command constraint overrides (TOML)" + )] + constraints: Option, + }, } -/// Helper function to get access credentials for workspace dependencies from netrc -async fn get_access_credentials_for_workspace( - workspace_dir: &camino::Utf8Path, -) -> Option { - // Load workspace config and check if any dependencies might need authentication - if let Ok(config) = load_workspace_config(workspace_dir) { - for dep in config.dependencies.values() { - if let Some(creds) = - get_access_credentials_for_repo(&dep.repo).await - { - return Some(creds); - } - } - } - None +#[derive(Subcommand)] +enum IpCommand { + #[command(about = "Generate an htcl wrapper from an IP-XACT component")] + Generate { + #[arg(help = "Path to an IP-XACT component XML file")] + input: Utf8PathBuf, + #[arg(short, long, help = "Output file (defaults to stdout)")] + output: Option, + #[arg( + long, + help = "Include parameters whose resolve attribute is not 'user'" + )] + include_internal: bool, + #[arg( + long = "preset", + value_name = "FILE", + help = "Supplementary Vivado preset XML file (`` format). May be given multiple times. The \ + declared values are merged into `@enum(...)` lists in the \ + generated wrapper, on top of the IP-XACT `` \ + entries." + )] + presets: Vec, + #[arg( + long, + help = "Skip auto-discovery of preset files under the Vivado \ + `data/versal/ps_pmc//` tree. Use this if the \ + discovered files are wrong or you only want the explicit \ + `--preset` ones." + )] + no_auto_presets: bool, + #[arg( + long, + value_name = "FILE", + help = "Per-IP TOML overrides file. Refines XML-derived \ + dict-schemas with `@enum(...)` restrictions and \ + per-field default overrides. Silently ignored when \ + the file doesn't exist." + )] + overrides: Option, + }, } +// Netrc credential lookup moved to `vw_lib` so `vw-vivado`'s +// RPC auto-update path can reuse it. See +// `vw_lib::get_access_credentials_for_workspace`. + #[tokio::main] async fn main() { let cli = Cli::parse(); @@ -194,7 +455,8 @@ async fn main() { ); } Commands::Update => { - let access_creds = get_access_credentials_for_workspace(&cwd).await; + let access_creds = + get_access_credentials_for_workspace(&cwd, false); match update_workspace_with_token(&cwd, access_creds).await { Ok(result) => { for dep in result.dependencies { @@ -233,7 +495,7 @@ async fn main() { recursive, sim_only, } => { - let access_creds = get_access_credentials_for_repo(&repo).await; + let access_creds = get_access_credentials_for_repo(&repo); match add_dependency_with_token( &cwd, repo.clone(), @@ -307,9 +569,8 @@ async fn main() { if deps.is_empty() { println!("No dependencies found in workspace"); } else { - println!("Dependencies:"); - for dep in deps { - let version_info = match dep.version { + let render = |dep: &vw_lib::DependencyInfo| { + let version_info = match &dep.version { VersionInfo::Branch { branch } => { format!(" (branch: {branch})") } @@ -319,15 +580,32 @@ async fn main() { VersionInfo::Locked { commit } => { format!(" ({})", &commit[..8.min(commit.len())]) } + VersionInfo::Local => " (local)".to_string(), VersionInfo::Unknown => String::new(), }; - println!( " {} - {}{}", dep.name.cyan(), - dep.repo, + dep.source, version_info.bright_black() ); + }; + let (test_deps, regular_deps): (Vec<_>, Vec<_>) = + deps.into_iter().partition(|d| d.is_test); + if !regular_deps.is_empty() { + println!("Dependencies:"); + for dep in ®ular_deps { + render(dep); + } + } + if !test_deps.is_empty() { + if !regular_deps.is_empty() { + println!(); + } + println!("Test dependencies:"); + for dep in &test_deps { + render(dep); + } } } } @@ -348,7 +626,7 @@ async fn main() { process::exit(1); } }, - Commands::Test { + Commands::Bench { testbench, std, list, @@ -452,5 +730,1988 @@ async fn main() { process::exit(1); } } + Commands::Run { + file, + check, + part, + variant, + log_level, + info_with_stack, + } => { + let resolved = match file { + Some(f) => f, + None => match discover_entry_file(&cwd) { + Ok(f) => f, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + }, + }; + if let Err(e) = run_htcl( + &resolved, + check, + part.as_deref(), + variant.as_deref(), + log_level.into(), + info_with_stack, + ) + .await + { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + Commands::Analyzer => { + init_analyzer_logging(); + vw_analyzer::run_stdio().await; + } + Commands::Repl { + log_level, + initial_load, + part, + variant, + info_with_stack, + } => { + // Same `design.htcl` auto-discovery as `vw run`: if the + // user didn't pass `--load`, look for a `design.htcl` + // in the enclosing workspace and source it up front. + // Silent no-op when there's no workspace or no + // `design.htcl` — the REPL still boots and the user + // can `:load` something explicitly. + let resolved_load = initial_load.or_else(|| { + vw_lib::find_workspace_dir(cwd.as_std_path()) + .and_then(|ws| vw_lib::find_design_file(&ws)) + }); + if let Err(e) = vw_repl::run(vw_repl::ReplOptions { + log_level: log_level.into(), + initial_load: resolved_load, + part, + variant, + info_with_stack, + }) + .await + { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + Commands::Check { + files, + part, + all_parts, + variant, + all_variants, + } => { + // Two shapes: + // `vw check FILE [FILE...]` — check the explicit list. + // `vw check` — discover from workspace: + // - `/module.htcl` in normal mode. + // - Every `/test/**/*.htcl` in test-mode + // (test-deps + self-injection visible so `src @` + // and `src @` both resolve). + let discovered = if files.is_empty() { + match discover_check_targets(&cwd) { + Ok(t) => t, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + } else { + files + .into_iter() + .map(|f| CheckTarget { + path: f, + include_test_deps: false, + }) + .collect() + }; + if discovered.is_empty() { + eprintln!( + "{} nothing to check — pass a file, or run from a \ + directory with a `vw.toml` that has a `design.htcl`, \ + `module.htcl`, or `test/*.htcl`", + "note:".bright_yellow(), + ); + return; + } + let mut had_errors = false; + // conflicts_with on the clap args guarantees at most + // one non-Default source at a time. Map both flag + // families to the same PartSelector; the resolver + // decides which config block applies based on the + // workspace shape. + let part_selector = if let Some(v) = variant.as_deref() { + PartSelector::Explicit(v.to_string()) + } else if all_variants { + PartSelector::All + } else { + PartSelector::from_flags(part.as_deref(), all_parts) + }; + for target in &discovered { + let res = check_htcl_with_mode( + &target.path, + target.include_test_deps, + &part_selector, + ) + .await; + match res { + Ok(file_errs) => { + if file_errs { + had_errors = true; + } + } + Err(e) => { + had_errors = true; + eprintln!( + "{} {}: {e}", + "error:".bright_red(), + target.path, + ); + } + } + } + if had_errors { + process::exit(1); + } + } + Commands::Test { + filter, + list, + test_threads, + part, + variant, + log_level, + info_with_stack, + } => { + if let Err(e) = htcl_test::run_htcl_tests( + &cwd, + filter, + list, + test_threads, + part.as_deref(), + variant.as_deref(), + log_level.into(), + info_with_stack, + ) + .await + { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + Commands::Ip(cmd) => match cmd { + IpCommand::Generate { + input, + output, + include_internal, + presets, + no_auto_presets, + overrides, + } => { + if let Err(e) = run_ip_generate( + &input, + output.as_deref(), + include_internal, + &presets, + no_auto_presets, + overrides.as_deref(), + ) { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + }, + Commands::HtclCmd(cmd) => match cmd { + HtclCmdCommand::Generate { + input, + output, + name, + constraints, + } => { + if let Err(e) = run_htcl_cmd_generate( + &input, + output.as_deref(), + name.as_deref(), + constraints.as_deref(), + ) { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + }, + } +} + +fn run_htcl_cmd_generate( + input: &Utf8Path, + output: Option<&Utf8Path>, + name: Option<&str>, + constraints_path: Option<&Utf8Path>, +) -> Result<(), String> { + let page = vw_htcl_cmd::load(input.as_std_path(), name) + .map_err(|e| format!("loading {input}: {e}"))?; + let constraints = match constraints_path { + Some(p) => vw_htcl_cmd::ConstraintsTable::load(p.as_std_path()) + .map_err(|e| format!("loading constraints: {e}"))?, + None => vw_htcl_cmd::ConstraintsTable::empty(), + }; + let opts = vw_htcl_cmd::GenerateOptions { + constraints, + ..Default::default() + }; + let text = vw_htcl_cmd::generate(&page, &opts); + match output { + Some(path) => std::fs::write(path, &text) + .map_err(|e| format!("writing {path}: {e}"))?, + None => print!("{text}"), + } + Ok(()) +} + +fn run_ip_generate( + input: &Utf8Path, + output: Option<&Utf8Path>, + include_internal: bool, + explicit_presets: &[Utf8PathBuf], + no_auto_presets: bool, + overrides_path: Option<&Utf8Path>, +) -> Result<(), String> { + let component = + vw_ip::load(input).map_err(|e| format!("loading {input}: {e}"))?; + + // Combine explicit `--preset` files with what we can auto-discover + // under Vivado's `data/versal/ps_pmc//` tree. + let mut preset_paths: Vec = explicit_presets + .iter() + .map(|p| std::path::PathBuf::from(p.as_str())) + .collect(); + if !no_auto_presets { + let discovered = + vw_ip::discover_presets(std::path::Path::new(input.as_str())); + for p in discovered { + if !preset_paths.contains(&p) { + preset_paths.push(p); + } + } + } + for p in &preset_paths { + eprintln!("{:>12} {}", "Sourcing".bright_green().bold(), p.display()); + } + let presets = if preset_paths.is_empty() { + vw_ip::PresetMap::new() + } else { + vw_ip::load_presets(&preset_paths) + .map_err(|e| format!("loading presets: {e}"))? + }; + + // Sub-proc schemas for Xilinx `structured_tcldict` parameters + // (PS_PMC_CONFIG, etc.). Empty when the component isn't a CIPS + // and doesn't have an accompanying schema tree. + let dict_schemas = + vw_ip::load_cips_dict_schemas(std::path::Path::new(input.as_str())); + for name in dict_schemas.keys() { + eprintln!( + "{:>12} schema for {name} ({} fields)", + "Loaded".bright_green().bold(), + dict_schemas[name].fields.len() + ); + } + + // Load per-IP TOML overrides. Missing file → empty overrides; + // the generator falls back to XML-only defaults everywhere. + let overrides = match overrides_path { + Some(p) => { + let ov = + vw_ip::overrides::OverridesFile::load_from(p.as_std_path()) + .map_err(|e| format!("{e}"))?; + if !ov.is_empty() { + eprintln!( + "{:>12} {} shape refinement(s) from {p}", + "Loaded".bright_green().bold(), + ov.shapes.len() + ); + } + ov + } + None => vw_ip::overrides::OverridesFile::default(), + }; + + let opts = vw_ip::GenerateOptions { + user_configurable_only: !include_internal, + overrides, + ..Default::default() + }; + let out = vw_ip::generate(&component, &presets, &dict_schemas, &opts); + match output { + Some(path) => { + let module_dir = match path.parent() { + Some(p) if !p.as_str().is_empty() => p.to_path_buf(), + _ => Utf8PathBuf::from("."), + }; + std::fs::write(path, &out.main) + .map_err(|e| format!("writing {path}: {e}"))?; + // Sibling `.htcl` files that the main module sources. + // For the split gtwiz-versal IP this is 8+5 = 13 files + // (one per intfN, quadN). For small IPs (cips, dcmac) + // the subfiles vec is empty and only main.htcl gets + // written. + for (basename, content) in &out.subfiles { + let sub_path = module_dir.join(basename); + std::fs::write(&sub_path, content) + .map_err(|e| format!("writing {sub_path}: {e}"))?; + } + if !out.subfiles.is_empty() { + eprintln!( + "{:>12} main + {} subfile(s)", + "Wrote".bright_green().bold(), + out.subfiles.len() + ); + } + // Generated IP modules need a workspace toml so `vw test`, + // `vw analyzer`, and the REPL can resolve `src @vivado-cmd` + // (and any other helpers the wrapper calls into). Seed a + // default one alongside `module.htcl` on first generation; + // never clobber an existing user-edited toml. + ensure_module_vw_toml(&module_dir)?; + // Extract `` from the source + // component.xml and write it into the module's vw.toml + // `[targets] supported` list. This lets downstream + // `vw check` catch device-family mismatches statically + // — no Vivado runtime dependency. + let targets = vw_ip::targets::extract_targets( + std::path::Path::new(input.as_str()), + ); + if !targets.supported.is_empty() + || !targets.not_supported.is_empty() + { + if let Err(e) = upsert_targets_in_vw_toml(&module_dir, &targets) + { + eprintln!( + "{} updating {}/vw.toml `[targets]`: {e}", + "warning:".bright_yellow(), + module_dir, + ); + } + } + } + None => { + // stdout mode has no way to represent multiple files; + // fall back to the concatenated single-file form so + // manual `vw ip generate ... > file.htcl` invocations + // still work. + print!("{}", out.into_single()); + } + } + Ok(()) +} + +/// Write a default `vw.toml` to `dir` if one doesn't already exist. +/// +/// Generated IP wrappers all `src @vivado-cmd` for the `ip::check` / +/// `log::error` / property-helper procs that drive their bodies, so +/// every module dir needs a workspace toml that points at the +/// vivado-cmd module. Without it, the analyzer and REPL flag +/// `undefined proc ip::check` on the first line of every freshly- +/// generated module. +/// +/// The function is idempotent: an existing `vw.toml` is left alone so +/// the user can edit it (add deps, rename the workspace) without +/// having their changes overwritten the next time `regenerate.sh` +/// runs. +/// Serialize a `[targets] supported = [...]` section into `dir/vw. +/// toml`, upserting: if the file already declares `[targets]`, +/// replace the `supported` list; otherwise append a fresh section. +/// Other sections (workspace, dependencies) are preserved verbatim. +/// +/// Uses a text-level upsert rather than a serde round-trip so +/// user-authored comments and formatting in other sections don't +/// get flattened. +fn upsert_targets_in_vw_toml( + dir: &Utf8Path, + targets: &vw_ip::targets::ExtractedTargets, +) -> Result<(), String> { + let toml_path = dir.join("vw.toml"); + let mut existing = std::fs::read_to_string(&toml_path) + .map_err(|e| format!("reading {toml_path}: {e}"))?; + // Render the new block once — same shape whether we're + // inserting or replacing. `not-supported` is written only when + // non-empty so IPs whose XML is uniformly blessed produce a + // tidy vw.toml with just `supported`. + let mut new_block = String::from("[targets]\nsupported = [\n"); + for t in &targets.supported { + new_block.push_str(&format!(" \"{t}\",\n")); + } + new_block.push_str("]\n"); + if !targets.not_supported.is_empty() { + new_block.push_str("not-supported = [\n"); + for t in &targets.not_supported { + new_block.push_str(&format!(" \"{t}\",\n")); + } + new_block.push_str("]\n"); + } + + // If a `[targets]` section already exists, replace it in place + // (from its header line up to but not including the next + // `[section]` header or EOF). Otherwise append at the bottom. + if let Some(section_start) = existing.find("[targets]") { + // Find end: the byte just before the next top-level + // section header, or EOF. + let after_hdr = section_start + "[targets]".len(); + let mut end = existing.len(); + let mut cur = after_hdr; + let bytes = existing.as_bytes(); + while cur < bytes.len() { + if bytes[cur] == b'\n' + && cur + 1 < bytes.len() + && bytes[cur + 1] == b'[' + { + end = cur + 1; + break; + } + cur += 1; + } + existing.replace_range(section_start..end, &new_block); + } else { + if !existing.ends_with('\n') { + existing.push('\n'); + } + if !existing.ends_with("\n\n") { + existing.push('\n'); + } + existing.push_str(&new_block); + } + std::fs::write(&toml_path, existing) + .map_err(|e| format!("writing {toml_path}: {e}"))?; + Ok(()) +} + +fn ensure_module_vw_toml(dir: &Utf8Path) -> Result<(), String> { + let toml_path = dir.join("vw.toml"); + if toml_path.exists() { + return Ok(()); + } + let name = dir + .canonicalize_utf8() + .ok() + .as_deref() + .and_then(|p| p.file_name()) + .or_else(|| dir.file_name()) + .unwrap_or("module") + .to_string(); + let (dep_line, note) = match discover_sibling_vivado_cmd(dir) { + Some(p) => (format!("path = \"{p}\""), None), + None => ( + "path = \"../vivado-cmd\"".to_string(), + Some("# TODO: adjust to your vivado-cmd module path"), + ), + }; + let mut content = format!( + "[workspace]\n\ + name = \"{name}\"\n\ + version = \"0.1.0\"\n\ + \n\ + [dependencies.vivado-cmd]\n" + ); + if let Some(n) = note { + content.push_str(n); + content.push('\n'); + } + content.push_str(&dep_line); + content.push('\n'); + std::fs::write(&toml_path, content) + .map_err(|e| format!("writing {toml_path}: {e}"))?; + eprintln!("{:>12} {}", "Created".bright_green().bold(), toml_path); + Ok(()) +} + +/// Walk up from `start` looking for a sibling `vivado-cmd/vw.toml`. +/// Returns the canonical absolute path to the directory if found. +/// Used by [`ensure_module_vw_toml`] to seed the dep path for +/// freshly-generated IP modules. +fn discover_sibling_vivado_cmd(start: &Utf8Path) -> Option { + let abs = start.canonicalize_utf8().ok()?; + let mut cur = abs.as_path(); + while let Some(parent) = cur.parent() { + let candidate = parent.join("vivado-cmd"); + if candidate.join("vw.toml").is_file() { + return Some(candidate); + } + cur = parent; + } + None +} + +/// Read `entry` and recursively resolve its `src` imports. Looks for +/// a `vw.toml` in the entry file's parent chain to discover the +/// workspace; falls back to an empty resolver (so relative/absolute +/// imports still work, but `@name/` imports fail with a clear error) +/// when no workspace is found. +/// +/// A [`CliObserver`] is attached so the loader's progress prints in +/// real time as `Sourcing …` / `Checking …` lines. +async fn load_htcl_program( + entry: &Utf8Path, +) -> Result> { + load_htcl_program_with_mode(entry, false).await +} + +/// Same as [`load_htcl_program`] but resolves +/// `[test-dependencies]` from the entry's workspace too. Used by +/// `vw test`. Cargo-parity: only the ENTRY workspace's test-deps +/// are pulled in — transitive workspaces don't leak their own +/// test-deps into the resolver. +#[allow(dead_code)] // wired via `crate::htcl_test` +pub(crate) async fn load_htcl_program_for_test( + entry: &Utf8Path, +) -> Result> { + load_htcl_program_with_mode(entry, true).await +} + +async fn load_htcl_program_with_mode( + entry: &Utf8Path, + include_test_deps: bool, +) -> Result> { + let entry_path = std::path::Path::new(entry.as_str()).to_path_buf(); + let workspace_dir = + entry_path.parent().and_then(vw_lib::find_workspace_dir); + let mut resolver = vw_htcl::Resolver::new(); + // Load the workspace config once — its `name` field feeds both + // the progress bar's label AND the self-injection at the + // bottom of this block. + let workspace_cfg = workspace_dir + .as_deref() + .and_then(|ws| vw_lib::load_workspace_config(ws).ok()); + if let Some(ws) = workspace_dir.as_deref() { + // Transitive resolution so a library's `src @other/...` + // import works even when the consumer hasn't redeclared + // `other` in their own `vw.toml`. + if let Ok(paths) = + vw_lib::transitive_dep_cache_paths_with_test(ws, include_test_deps) + { + for (name, path) in paths { + resolver = resolver.with_dep(name, path); + } + } + // Cargo-parity self-reference: a library named `foo` can + // `src @foo/bar` to reach its own siblings without the + // user having to declare `foo` as a dep of itself. Uses + // `with_dep_if_absent` so a legitimately-declared external + // `foo` still wins. + if let Some(cfg) = &workspace_cfg { + resolver = resolver.with_dep_if_absent( + cfg.workspace.name.clone(), + ws.as_std_path().to_path_buf(), + ); + } + } + let dep_paths: Vec<(String, std::path::PathBuf)> = workspace_dir + .as_deref() + .and_then(|ws| vw_lib::transitive_dep_cache_paths(ws).ok()) + .map(|paths| paths.into_iter().collect()) + .unwrap_or_default(); + // Pick up the workspace name from `vw.toml` so the local + // (non-@dep) files' bar shows `Checking metroid` instead of + // `Checking workspace`. Falls back to the literal `workspace` + // when there's no vw.toml or no `name = "…"` field. + let workspace_label = workspace_cfg + .as_ref() + .map(|cfg| cfg.workspace.name.clone()) + .unwrap_or_else(|| "workspace".to_string()); + let observer = std::sync::Arc::new( + parallel_load::MultiProgressObserver::new(dep_paths, workspace_label), + ); + let obs_for_load: std::sync::Arc = + observer.clone(); + let result = parallel_load::load_parallel( + &entry_path, + std::sync::Arc::new(resolver), + obs_for_load, + std::collections::HashMap::new(), + ) + .await; + observer.finish(); + Ok(result?) +} + +/// `entry`. Used by `check_htcl` to pre-flight `src @` +/// imports and produce spanned diagnostics before the loader's +/// hard-abort path fires. Returns an empty set when `entry` isn't +/// inside a workspace or the dep cache can't be read — the caller +/// treats empty as "skip the check", matching the validator's own +/// short-circuit. +#[allow(dead_code)] // legacy entry — new callers use `_with_mode` +fn collect_dep_names(entry: &Utf8Path) -> std::collections::HashSet { + collect_dep_names_with_mode(entry, false) +} + +fn collect_dep_names_with_mode( + entry: &Utf8Path, + include_test_deps: bool, +) -> std::collections::HashSet { + let entry_path = std::path::Path::new(entry.as_str()); + let Some(ws) = entry_path.parent().and_then(vw_lib::find_workspace_dir) + else { + return std::collections::HashSet::new(); + }; + let Ok(paths) = + vw_lib::transitive_dep_cache_paths_with_test(&ws, include_test_deps) + else { + return std::collections::HashSet::new(); + }; + let mut names: std::collections::HashSet = + paths.into_keys().collect(); + // Mirror `load_htcl_program`'s self-injection: a library named + // `foo` can `src @foo/bar` at check time even though `foo` + // isn't in its own [dependencies]. + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + names.insert(cfg.workspace.name); + } + names +} + +fn init_analyzer_logging() { + // Silent by default — see the matching note in vw-analyzer's main. + let filter = tracing_subscriber::EnvFilter::try_from_env("VW_ANALYZER_LOG") + .unwrap_or_else(|_| "vw_analyzer=off".into()); + let _ = tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_ansi(false) + .with_env_filter(filter) + .try_init(); +} + +/// Run parse + signature validation on `file`. Returns `Ok(true)` +/// if any error-severity diagnostics were reported, `Ok(false)` for +/// clean. Warnings don't flip the return value but still print. +#[allow(dead_code)] // legacy entry — new callers use `_with_mode` +async fn check_htcl( + file: &camino::Utf8Path, +) -> Result> { + check_htcl_with_mode(file, false, &PartSelector::Default).await +} + +/// One entry in the `vw check` (no-args) discovery result. Test +/// files get `include_test_deps = true` so the validator can see +/// `@` and `@` imports without the user +/// spelling them out in the vw.toml `[dependencies]` section. +struct CheckTarget { + path: Utf8PathBuf, + include_test_deps: bool, +} + +/// Discover files to check when the user runs `vw check` from a +/// workspace directory with no explicit file list. Picks up +/// `/design.htcl` (project entry) and `/module.htcl` +/// (library entry) — either or both may be present — plus every +/// `/test/**/*.htcl` (checked in test-mode). +/// +/// Errors when we can't find the enclosing workspace at all — +/// otherwise returns an empty vec, letting the caller print +/// "nothing to check" without treating it as a hard failure. +/// Resolve the CLI's `--part` / `--variant` flags against the +/// workspace's declared parts / variants and return the +/// `(auto_project, active_variant)` pair the RPC and Vivado +/// spawn need. Applies mutual-exclusion / mode-mismatch rules: +/// +/// - `--variant` on a part-mode workspace (no +/// `[[workspace.variants]]`) → error. +/// - `--part` on a variant-mode workspace → error (variants own +/// their parts). +/// - Neither flag → workspace default (either the default +/// target-part or the default variant's `.part`). +/// +/// The `active_variant` slot flows into the RPC session so +/// `vw::vhdl_design_sources` filters by the CLI-picked variant +/// automatically, without design.htcl having to name it. +pub(crate) fn resolve_workspace_selection( + ws: &camino::Utf8Path, + part: Option<&str>, + variant: Option<&str>, +) -> Result<(Option, Option), String> { + let Ok(cfg) = vw_lib::load_workspace_config(ws) else { + return Ok((None, None)); + }; + let ws_info = &cfg.workspace; + // Mode-mismatch guards. + if variant.is_some() && ws_info.variants.is_empty() { + return Err(format!( + "workspace at {ws} has no `[[workspace.variants]]` block; \ + remove `--variant` or add variants to vw.toml", + )); + } + if part.is_some() && !ws_info.variants.is_empty() { + return Err(format!( + "workspace at {ws} is variant-mode (has \ + `[[workspace.variants]]`); use `--variant ` instead of \ + `--part` — variants own their parts inline", + )); + } + // Resolve to a specific variant (variant mode) or part (part mode). + if !ws_info.variants.is_empty() { + let selected = + ws_info.select_variant(variant).map_err(|e| e.to_string())?; + let Some(v) = selected else { + return Ok((None, None)); + }; + let persist_dir = prepare_persist_dir(ws, &ws_info.name)?; + Ok(( + Some(vw_vivado::AutoProject { + name: ws_info.name.clone(), + part: v.part.clone(), + persist_dir, + }), + Some(v.name.clone()), + )) + } else { + let selected = ws_info + .select_target_part(part) + .map_err(|e| e.to_string())?; + let persist_dir = prepare_persist_dir(ws, &ws_info.name)?; + Ok(( + selected.map(|p| vw_vivado::AutoProject { + name: ws_info.name.clone(), + part: p.to_string(), + persist_dir: persist_dir.clone(), + }), + None, + )) + } +} + +/// Shared bootstrap for the on-disk Vivado project dir used by +/// `vw run` and `vw repl`. Runs the one-shot legacy IP-cache +/// cleanup + staleness wipe, then returns +/// `Some(/target/vw-project)` on success. +/// +/// A failure here is not fatal — we log the reason and fall back +/// to `persist_dir: None` (in-memory project), so the user still +/// gets a working session. That covers e.g. a read-only workspace +/// or the `target/` dir being held by another process. +fn prepare_persist_dir( + ws: &camino::Utf8Path, + name: &str, +) -> Result, String> { + match vw_lib::prepare_vw_project_dir(ws, name) { + Ok(prep) => { + if prep.legacy_cache_removed > 0 { + eprintln!( + "{} removed {} legacy IP cache entr{y} under \ + {ws}/target/ip — replaced by on-disk Vivado project", + "info:".cyan(), + prep.legacy_cache_removed, + y = if prep.legacy_cache_removed == 1 { + "y" + } else { + "ies" + }, + ); + } + if let Some(wiped) = &prep.wiped_project { + eprintln!( + "{} wiped stale Vivado project at {wiped} \ + (source fingerprint changed or manifest missing)", + "info:".cyan(), + ); + } + Ok(Some(prep.project_dir.into_std_path_buf())) + } + Err(e) => { + eprintln!( + "{} failed to prepare on-disk Vivado project dir under \ + {ws}/target/vw-project ({e}); falling back to in-memory \ + project (state won't persist across sessions)", + "warning:".bright_yellow(), + ); + Ok(None) + } + } +} + +/// Locate the workspace's `design.htcl` for a bare `vw run` with +/// no file argument. Errors with a targeted message when the +/// workspace doesn't have one — the user's next move is either +/// `vw run ` or creating a `design.htcl`. +fn discover_entry_file( + cwd: &Utf8Path, +) -> Result> { + let ws = vw_lib::find_workspace_dir(cwd.as_std_path()) + .ok_or("not in a vw workspace (no vw.toml in the parent chain)")?; + vw_lib::find_design_file(&ws).ok_or_else(|| { + format!( + "no `design.htcl` in {ws}; pass a file explicitly \ + (`vw run `) or create a `design.htcl`", + ) + .into() + }) +} + +fn discover_check_targets( + cwd: &Utf8Path, +) -> Result, Box> { + let ws = vw_lib::find_workspace_dir(cwd.as_std_path()) + .ok_or("not in a vw workspace (no vw.toml in the parent chain)")?; + let mut targets = Vec::new(); + if let Some(design) = vw_lib::find_design_file(&ws) { + targets.push(CheckTarget { + path: design, + include_test_deps: false, + }); + } + let module = ws.join("module.htcl"); + if module.is_file() { + targets.push(CheckTarget { + path: module, + include_test_deps: false, + }); + } + for path in vw_lib::list_htcl_tests(&ws)? { + let Ok(path) = Utf8PathBuf::from_path_buf(path) else { + continue; + }; + targets.push(CheckTarget { + path, + include_test_deps: true, + }); + } + Ok(targets) +} + +/// How `vw check` decides which `[[target-parts]]` entries to +/// evaluate. Constructed from the CLI's `--part` / `--all-parts` +/// flags; consumed by [`check_htcl_with_mode`] to iterate the +/// selected parts through the compatibility check. +#[derive(Debug, Clone)] +enum PartSelector { + /// No flag → use the workspace's default part. + Default, + /// `--part ` → use the matching entry. + Explicit(String), + /// `--all-parts` → iterate every entry. + All, +} + +impl PartSelector { + fn from_flags(part: Option<&str>, all_parts: bool) -> Self { + if all_parts { + Self::All + } else if let Some(p) = part { + Self::Explicit(p.to_string()) + } else { + Self::Default + } + } + + /// Resolve to a concrete list of part strings against `ws_info`. + /// Handles both the legacy `[[target-parts]]` shape and the + /// new `[[workspace.variants]]` shape (variants own their + /// parts inline; the compat check treats them the same way + /// once the part is extracted). Returns an empty vec for a + /// library workspace (neither block populated) — caller + /// treats that as "no compat check." + fn resolve<'a>( + &self, + ws_info: &'a vw_lib::WorkspaceInfo, + ) -> std::result::Result, String> { + if !ws_info.variants.is_empty() { + return self.resolve_variant_mode(ws_info); + } + if ws_info.target_parts.is_empty() { + return Ok(Vec::new()); + } + match self { + Self::Default => Ok(ws_info + .default_target_part() + .map_err(|e| e.to_string())? + .into_iter() + .collect()), + Self::Explicit(q) => Ok(ws_info + .select_target_part(Some(q)) + .map_err(|e| e.to_string())? + .into_iter() + .collect()), + Self::All => Ok(ws_info + .target_parts + .iter() + .map(|p| p.part.as_str()) + .collect()), + } + } + + fn resolve_variant_mode<'a>( + &self, + ws_info: &'a vw_lib::WorkspaceInfo, + ) -> std::result::Result, String> { + match self { + Self::Default => Ok(ws_info + .default_variant() + .map_err(|e| e.to_string())? + .map(|v| v.part.as_str()) + .into_iter() + .collect()), + Self::Explicit(q) => Ok(ws_info + .select_variant(Some(q)) + .map_err(|e| e.to_string())? + .map(|v| v.part.as_str()) + .into_iter() + .collect()), + Self::All => { + Ok(ws_info.variants.iter().map(|v| v.part.as_str()).collect()) + } + } + } +} + +async fn check_htcl_with_mode( + file: &camino::Utf8Path, + include_test_deps: bool, + part_selector: &PartSelector, +) -> Result> { + // Pre-flight: run the validator's src-import check on the entry + // file BEFORE handing to `load_htcl_program`, which would + // hard-abort on the first unresolved `src @` with a bare + // non-span error. The pre-flight produces the same spanned + // diagnostics the LSP shows, so `vw check` and the editor agree + // on where the missing dep is and how to fix it. Only kicks in + // when the workspace has a `vw.toml` (otherwise dep-names is + // empty and the check is a no-op). + let dep_names = collect_dep_names_with_mode(file, include_test_deps); + if !dep_names.is_empty() { + let entry_text = std::fs::read_to_string(file.as_str())?; + let entry_parsed = vw_htcl::parse(&entry_text); + let pre_diags = vw_htcl::validate_with_all_extras_and_vars( + &entry_parsed.document, + &entry_text, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + &std::collections::HashSet::new(), + &dep_names, + ); + let src_errs: Vec<_> = pre_diags + .iter() + .filter(|d| { + d.severity == vw_htcl::Severity::Error + && d.message.starts_with("unknown src module") + }) + .collect(); + if !src_errs.is_empty() { + let idx = vw_htcl::LineIndex::new(&entry_text); + let cwd_owned = std::env::current_dir().ok(); + let cwd = cwd_owned.as_deref(); + let display_path = + render_path(std::path::Path::new(file.as_str()), cwd); + for d in &src_errs { + let (start, _) = idx.range(d.span); + eprintln!( + "{} {display_path}:{}:{}: {}", + "error:".bright_red(), + start.line + 1, + start.character + 1, + d.message, + ); + } + eprintln!("{file}: {} error(s), 0 warning(s)", src_errs.len()); + return Ok(true); + } + } + + let program = if include_test_deps { + load_htcl_program_for_test(file).await? + } else { + load_htcl_program(file).await? + }; + let parsed = vw_htcl::parse(&program.source); + let validator_diags = vw_htcl::validate(&parsed.document, &program.source); + + // Build a per-file `LineIndex` lazily so we only pay for files + // that actually have diagnostics. Keyed by `file_index`. + let cwd_owned = std::env::current_dir().ok(); + let cwd = cwd_owned.as_deref(); + let mut indices: std::collections::HashMap = + std::collections::HashMap::new(); + + let mut error_count = 0usize; + let mut warning_count = 0usize; + let mut emit = |severity: Option, + message: &str, + span: vw_htcl::Span| { + let level = match severity { + None | Some(vw_htcl::Severity::Error) => { + error_count += 1; + "error:".bright_red() + } + Some(vw_htcl::Severity::Warning) => { + warning_count += 1; + "warning:".bright_yellow() + } + }; + // Map the span back to its originating file's line/col so the + // displayed location is the file the user actually wrote — not + // the flat dependency-concatenated source the loader produced. + let (display_path, line, col) = match program.locate_span(span) { + Some((idx, file_span)) => { + let loaded = &program.files[idx]; + let index = indices + .entry(idx) + .or_insert_with(|| vw_htcl::LineIndex::new(&loaded.source)); + let (start, _) = index.range(file_span); + ( + render_path(&loaded.path, cwd), + start.line + 1, + start.character + 1, + ) + } + None => (file.to_string(), 0, 0), + }; + eprintln!("{} {display_path}:{line}:{col}: {message}", level); + }; + + for err in &parsed.errors { + emit(None, &err.message, err.span); + } + for d in &validator_diags { + emit(Some(d.severity), &d.message, d.span); + } + + // Target-compatibility check. Iterates over the parts the + // selector picked (default, explicit `--part`, or every + // `[[target-parts]]` entry with `--all-parts`). Library + // workspaces (no target parts) skip silently. + let entry_path = std::path::Path::new(file.as_str()); + if let Some(ws) = entry_path.parent().and_then(vw_lib::find_workspace_dir) { + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + let parts = match part_selector.resolve(&cfg.workspace) { + Ok(p) => p, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + error_count += 1; + Vec::new() + } + }; + if !parts.is_empty() { + let dep_targets = vw_lib::collect_dep_targets(&ws); + for (dep, err) in &dep_targets.errors { + eprintln!( + "{} bad `[targets]` pattern in dep `{dep}`: {err}", + "warning:".bright_yellow(), + ); + warning_count += 1; + } + for target_part in &parts { + let mismatches = vw_lib::check_target_compatibility( + Some(target_part), + &dep_targets, + ); + for m in &mismatches { + match m.kind { + vw_lib::TargetMismatchKind::NotSupported => { + eprintln!( + "{} target-part `{}` matches dep `{}`'s \ + `not-supported` list — Xilinx has \ + attested the IP is not usable on this \ + part ({})", + "error:".bright_red(), + m.target_part, + m.dep, + target_mismatch_families_hint(m), + ); + error_count += 1; + } + vw_lib::TargetMismatchKind::Unblessed => { + eprintln!( + "{} target-part `{}` isn't blessed by \ + dep `{}` ({}); the IP may still work \ + but Xilinx hasn't blessed the \ + combination", + "warning:".bright_yellow(), + m.target_part, + m.dep, + target_mismatch_families_hint(m), + ); + warning_count += 1; + } + } + } + } + } + } + } + + if error_count > 0 || warning_count > 0 { + eprintln!("{file}: {error_count} error(s), {warning_count} warning(s)"); + } + Ok(error_count > 0) +} + +/// Render `path` relative to `cwd` when it sits underneath, otherwise +/// fall back to the absolute path. Keeps diagnostic locations short +/// and click-through-able in editors / terminals. +fn render_path( + path: &std::path::Path, + cwd: Option<&std::path::Path>, +) -> String { + if let Some(cwd) = cwd { + if let Ok(rel) = path.strip_prefix(cwd) { + return rel.display().to_string(); + } + } + path.display().to_string() +} + +/// Human-readable summary of what a dep declared in its +/// `[targets]` block. Splits blessed vs. banned families so a dep +/// with only a `not-supported` list (e.g. clk-wizard v1.0, where +/// every entry is `Not-Supported`) doesn't get misreported as +/// "declared families: versal" — the versal families are BANNED +/// there, not blessed. +fn target_mismatch_families_hint(m: &vw_lib::TargetMismatch) -> String { + match ( + m.supported_families.is_empty(), + m.not_supported_families.is_empty(), + ) { + (true, true) => { + "no `[targets]` families declared — the dep has patterns \ + but none carry family names" + .to_string() + } + (false, true) => { + format!("blessed families: {}", m.supported_families.join(", "),) + } + (true, false) => { + format!( + "no blessed families — only `not-supported` entries for {}", + m.not_supported_families.join(", "), + ) + } + (false, false) => { + format!( + "blessed families: {}; also `not-supported` entries for {}", + m.supported_families.join(", "), + m.not_supported_families.join(", "), + ) + } + } +} + +/// For every monomorphized generic encountered while walking `ty` +/// (recursing through user-newtype underlyings), emit its repr to +/// the backend exactly once. Dedup is owned by the caller so +/// repeated invocations across signatures don't re-ship the same +/// proc. +/// Turn a sequence of [`vw_repl::highlight::Piece`]s (a repr-line +/// highlight result) into an ANSI-colored string using the same +/// palette the REPL's ratatui renderer uses. +/// +/// Emits raw 24-bit truecolor escapes (`\x1b[38;2;R;G;Bm`) +/// directly instead of going through `colored`'s `.truecolor()` +/// — `colored` 2.0 downgrades RGB to the nearest ANSI-16 code +/// unless `COLORTERM` explicitly announces truecolor, which +/// happens to map our REPL palette to shades that read as grey +/// (e.g. RGB(120, 200, 120) → `\e[90m` bright_black). The REPL's +/// ratatui backend emits raw truecolor unconditionally and looks +/// correct on every modern terminal; matching its behavior here +/// keeps `vw run` visually aligned with the REPL. +/// +/// Suppression on `NO_COLOR` / non-TTY stdout is preserved via +/// `colored`'s global gate. +fn ansi_from_pieces(pieces: &[vw_repl::highlight::Piece]) -> String { + use vw_repl::highlight::StyleKind; + let colorize = colored::control::SHOULD_COLORIZE.should_colorize(); + let mut out = String::new(); + for p in pieces { + if !colorize { + out.push_str(&p.text); + continue; + } + // Palette constants mirror the ratatui RGB values in + // `vw-repl/src/highlight.rs::{key_style, variant_style, + // scalar_style}` and the DIM modifier on `punct_style`. + // Keep the two backends in sync — if the ratatui palette + // changes, this needs the same edit. + let escape = match p.kind { + StyleKind::Plain => None, + StyleKind::Key => Some("\x1b[38;2;80;150;255m"), + StyleKind::Variant => Some("\x1b[38;2;100;200;200m"), + StyleKind::Punct => Some("\x1b[2m"), + StyleKind::Scalar => Some("\x1b[38;2;120;200;120m"), + }; + match escape { + None => out.push_str(&p.text), + Some(prefix) => { + out.push_str(prefix); + out.push_str(&p.text); + out.push_str("\x1b[0m"); + } + } + } + out +} + +/// Encode a [`vw_vivado::Severity`] as a `u8` matching the ladder's +/// `PartialOrd`. Used for `AtomicU8::fetch_max` on the "worst seen +/// during this session" counter — atomic comparison over the raw +/// integer is thread-safe without needing a mutex, and the mapping +/// is a static one-liner rather than a lookup table. +fn severity_as_u8(s: vw_vivado::Severity) -> u8 { + match s { + vw_vivado::Severity::None => 0, + vw_vivado::Severity::Info => 1, + vw_vivado::Severity::Warning => 2, + vw_vivado::Severity::CriticalWarning => 3, + vw_vivado::Severity::Error => 4, + } +} + +/// Render one classified [`vw_vivado::Block`] to stdout, applying the +/// user-selected log level: +/// +/// - `Block::Diagnostic { severity, .. }` — passes through +/// [`render_chunk`] with the block joined back into a single +/// chunk, provided the severity clears `log_level`. +/// - `Block::None { .. }` — at `LogLevel::Debug`, renders raw via +/// the Stdout path so users see every byte Vivado emitted. At +/// Info+, prints each line dimmed so users still know noise +/// flowed through without it dominating the console. (The REPL's +/// version of this path renders NONE blocks as a togglable +/// collapsed placeholder — vw run's flat output can't collapse, +/// so it dims instead.) +fn render_block( + block: &vw_vivado::Block, + log_level: vw_vivado::LogLevel, + proc_table: &std::collections::HashMap, + origin: Option<&vw_repl::Origin>, + input_file: Option<&std::path::Path>, +) { + use colored::Colorize; + match block { + vw_vivado::Block::Diagnostic { severity, lines } => { + if !log_level.allows(*severity) { + return; + } + let kind = vw_vivado::stream_kind_for(*severity); + let joined = lines.join("\n"); + render_chunk(kind, &joined, proc_table, origin, input_file); + } + vw_vivado::Block::None { lines } => { + if matches!(log_level, vw_vivado::LogLevel::Debug) { + // Raw pass-through — same style as if it hadn't been + // block-grouped at all. + let joined = lines.join("\n"); + render_chunk( + vw_vivado::StreamKind::Stdout, + &joined, + proc_table, + origin, + input_file, + ); + } else { + use std::io::Write; + let mut out = std::io::stdout().lock(); + for line in lines { + let _ = writeln!(out, "{}", line.dimmed()); + } + let _ = out.flush(); + } + } + } +} + +/// Stream-sink rendering for `vw run`. Mirrors the REPL's +/// scrollback colors + stack-frame rewriting so both surfaces +/// look the same: +/// +/// - **Stream kind → ANSI color/prefix** +/// - `Error` → `✗ ` red bold +/// - `Warning` → `⚠ ` orange (Rgb 255,140,0) +/// - `Info` → `· ` dark gray +/// - `Stdout` → no prefix, no color +/// +/// - **Stack-frame rewriting**: lines matching ` at :N +/// in ::proc` are mapped to the real htcl source via +/// [`vw_repl::resolve_stack_frames_with`] + `proc_table`. +/// Adjacent frames pointing at the same proc collapse to one. +/// +/// - **Origin tagging**: warnings/errors that arrive without an +/// `\n at …` trace get one appended pointing at the +/// currently-executing top-level statement (`origin`). Mirrors +/// the REPL's `tag_streamed_message` — Vivado C++ paths +/// bypass `::common::send_msg_id` and emit traceless messages +/// we'd otherwise have no anchor for. +fn render_chunk( + kind: vw_vivado::StreamKind, + chunk: &str, + proc_table: &std::collections::HashMap, + origin: Option<&vw_repl::Origin>, + input_file: Option<&std::path::Path>, +) { + use colored::Colorize; + use std::io::Write; + // Drop a single trailing newline so the per-message layout + // doesn't insert a blank gap. The shim's `puts` already + // preserves user-side newlines inside the message. + let trimmed = chunk.trim_end_matches('\n'); + if trimmed.is_empty() { + return; + } + let resolved = vw_repl::resolve_stack_frames_with( + trimmed, + |name| proc_table.get(name).cloned(), + input_file, + ); + // Tag traceless warnings/errors with the currently-executing + // statement's origin. + let tagged = match kind { + vw_vivado::StreamKind::Warning | vw_vivado::StreamKind::Error + if !resolved.contains("\n at ") => + { + match origin { + Some(o) => { + let path = o + .file + .as_deref() + .map(vw_repl::display_path) + .unwrap_or_else(|| { + input_file + .map(vw_repl::display_path) + .unwrap_or_else(|| "".into()) + }); + format!("{resolved}\n at {path}:{}", o.line) + } + None => resolved, + } + } + _ => resolved, + }; + let prefix = match kind { + vw_vivado::StreamKind::Error + | vw_vivado::StreamKind::CriticalWarning => "✗ ", + vw_vivado::StreamKind::Warning => "⚠ ", + vw_vivado::StreamKind::Info => "· ", + vw_vivado::StreamKind::Stdout => "", + }; + let mut out = std::io::stdout().lock(); + for (i, line) in tagged.lines().enumerate() { + let leading = if i == 0 || prefix.is_empty() { + prefix + } else { + " " + }; + let styled_prefix: String = match kind { + vw_vivado::StreamKind::Error + | vw_vivado::StreamKind::CriticalWarning => { + leading.red().bold().to_string() + } + vw_vivado::StreamKind::Warning => { + leading.truecolor(255, 140, 0).bold().to_string() + } + vw_vivado::StreamKind::Info => leading.bright_black().to_string(), + vw_vivado::StreamKind::Stdout => leading.to_string(), + }; + let styled_line: String = match kind { + vw_vivado::StreamKind::Error + | vw_vivado::StreamKind::CriticalWarning => line.red().to_string(), + vw_vivado::StreamKind::Warning => { + line.truecolor(255, 140, 0).to_string() + } + vw_vivado::StreamKind::Info => line.bright_black().to_string(), + // Stdout is where the shim's `puts` output lands, + // including compiler-emitted enum reprs like + // CPM_PCIE0_MODES Scalar(None) + // CONFIG Nested( + // … + // ) + // Route it through the REPL's shape-based highlighter + // so `vw run` and the REPL scrollback look identical + // for repr-shaped lines. Non-repr text (plain `puts` + // messages, error messages that reach Stdout, etc.) + // fails the parse and falls through to the raw line. + // The `colored` crate that underpins ansi_from_pieces + // already respects NO_COLOR + tty-detection, so the + // integration inherits the standard "quiet when piped + // or NO_COLOR is set" behavior for free. + vw_vivado::StreamKind::Stdout => { + match vw_repl::highlight::highlight_line_pieces(line) { + Some(pieces) => ansi_from_pieces(&pieces), + None => line.to_string(), + } + } + }; + let _ = writeln!(out, "{styled_prefix}{styled_line}"); + } + let _ = out.flush(); +} + +async fn ship_generic_reprs( + backend: &mut vw_vivado::VivadoBackend, + ty: &vw_htcl::TypeExpr, + types: &std::collections::HashMap, + emitted: &mut std::collections::HashSet, +) -> Result<(), Box> { + // TypeExpr::Qualified appears only on overloaded-handler + // first-args; the validator forbids it anywhere else, and + // codegen doesn't need a repr for it. + if matches!(ty, vw_htcl::TypeExpr::Qualified { .. }) { + return Ok(()); + } + let emission = vw_htcl::emit_repr_with_types(ty, types); + for p in &emission.procs { + // The procs are emitted in dependency order; the body of + // each instantiation may reference earlier ones in the + // same emission, so we ship them sequentially through + // the same eval channel. + if emitted.insert(p.clone()) { + backend.eval(p).await?; + } + } + Ok(()) +} + +/// Mirror of `vw-repl/src/lower.rs::overload_specialization_mangle`. +/// If `cmd` is a top-level `proc` whose name is an overload public +/// name AND whose first arg is a qualified-variant annotation, +/// return the mangled internal name to lower it under. Keeps +/// `vw run` in step with the REPL's specialization-rerouting. +fn overload_specialization_mangle( + cmd: &vw_htcl::Command, + overloads: &vw_htcl::OverloadTable, +) -> Option { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + return None; + }; + let name = proc.name.as_deref()?; + if !overloads.contains_key(name) { + return None; + } + let sig = proc.signature.as_ref()?; + let first = sig.args.first()?; + let vw_htcl::TypeExpr::Qualified { variant, .. } = + first.type_annotation.as_ref()? + else { + return None; + }; + Some(vw_htcl::mangle_specialization(name, variant)) +} + +async fn run_htcl( + file: &camino::Utf8Path, + check_only: bool, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, +) -> Result<(), Box> { + // Under the hood the block renderer decides what to show; the + // backend's `verbose` toggle still gates the unclassified PTY + // firehose (banner, source echo, idle chatter). At log_level = + // Debug the user wants everything, so verbose stays on; at + // higher levels it stays off, and the classifier's NONE-block + // path is the only source of non-diagnostic content — which the + // renderer then dims or collapses. + let verbose = matches!(log_level, vw_vivado::LogLevel::Debug); + let program = load_htcl_program(file).await?; + // Keep `program` alive — the stack-frame rewriting needs the + // LoadedProgram for body-span resolution. We borrow `source` + // from it instead of moving. + let source = program.source.clone(); + let parsed = vw_htcl::parse(&source); + let line_index = vw_htcl::LineIndex::new(&source); + // Compile-time `putr` rewrite map: every `putr ` command + // in the document gets a replacement Tcl string keyed by its + // span. `vw_htcl::lower_command_with_putr` consults the map at + // emit time. See `vw-htcl/src/putr.rs` for the walker. + let putr_map = vw_htcl::putr::rewrite(&source, &parsed.document); + + let mut had_errors = false; + for err in &parsed.errors { + had_errors = true; + let (start, _end) = line_index.range(err.span); + eprintln!( + "{} {}:{}:{}: {}", + "error:".bright_red(), + file, + start.line + 1, + start.character + 1, + err.message + ); + } + if had_errors { + return Err(format!( + "{} parse error(s); aborting", + parsed.errors.len() + ) + .into()); + } + + // Validator gate. `vw check prime.htcl` runs `vw_htcl::validate` + // and returns non-zero on any error; `vw run prime.htcl` used + // to skip the validator entirely and hand the (possibly + // type-broken) program to Vivado, letting silent runtime + // divergence hide real bugs the checker had already found. Run + // the same validator here and abort on any error before + // spawning Vivado — same behavior `check` shows, same exit + // path. Warnings still emit but don't gate execution. + let validator_diags = vw_htcl::validate(&parsed.document, &source); + let mut error_count = 0usize; + let mut warning_count = 0usize; + let cwd_owned = std::env::current_dir().ok(); + let cwd = cwd_owned.as_deref(); + let mut indices: std::collections::HashMap = + std::collections::HashMap::new(); + for d in &validator_diags { + let (display_path, line, col) = match program.locate_span(d.span) { + Some((idx, file_span)) => { + let loaded = &program.files[idx]; + let index = indices + .entry(idx) + .or_insert_with(|| vw_htcl::LineIndex::new(&loaded.source)); + let (start, _) = index.range(file_span); + ( + render_path(&loaded.path, cwd), + start.line + 1, + start.character + 1, + ) + } + None => (file.to_string(), 0, 0), + }; + match d.severity { + vw_htcl::Severity::Error => { + error_count += 1; + eprintln!( + "{} {display_path}:{line}:{col}: {}", + "error:".bright_red(), + d.message + ); + } + vw_htcl::Severity::Warning => { + warning_count += 1; + eprintln!( + "{} {display_path}:{line}:{col}: {}", + "warning:".bright_yellow(), + d.message + ); + } + } + } + if error_count > 0 { + eprintln!("{file}: {error_count} error(s), {warning_count} warning(s)"); + return Err( + format!("{error_count} validation error(s); aborting",).into() + ); + } + + if check_only { + let cmd_count = parsed + .document + .stmts + .iter() + .filter(|s| matches!(s, vw_htcl::Stmt::Command(_))) + .count(); + println!( + "{} {file}: parsed OK ({cmd_count} command(s))", + "✓".bright_green() + ); + return Ok(()); + } + + // RPC handler — serves htcl `vw::…` calls whose answers live + // on the tool side (workspace root, design source list, …). + // Constructed with whatever workspace state we can discover + // from the entry file; when no `vw.toml` is found the + // handler still exists but returns an error for `workspace_ + // root`, matching how the same call behaves in the LSP. + let file_path = std::path::Path::new(file.as_str()); + let rpc_workspace_root: Option = file_path + .parent() + .and_then(vw_lib::find_workspace_dir) + .map(|p| p.into_std_path_buf()); + // Auto-project bootstrap: if the enclosing workspace declares + // `[[target-parts]]` or `[[workspace.variants]]`, ship a + // `create_project -in_memory` down the wire before user code + // runs. `--variant ` picks a variant (variant-mode + // workspaces); `--part ` picks a target-parts entry + // (part-mode workspaces). Otherwise the workspace default + // wins. Eliminates the "no open project" failure mode for + // `ip::check` / `get_ipdefs` / etc., and gives the whole + // session a stable part context for downstream + // implementation/timing steps. + let ws_utf8 = rpc_workspace_root + .as_deref() + .and_then(camino::Utf8Path::from_path) + .map(|p| p.to_path_buf()); + let (auto_project, active_variant) = match ws_utf8.as_deref() { + Some(ws) => resolve_workspace_selection(ws, part, variant)?, + None => (None, None), + }; + // Seed the RPC handler's preload map with everything the + // entry-file load pulled in. `vw run` is single-shot — no + // batches commit after this — so the initial population IS + // the final state. `compile_htcl_module` (called from + // `vw::configure_ip` if the user's htcl invokes it) then + // skips re-shipping files whose procs are already installed. + let preload: vw_vivado::SharedPreload = { + let mut m = std::collections::HashMap::new(); + for f in &program.files { + if let Some(t) = f.mtime { + m.insert(f.path.clone(), t); + } + } + std::sync::Arc::new(std::sync::RwLock::new(m)) + }; + // Shared CRITICAL WARNING counter. Bumped by the stream sink + // below on each `Severity::CriticalWarning` chunk; read via + // the `critical_warning_count` RPC by `vw::synth` / `vw::place` + // to gate checkpoint writes on a CW-clean phase. Cloned into + // the handler and the sink so both sides see the same atomic. + let cw_count: vw_vivado::SharedCriticalWarningCount = + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let rpc_handler = vw_vivado::make_handler_full( + rpc_workspace_root.clone(), + active_variant, + preload, + cw_count.clone(), + ); + // Raw byte-log: `/target/logs/vivado-.log`. + // Failure to create the directory demotes to no-log rather than + // aborting the run — the log is a diagnostic aid, and a + // misconfigured target/ dir shouldn't block a synth flow. + let raw_log = rpc_workspace_root.as_deref().and_then(|ws| { + match vw_vivado::raw_log_path_for_workspace(ws) { + Ok(p) => { + eprintln!( + "{} {}", + "raw vivado log:".bright_black(), + p.display().to_string().bright_black(), + ); + Some(p) + } + Err(e) => { + eprintln!("{} raw log unavailable: {e}", "warning:".yellow()); + None + } + } + }); + let mut backend = + vw_vivado::VivadoBackend::spawn(vw_vivado::VivadoConfig { + verbose, + info_with_stack, + rpc_handler: Some(rpc_handler), + auto_project, + raw_log, + ..Default::default() + }) + .await + .map_err(|e| format!("failed to start Vivado worker: {e}"))?; + + // Build the proc-location table the stream sink uses to map + // Tcl `:N in ::proc` frames back to real htcl source. + // Mirrors what `vw-repl` does per batch — we use the same + // shared helpers (`vw_repl::trace::*`) so REPL and CLI render + // the same. The entry file IS the scratch from build_proc_locations' + // perspective. + let entry_std_path = std::path::Path::new(file.as_str()).to_path_buf(); + let proc_table = std::sync::Arc::new(vw_repl::build_proc_locations( + &parsed.document, + &program, + &entry_std_path, + )); + let input_file_for_stack = std::sync::Arc::new(entry_std_path.clone()); + // Shared between the main loop (writes the current origin + // before each eval) and the stream sink (reads it to tag + // unattributed warnings — e.g. Vivado IP-Flow C++ messages + // that bypass `::common::send_msg_id`). Same trick the REPL + // uses with `pending_origins[pending_eval_index]`. + let current_origin = + std::sync::Arc::new(std::sync::Mutex::new(None::)); + // Segmenter state — shared between the sink (which pushes into + // it per chunk) and the outer scope (which flushes any trailing + // NONE after the last eval returns so the tail doesn't get + // silently dropped when Vivado's last write is non-diagnostic). + let block_acc = std::sync::Arc::new(std::sync::Mutex::new( + vw_vivado::BlockAccumulator::new(), + )); + // Track the highest severity classified during the session. We + // check `kind` — not the rendered block — so `--log-level=error` + // can't silence a CRITICAL WARNING into a false-pass exit code. + // Encoded as u8 (0..=4) matching the Severity ladder so a plain + // atomic fetch_max gets us thread-safe "worst so far" semantics + // without a mutex. + let worst_severity = std::sync::Arc::new(std::sync::atomic::AtomicU8::new( + severity_as_u8(vw_vivado::Severity::None), + )); + { + let procs = std::sync::Arc::clone(&proc_table); + let input_file = std::sync::Arc::clone(&input_file_for_stack); + let origin = std::sync::Arc::clone(¤t_origin); + let acc = std::sync::Arc::clone(&block_acc); + let worst = std::sync::Arc::clone(&worst_severity); + let cw = std::sync::Arc::clone(&cw_count); + backend.set_stdout_sink(move |kind, chunk: &str| { + let cur_origin = origin.lock().ok().and_then(|g| g.clone()); + worst.fetch_max( + severity_as_u8(vw_vivado::severity_of(kind)), + std::sync::atomic::Ordering::Relaxed, + ); + // Bump the CW counter exposed via the + // `critical_warning_count` RPC. Only count exact + // CriticalWarning (not Error) — htcl checkpoint gates + // want CWs specifically; errors already abort the + // eval before the checkpoint-write branch is reached. + if matches!(kind, vw_vivado::StreamKind::CriticalWarning) { + cw.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + let blocks = acc + .lock() + .map(|mut a| a.push(kind, chunk)) + .unwrap_or_default(); + for block in blocks { + render_block( + &block, + log_level, + &procs, + cur_origin.as_ref(), + Some(input_file.as_path()), + ); + } + }); + } + + // Lower structured proc declarations and call sites to plain Tcl + // before sending. Generic commands pass through unchanged. + let table = vw_htcl::signature_table(&parsed.document); + // Ship enum preludes + overload dispatchers before any user + // statements run — same shape as the REPL's prepare path. + // Without these, calls to `Property::Scalar` or to an + // overloaded `handle` would fail at runtime with `invalid + // command name`. + let mut _ignored = Vec::new(); + let enum_decl_table = + vw_htcl::build_enum_decl_table(&parsed.document, &mut _ignored); + let type_decl_table = + vw_htcl::build_type_decl_table(&parsed.document, &mut _ignored); + let type_decl_names: std::collections::HashSet = + type_decl_table.keys().cloned().collect(); + let (full_sigs, overload_table) = + vw_htcl::build_signature_table_with_overloads( + &parsed.document, + &type_decl_names, + &mut _ignored, + ); + // Always ship the primitive prelude so user-written newtype + // reprs can call e.g. `string::repr -v $v` for their inner + // values. + for p in vw_htcl::emit_primitive_prelude() { + let _ = backend.eval(&p).await?; + } + for ed in enum_decl_table.values() { + let prelude = vw_htcl::emit_enum_prelude(ed); + if !prelude.trim().is_empty() { + let _ = backend.eval(&prelude).await?; + } + } + for info in overload_table.values() { + let dispatcher = vw_htcl::emit_dispatcher(info); + let _ = backend.eval(&dispatcher).await?; + } + // Ship monomorphized generic reprs for every type expression + // referenced in any proc signature. This covers user newtypes + // that delegate to a generic repr (e.g. `Properties::repr` + // delegates to `dict_string_Property::repr`); without these + // the user's repr body errors at runtime with `invalid + // command name`. + let mut emitted_generics: std::collections::HashSet = + std::collections::HashSet::new(); + for sig in full_sigs.values() { + if let Some(ret) = sig.return_type.as_ref() { + ship_generic_reprs( + &mut backend, + ret, + &type_decl_table, + &mut emitted_generics, + ) + .await?; + } + for arg in &sig.args { + if let Some(ty) = arg.type_annotation.as_ref() { + ship_generic_reprs( + &mut backend, + ty, + &type_decl_table, + &mut emitted_generics, + ) + .await?; + } + } + } + // Per-file LineIndex cache so traceless-warning origins can be + // reported in the *originating file's* coordinates rather than the + // flattened LoadedProgram's. Without this, a warning anchored at + // the entry file ends up rendered at the merged-source line + // (something like `prime.htcl:119767`), which is meaningless. + let merged_line_index = vw_htcl::LineIndex::new(&source); + let mut per_file_line_index: std::collections::HashMap< + usize, + vw_htcl::LineIndex, + > = std::collections::HashMap::new(); + for stmt in &parsed.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + // Snapshot the origin of THIS statement before shipping + // it, so the stream sink can tag any traceless warning + // Vivado emits during the eval with the right "what was + // running" anchor. Mirrors the REPL's pending_origins + + // pending_eval_index mechanism. + let stmt_origin = { + let (file_path, line, snippet) = match program.locate_span(cmd.span) + { + Some((idx, local)) => { + let file_src = &program.files[idx].source; + let li = per_file_line_index + .entry(idx) + .or_insert_with(|| vw_htcl::LineIndex::new(file_src)); + let (lc, _) = li.range(local); + let snippet = file_src + [local.start as usize..local.end as usize] + .lines() + .next() + .unwrap_or("") + .to_string(); + ( + Some(program.files[idx].path.clone()), + lc.line + 1, + snippet, + ) + } + None => { + // Synthetic span that doesn't lie in any loaded + // file (e.g. a generated dispatcher). Fall back to + // the merged-source line; the entry-file label is + // attached by the renderer when `file` is None. + let (lc, _) = merged_line_index.range(cmd.span); + let snippet = source + [cmd.span.start as usize..cmd.span.end as usize] + .lines() + .next() + .unwrap_or("") + .to_string(); + (None, lc.line + 1, snippet) + } + }; + let origin = vw_repl::Origin { + file: file_path, + line, + snippet, + via: Vec::new(), + }; + if let Ok(mut g) = current_origin.lock() { + *g = Some(origin.clone()); + } + origin + }; + // Overload specializations lower under their mangled + // names so the dispatcher's switch arms can find them. + let lowered = match overload_specialization_mangle(cmd, &overload_table) + { + Some(mangled) => { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + unreachable!() + }; + vw_htcl::lower_proc_decl_with_name_and_index( + proc, + &source, + &table, + Some(&mangled), + &putr_map, + &merged_line_index, + ) + } + None => vw_htcl::lower_command_with_putr_and_index( + cmd, + &source, + &table, + &putr_map, + &merged_line_index, + ), + }; + // Rewrite `extern::name` → `::name` (the textual pass the + // REPL also runs) so calls to runtime-Tcl/Vivado procs + // reach Vivado as the bare native name instead of the + // literal `extern::` text — without this, every wrapper + // body that forwards via `extern::` errors out at runtime + // with `invalid command name "extern::create_project"`. + let tcl = vw_htcl::rewrite_externs(&lowered).text; + // Wrap with a shim-side origin marker so any traceless + // warning emitted during THIS eval stays anchored to + // `stmt_origin` — see [`vw_repl::wrap_tcl_with_origin_marker`] + // for the race this fixes. + let tcl = vw_repl::wrap_tcl_with_origin_marker(&tcl, &stmt_origin); + // `set VAR ` is a binding — the user asked to name a + // value, not to display it. Vivado's Tcl returns the + // bound value as the eval result, and echoing that would + // leak the raw internal form (e.g. `metroid` from + // `set proj [create_project … -name metroid]` in + // `project.htcl`). Suppress the result echo for set + // bindings so the batch path matches the REPL's + // `is_set_binding` policy (`vw-repl/src/app.rs:2437`). + let is_set_binding = matches!(cmd.kind, vw_htcl::CommandKind::Set); + match backend.eval(&tcl).await { + Ok(out) => { + // Puts output already streamed to stdout via the + // sink; `out.stdout` is empty here by contract. The + // eval's return value gets a newline only when it's + // not already empty AND the source command wasn't a + // set binding. + if !out.value.is_empty() && !is_set_binding { + println!("{}", out.value); + } + } + Err(vw_eda::BackendError::Tcl { message, .. }) => { + eprintln!("{} {message}", "vivado:".bright_red()); + } + Err(e) => { + eprintln!("{} {e}", "vivado:".bright_red()); + } + } + } + let _ = backend.shutdown().await; + // Flush any trailing NONE block the last chunk left in the + // accumulator. Vivado's final write is often non-diagnostic + // (banner, resource summary, "synth_design completed" table); + // without this the very end of the session would sit unemitted + // when the caller's eval returns and the sink stops being + // called. render_block honors the log-level here too, so + // trailing noise still gets dimmed at Info+ and shown raw at + // Debug. + let trailing = block_acc.lock().map(|mut a| a.flush()).unwrap_or_default(); + let cur_origin = current_origin.lock().ok().and_then(|g| g.clone()); + for block in trailing { + render_block( + &block, + log_level, + &proc_table, + cur_origin.as_ref(), + Some(input_file_for_stack.as_path()), + ); + } + // Severity-driven exit code. If ANY chunk classified as + // CRITICAL WARNING or ERROR during the session, propagate a + // non-zero exit so CI wrappers / scripts can gate on it. We + // read the classification counter — NOT the render decision — + // so a user passing `--log-level=error` can't hide a + // CRITICAL WARNING into a false-pass. The diagnostic itself + // has already streamed through the log-level filter (or was + // suppressed at the caller's request); this is just about the + // process return code. + let worst = worst_severity.load(std::sync::atomic::Ordering::Relaxed); + if worst >= severity_as_u8(vw_vivado::Severity::CriticalWarning) { + let label = if worst >= severity_as_u8(vw_vivado::Severity::Error) { + "error" + } else { + "critical warning" + }; + return Err(format!("session emitted at least one {label}").into()); } + Ok(()) } diff --git a/vw-cli/src/parallel_load.rs b/vw-cli/src/parallel_load.rs new file mode 100644 index 0000000..84351d1 --- /dev/null +++ b/vw-cli/src/parallel_load.rs @@ -0,0 +1,740 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Parallel htcl loader with per-dep progress rows. +//! +//! Wraps the sync `vw_htcl` load primitives with an async coordinator +//! that: +//! +//! - Recursively discovers and parses files, with `tokio::task:: +//! spawn_blocking` for each file's read+parse so N cores work in +//! parallel. +//! - Stitches the results back into the SAME `LoadedProgram` shape +//! the serial loader produces (flat `source` + `regions`), in the +//! same deterministic DFS-from-entry order — so every downstream +//! consumer (validator, putr, lower, LSP) works unchanged. +//! - Fires observer events (`on_source`, `on_parsed`, `on_dep_ +//! completed`) as parses complete, so the CLI's +//! [`MultiProgress`] UI can render one live row per top-level +//! dep and commit each row when its subtree is done. +//! +//! The loader is dependency-graph aware via `petgraph`. Cycles are +//! detected and rejected with `LoadError::Cycle`. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; + +use futures::future::BoxFuture; +use futures::FutureExt; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use tokio::task::JoinError; +use vw_htcl::{ + parse, CommandKind, Document, ImportEdge, LoadError, LoadedFile, + LoadedProgram, Resolver, SourceRegion, Span, Stmt, +}; + +/// Extended observer trait for the parallel loader. Adds +/// [`on_dep_completed`] so the UI can commit a top-level dep's +/// progress row when its whole subtree has finished parsing. +/// +/// The base `LoadObserver` methods still fire (once per file, from +/// blocking-thread contexts so implementations must be Send + Sync). +pub trait ParallelObserver: Send + Sync { + fn on_source(&self, _raw: &str, _resolved: &Path) {} + fn on_parsed(&self, _file: &Path, _raw: Option<&str>) {} + /// A top-level dep name (`@vivado-cmd`, `@cpm5`, ...) has + /// finished parsing its whole subtree. Used by the CLI to + /// commit the dep's progress row to `Checking @`. + fn on_dep_completed(&self, _dep_name: &str) {} +} + +/// The result of parsing one file. Shared across all consumers via +/// `Arc` so multiple parts of the stitch phase can hold references +/// without cloning the source text. +struct ParsedFile { + path: PathBuf, + source: String, + document: Document, + mtime: Option, + /// Import descriptors in source order, each resolved to a + /// canonical path. Preserving order matters for the stitch + /// phase's chunking output. + imports: Vec, +} + +#[derive(Clone)] +struct ImportInfo { + raw: String, + resolved: PathBuf, + /// Span of the `src` command in this file's local source + /// (the source the parse was against, not the flat source). + src_span: Span, +} + +/// Shared parse-cache. One slot per canonical path; the slot's +/// `Notify` fires when the parse completes, so a second discovery +/// of the same path awaits the first task's result instead of +/// re-parsing. +type ParseCache = Arc>>; + +#[derive(Clone)] +enum ParseSlot { + /// In-progress. Awaiters watch the notify. + Pending(Arc), + /// Completed successfully. + Done(Arc), + /// Failed. Error is stringified because `LoadError` isn't `Clone`. + Failed(String), + /// Preloaded and unchanged (mtime match). Skip; downstream + /// treats this file as if it wasn't there (it's already in a + /// prior batch's LoadedProgram). + Skipped, +} + +/// Public entry point for the parallel loader. Same contract as +/// `vw_htcl::load_program`: reads the entry file, recursively +/// resolves every `src` statement, and returns a `LoadedProgram` +/// whose flat source is DFS-ordered exactly as the serial loader +/// would produce. +/// +/// `preloaded` mirrors the sync loader's cross-batch cache — files +/// listed here whose current mtime matches the stored one are +/// skipped as if already sourced. +pub async fn load_parallel( + entry: &Path, + resolver: Arc, + observer: Arc, + preloaded: HashMap, +) -> Result { + let entry = entry.canonicalize().unwrap_or_else(|_| entry.to_path_buf()); + let cache: ParseCache = Arc::new(Mutex::new(HashMap::new())); + let preloaded = Arc::new(preloaded); + let deps_in_flight: Arc>> = + Arc::new(Mutex::new(HashMap::new())); + + // Recursive parallel discovery+parse. The entry file has no + // dep context; every reachable file inherits from its + // importer's dep bucket (see `bucket_for_path` used by the + // observer wiring). + schedule_parse( + entry.clone(), + None, + resolver.clone(), + observer.clone(), + cache.clone(), + preloaded.clone(), + deps_in_flight.clone(), + ) + .await?; + + // Serially stitch the flat source in the same DFS order the + // sync loader produces. This is what makes the output + // interchangeable with `vw_htcl::load_program` for downstream + // consumers. + stitch(&entry, &cache) +} + +/// Await the parse of `path` (kicking it off first if no other +/// task has). Recurses into imports concurrently, so multiple +/// file trees load in parallel. +fn schedule_parse( + path: PathBuf, + imported_via_raw: Option, + resolver: Arc, + observer: Arc, + cache: ParseCache, + preloaded: Arc>, + deps_in_flight: Arc>>, +) -> BoxFuture<'static, Result<(), LoadError>> { + async move { + // Fast path: skip preloaded files whose mtime hasn't + // changed. Same cross-batch semantics as the sync loader. + if let Some(stored_mtime) = preloaded.get(&path) { + let current_mtime = std::fs::metadata(&path) + .ok() + .and_then(|m| m.modified().ok()); + if current_mtime == Some(*stored_mtime) { + let mut guard = cache.lock().unwrap(); + guard.entry(path.clone()).or_insert(ParseSlot::Skipped); + return Ok(()); + } + } + + // Claim the slot. If someone else already claimed, await + // their notify; if already done, return. + enum Claim { + Done, + Failed(String), + AwaitPending(Arc), + Owned(Arc), + } + let claim = { + let mut guard = cache.lock().unwrap(); + match guard.get(&path).cloned() { + Some(ParseSlot::Done(_)) | Some(ParseSlot::Skipped) => { + Claim::Done + } + Some(ParseSlot::Failed(msg)) => Claim::Failed(msg), + Some(ParseSlot::Pending(n)) => Claim::AwaitPending(n), + None => { + let n = Arc::new(tokio::sync::Notify::new()); + guard.insert(path.clone(), ParseSlot::Pending(n.clone())); + Claim::Owned(n) + } + } + }; + let notify = match claim { + Claim::Done => return Ok(()), + Claim::Failed(msg) => { + return Err(LoadError::Io { + path: path.clone(), + source: std::io::Error::other(msg), + }); + } + Claim::AwaitPending(n) => { + n.notified().await; + return Ok(()); + } + Claim::Owned(n) => n, + }; + + // Do read+parse on a blocking thread. Reading is I/O + // bound; parsing is CPU bound. spawn_blocking is the + // right primitive. + let parsed_path = path.clone(); + let parse_task = + tokio::task::spawn_blocking(move || read_and_parse(&parsed_path)); + + let parsed_result = parse_task.await.map_err(join_err_to_load_err)?; + let parsed = match parsed_result { + Ok(p) => Arc::new(p), + Err(e) => { + let msg = format!("{e}"); + let mut guard = cache.lock().unwrap(); + guard.insert(path.clone(), ParseSlot::Failed(msg)); + notify.notify_waiters(); + return Err(e); + } + }; + + // Extract src operands from the parsed doc, resolve + // each, and schedule parallel parses for the targets. + // The `imports` field on ParsedFile carries the ordered + // (raw, resolved) pairs; we resolve here so we can + // emit the on_source event with the resolved path. + let parent_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let mut imports: Vec = Vec::new(); + for stmt in &parsed.document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(import) = &cmd.kind else { + continue; + }; + let Some(raw) = import.path.as_deref() else { + let line = line_of(&parsed.source, cmd.span.start) + 1; + let err = LoadError::DynamicPath { + importer: path.clone(), + line, + }; + let mut guard = cache.lock().unwrap(); + guard.insert(path.clone(), ParseSlot::Failed(format!("{err}"))); + notify.notify_waiters(); + return Err(err); + }; + let resolved = + resolver.resolve(&parent_dir, raw).map_err(|source| { + LoadError::Resolve { + importer: path.clone(), + raw: raw.to_string(), + source, + } + })?; + imports.push(ImportInfo { + raw: raw.to_string(), + resolved, + src_span: cmd.span, + }); + } + + // Attach imports to the parsed file so the stitch phase + // can reproduce the byte order. Use the `Arc` trick: + // wrap in a new Arc after mutating. + let parsed_with_imports = Arc::new(ParsedFile { + path: parsed.path.clone(), + source: parsed.source.clone(), + document: parsed.document.clone(), + mtime: parsed.mtime, + imports: imports.clone(), + }); + + // Emit on_source for each not-yet-seen target BEFORE + // recursing. The observer sees Sourcing events in the + // same conceptual order the serial loader would. + for imp in &imports { + let already_seen = cache + .lock() + .unwrap() + .get(&imp.resolved) + .map(|s| !matches!(s, ParseSlot::Failed(_))) + .unwrap_or(false); + if !already_seen { + observer.on_source(&imp.raw, &imp.resolved); + // Track top-level dep in-flight count for + // on_dep_completed firing. + if let Some(dep) = extract_dep_from_raw(&imp.raw) { + let mut deps = deps_in_flight.lock().unwrap(); + *deps.entry(dep).or_insert(0) += 1; + } + } + } + + // Spawn recursive parses concurrently. + let mut recurse_tasks = Vec::new(); + for imp in imports.iter() { + let fut = schedule_parse( + imp.resolved.clone(), + Some(imp.raw.clone()), + resolver.clone(), + observer.clone(), + cache.clone(), + preloaded.clone(), + deps_in_flight.clone(), + ); + recurse_tasks.push(fut); + } + for result in futures::future::join_all(recurse_tasks).await { + result?; + } + + // All this file's imports are done. Publish the parse + // result and notify awaiters. + { + let mut guard = cache.lock().unwrap(); + guard.insert( + path.clone(), + ParseSlot::Done(parsed_with_imports.clone()), + ); + } + notify.notify_waiters(); + + observer.on_parsed(&path, imported_via_raw.as_deref()); + + // If this file is the last outstanding file of a + // top-level dep bucket, fire on_dep_completed. + if let Some(via_raw) = imported_via_raw.as_deref() { + if let Some(dep) = extract_dep_from_raw(via_raw) { + let mut deps = deps_in_flight.lock().unwrap(); + if let Some(count) = deps.get_mut(&dep) { + *count = count.saturating_sub(1); + if *count == 0 { + deps.remove(&dep); + drop(deps); + observer.on_dep_completed(&dep); + } + } + } + } + + Ok(()) + } + .boxed() +} + +/// Sync read+parse for one file. Runs on a `spawn_blocking` +/// thread. Failures propagate as `LoadError`. +fn read_and_parse(path: &Path) -> Result { + let source = std::fs::read_to_string(path).map_err(|e| LoadError::Io { + path: path.to_path_buf(), + source: e, + })?; + let mtime = std::fs::metadata(path).ok().and_then(|m| m.modified().ok()); + let parsed = parse(&source); + if !parsed.errors.is_empty() { + return Err(LoadError::Parse { + path: path.to_path_buf(), + errors: parsed.errors, + }); + } + Ok(ParsedFile { + path: path.to_path_buf(), + source, + document: parsed.document, + mtime, + imports: Vec::new(), + }) +} + +/// Extract the top-level `@name` prefix from a src operand, so +/// the observer can bucket files by top-level dep. `./relative` +/// operands (dep-internal imports) return None — the CLI-side +/// observer resolves them via the loader-observer path. +fn extract_dep_from_raw(raw: &str) -> Option { + let rest = raw.strip_prefix('@')?; + let name = match rest.split_once('/') { + Some((n, _)) => n, + None => rest, + }; + Some(format!("@{name}")) +} + +/// Serially stitch the flat `LoadedProgram` from the parsed +/// cache. Walks in the same DFS-from-entry order the sync +/// loader uses so byte offsets line up with what downstream +/// consumers expect. +fn stitch( + entry: &Path, + cache: &ParseCache, +) -> Result { + let mut program = LoadedProgram::default(); + let mut loaded_files: HashMap = HashMap::new(); + let mut in_progress: HashSet = HashSet::new(); + stitch_file( + entry, + None, + cache, + &mut program, + &mut loaded_files, + &mut in_progress, + )?; + if !program.source.ends_with('\n') { + program.source.push('\n'); + } + Ok(program) +} + +/// Recursive DFS stitch. Same shape as the sync loader's +/// `load_file` but reads pre-parsed content from the cache. +fn stitch_file( + path: &Path, + imported_via: Option, + cache: &ParseCache, + program: &mut LoadedProgram, + loaded_files: &mut HashMap, + in_progress: &mut HashSet, +) -> Result<(), LoadError> { + if loaded_files.contains_key(path) || in_progress.contains(path) { + return Ok(()); + } + let parsed = { + let guard = cache.lock().unwrap(); + match guard.get(path).cloned() { + Some(ParseSlot::Done(p)) => p, + Some(ParseSlot::Skipped) => return Ok(()), + Some(ParseSlot::Failed(msg)) => { + return Err(LoadError::Io { + path: path.to_path_buf(), + source: std::io::Error::other(msg), + }); + } + Some(ParseSlot::Pending(_)) | None => { + return Err(LoadError::Io { + path: path.to_path_buf(), + source: std::io::Error::new( + std::io::ErrorKind::NotFound, + "parallel loader lost parse for this path", + ), + }); + } + } + }; + in_progress.insert(path.to_path_buf()); + + let file_index = program.files.len() as u32; + program.files.push(LoadedFile { + path: path.to_path_buf(), + source: parsed.source.clone(), + imported_via, + mtime: parsed.mtime, + }); + loaded_files.insert(path.to_path_buf(), file_index as usize); + + // Emit chunks + regions in span order, recursing into each + // src target between chunks. + let mut cursor = 0usize; + for imp in &parsed.imports { + // Find this src stmt's span in the doc so we chunk + // around it. `imp.src_span` was recorded at parse time. + let cmd_start = imp.src_span.start as usize; + let cmd_end = imp.src_span.end as usize; + emit_chunk(program, &parsed.source, cursor, cmd_start, file_index); + cursor = cmd_end; + if parsed.source.as_bytes().get(cursor) == Some(&b'\n') { + cursor += 1; + } + stitch_file( + &imp.resolved, + Some(ImportEdge { + importer_file: file_index as usize, + src_span: imp.src_span, + }), + cache, + program, + loaded_files, + in_progress, + )?; + } + emit_chunk( + program, + &parsed.source, + cursor, + parsed.source.len(), + file_index, + ); + if !program.source.ends_with('\n') { + program.source.push('\n'); + } + in_progress.remove(path); + Ok(()) +} + +fn emit_chunk( + program: &mut LoadedProgram, + source: &str, + start: usize, + end: usize, + file_index: u32, +) { + if start >= end { + return; + } + let flat_start = program.source.len() as u32; + program.source.push_str(&source[start..end]); + let flat_end = program.source.len() as u32; + program.regions.push(SourceRegion { + flat_start, + flat_end, + file_index, + file_offset: start as u32, + }); +} + +fn line_of(source: &str, byte: u32) -> u32 { + source[..(byte as usize).min(source.len())] + .bytes() + .filter(|b| *b == b'\n') + .count() as u32 +} + +fn join_err_to_load_err(e: JoinError) -> LoadError { + LoadError::Io { + path: PathBuf::new(), + source: std::io::Error::other(e.to_string()), + } +} + +/// Observer that drives an `indicatif::MultiProgress` panel — one +/// live [`ProgressBar`] per top-level dep, plus one for local / +/// workspace files. +/// +/// On a TTY, each bar's message updates as its inner files fly by. +/// When [`on_dep_completed`] fires, the bar commits with +/// `Checking @` and stays in scrollback. Non-TTY stdout is +/// handled natively by `indicatif` — bars degrade to no-op and +/// we fall back to the `println` path for each event. +/// +/// The observer holds bars behind an internal `Mutex` because +/// [`ProgressBar`] is `Send + Sync` but our bookkeeping around +/// bar creation (first-sight-per-dep) needs synchronized +/// access. Bar updates otherwise happen from many blocking- +/// thread contexts concurrently. +/// +/// **Terminal-side note.** Committed rows are emitted via +/// [`MultiProgress::println`] which uses cursor manipulation to +/// insert content above the bar area rather than a natural scroll. +/// On terminals whose Ctrl-L handler pushes visible content into +/// scrollback before clearing (iTerm2, most xterm-family +/// emulators, tmux), this behaves correctly. On terminals whose +/// Ctrl-L just erases the visible display without preserving +/// unscrolled content (some recent ghostty builds), the committed +/// rows can vanish. That's a terminal behavior, not something the +/// loader can fix at emission time — the tradeoff of the multi- +/// row UI. +pub struct MultiProgressObserver { + multi: MultiProgress, + /// `(name, ProgressBar)` — one per top-level dep. `name` is + /// the `@dep` prefix or the workspace label for local files. + /// Insertion order preserved so scrollback matches source + /// order. + bars: Mutex>, + /// `(depname, cache-directory-abs-path)` pairs used to rewrite + /// resolved paths back into `@dep/relative` labels. + dep_paths: Vec<(String, PathBuf)>, + /// True when stdout is a real terminal. Bars only render + /// when true; non-TTY falls back to plain `println!`. + stdout_is_tty: bool, + /// Human-friendly label for the local workspace's bar. Picked + /// up from `vw.toml`'s `name = "…"` field when available, so + /// `Checking workspace` becomes `Checking metroid` for the + /// metroid workspace. Falls back to the literal `workspace` + /// when no name is configured. + workspace_label: String, +} + +impl MultiProgressObserver { + pub fn new( + dep_paths: Vec<(String, PathBuf)>, + workspace_label: String, + ) -> Self { + use std::io::IsTerminal; + let multi = MultiProgress::new(); + Self { + multi, + bars: Mutex::new(Vec::new()), + dep_paths, + stdout_is_tty: std::io::stdout().is_terminal(), + workspace_label, + } + } + + /// Format a `src` label. `@/` when the + /// path is inside a known dep-cache directory; otherwise + /// strip `@` sigil and `.htcl` suffix. + fn friendly_label(&self, raw: &str, resolved: Option<&Path>) -> String { + if let Some(resolved) = resolved { + let canonical = resolved + .canonicalize() + .unwrap_or_else(|_| resolved.to_path_buf()); + for (name, dep_path) in &self.dep_paths { + let dep_canonical = dep_path + .canonicalize() + .unwrap_or_else(|_| dep_path.clone()); + if let Ok(rel) = canonical.strip_prefix(&dep_canonical) { + let rel_str = rel.display().to_string(); + let rel_str = rel_str.trim_end_matches(".htcl"); + return if rel_str.is_empty() { + format!("@{name}") + } else { + format!("@{name}/{rel_str}") + }; + } + } + } + if !raw.is_empty() { + return raw + .trim_start_matches('@') + .trim_end_matches(".htcl") + .to_string(); + } + resolved + .and_then(|p| p.file_stem()) + .and_then(|s| s.to_str()) + .unwrap_or("?") + .to_string() + } + + /// Bucket a `label` into a top-level dep name for bar routing. + /// `@foo/bar` → `@foo`; `foo/bar` or `foo` → the workspace + /// label (typically the `vw.toml` name). + fn bucket_of(&self, label: &str) -> String { + if let Some(rest) = label.strip_prefix('@') { + let name = match rest.split_once('/') { + Some((n, _)) => n, + None => rest, + }; + format!("@{name}") + } else { + self.workspace_label.clone() + } + } + + /// Return the bar for `bucket`, creating it on first sight. + /// Access is serialized on the bars mutex so parallel task + /// contexts don't race. + fn bar_for(&self, bucket: &str) -> ProgressBar { + let mut guard = self.bars.lock().unwrap(); + if let Some((_, bar)) = guard.iter().find(|(n, _)| n == bucket) { + return bar.clone(); + } + let bar = self.multi.add(ProgressBar::new_spinner()); + bar.set_style( + ProgressStyle::with_template("{prefix:>12.bold.green} {msg}") + .expect("static template compiles"), + ); + bar.set_prefix("Sourcing"); + bar.set_message(bucket.to_string()); + // Steady-tick so spinner-shaped bars animate without needing + // explicit ticks between updates. + bar.enable_steady_tick(std::time::Duration::from_millis(100)); + guard.push((bucket.to_string(), bar.clone())); + bar + } +} + +impl ParallelObserver for MultiProgressObserver { + fn on_source(&self, raw: &str, resolved: &Path) { + let label = self.friendly_label(raw, Some(resolved)); + let bucket = self.bucket_of(&label); + if !self.stdout_is_tty { + println!("{:>12} {label}", "Sourcing"); + return; + } + let bar = self.bar_for(&bucket); + bar.set_prefix("Sourcing"); + bar.set_message(label); + } + + fn on_parsed(&self, file: &Path, raw: Option<&str>) { + let label = self.friendly_label(raw.unwrap_or(""), Some(file)); + let bucket = self.bucket_of(&label); + if !self.stdout_is_tty { + println!("{:>12} {label}", "Checking"); + return; + } + let bar = self.bar_for(&bucket); + bar.set_prefix("Checking"); + bar.set_message(label); + } + + fn on_dep_completed(&self, dep_name: &str) { + use colored::Colorize; + if !self.stdout_is_tty { + return; + } + // Print the committed row THROUGH the MultiProgress. Using + // `multi.println` inserts the text ABOVE the bar area so + // the row settles into normal scrollback flow. + let prefix = "Checking".bright_green().bold(); + let _ = self.multi.println(format!("{:>12} {}", prefix, dep_name)); + // Now remove the bar from the display so it doesn't double- + // render on the next tick. + let mut guard = self.bars.lock().unwrap(); + if let Some(pos) = guard.iter().position(|(n, _)| n == dep_name) { + let (_, bar) = guard.remove(pos); + self.multi.remove(&bar); + } + } +} + +impl MultiProgressObserver { + /// Finalize any bar that wasn't committed by an + /// [`on_dep_completed`] — the local-workspace bar covering + /// non-`@dep` files, and the entry file's own bar — by + /// routing each row through [`MultiProgress::println`] so it + /// enters the normal output stream ABOVE the (now-cleared) + /// bar area. + pub fn finish(&self) { + use colored::Colorize; + if !self.stdout_is_tty { + return; + } + let mut guard = self.bars.lock().unwrap(); + let prefix = "Checking".bright_green().bold(); + // Snapshot the remaining bar names in the current order so + // we can commit + remove without mutating the vec mid-scan. + let remaining: Vec = + guard.iter().map(|(n, _)| n.clone()).collect(); + for name in remaining { + let _ = self.multi.println(format!("{:>12} {}", prefix, name)); + if let Some(pos) = guard.iter().position(|(n, _)| n == &name) { + let (_, bar) = guard.remove(pos); + self.multi.remove(&bar); + } + } + let _ = self.multi.clear(); + } +} diff --git a/vw-eda/Cargo.toml b/vw-eda/Cargo.toml new file mode 100644 index 0000000..f66da6c --- /dev/null +++ b/vw-eda/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "vw-eda" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "EDA backend trait and wire protocol for driving vendor TCL interpreters (Vivado, Quartus, ...)" + +[dependencies] +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +async-trait.workspace = true diff --git a/vw-eda/src/lib.rs b/vw-eda/src/lib.rs new file mode 100644 index 0000000..d86a4b1 --- /dev/null +++ b/vw-eda/src/lib.rs @@ -0,0 +1,105 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! EDA backend abstraction. +//! +//! Defines the trait that every vendor-specific TCL worker implements +//! and the wire protocol used to talk to it. `vw-vivado` is the first +//! implementation; future `vw-quartus` / `vw-synopsys` crates will +//! implement the same trait, and consumers (`vw run`, `vw repl`, the +//! analyzer) talk only to this abstraction. +//! +//! The protocol is intentionally small: newline-delimited JSON +//! requests, one response per request, monotonic IDs. See the project +//! plan's "Wire protocol" section for the design rationale. + +pub mod protocol; + +use async_trait::async_trait; +use thiserror::Error; + +pub use protocol::{ + ErrorPayload, Request, RequestOp, Response, ResponseResult, StreamMessage, + WireMessage, +}; + +/// Errors returned by an [`EdaBackend`]. +#[derive(Debug, Error)] +pub enum BackendError { + /// The worker process exited or could not be started. + #[error("worker process error: {0}")] + Worker(String), + + /// I/O error while reading or writing the wire protocol. + #[error("wire I/O error: {0}")] + Io(#[from] std::io::Error), + + /// Wire protocol message could not be serialized or parsed. + #[error("wire protocol error: {0}")] + Protocol(#[from] serde_json::Error), + + /// The backend reported a TCL-level error in response to a command. + /// `stdout` carries any output the command produced before erroring, + /// so callers can show context. + #[error("TCL error: {message}")] + Tcl { + message: String, + code: Option, + info: Option, + stdout: String, + }, + + /// Catch-all for backend-specific failures. + #[error("{0}")] + Other(String), +} + +/// Result of an [`EdaBackend::eval`] call. +#[derive(Clone, Debug, Default)] +pub struct EvalOutput { + /// The TCL expression's return value, as a string. + pub value: String, + /// stdout captured during this eval (puts to stdout from the user + /// TCL while the shim's capturing flag was set). Always present + /// and may be empty; trailing newlines are preserved as written. + pub stdout: String, +} + +/// A long-lived TCL worker driven by `vw`. +/// +/// Implementations spawn the vendor process (Vivado, Quartus, ...), +/// inject a small shim that speaks the wire protocol, and translate +/// [`Request`]s into [`Response`]s. The trait is intentionally narrow: +/// callers issue commands, the backend runs them, and the protocol is +/// the contract. +#[async_trait] +pub trait EdaBackend: Send { + /// Human-readable backend name, e.g. `"vivado"`. + fn name(&self) -> &str; + + /// Evaluate a TCL command string and return its result plus any + /// stdout the command produced. + /// + /// Equivalent to issuing a [`RequestOp::Eval`] request and + /// extracting both the return value and the captured-puts payload. + /// Most callers should use this in preference to + /// [`EdaBackend::send`] until the structured-eval machinery + /// (phase 4) lands. + async fn eval(&mut self, tcl: &str) -> Result; + + /// Issue an arbitrary request and return the raw response. + /// + /// The default implementation in concrete backends is the + /// preferred place to add `eval_structured` and future ops without + /// changing the trait surface. + async fn send( + &mut self, + request: Request, + ) -> Result; + + /// Cleanly shut the worker down. Backends should make this + /// idempotent so that `Drop` can fall back to it without + /// double-shutdown errors. + async fn shutdown(&mut self) -> Result<(), BackendError>; +} diff --git a/vw-eda/src/protocol.rs b/vw-eda/src/protocol.rs new file mode 100644 index 0000000..0559eec --- /dev/null +++ b/vw-eda/src/protocol.rs @@ -0,0 +1,244 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Newline-delimited JSON wire protocol between `vw` and a vendor +//! TCL worker. +//! +//! v0 implements the `eval` op only. The `eval_structured` op (Phase 4 +//! of the project plan) will land as an additional [`RequestOp`] +//! variant without breaking the wire format. + +use serde::{Deserialize, Serialize}; + +/// A request sent from `vw` to the worker. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Request { + /// Monotonic request id chosen by the sender. The worker echoes + /// it in the matching [`Response`]. + pub id: u64, + #[serde(flatten)] + pub op: RequestOp, +} + +/// The operation a [`Request`] performs. +/// +/// Serialized with `op` as the discriminator (`{"op": "eval", "tcl": +/// "..."}`), matching the project plan's spec. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum RequestOp { + /// Evaluate a TCL command in the worker's interpreter and return + /// the result as a string. + Eval { tcl: String }, + /// Cleanly shut the worker down. Issued by [`crate::EdaBackend::shutdown`]. + Shutdown, +} + +/// A response from the worker for a single request. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Response { + pub id: u64, + #[serde(flatten)] + pub result: ResponseResult, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ResponseResult { + Ok { + ok: OkMarker, + #[serde(default)] + result: serde_json::Value, + }, + Err { + ok: ErrMarker, + error: ErrorPayload, + }, +} + +/// Streaming notification emitted by the worker between request and +/// response. `puts` writes from inside an eval are forwarded as these +/// so callers can show output live rather than waiting for the eval +/// to complete (necessary for any long-running synthesis or +/// implementation command). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StreamMessage { + /// Id of the in-flight request this stream chunk belongs to. + pub id: u64, + /// `"stdout"` today; reserved for `"stderr"` etc. later. + pub stream: String, + /// The chunk's bytes, including any trailing newline as written. + pub data: String, +} + +/// One wire-level message read from the worker. Either a streaming +/// chunk for an in-flight request, the request's final response, or +/// an unsolicited RPC call FROM the worker asking `vw` (Rust side) to +/// compute a value. Discriminated by structural inspection: +/// - stream chunks have a `stream` field, +/// - responses have `ok`, +/// - RPC calls have `rpc` set to `true` plus a `method` field. +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum WireMessage { + Stream(StreamMessage), + Response(Response), + Rpc(RpcCall), +} + +/// An RPC call FROM the worker (shim) TO `vw`. The shim's htcl +/// library uses this to reach Rust-implemented externs like +/// `vw::workspace_root` and `vw::design_sources` — anything whose +/// answer lives on the tool side, not in Vivado. +/// +/// Wire shape: `{"id": M, "rpc": true, "method": "...", "args": ...}` +/// The `rpc: true` marker keeps the untagged `WireMessage` union +/// unambiguous — plain responses have `ok`, streams have `stream`, +/// RPC calls have `rpc`. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RpcCall { + pub id: u64, + /// Always `true` — used as an untagged-enum discriminator. See + /// [`RpcMarker`] for the deserializer. + pub rpc: RpcMarker, + pub method: String, + #[serde(default)] + pub args: serde_json::Value, +} + +/// Marker that always serializes to the literal `true`. Same +/// pattern as [`OkMarker`] — distinguishes RPC calls from Responses +/// in the untagged `WireMessage` union. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +pub struct RpcMarker(#[serde(deserialize_with = "deserialize_true")] pub bool); + +impl RpcMarker { + pub const TRUE: RpcMarker = RpcMarker(true); +} + +/// Marker that always serializes to the literal `true`. Lets us use +/// the same `ok` field as a discriminator without a custom serializer. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +pub struct OkMarker(#[serde(deserialize_with = "deserialize_true")] pub bool); + +impl OkMarker { + pub const TRUE: OkMarker = OkMarker(true); +} + +fn deserialize_true<'de, D: serde::Deserializer<'de>>( + de: D, +) -> Result { + let v = bool::deserialize(de)?; + if v { + Ok(true) + } else { + Err(serde::de::Error::custom("expected `true`")) + } +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +pub struct ErrMarker(#[serde(deserialize_with = "deserialize_false")] pub bool); + +impl ErrMarker { + pub const FALSE: ErrMarker = ErrMarker(false); +} + +fn deserialize_false<'de, D: serde::Deserializer<'de>>( + de: D, +) -> Result { + let v = bool::deserialize(de)?; + if !v { + Ok(false) + } else { + Err(serde::de::Error::custom("expected `false`")) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ErrorPayload { + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub info: Option, +} + +impl Response { + pub fn ok(id: u64, result: serde_json::Value) -> Self { + Self { + id, + result: ResponseResult::Ok { + ok: OkMarker::TRUE, + result, + }, + } + } + + pub fn err(id: u64, error: ErrorPayload) -> Self { + Self { + id, + result: ResponseResult::Err { + ok: ErrMarker::FALSE, + error, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip_eval_request() { + let req = Request { + id: 1, + op: RequestOp::Eval { + tcl: "puts hi".into(), + }, + }; + let s = serde_json::to_string(&req).unwrap(); + assert!(s.contains("\"op\":\"eval\"")); + assert!(s.contains("\"tcl\":\"puts hi\"")); + let back: Request = serde_json::from_str(&s).unwrap(); + match back.op { + RequestOp::Eval { tcl } => assert_eq!(tcl, "puts hi"), + _ => panic!(), + } + } + + #[test] + fn round_trip_ok_response() { + let r = Response::ok(7, serde_json::json!("hi")); + let s = serde_json::to_string(&r).unwrap(); + let back: Response = serde_json::from_str(&s).unwrap(); + match back.result { + ResponseResult::Ok { result, .. } => { + assert_eq!(result, serde_json::json!("hi")) + } + _ => panic!(), + } + } + + #[test] + fn round_trip_err_response() { + let r = Response::err( + 8, + ErrorPayload { + message: "boom".into(), + code: Some("E1".into()), + info: None, + }, + ); + let s = serde_json::to_string(&r).unwrap(); + let back: Response = serde_json::from_str(&s).unwrap(); + match back.result { + ResponseResult::Err { error, .. } => { + assert_eq!(error.message, "boom"); + assert_eq!(error.code.as_deref(), Some("E1")); + } + _ => panic!(), + } + } +} diff --git a/vw-htcl-cmd/Cargo.toml b/vw-htcl-cmd/Cargo.toml new file mode 100644 index 0000000..ad3a891 --- /dev/null +++ b/vw-htcl-cmd/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "vw-htcl-cmd" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Generate documented htcl command wrappers from Vivado man-page references" + +[dependencies] +vw-htcl = { path = "../vw-htcl" } +serde.workspace = true +thiserror.workspace = true +toml.workspace = true +winnow.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-htcl-cmd/src/constraints.rs b/vw-htcl-cmd/src/constraints.rs new file mode 100644 index 0000000..a683422 --- /dev/null +++ b/vw-htcl-cmd/src/constraints.rs @@ -0,0 +1,212 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Per-command signature augmentations layered on top of the +//! man-page-derived wrapper. +//! +//! UG835 gives us each command's flag/positional list and types, but +//! it has no language for the semantic refinements an htcl wrapper +//! benefits from — mutually-exclusive call modes (`set_property`'s +//! `-dict` vs `-name/-value/-objects` pair), inter-argument +//! requirements (`tuser_width @requires has_tuser`), reclassifying +//! a positional into a keyword-form arg with a default. Those live +//! in a TOML file the wrapper-module author hand-maintains alongside +//! the auto-generated `cmd/*.htcl` files. +//! +//! File shape: +//! +//! ```toml +//! [.args.] +//! default = "..." # adds/replaces @default(...) +//! enum = ["a", "b"] # adds/replaces @enum(a, b) +//! clear_enum = true # drops any @enum the man-page emitted +//! one_of = ["other"] # adds @one_of(other) +//! requires = ["a", "b"] # adds @requires(a, b) +//! conflicts = ["a"] # adds @conflicts(a) +//! ``` +//! +//! The generator applies overrides during signature emission. The +//! body emission then follows the post-override arg classification +//! — flipping a flag from `@enum(0, 1)` to `@default("")` makes it +//! a value-taking arg, and the body forwards `-flag $value` +//! instead of `if {$flag} { lappend cmd -flag }`. + +use std::collections::HashMap; +use std::path::Path; + +use serde::Deserialize; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ConstraintsError { + #[error("reading {path}: {source}")] + Io { + path: std::path::PathBuf, + #[source] + source: std::io::Error, + }, + #[error("parsing {path}: {source}")] + Parse { + path: std::path::PathBuf, + #[source] + source: toml::de::Error, + }, +} + +/// Per-arg overrides for one command. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +pub struct ArgOverride { + /// New `@default(...)` value. Replaces any inherited default. + #[serde(default)] + pub default: Option, + /// New `@enum(...)` choices. Replaces any inherited enum. + #[serde(default, rename = "enum")] + pub enum_: Option>, + /// Drop any inherited `@enum`. Use when the man-page parsing + /// modeled an arg as `@enum(0, 1)` (boolean toggle) but it's + /// actually value-taking. + #[serde(default)] + pub clear_enum: bool, + /// `@one_of(...)` declarations to add. Empty means no addition. + #[serde(default)] + pub one_of: Vec, + /// `@requires(...)` declarations to add. + #[serde(default)] + pub requires: Vec, + /// `@conflicts(...)` declarations to add. + #[serde(default)] + pub conflicts: Vec, + /// Override whether this arg carries a Vivado typed Tcl_Obj + /// handle (e.g. a bd_cell, get_bd_pins result). `None` keeps + /// the generator default (a name-based allowlist of well-known + /// typed-arg names like `objects`/`cells`/`pin`/...). `Some(true)` + /// forces this arg to be treated as typed; `Some(false)` forces + /// it to be treated as a plain string. + /// + /// Typed args are passed directly via `-flag $value` in the + /// wrapper body — never through `[list]` or `lappend` — because + /// list-construction shimmers Vivado's internal typed Tcl_Obj to + /// a plain string, and downstream consumers like + /// `set_property -objects` reject the stringified path. + #[serde(default)] + pub typed: Option, + + /// Per-arg type annotation: any valid htcl type expression + /// (`bd_cell`, `list`, `string`, etc.). Wins over + /// the inferred type from the typed-handle name table when + /// both are present. Set when an arg's name doesn't match + /// the table's plural-aware heuristic, or when the man-page + /// `object` placeholder is actually a specific type. + #[serde(default, rename = "type")] + pub arg_type: Option, +} + +/// All overrides for one command, indexed by arg ident. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +pub struct CommandOverride { + /// Per-arg overrides. The key is the htcl proc-arg identifier + /// (matches `Argument::ident`). + #[serde(default)] + pub args: HashMap, + /// Override the command's return type. The string is taken + /// verbatim and emitted as the proc's 4th-word annotation + /// (`proc NAME { args } { body }`), so any valid + /// htcl type expression works: `bd_cell`, `list`, + /// `dict`, `unit`. Use this when the Returns: + /// phrase auto-mapping is ambiguous or wrong for a specific + /// command. + #[serde(default)] + pub returns: Option, +} + +/// The complete set of overrides loaded from the constraints file. +/// Lookups are by command name (`set_property`, `create_project`, +/// …) — missing entries return `None` and the generator emits the +/// pure man-page-derived wrapper. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(transparent)] +pub struct ConstraintsTable { + commands: HashMap, +} + +impl ConstraintsTable { + /// Empty table — every command falls back to the pure man-page + /// signature. Used when no `--constraints` was passed. + pub fn empty() -> Self { + Self::default() + } + + /// Load from a TOML file at `path`. + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path).map_err(|e| { + ConstraintsError::Io { + path: path.to_path_buf(), + source: e, + } + })?; + toml::from_str(&text).map_err(|e| ConstraintsError::Parse { + path: path.to_path_buf(), + source: e, + }) + } + + /// Per-command overrides, or `None` when nothing is declared + /// for `command`. + pub fn for_command(&self, command: &str) -> Option<&CommandOverride> { + self.commands.get(command) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_table_returns_no_overrides() { + let t = ConstraintsTable::empty(); + assert!(t.for_command("set_property").is_none()); + } + + #[test] + fn parses_full_arg_override_block() { + let toml = r#" + [set_property.args.dict] + default = "" + clear_enum = true + one_of = ["name"] + requires = ["objects"] + + [set_property.args.name] + default = "" + one_of = ["dict"] + requires = ["value", "objects"] + "#; + let t: ConstraintsTable = toml::from_str(toml).unwrap(); + let sp = t.for_command("set_property").unwrap(); + let dict = sp.args.get("dict").unwrap(); + assert_eq!(dict.default.as_deref(), Some("")); + assert!(dict.clear_enum); + assert_eq!(dict.one_of, vec!["name".to_string()]); + assert_eq!(dict.requires, vec!["objects".to_string()]); + + let name = sp.args.get("name").unwrap(); + assert_eq!(name.default.as_deref(), Some("")); + assert_eq!(name.one_of, vec!["dict".to_string()]); + assert_eq!( + name.requires, + vec!["value".to_string(), "objects".to_string()] + ); + } + + #[test] + fn missing_command_returns_none() { + let toml = r#" + [set_property.args.dict] + default = "" + "#; + let t: ConstraintsTable = toml::from_str(toml).unwrap(); + assert!(t.for_command("create_project").is_none()); + assert!(t.for_command("set_property").is_some()); + } +} diff --git a/vw-htcl-cmd/src/generate.rs b/vw-htcl-cmd/src/generate.rs new file mode 100644 index 0000000..15c67fb --- /dev/null +++ b/vw-htcl-cmd/src/generate.rs @@ -0,0 +1,953 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Emit an htcl wrapper proc for a parsed [`ManPage`]. +//! +//! Shape, for a command `add_files`: +//! +//! ```htcl +//! # Preserve the underlying Vivado command so the wrapper can forward +//! # to it after shadowing the global name. +//! if {[info commands __viv_add_files] eq "" && [info commands add_files] ne ""} { +//! rename add_files __viv_add_files +//! } +//! +//! ## Adds one or more source files ... +//! proc add_files { +//! ## (Optional) The fileset to add to. +//! @default("") fileset +//! ## (Optional) Do not recurse ... +//! @enum(0, 1) @default(0) norecurse +//! ## Positional operands ... +//! @default("") operands +//! } { +//! set cmd [list __viv_add_files] +//! if {$fileset ne ""} { lappend cmd -fileset $fileset } +//! if {$norecurse} { lappend cmd -norecurse } +//! if {$operands ne ""} { lappend cmd {*}$operands } +//! return [{*}$cmd] +//! } +//! ``` +//! +//! The wrapper keeps the command's natural name and shadows the +//! builtin; a guarded `rename` stashes the original under +//! `` so the body forwards to it without recursing. All +//! arguments are addressed by keyword (`-fileset value`); boolean flags +//! take a `0`/`1` value at the htcl layer and lower to flag +//! presence/absence on the Vivado command line. + +use std::fmt::Write; + +use vw_htcl::emit::{Command, Doc, Item, Word}; + +use crate::constraints::{ArgOverride, ConstraintsTable}; +use crate::model::{ArgKind, Argument, ManPage}; + +/// Arg names whose values are Vivado typed `Tcl_Obj` handles — +/// `bd_cell`, `bd_pin`, etc. — and therefore must be passed to the +/// underlying command **directly** (`-flag $value`) rather than +/// through `[list]` or `lappend`. List construction shimmers Tcl's +/// internal typed representation away, leaving the handle as a +/// plain path string; downstream code paths in Vivado (notably +/// `set_property -objects`) reject the stringified path with +/// `[Common 17-161] Invalid option value`. +/// +/// Curated list of the obvious cases. Per-arg override via +/// `cmd-constraints.toml`'s `typed = true|false` covers the long +/// tail. +const TYPED_ARG_NAMES: &[&str] = &[ + "object", + "objects", + "of_objects", + "cell", + "cells", + "pin", + "pins", + "port", + "ports", + "intf_pin", + "intf_pins", + "intf_port", + "intf_ports", + "net", + "nets", + "intf_net", + "intf_nets", + // File-object handles: `get_files` returns a Tcl_Obj carrying + // Vivado's internal file representation. Passing that through + // `lappend flags {*}$files` shimmers the internal rep to a plain + // path string, which `make_wrapper -files` (and any downstream + // command taking a file-object list) then rejects with + // `[Common 17-161] Invalid option value`. + "file", + "files", + // Filesets: `get_filesets` returns fileset objects. Same + // shimmer pitfall as files/cells. + "fileset", + "filesets", +]; + +fn is_typed_arg(name: &str, override_: Option) -> bool { + match override_ { + Some(t) => t, + None => TYPED_ARG_NAMES.contains(&name), + } +} + +/// Map a typed-arg name to its concrete `TypeExpr` text, when +/// known. Drives the `name: TYPE` annotation emitted in the +/// generated proc args. Plural names (`cells`, `pins`) map to +/// `list`; singulars (`cell`, `pin`) map to the bd_* type +/// directly. Generic catch-alls (`object`, `objects`, +/// `of_objects`) can refer to any Vivado handle class, so we +/// leave them untyped at this layer — the type system doesn't +/// have unions in v1. +/// +/// Returning `None` means "the arg is typed (don't list-wrap in +/// the body) but we don't have a precise type expression for +/// it" — the generator emits the arg without an annotation. +fn typed_arg_type(name: &str) -> Option<&'static str> { + match name { + "cell" => Some("bd_cell"), + "cells" => Some("list"), + "pin" => Some("bd_pin"), + "pins" => Some("list"), + "port" => Some("bd_port"), + "ports" => Some("list"), + "net" => Some("bd_net"), + "nets" => Some("list"), + "intf_pin" => Some("bd_intf_pin"), + "intf_pins" => Some("list"), + "intf_port" => Some("bd_intf_port"), + "intf_ports" => Some("list"), + "intf_net" => Some("bd_intf_net"), + "intf_nets" => Some("list"), + // object / objects / of_objects: any handle class — no + // precise type until we have unions. + // + // file / files / fileset / filesets: no concrete newtype + // in the current type-decl set — leave the annotation off + // (returned as `string` today from `get_files` / + // `get_filesets`); still routed through the typed-arg + // fast path so we don't shimmer the internal rep away. + _ => None, + } +} + +#[derive(Clone, Debug)] +pub struct GenerateOptions { + /// Prefix for the stashed original command (`rename add_files + /// __viv_add_files`). Kept for backwards compatibility — the + /// lowering pass now generates the rename plumbing, so this + /// field has no effect. + pub rename_prefix: String, + /// Emit each command's `See Also` list as a doc-comment footer. + pub include_see_also: bool, + /// Per-command signature augmentations loaded from + /// `cmd-constraints.toml`. The generator merges these onto the + /// man-page-derived shape so wrapper authors can declare + /// mutually-exclusive call modes, value-taking flags + /// misclassified by the man page, etc., without hand-editing + /// the generated files. + pub constraints: ConstraintsTable, +} + +impl Default for GenerateOptions { + fn default() -> Self { + Self { + rename_prefix: "__viv_".to_string(), + include_see_also: true, + constraints: ConstraintsTable::empty(), + } + } +} + +/// Generate the htcl wrapper text for `page`. +pub fn generate(page: &ManPage, opts: &GenerateOptions) -> String { + let cmd = &page.name; + // Wrapper body forwards to the underlying Vivado proc via + // `extern::` (which the lowering rewrites to the bare native + // name). The wrapper itself lives inside `namespace eval + // vivado { ... }` so it doesn't shadow the global name the + // body is forwarding to — that's what frees Vivado's own + // internal Tcl from accidentally hitting our typed wrappers + // when it calls a sibling builtin. + let forwarded = format!("extern::{}", page.name); + + let overrides = opts.constraints.for_command(&page.name); + let effective = effective_args(page, overrides); + + let mut out = String::new(); + writeln!( + out, + "# Generated by `vw htcl-cmd generate` from the Vivado command \ + reference." + ) + .unwrap(); + writeln!(out, "# Do not edit by hand.").unwrap(); + writeln!(out).unwrap(); + + // Wrappers live in `vivado_cmd::`, NOT `vivado::`. Vivado has + // its own internal `::vivado` namespace and code paths that + // behave differently depending on the calling namespace — + // notably `set_property -dict -objects ...` rejects valid cell + // handles when invoked from inside `::vivado`. Picking a name + // Vivado doesn't use means our wrapper bodies never collide + // with Vivado-internal namespace state. + writeln!(out, "namespace eval vivado_cmd {{").unwrap(); + writeln!(out).unwrap(); + + // Proc doc comment: the command Description, then a See-Also footer. + emit_proc_doc(&mut out, page, opts); + + // Proc args (structured) and body (compact Tcl). + let args = build_args(page, &effective); + let body = build_body(&forwarded, &effective); + // Resolve return type. Priority: + // 1. Explicit override in `cmd-constraints.toml`. + // 2. The page's `Returns:` section, if present. + // 3. Phrases in the `Description:` section — Vivado very + // rarely uses a dedicated Returns: header, so this is + // actually the common path. The phrase table is the same + // either way. + // 4. Fallback to `string`. The emitted body ALWAYS ends with + // `return [extern::_vw_global_call ...]`, which in Tcl + // yields whatever the underlying command returns — a + // string, possibly empty. The htcl validator rejects + // value-returning procs with no return-type annotation, + // so we must always emit one. `string` is the safe + // universal fallback for the commands whose Returns: + // prose doesn't match any of the specific phrases. + let return_type = overrides + .and_then(|o| o.returns.as_deref()) + .map(String::from) + .or_else(|| infer_return_type(page.returns.as_deref())) + .or_else(|| infer_return_type(Some(page.description.as_slice()))) + .or_else(|| Some("string".to_string())); + emit_proc(&mut out, cmd, &args, return_type.as_deref(), &body); + + writeln!(out).unwrap(); + writeln!(out, "}}").unwrap(); + + // Trim trailing whitespace line-by-line (empty doc comments emit a + // trailing space) and guarantee a single trailing newline. + let mut cleaned: String = out + .lines() + .map(str::trim_end) + .collect::>() + .join("\n"); + cleaned.push('\n'); + cleaned +} + +/// Write the proc-level doc comments above the `proc` line so they +/// attach to it. The output is structured as +/// +/// ```text +/// ## +/// ## +/// ## +/// ## +/// ## +/// ``` +/// +/// where the summary is the first sentence of the source description +/// (LSP-clients use it for inline annotations like +/// `CompletionItem::detail`) and the body is everything after, +/// rendered as separate paragraphs. The body paragraphs are +/// re-wrapped at ~78 columns so the on-disk file stays readable +/// without preserving the man-page's source wrap. +fn emit_proc_doc(out: &mut String, page: &ManPage, opts: &GenerateOptions) { + let raw: Vec = + page.description.iter().map(|l| sanitize_doc(l)).collect(); + let summary = vw_htcl::doc::brief(&raw); + let extended = vw_htcl::doc::extended(&raw); + + match summary { + None => { + writeln!(out, "## Wrapper for the Vivado `{}` command.", page.name) + .unwrap(); + } + Some(s) => { + emit_paragraph_lines(out, &s, "## ", 78); + } + } + if let Some(body) = extended { + for paragraph in body.split("\n\n") { + writeln!(out, "##").unwrap(); + emit_paragraph_lines(out, paragraph, "## ", 78); + } + } + + if opts.include_see_also && !page.see_also.is_empty() { + writeln!(out, "##").unwrap(); + writeln!(out, "## See also: {}", page.see_also.join(", ")).unwrap(); + } +} + +fn emit_paragraph_lines( + out: &mut String, + text: &str, + prefix: &str, + width: usize, +) { + let body_width = width.saturating_sub(prefix.len()); + for line in vw_htcl::doc::wrap_paragraph(text, body_width) { + writeln!(out, "{prefix}{line}").unwrap(); + } +} + +/// One argument plus whatever overrides from `cmd-constraints.toml` +/// apply to it. `default`, `enum_values`, `one_of`, `requires`, +/// `conflicts` are the *final* values the wrapper should emit; +/// constraint resolution has already happened. +/// +/// `kind` is derived: a constraint that clears the enum and adds a +/// default to a man-page-Boolean arg flips it to value-taking, so +/// the body-emitter forwards `-flag $value` instead of `if {$f} { +/// lappend cmd -flag }`. +#[derive(Clone, Debug)] +struct EffectiveArg { + ident: String, + flag: Option, + kind: ArgKind, + /// `None` → no default (required); `Some(text)` → emit `@default(text)`. + default: Option, + /// `None` → no enum; `Some(vec)` → emit `@enum(...)`. + enum_values: Option>, + one_of: Vec, + requires: Vec, + conflicts: Vec, + description: Vec, + /// True when this arg carries a Vivado typed `Tcl_Obj` handle + /// — body emission passes it directly (`-flag $value`) rather + /// than threading it through a list. See [`TYPED_ARG_NAMES`]. + typed: bool, + /// The arg's declared type expression, if known. Emitted in + /// the proc args as `name: TYPE`. Set from + /// [`typed_arg_type`] for the typed-handle allowlist; an + /// explicit per-arg `type = "..."` override in + /// `cmd-constraints.toml` wins over the inferred value. + arg_type: Option, +} + +fn effective_args( + page: &ManPage, + overrides: Option<&crate::constraints::CommandOverride>, +) -> Vec { + page.arguments + .iter() + .map(|arg| { + effective_arg(arg, overrides.and_then(|o| o.args.get(&arg.ident))) + }) + .collect() +} + +/// Translate legacy `"0"`/`"1"` boolean defaults into `"false"`/ +/// `"true"` when the arg is still typed as a bool (i.e., no +/// `clear_enum` or explicit type override that would fall through +/// to the value-taking path). Anything else passes through verbatim. +fn bool_translate_if_needed( + raw: &str, + kind: &ArgKind, + over: &ArgOverride, +) -> String { + let is_bool_arg = matches!(kind, ArgKind::Boolean) + && !over.clear_enum + && over.arg_type.is_none(); + if !is_bool_arg { + return raw.to_string(); + } + match raw { + "0" => "false".to_string(), + "1" => "true".to_string(), + _ => raw.to_string(), + } +} + +fn effective_arg(arg: &Argument, over: Option<&ArgOverride>) -> EffectiveArg { + let empty = ArgOverride::default(); + let over = over.unwrap_or(&empty); + + // Default value: explicit override wins; else man-page heuristic. + // Boolean args now emit as `@default(false) name: bool` instead + // of `@enum(0, 1) @default(0) name` — Vivado's man pages + // universally document them as toggles, and htcl already has a + // `bool` type. Callers write `-quiet true` instead of + // `-quiet 1`. + let mut default: Option = match &arg.kind { + ArgKind::Boolean => Some("false".to_string()), + ArgKind::Value | ArgKind::Positional => { + (!arg.required).then(|| "".to_string()) + } + }; + if let Some(d) = over.default.as_deref() { + // Translate `"0"`/`"1"` in an override to `false`/`true` if + // the arg is still typed as a bool. Otherwise the override + // wins verbatim. + default = Some(bool_translate_if_needed(d, &arg.kind, over)); + } + + // Enum: no `@enum` for booleans anymore — the `bool` type + // annotation carries the constraint. Non-boolean args still + // pick up whatever the override declares. + let mut enum_values: Option> = None; + if let Some(v) = &over.enum_ { + enum_values = Some(v.clone()); + } + + // Kind: an override that clears the (former) `@enum(0, 1)` on a + // man-page-Boolean and gives a string-typed default is signaling + // "actually value-taking" — body-emit should forward + // `-flag $value`, not `if {$flag} { ... }`. This is the exact + // shape `set_property -dict` needs. The signal is `clear_enum` + // in the override; without it we keep the Boolean shape and + // emit as a bool toggle. + let kind = if matches!(arg.kind, ArgKind::Boolean) && over.clear_enum { + if arg.flag.is_some() { + ArgKind::Value + } else { + ArgKind::Positional + } + } else { + arg.kind + }; + + let typed = is_typed_arg(&arg.ident, over.typed); + let arg_type = over + .arg_type + .clone() + .or_else(|| typed_arg_type(&arg.ident).map(String::from)) + // Default booleans to `bool` when there's no override and + // no allowlist entry. + .or_else(|| { + matches!(kind, ArgKind::Boolean).then(|| "bool".to_string()) + }); + + EffectiveArg { + ident: arg.ident.clone(), + flag: arg.flag.clone(), + kind, + default, + enum_values, + one_of: over.one_of.clone(), + requires: over.requires.clone(), + conflicts: over.conflicts.clone(), + description: arg.description.clone(), + typed, + arg_type, + } +} + +/// Build the structured arg list as an emit [`Doc`]: per-argument doc +/// comments followed by an `@attr… ident` declaration. The doc +/// comments follow the same `summary, blank, body` shape the +/// proc-level docs use, so LSP clients can split brief/detail from +/// extended documentation consistently. +fn build_args(_page: &ManPage, effective: &[EffectiveArg]) -> Doc { + let mut doc = Doc::new(); + for (i, arg) in effective.iter().enumerate() { + if i > 0 { + doc.push(Item::Blank); + } + let raw: Vec = + arg.description.iter().map(|l| sanitize_doc(l)).collect(); + let summary = vw_htcl::doc::brief(&raw); + let extended = vw_htcl::doc::extended(&raw); + + let body_width = 76usize; + if let Some(s) = summary.as_deref() { + for line in vw_htcl::doc::wrap_paragraph(s, body_width) { + doc.push(Item::DocComment(line)); + } + } + if let Some(body) = extended { + for paragraph in body.split("\n\n") { + doc.push(Item::DocComment(String::new())); + for line in vw_htcl::doc::wrap_paragraph(paragraph, body_width) + { + doc.push(Item::DocComment(line)); + } + } + } + + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: effective_attr_words(arg), + body: None, + })); + } + doc +} + +/// The attribute words + identifier for one effective argument. +fn effective_attr_words(arg: &EffectiveArg) -> Vec { + let mut words = Vec::new(); + if let Some(values) = &arg.enum_values { + let inner = values + .iter() + .map(|v| format_attribute_value(v)) + .collect::>() + .join(", "); + words.push(Word::Raw(format!("@enum({inner})"))); + } + if let Some(default) = &arg.default { + words.push(Word::Raw(format!( + "@default({})", + format_attribute_value(default) + ))); + } + if !arg.one_of.is_empty() { + words.push(Word::Raw(format!("@one_of({})", arg.one_of.join(", ")))); + } + if !arg.requires.is_empty() { + words + .push(Word::Raw(format!("@requires({})", arg.requires.join(", ")))); + } + if !arg.conflicts.is_empty() { + words.push(Word::Raw(format!( + "@conflicts({})", + arg.conflicts.join(", ") + ))); + } + match arg.arg_type.as_deref() { + Some(ty) => { + // Emit `name: TYPE` as two adjacent bare words. The + // proc-args parser tokenizes `name`, `:`, and TYPE + // independently — the layout reads as the user would + // write it. + words.push(Word::Bare(format!("{}:", arg.ident))); + words.push(Word::Bare(ty.to_string())); + } + None => { + words.push(Word::Bare(arg.ident.clone())); + } + } + words +} + +fn format_attribute_value(s: &str) -> String { + let is_int = !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()); + let is_ident = !s.is_empty() + && s.bytes().enumerate().all(|(i, b)| { + if i == 0 { + b.is_ascii_alphabetic() || b == b'_' + } else { + b.is_ascii_alphanumeric() || b == b'_' + } + }); + if is_int || is_ident { + s.to_string() + } else { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } +} + +/// Build the proc body. +/// +/// Args are split into two cohorts: +/// +/// - **Non-typed args** (booleans, strings, positionals whose name +/// isn't in the typed-handle allowlist) accumulate into a `flags` +/// list via `lappend`. Each arg's kind drives its emit form — +/// `Boolean` → `if {$x} { lappend flags -flag }`, `Value` → +/// `if {$x ne ""} { lappend flags -flag $x }`, `Positional` → +/// `if {$x ne ""} { lappend flags {*}$x }`. These values are all +/// strings, so the lappend / `{*}`-expansion that follows is +/// safe — string values don't have a typed Tcl_Obj to shimmer. +/// - **Typed-handle args** (`-objects`/`-cell`/etc., per +/// [`TYPED_ARG_NAMES`] or per-arg `typed = true` override) are +/// passed **directly** to the underlying command via +/// `-flag $value`. Putting them through `[list]` or `lappend` +/// would shimmer Vivado's internal typed Tcl_Obj to a string, +/// and downstream code paths like `set_property -objects` reject +/// the stringified path with `[Common 17-161] Invalid option +/// value '...' specified for 'objects'`. +/// +/// The invocation site branches on which typed args are present so +/// no typed-arg flag appears in the call when its variable is +/// empty. With N typed args this is 2^N branches; in practice N is +/// 0 or 1 for almost every Vivado command, and never more than 2. +fn build_body(orig: &str, effective: &[EffectiveArg]) -> String { + let mut body = String::new(); + + let non_typed: Vec<&EffectiveArg> = + effective.iter().filter(|a| !a.typed).collect(); + let typed: Vec<&EffectiveArg> = + effective.iter().filter(|a| a.typed).collect(); + + // Non-typed accumulator. `flags` is a plain Tcl list — only + // ever contains string values, so list-construction shimmer is + // a non-issue. + writeln!(body, "set flags [list]").unwrap(); + for arg in &non_typed { + let id = &arg.ident; + let required = arg.default.is_none(); + match arg.kind { + ArgKind::Boolean => { + let flag = arg.flag.as_deref().unwrap_or(id); + writeln!(body, "if {{${id}}} {{ lappend flags -{flag} }}") + .unwrap(); + } + ArgKind::Value => { + let flag = arg.flag.as_deref().unwrap_or(id); + if required { + writeln!(body, "lappend flags -{flag} ${id}").unwrap(); + } else { + writeln!( + body, + "if {{${id} ne \"\"}} {{ lappend flags -{flag} ${id} }}" + ) + .unwrap(); + } + } + ArgKind::Positional => { + if required { + writeln!(body, "lappend flags {{*}}${id}").unwrap(); + } else { + writeln!( + body, + "if {{${id} ne \"\"}} \ + {{ lappend flags {{*}}${id} }}" + ) + .unwrap(); + } + } + } + } + + // Typed-arg branching. Direct invocation per combination of + // typed args that are non-empty, so the typed values never + // touch a Tcl list. + emit_typed_invocation(&mut body, orig, &typed, 0); + + body +} + +/// Emit the typed-arg branch tree. At each level we split on +/// "this typed arg present?" and recurse; at the leaves we emit +/// `return [extern::_vw_global_call extern:: {*}$flags …]` +/// with whatever subset of typed args was present. The `extern::` +/// prefix on the helper is what the htcl analyzer sees — the +/// lowerer's `rewrite_externs` pass strips it, so the actual +/// runtime call is `::_vw_global_call :: …`. +/// +/// The `::_vw_global_call` helper (defined in the shim at +/// `::` namespace) is what keeps Vivado's internal Tcl (XDC +/// parsing, OOC synth flows) from resolving unqualified commands +/// to *our* wrappers just because we're currently inside +/// `::vivado_cmd::`. Without a global-namespace call, a +/// wrapper that forwards to `synth_ip` — which itself sources XDC +/// files whose scripts call `create_clock`, `get_ports`, … +/// positionally — would put those XDC scripts in the +/// `::vivado_cmd::` namespace context. The XDC's unqualified +/// `create_clock` would then land on our typed wrapper (which +/// expects `-flag` args) and its kwargs prologue would crash on +/// the positional port name. +/// +/// We use the helper rather than `namespace eval ::` / +/// `namespace inscope ::` / `uplevel #0`, all of which internally +/// serialize their args to a script string via `concat` and +/// re-parse them. That round-trip loses Tcl_Obj internal reps — +/// bd_cell handles become plain paths like `/cpm5`, which +/// Vivado's `set_property -objects` then rejects as "Invalid +/// option value". The helper's `{*}$cmd {*}$args` expansion is a +/// direct arg-passing, not a script re-parse, so object identity +/// is preserved end-to-end. +fn emit_typed_invocation( + body: &mut String, + orig: &str, + typed: &[&EffectiveArg], + depth: usize, +) { + let indent = " ".repeat(depth); + if typed.is_empty() { + writeln!( + body, + "{indent}return [extern::_vw_global_call {orig} {{*}}$flags]" + ) + .unwrap(); + return; + } + if let Some((first, rest)) = typed.split_first() { + // Required typed args have no `ne ""` guard — they're + // always passed. Optional typed args branch on presence. + let id = &first.ident; + let flag = first.flag.as_deref().unwrap_or(id); + let required = first.default.is_none(); + if required { + emit_typed_invocation_with( + body, + orig, + rest, + &[(*first, flag)], + depth, + ); + } else { + writeln!(body, "{indent}if {{${id} ne \"\"}} {{").unwrap(); + emit_typed_invocation_with( + body, + orig, + rest, + &[(*first, flag)], + depth + 1, + ); + writeln!(body, "{indent}}} else {{").unwrap(); + emit_typed_invocation(body, orig, rest, depth + 1); + writeln!(body, "{indent}}}").unwrap(); + } + } +} + +/// Inner: we've decided to include `included` typed args; the +/// remaining `rest` still need branching. At the leaf we emit a +/// `return` with `{*}$flags` and each included typed arg as +/// `-flag $var`. +fn emit_typed_invocation_with( + body: &mut String, + orig: &str, + rest: &[&EffectiveArg], + included: &[(&EffectiveArg, &str)], + depth: usize, +) { + let indent = " ".repeat(depth); + if rest.is_empty() { + // Same `extern::_vw_global_call` helper as the no-typed + // leaf — see [`emit_typed_invocation`] for the rationale + // (including why the `extern::` prefix is on the helper). + let mut line = format!( + "{indent}return [extern::_vw_global_call {orig} {{*}}$flags" + ); + for (arg, flag) in included { + match arg.kind { + ArgKind::Positional => { + // Typed positional args carry Vivado object + // handles (`get_files`, `get_filesets`, + // `get_cells`, …). At the Tcl call level + // Vivado's commands universally take these + // via `- $value`, not positional — see + // the `make_wrapper -files [get_files ...]` + // example in the man page. Emit the flag + // form so both shimmer avoidance AND flag + // routing land at once; a bare positional + // yields `[Common 17-161] Invalid option + // value` because Vivado can't tell which + // slot it was intended for. + write!(line, " -{flag} ${id}", id = arg.ident).unwrap(); + } + _ => { + write!(line, " -{flag} ${id}", id = arg.ident).unwrap(); + } + } + } + line.push(']'); + writeln!(body, "{line}").unwrap(); + return; + } + if let Some((first, more)) = rest.split_first() { + let id = &first.ident; + let flag = first.flag.as_deref().unwrap_or(id); + let required = first.default.is_none(); + if required { + let mut new_included = included.to_vec(); + new_included.push((*first, flag)); + emit_typed_invocation_with(body, orig, more, &new_included, depth); + } else { + writeln!(body, "{indent}if {{${id} ne \"\"}} {{").unwrap(); + let mut new_included = included.to_vec(); + new_included.push((*first, flag)); + emit_typed_invocation_with( + body, + orig, + more, + &new_included, + depth + 1, + ); + writeln!(body, "{indent}}} else {{").unwrap(); + emit_typed_invocation_with(body, orig, more, included, depth + 1); + writeln!(body, "{indent}}}").unwrap(); + } + } +} + +/// Emit `proc { } ? { }` with args and +/// body each indented two spaces. When `return_type` is Some, emits +/// it as the 4th htcl word between the args block and the body — +/// brace-wrapping if the type expression contains whitespace so it +/// parses as a single word. +fn emit_proc( + out: &mut String, + name: &str, + args: &Doc, + return_type: Option<&str>, + body: &str, +) { + let args_text = args.to_string(); + writeln!(out, "proc {name} {{").unwrap(); + for line in args_text.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + match return_type { + Some(ty) => { + // Wrap with `{ … }` if the type expression contains + // whitespace (the htcl parser would otherwise see + // multiple words). + let needs_brace = ty.chars().any(char::is_whitespace); + if needs_brace { + writeln!(out, "}} {{{ty}}} {{").unwrap(); + } else { + writeln!(out, "}} {ty} {{").unwrap(); + } + } + None => { + writeln!(out, "}} {{").unwrap(); + } + } + for line in body.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + writeln!(out, "}}").unwrap(); +} + +/// Infer a return-type annotation from the Vivado man-page's +/// `Returns:` prose. The phrase-table is intentionally small — +/// matches the recurring shapes Vivado uses across hundreds of +/// commands. Unmatched phrasings return `None`; the +/// `cmd-constraints.toml` `returns = "…"` override picks up +/// whatever doesn't match. +/// +/// Matched on the joined, lowercased text — Vivado's prose is +/// short (usually one or two lines) so we don't need a real NLP +/// pipeline. +fn infer_return_type(returns: Option<&[String]>) -> Option { + let lines = returns?; + let joined = lines.join(" ").to_ascii_lowercase(); + let text = joined.trim(); + if text.is_empty() { + return None; + } + // Order matters: more-specific phrases first. Each entry is + // (substring needle, type). A real future implementation + // could swap in regex; substring search is good enough for + // the v1 phrase set. + let table: &[(&str, &str)] = &[ + // Singular creator/current handles. These fire BEFORE the + // `nothing` catchall so a page like "returns the name of + // the newly created cell object, or returns nothing if + // the command fails" gets the meaningful type instead of + // the failure-sentinel `unit`. + // + // Ordering within this group: exact BD types before the + // generic `cell/pin/port/net` forms so `intf_port` + // outranks `port`, etc. + ("newly created interface port object", "bd_intf_port"), + ("newly created interface pin object", "bd_intf_pin"), + ("newly created interface net object", "bd_intf_net"), + ("current interface port object", "bd_intf_port"), + ("current interface pin object", "bd_intf_pin"), + ("current interface net object", "bd_intf_net"), + ("newly created cell object", "bd_cell"), + ("newly created pin object", "bd_pin"), + ("newly created port object", "bd_port"), + ("newly created net object", "bd_net"), + ("newly created master address segment object", "bd_addr_seg"), + ("newly created address segment object", "bd_addr_seg"), + ("newly created address segment", "bd_addr_seg"), + ("current ip integrator cell instance object", "bd_cell"), + ("current cell object", "bd_cell"), + ("current pin object", "bd_pin"), + ("current port object", "bd_port"), + ("current net object", "bd_net"), + ("current instance object", "bd_cell"), + // Report-shaped commands whose prose starts "Returns a + // list of strings …" — `list` even when the rest + // of the description mentions "nothing". + ("returns a list of strings", "list"), + // Name-of-object pages return `string` (a path/name). + ("returns the name of the design object", "string"), + ("name of the design object", "string"), + // "nothing" / "Tcl_OK on success" — side-effecting commands. + // Placed AFTER the creator/current patterns so those + // pull the actual return type from the descriptive prose + // before falling to the failure-sentinel. + ("returns nothing", "unit"), + ("nothing", "unit"), + // Lists of typed handles. Order matters within each type + // family: more-specific "intf" phrasings first so a plain + // "list of pins" doesn't shadow "list of intf_pins" for a + // page whose prose mentions both. + // + // The `list of objects` variants catch phrasing + // Vivado uses on the `get_bd_*` pages ("Gets a list of pin + // objects", "Gets a list of net objects", …). Without them, + // those procs land untyped and the REPL renders their + // return value as one wrapped wall of text instead of + // one-per-line via `list::repr`. + ("a list of intf_pins", "list"), + ("a list of interface pins", "list"), + ("list of intf_pin objects", "list"), + ("list of interface pin objects", "list"), + ("a list of intf_ports", "list"), + ("a list of interface ports", "list"), + ("list of intf_port objects", "list"), + ("list of interface port objects", "list"), + ("a list of intf_nets", "list"), + ("a list of interface nets", "list"), + ("list of intf_net objects", "list"), + ("list of interface net objects", "list"), + ("a list of cells", "list"), + ("a list of bd_cells", "list"), + ("list of cell objects", "list"), + ("a list of pins", "list"), + ("a list of bd_pins", "list"), + ("list of pin objects", "list"), + ("a list of ports", "list"), + ("list of port objects", "list"), + ("a list of nets", "list"), + ("list of net objects", "list"), + // Singular handles. + ("the cell created", "bd_cell"), + ("the new cell", "bd_cell"), + ("the pin created", "bd_pin"), + ("the port created", "bd_port"), + ("the net created", "bd_net"), + // Property values. + ("the property value", "string"), + ("the value of the property", "string"), + ("a list of properties", "list"), + // Generic strings (catch-all when prose says "string" + // explicitly). + ("returns a string", "string"), + ]; + for (needle, ty) in table { + if text.contains(needle) { + return Some((*ty).into()); + } + } + None +} + +/// Make doc-comment text safe to embed inside the proc arg-list braces. +/// +/// The htcl parser captures a proc's arg list as a braced word and +/// brace-matches it raw (only `{`, `}`, `\` are special); a per-arg +/// `##` doc comment with an unbalanced brace or a stray backslash would +/// corrupt that match. Neutralize the three offenders — braces become +/// parentheses, backslashes become slashes — which keeps the prose +/// legible while guaranteeing the generated wrapper parses. +fn sanitize_doc(s: &str) -> String { + s.replace('\\', "/") + .replace('{', "(") + .replace('}', ")") + .trim_end() + .to_string() +} diff --git a/vw-htcl-cmd/src/lib.rs b/vw-htcl-cmd/src/lib.rs new file mode 100644 index 0000000..d90c42c --- /dev/null +++ b/vw-htcl-cmd/src/lib.rs @@ -0,0 +1,77 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Vivado command reference → htcl wrapper generation. +//! +//! The dual of [`vw_ip`]: where that crate turns an IP-XACT component +//! into a configuration-interface proc, this one turns a Vivado Tcl +//! command's plain-text reference page (under +//! `/doc/eng/man`) into a documented, typed htcl wrapper for +//! that command. +//! +//! Each generated wrapper keeps the command's natural name and shadows +//! the Vivado builtin, forwarding to a `rename`-stashed copy of the +//! original. The payoff is the htcl surface: hover documentation drawn +//! from the man page, `@enum`/`@default` validation on flags, and +//! keyword call sites the analyzer can check — all on the real command +//! names. +//! +//! ```no_run +//! let page = vw_htcl_cmd::load("/opt/Vivado/doc/eng/man/add_files", None)?; +//! let htcl = vw_htcl_cmd::generate(&page, &Default::default()); +//! print!("{htcl}"); +//! # Ok::<(), vw_htcl_cmd::Error>(()) +//! ``` + +pub mod constraints; +pub mod generate; +pub mod model; +pub mod parse; + +pub use constraints::{ + ArgOverride, CommandOverride, ConstraintsError, ConstraintsTable, +}; +pub use generate::{generate, GenerateOptions}; +pub use model::{ArgKind, Argument, ManPage}; +pub use parse::parse_man_page; + +use std::path::Path; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("reading man page `{path}`: {source}")] + Io { + path: String, + #[source] + source: std::io::Error, + }, + #[error("cannot derive a command name from `{0}` (no file stem)")] + NoName(String), +} + +pub type Result = std::result::Result; + +/// Load and parse a man page from disk. +/// +/// The command name comes from `name_override` when given, otherwise +/// from the file stem (`.../man/add_files` → `add_files`). +pub fn load( + path: impl AsRef, + name_override: Option<&str>, +) -> Result { + let path = path.as_ref(); + let text = std::fs::read_to_string(path).map_err(|source| Error::Io { + path: path.display().to_string(), + source, + })?; + let name = match name_override { + Some(n) => n.to_string(), + None => path + .file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(|| Error::NoName(path.display().to_string()))? + .to_string(), + }; + Ok(parse_man_page(&name, &text)) +} diff --git a/vw-htcl-cmd/src/model.rs b/vw-htcl-cmd/src/model.rs new file mode 100644 index 0000000..4034f67 --- /dev/null +++ b/vw-htcl-cmd/src/model.rs @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! The structured model of a Vivado command reference ("man page"). +//! +//! Vivado ships a plain-text reference page per Tcl command under +//! `doc/eng/man`. Each page follows a regular shape: +//! +//! ```text +//! Description: +//! +//! +//! +//! Arguments: +//! +//! -fileset - (Optional) +//! -norecurse - (Optional) +//! - (Required) +//! +//! Examples: +//! ... +//! +//! See Also: +//! +//! * import_files +//! * read_ip +//! ``` +//! +//! [`crate::parse`] turns that text into a [`ManPage`]; [`crate::generate`] +//! turns a [`ManPage`] into an htcl wrapper proc. + +/// A parsed Vivado command reference page. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManPage { + /// The command name (e.g. `add_files`). Derived from the source + /// file stem, not the page body — the body never repeats it. + pub name: String, + /// The `Description:` section, de-indented, one entry per source + /// line. Empty lines are preserved as empty strings so paragraph + /// breaks survive into the emitted doc comment. + pub description: Vec, + /// The `Arguments:` section, one entry per documented flag or + /// positional operand, in declared order. + pub arguments: Vec, + /// Command names listed under `See Also:`. + pub see_also: Vec, + /// The raw `Returns:` prose, one entry per source line. Many + /// Vivado man pages don't include this section — it's `None` + /// in that case, and the generator emits the wrapper without + /// a return-type annotation (the REPL falls back to the + /// untyped heuristic for those). + pub returns: Option>, +} + +/// How an argument maps onto the underlying Vivado command line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ArgKind { + /// A `-flag` with no value placeholder: a boolean toggle. Emitted + /// as `@enum(0, 1) @default(0)` and forwarded as a bare `-flag` + /// when set. + Boolean, + /// A `-flag `: forwarded as `-flag $value` when non-empty. + Value, + /// A trailing positional operand (``, ``, …): + /// forwarded by list-expansion (`{*}$operands`) at the end of the + /// command line. + Positional, +} + +/// One documented argument of a command. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Argument { + pub kind: ArgKind, + /// The htcl proc-arg identifier the caller uses as `-`. + /// Equal to `flag` for flags; derived from the `` for + /// positionals. May be de-collided with a suffix. + pub ident: String, + /// The underlying Vivado flag name without its leading dash + /// (`fileset`, `norecurse`). `None` for positionals, which have no + /// flag on the command line. + pub flag: Option, + /// Whether the man page marked the argument `(Required)`. Required + /// arguments are emitted without an `@default`, so htcl forces the + /// caller to supply them. + pub required: bool, + /// Whether this is a generic operand placeholder synthesized by the + /// generator (the page documented no positional), rather than one + /// taken from the page text. + pub synthesized: bool, + /// The argument's prose description, de-indented, one entry per + /// source line (empty strings preserve paragraph breaks). + pub description: Vec, +} + +impl Argument { + /// `true` for flags (`-flag` / `-flag `), `false` for + /// positionals. + pub fn is_flag(&self) -> bool { + matches!(self.kind, ArgKind::Boolean | ArgKind::Value) + } +} diff --git a/vw-htcl-cmd/src/parse.rs b/vw-htcl-cmd/src/parse.rs new file mode 100644 index 0000000..a2639aa --- /dev/null +++ b/vw-htcl-cmd/src/parse.rs @@ -0,0 +1,554 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Parse a Vivado command reference page into a [`ManPage`]. +//! +//! Following the convention in [`vw_htcl::parser`], the outer loop is +//! hand-rolled — it owns line grouping and section recovery, which a +//! pure combinator grammar models awkwardly for free-form reference +//! text — while the structural inner pieces (an argument header's +//! `-flag - (marker)` shape, a `See Also` bullet) are +//! parsed with [`winnow`]. +//! +//! The grammar the inner parsers recognize, per argument block: +//! +//! ```text +//! flag-header := '-' ident placeholder? ' - ' marker? prose +//! pos-header := '<' .. '>' '...'? ' - ' marker? prose +//! marker := '(' ('Optional' | 'Required') ')' +//! ``` +//! +//! A `placeholder` (anything between the flag name and the ` - ` +//! separator) makes a flag a *value* flag; its absence makes it a +//! *boolean* toggle. + +use winnow::ascii::space0; +use winnow::combinator::{opt, preceded}; +use winnow::token::take_while; +use winnow::ModalResult; +use winnow::Parser; + +use crate::model::{ArgKind, Argument, ManPage}; + +/// Parse `text` (the contents of one man page) into a [`ManPage`]. +/// `name` is the command name (the source file stem). +pub fn parse_man_page(name: &str, text: &str) -> ManPage { + let normalized = text.replace('\r', ""); + let sections = split_sections(&normalized); + + let description = section_lines(§ions, "Description") + .map(dedent_block) + .unwrap_or_default(); + + let arguments = section_lines(§ions, "Arguments") + .map(|lines| parse_arguments(&dedent_block(lines))) + .unwrap_or_default(); + + let see_also = section_lines(§ions, "See Also") + .or_else(|| section_lines(§ions, "See also")) + .map(parse_see_also) + .unwrap_or_default(); + + // The `Returns:` section is optional and usually one or two + // lines. Some pages spell it `Return Value` or `Return value` + // — accept both. + let returns = section_lines(§ions, "Returns") + .or_else(|| section_lines(§ions, "Return Value")) + .or_else(|| section_lines(§ions, "Return value")) + .map(dedent_block); + + let mut page = ManPage { + name: name.to_string(), + description, + arguments, + see_also, + returns, + }; + finalize_arguments(&mut page); + page +} + +// --------------------------------------------------------------------------- +// Sectioning (hand-rolled outer loop). +// --------------------------------------------------------------------------- + +/// A man page is a flat list of `Header:` sections. Returns each +/// section's title (without the trailing colon) paired with its raw +/// body lines, in document order. +fn split_sections(text: &str) -> Vec<(String, Vec)> { + let mut sections: Vec<(String, Vec)> = Vec::new(); + for line in text.lines() { + if let Some(title) = section_header(line) { + sections.push((title, Vec::new())); + } else if let Some((_, body)) = sections.last_mut() { + body.push(line.to_string()); + } + // Lines before the first header (a leading blank line, usually) + // are dropped. + } + sections +} + +/// Recognize a section header line — a capitalized label at column +/// zero ending in a colon, e.g. `Arguments:` or `See Also:`. Returns +/// the label without the colon. +fn section_header(line: &str) -> Option { + // Headers sit flush left; body text is indented. Cheap reject + // first. + if line.is_empty() || line.starts_with(' ') { + return None; + } + let stripped = line.strip_suffix(':')?; + if stripped.is_empty() + || !stripped + .chars() + .all(|c| c.is_ascii_alphabetic() || c == ' ') + || !stripped.starts_with(|c: char| c.is_ascii_uppercase()) + { + return None; + } + Some(stripped.to_string()) +} + +/// The body lines of the first section whose title equals `title`. +fn section_lines<'a>( + sections: &'a [(String, Vec)], + title: &str, +) -> Option<&'a [String]> { + sections + .iter() + .find(|(t, _)| t == title) + .map(|(_, body)| body.as_slice()) +} + +/// Strip the uniform two-space indent man-page bodies carry, leaving +/// any deeper (bullet / code) indentation intact, and drop leading and +/// trailing blank lines. Interior blank lines are preserved. +fn dedent_block(lines: &[String]) -> Vec { + let mut out: Vec = lines + .iter() + .map(|l| l.strip_prefix(" ").unwrap_or(l).trim_end().to_string()) + .collect(); + while out.first().is_some_and(|l| l.is_empty()) { + out.remove(0); + } + while out.last().is_some_and(|l| l.is_empty()) { + out.pop(); + } + out +} + +// --------------------------------------------------------------------------- +// Arguments. +// --------------------------------------------------------------------------- + +/// Group the (already de-indented) argument-section lines into blocks +/// — runs of consecutive non-blank lines — then turn each block into +/// an [`Argument`]. Blocks that aren't an argument header (`Note:`, +/// `Tip:`, free prose) are folded into the preceding argument's +/// description. +fn parse_arguments(lines: &[String]) -> Vec { + let mut args: Vec = Vec::new(); + for block in blocks(lines) { + let first = &block[0]; + match parse_arg_header(first) { + Some(header) => { + let mut description = Vec::new(); + let head = header.prose.trim().to_string(); + if !head.is_empty() { + description.push(head); + } + for line in &block[1..] { + description.push(line.trim().to_string()); + } + args.push(Argument { + kind: header.kind, + // Provisional: the positional's placeholder name, or + // empty for a flag. `finalize_arguments` sanitizes + // and de-collides it into the final identifier. + ident: header.name_hint.unwrap_or_default(), + flag: header.flag, + required: header.required, + synthesized: false, + description, + }); + } + None => { + // A `Note:` / `Tip:` / prose continuation block. Attach + // it to the most recent argument, separated by a blank + // line, so the context survives into hover. + if let Some(prev) = args.last_mut() { + prev.description.push(String::new()); + for line in &block { + prev.description.push(line.trim().to_string()); + } + } + } + } + } + args +} + +/// Split lines into blocks of consecutive non-empty lines. +fn blocks(lines: &[String]) -> Vec> { + let mut out: Vec> = Vec::new(); + let mut cur: Vec = Vec::new(); + for line in lines { + if line.trim().is_empty() { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + } else { + cur.push(line.clone()); + } + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +/// The structured outcome of parsing an argument header's first line. +struct ArgHeader { + kind: ArgKind, + /// Flag name without the dash, or `None` for a positional. + flag: Option, + /// A positional's identifier hint, recovered from its `` + /// (e.g. `` → `hw_sio_linkgroups`). `None` for a + /// flag, whose identifier comes from its flag name. + name_hint: Option, + required: bool, + /// The description text that followed the ` - ` separator on the + /// header line. + prose: String, +} + +/// Parse the first line of an argument block. Returns `None` when the +/// line is not a flag/positional header (so the caller treats the +/// block as a note attached to the previous argument). +fn parse_arg_header(line: &str) -> Option { + let mut input = line; + if let Ok(flag) = flag_lead.parse_next(&mut input) { + let (placeholder, prose) = split_separator(input); + let kind = if placeholder.trim().is_empty() { + ArgKind::Boolean + } else { + ArgKind::Value + }; + return Some(ArgHeader { + kind, + flag: Some(flag.to_string()), + name_hint: None, + required: is_required(&prose), + prose, + }); + } + if let Ok(inner) = positional_lead.parse_next(&mut input) { + let (_ellipsis, prose) = split_separator(input); + return Some(ArgHeader { + kind: ArgKind::Positional, + flag: None, + name_hint: first_ident_token(inner), + required: is_required(&prose), + prose, + }); + } + None +} + +/// The first `[A-Za-z_][A-Za-z0-9_]*` token in a positional's +/// placeholder text (`hw_sio_linkgroups`, `arg1 arg2 ...` → `arg1`). +/// `None` when the placeholder has no identifier-shaped run (`[0:750]`). +fn first_ident_token(inner: &str) -> Option { + let mut chars = inner.char_indices().peekable(); + while let Some(&(start, c)) = chars.peek() { + if c.is_ascii_alphabetic() || c == '_' { + let mut end = start; + for (i, c) in inner[start..].char_indices() { + if c.is_ascii_alphanumeric() || c == '_' { + end = start + i + c.len_utf8(); + } else { + break; + } + } + return Some(inner[start..end].to_string()); + } + chars.next(); + } + None +} + +/// `-ident` — consume the dash and flag name, leaving the rest of the +/// line in `input`. Returns the flag name without the dash. +fn flag_lead<'s>(input: &mut &'s str) -> ModalResult<&'s str> { + preceded('-', ident).parse_next(input) +} + +/// `<...>` (with an optional trailing `...`) — consume the angle-bracket +/// placeholder that introduces a positional operand, leaving the rest +/// of the line in `input`. Returns the text inside the brackets. +fn positional_lead<'s>(input: &mut &'s str) -> ModalResult<&'s str> { + '<'.parse_next(input)?; + let inner = take_while(0.., |c: char| c != '>').parse_next(input)?; + '>'.parse_next(input)?; + let _ = opt("...").parse_next(input)?; + // Note: do not consume the space after `>` — it is part of the + // ` - ` separator that `split_separator` looks for. + Ok(inner) +} + +/// An htcl-identifier run: `[A-Za-z0-9_]+`. +fn ident<'s>(input: &mut &'s str) -> ModalResult<&'s str> { + take_while(1.., |c: char| c.is_ascii_alphanumeric() || c == '_') + .parse_next(input) +} + +/// Split a header remainder on its first ` - ` separator into the +/// (placeholder, description) halves. With no separator the whole +/// remainder is taken as the description (and the placeholder is +/// empty), which makes a bare `-flag` a boolean toggle. +fn split_separator(rest: &str) -> (String, String) { + match rest.find(" - ") { + Some(idx) => ( + rest[..idx].trim().to_string(), + rest[idx + 3..].trim().to_string(), + ), + None => (String::new(), rest.trim().to_string()), + } +} + +/// Whether an argument's description marks it `(Required)`. +fn is_required(prose: &str) -> bool { + prose + .trim_start() + .to_ascii_lowercase() + .starts_with("(required") +} + +// --------------------------------------------------------------------------- +// See Also. +// --------------------------------------------------------------------------- + +/// Extract the command names from `See Also` bullet lines +/// (` * get_clocks`). +fn parse_see_also(lines: &[String]) -> Vec { + let mut out = Vec::new(); + for line in lines { + if let Ok(name) = see_also_entry.parse_next(&mut line.as_str()) { + if !name.is_empty() { + out.push(name.to_string()); + } + } + } + out +} + +/// ` * ` — a See-Also bullet. Returns the command name. +fn see_also_entry<'s>(input: &mut &'s str) -> ModalResult<&'s str> { + space0.parse_next(input)?; + '*'.parse_next(input)?; + space0.parse_next(input)?; + take_while(1.., |c: char| c.is_ascii_alphanumeric() || c == '_') + .parse_next(input) +} + +// --------------------------------------------------------------------------- +// Finalization: identifiers, de-collision, synthesized operands. +// --------------------------------------------------------------------------- + +/// Assign final htcl identifiers, de-collide duplicates, and synthesize +/// a generic trailing operand when the page documented no positional. +fn finalize_arguments(page: &mut ManPage) { + let mut used: std::collections::HashSet = + std::collections::HashSet::new(); + let mut has_positional = false; + + let args = std::mem::take(&mut page.arguments); + for mut arg in args { + let base = match arg.kind { + ArgKind::Positional => { + has_positional = true; + if arg.ident.is_empty() { + "operands".to_string() + } else { + arg.ident.clone() + } + } + _ => arg.flag.clone().unwrap_or_else(|| "arg".to_string()), + }; + let base = sanitize_ident(&base); + // A duplicate flag (the page listing `-foo` twice) is dropped; + // a positional that collides with a flag is renamed. + if used.contains(&base) { + if arg.is_flag() { + continue; + } + arg.ident = unique_ident(&base, &mut used); + } else { + used.insert(base.clone()); + arg.ident = base; + } + page.arguments.push(arg); + } + + if !has_positional { + let ident = unique_ident("operands", &mut used); + page.arguments.push(Argument { + kind: ArgKind::Positional, + ident, + flag: None, + required: false, + synthesized: true, + description: vec![ + "Positional operands passed through to the underlying \ + command (object patterns, names, files, …)." + .to_string(), + ], + }); + } +} + +/// First unused identifier in the `base`, `base_2`, `base_3`, … family. +fn unique_ident( + base: &str, + used: &mut std::collections::HashSet, +) -> String { + let base = sanitize_ident(base); + if used.insert(base.clone()) { + return base; + } + for n in 2.. { + let candidate = format!("{base}_{n}"); + if used.insert(candidate.clone()) { + return candidate; + } + } + unreachable!("exhausted identifier suffixes") +} + +/// Coerce an arbitrary string into the htcl proc-arg grammar +/// (`[A-Za-z_][A-Za-z0-9_]*`). Non-conforming characters become +/// underscores; a digit-leading or empty result gets a leading +/// underscore. Never produces the Tcl-reserved varargs name `args`. +fn sanitize_ident(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 1); + for c in s.chars() { + if c.is_ascii_alphanumeric() || c == '_' { + out.push(c); + } else { + out.push('_'); + } + } + let needs_lead = out + .as_bytes() + .first() + .map(|b| b.is_ascii_digit()) + .unwrap_or(true); + if needs_lead { + out.insert(0, '_'); + } + if out == "args" { + out = "args_".to_string(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn header(line: &str) -> ArgHeader { + parse_arg_header(line).expect("expected an argument header") + } + + #[test] + fn classifies_value_flag() { + let h = header("-fileset - (Optional) The fileset."); + assert_eq!(h.kind, ArgKind::Value); + assert_eq!(h.flag.as_deref(), Some("fileset")); + assert!(!h.required); + assert!(h.prose.starts_with("(Optional)")); + } + + #[test] + fn classifies_boolean_flag() { + let h = header("-norecurse - (Optional) Do not recurse."); + assert_eq!(h.kind, ArgKind::Boolean); + assert_eq!(h.flag.as_deref(), Some("norecurse")); + } + + #[test] + fn required_value_flag() { + let h = header("-period - (Required) The period."); + assert_eq!(h.kind, ArgKind::Value); + assert!(h.required); + } + + #[test] + fn multiword_placeholder_is_value_flag() { + let h = header("-waveform - (Optional) Edges."); + assert_eq!(h.kind, ArgKind::Value); + } + + #[test] + fn classifies_positional_and_recovers_name() { + let h = header(" - (Required) Objects to remove."); + assert_eq!(h.kind, ArgKind::Positional); + assert_eq!(h.name_hint.as_deref(), Some("hw_sio_linkgroups")); + assert!(h.required); + } + + #[test] + fn positional_without_marker_is_optional() { + let h = header(" - Version of the library."); + assert_eq!(h.kind, ArgKind::Positional); + assert!(!h.required); + } + + #[test] + fn non_header_block_is_rejected() { + assert!(parse_arg_header("Note: this is a note.").is_none()); + assert!(parse_arg_header("Plain prose continuation.").is_none()); + } + + #[test] + fn first_ident_token_extraction() { + assert_eq!(first_ident_token("name").as_deref(), Some("name")); + assert_eq!(first_ident_token("arg1 arg2 ...").as_deref(), Some("arg1")); + assert_eq!(first_ident_token("[0:750]"), None); + } + + #[test] + fn sanitize_ident_never_yields_varargs() { + assert_eq!(sanitize_ident("args"), "args_"); + assert_eq!(sanitize_ident("64bit"), "_64bit"); + assert_eq!(sanitize_ident("a-b.c"), "a_b_c"); + } + + #[test] + fn de_collides_positional_against_flag() { + // A flag `-name` and a positional `` must not collide. + let page = parse_man_page( + "demo", + "\nArguments:\n\n -name - (Optional) The flag.\n\n \ + - (Required) The operand.\n", + ); + let idents: Vec<&str> = + page.arguments.iter().map(|a| a.ident.as_str()).collect(); + assert!(idents.contains(&"name")); + assert!(idents.contains(&"name_2")); + } + + #[test] + fn drops_duplicate_flag() { + let page = parse_man_page( + "demo", + "\nArguments:\n\n -quiet - (Optional) Quietly.\n\n \ + -quiet - (Optional) Quietly again.\n", + ); + let quiets = + page.arguments.iter().filter(|a| a.ident == "quiet").count(); + assert_eq!(quiets, 1); + } +} diff --git a/vw-htcl-cmd/tests/generate.rs b/vw-htcl-cmd/tests/generate.rs new file mode 100644 index 0000000..2794570 --- /dev/null +++ b/vw-htcl-cmd/tests/generate.rs @@ -0,0 +1,310 @@ +// Integration tests: parse synthetic and real man pages, then prove the +// generated htcl re-parses cleanly through `vw_htcl` (the same parser +// `vw check` and the LSP use). + +use std::path::Path; + +use vw_htcl_cmd::{generate, parse_man_page, ArgKind, GenerateOptions}; + +/// A man page exercising every argument shape: required value flag, +/// optional value flag, boolean flag, a multi-word placeholder, +/// required positional, optional positional, and a `Note:` block that +/// must fold into the preceding argument. +const SAMPLE: &str = " +Description: + + Creates a thing. Pass it a list like {a b c} and it just works. + + Returns the created thing, or an error if it fails. + +Arguments: + + -period - (Required) The period, must be > 0. + + -name - (Optional) The name of the thing. + + -waveform - (Optional) Edge times. + + -add - (Optional) Add instead of replace. + + -quiet - (Optional) Execute quietly. + + Note: errors on the command line are still returned. + + - (Required) The source objects. + +Examples: + + make_thing -period 10 + +See Also: + + * destroy_thing + * get_things +"; + +fn assert_reparses(htcl: &str) { + let parsed = vw_htcl::parse(htcl); + assert!( + parsed.errors.is_empty(), + "generated htcl failed to parse: {:#?}\n---\n{htcl}", + parsed.errors + ); +} + +#[test] +fn parses_every_argument_shape() { + let page = parse_man_page("make_thing", SAMPLE); + + assert_eq!(page.name, "make_thing"); + assert_eq!(page.see_also, vec!["destroy_thing", "get_things"]); + + let by_ident = |id: &str| { + page.arguments + .iter() + .find(|a| a.ident == id) + .unwrap_or_else(|| panic!("missing arg {id}")) + }; + + let period = by_ident("period"); + assert_eq!(period.kind, ArgKind::Value); + assert!(period.required); + + let name = by_ident("name"); + assert_eq!(name.kind, ArgKind::Value); + assert!(!name.required); + + let waveform = by_ident("waveform"); + assert_eq!(waveform.kind, ArgKind::Value, "multi-word placeholder"); + + let add = by_ident("add"); + assert_eq!(add.kind, ArgKind::Boolean); + + // The `Note:` block must have folded into -quiet's description. + let quiet = by_ident("quiet"); + assert!( + quiet + .description + .iter() + .any(|l| l.contains("still returned")), + "Note block did not fold into -quiet: {:?}", + quiet.description + ); + + let objects = by_ident("objects"); + assert_eq!(objects.kind, ArgKind::Positional); + assert!(objects.required); + // A positional was documented, so none is synthesized. + assert!(page.arguments.iter().all(|a| !a.synthesized)); +} + +#[test] +fn generated_wrapper_reparses() { + let page = parse_man_page("make_thing", SAMPLE); + let htcl = generate(&page, &GenerateOptions::default()); + + // Doc braces are neutralized so the arg-list brace match survives. + assert!( + htcl.contains("{a b c}".replace('{', "(").replace('}', ")").as_str()) + ); + // Natural name + extern-prefixed forward (lowering autogen + // produces the rename plumbing at session startup). + assert!(htcl.contains("proc make_thing {")); + assert!(!htcl.contains("rename")); + // New body shape: non-typed args accumulate into `flags` via + // lappend (safe — strings only). Typed args (objects, cell, + // pin, ...) are passed directly via `-flag $value` so Vivado's + // typed Tcl_Obj survives. + assert!( + htcl.contains("lappend flags -period $period"), + "non-typed value-flag should lappend into flags: {htcl}" + ); + assert!( + htcl.contains("if {$add} { lappend flags -add }"), + "boolean should lappend into flags only when true: {htcl}" + ); + // `objects` is in TYPED_ARG_NAMES → must NOT be lappended + // (would shimmer the typed handle). Must appear in a direct + // invocation as `-objects $objects` or as positional `$objects`. + assert!( + !htcl.contains("lappend flags {*}$objects"), + "typed arg `objects` must not be lappended: {htcl}" + ); + assert!( + !htcl.contains("lappend cmd"), + "old `cmd`-accumulator shape should be gone: {htcl}" + ); + assert!( + htcl.contains("extern::make_thing {*}$flags"), + "direct invocation with {{*}}$flags expected: {htcl}" + ); + assert!( + htcl.contains("$objects"), + "objects must be referenced in the invocation: {htcl}" + ); + + assert_reparses(&htcl); +} + +#[test] +fn synthesizes_operand_when_no_positional() { + let page = parse_man_page( + "current_thing", + "\nDescription:\n\n Gets the current thing.\n\nArguments:\n\n \ + -quiet - (Optional) Quietly.\n", + ); + let synth: Vec<_> = + page.arguments.iter().filter(|a| a.synthesized).collect(); + assert_eq!(synth.len(), 1, "exactly one synthesized operand"); + assert_eq!(synth[0].kind, ArgKind::Positional); + assert!(!synth[0].required); + + assert_reparses(&generate(&page, &GenerateOptions::default())); +} + +#[test] +fn empty_man_page_still_generates_valid_wrapper() { + // No Description, no Arguments — the generator must still emit a + // parseable, self-contained wrapper. + let page = parse_man_page("noop", ""); + let htcl = generate(&page, &GenerateOptions::default()); + assert!(htcl.contains("proc noop {")); + assert_reparses(&htcl); +} + +/// Smoke test over the real Vivado man pages when a local install is +/// present: every page must generate htcl that re-parses cleanly. +#[test] +fn real_man_pages_reparse() { + let dir = "/home/ry/Xilinx/2025.1/Vivado/doc/eng/man"; + if !Path::new(dir).exists() { + eprintln!("skipping: {dir} not present"); + return; + } + let mut checked = 0; + let mut failures = Vec::new(); + let mut stack = vec![std::path::PathBuf::from(dir)]; + while let Some(d) = stack.pop() { + for entry in std::fs::read_dir(&d).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + continue; + } + let stem = match path.file_name().and_then(|s| s.to_str()) { + Some(s) + if s.chars().all(|c| { + c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' + }) => + { + s + } + _ => continue, // skip tmp.* / *_Copy junk + }; + let text = std::fs::read_to_string(&path).unwrap(); + let page = parse_man_page(stem, &text); + let htcl = generate(&page, &GenerateOptions::default()); + let parsed = vw_htcl::parse(&htcl); + if !parsed.errors.is_empty() { + failures.push(format!("{stem}: {:?}", parsed.errors)); + } + checked += 1; + } + } + eprintln!("checked {checked} real man pages"); + assert!(checked > 500, "expected many man pages, saw {checked}"); + assert!( + failures.is_empty(), + "{} man pages produced unparseable htcl:\n{}", + failures.len(), + failures.join("\n") + ); +} + +// --- return-type emission (step 5) ----------------------------------------- + +#[test] +fn returns_section_emits_type_annotation() { + let src = " +Description: + + Returns a list of cells matching the search. + +Arguments: + + -hierarchical - (Optional) Search hierarchically. + +Returns: + + a list of cells + +See Also: + + * get_cells +"; + let page = parse_man_page("get_things", src); + assert!(page.returns.is_some(), "Returns: section should be parsed"); + let htcl = generate(&page, &GenerateOptions::default()); + assert_reparses(&htcl); + // The `proc get_things { … } list { … }` shape. + assert!( + htcl.contains("list {"), + "expected return-type annotation in: {htcl}" + ); +} + +#[test] +fn returns_section_nothing_emits_unit() { + let src = " +Description: + + Sets things. + +Arguments: + + -quiet - (Optional) Quiet. + +Returns: + + Returns nothing. + +See Also: + + * unset_things +"; + let page = parse_man_page("set_things", src); + let htcl = generate(&page, &GenerateOptions::default()); + assert_reparses(&htcl); + assert!( + htcl.contains(" unit {"), + "expected `unit` return annotation in: {htcl}" + ); +} + +#[test] +fn page_without_returns_section_emits_no_annotation() { + let src = " +Description: + + Does a thing. + +Arguments: + + -x - (Required) Thing. + +See Also: + + * other +"; + let page = parse_man_page("do_a_thing", src); + assert!(page.returns.is_none()); + let htcl = generate(&page, &GenerateOptions::default()); + assert_reparses(&htcl); + // No return-type annotation present. + assert!( + !htcl.contains("unit {") + && !htcl.contains("bd_cell {") + && !htcl.contains("list<"), + "unannotated page should not synthesize a return type: {htcl}" + ); +} diff --git a/vw-htcl/Cargo.toml b/vw-htcl/Cargo.toml new file mode 100644 index 0000000..91aecab --- /dev/null +++ b/vw-htcl/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "vw-htcl" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "htcl language layer: parser, AST, name resolution, signature checking, TCL emission" + +[dependencies] +serde.workspace = true +thiserror.workspace = true +winnow.workspace = true +camino.workspace = true +vw-quote = { path = "../vw-quote" } + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-htcl/src/ast.rs b/vw-htcl/src/ast.rs new file mode 100644 index 0000000..a533063 --- /dev/null +++ b/vw-htcl/src/ast.rs @@ -0,0 +1,585 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Concrete syntax tree for htcl. +//! +//! Every node carries a [`Span`] so the same tree drives diagnostics, +//! hover, navigation, and source-faithful lowering back to TCL. The +//! tree is concrete in the sense that it retains enough information to +//! recover the original source (comments, blank lines, word forms); +//! later passes derive a stripped AST for analysis. + +use crate::span::Span; + +#[derive(Clone, Debug)] +pub struct Document { + pub stmts: Vec, + pub span: Span, +} + +// `Stmt::Command(Command)` is ~320 bytes while the other variants +// are <50; clippy flags the size disparity and suggests boxing +// `Command`. We don't box because: +// - Commands are by far the most common variant (often >95% of +// Stmt instances in real source), so the boxed-pointer +// indirection on the hot path would cost more than the +// wasted bytes in rare Comment/Error variants. +// - Boxing would ripple through ~50 pattern-match sites +// (`let Stmt::Command(cmd) = ...`) and complicate the AST's +// "by-value clone-and-mutate" rewrite passes. +#[derive(Clone, Debug)] +#[allow(clippy::large_enum_variant)] +pub enum Stmt { + Command(Command), + Comment(Comment), + Error(ParseFailure), +} + +impl Stmt { + pub fn span(&self) -> Span { + match self { + Stmt::Command(c) => c.span, + Stmt::Comment(c) => c.span, + Stmt::Error(e) => e.span, + } + } +} + +/// A single TCL command — a whitespace-separated sequence of words, +/// terminated by newline, semicolon, or EOF. +#[derive(Clone, Debug)] +pub struct Command { + pub words: Vec, + pub span: Span, + pub kind: CommandKind, + /// Doc comments (`##`) immediately preceding the command, in + /// source order with the `##` prefix stripped. + pub doc_comments: Vec, + /// Source span covering the whole `##` block — from the first + /// `#` of the first line to the newline after the last line. + /// `None` when the command has no doc comments. Used by the + /// analyzer to answer "is the cursor inside this command's doc + /// block?" so `[NAME]` references inside `##` text can resolve + /// via goto/hover. + pub doc_comments_span: Option, +} + +/// Recognized command shapes. Generic covers any unrecognized command; +/// specific variants exist so downstream passes (symbol tables, the +/// LSP, the structured-proc work in Phase 2) can act on them without +/// re-parsing. +#[derive(Clone, Debug)] +pub enum CommandKind { + Generic, + Set, + Proc(Proc), + Src(SrcImport), + NamespaceEval(NamespaceEval), + /// A `type = ` declaration. Compile-time only + /// — never lowered to Tcl. Together with the required + /// `::repr` / `from` / `to` procs (enforced by the + /// validator), introduces a new newtype the rest of the program + /// can reference in return-type annotations and (later) arg + /// annotations. + TypeDecl(TypeDecl), + /// An `enum = { }` declaration. Compile-time + /// only — the lowerer emits the auto-generated constructor / + /// repr / accessor procs through the repr-codegen path, NOT via + /// shipping the source verbatim. Variants are + /// `IDENT (':' TYPE)?`; the optional `:TYPE` payload makes + /// empty-payload variants first-class. + EnumDecl(EnumDecl), +} + +/// A `namespace eval { }` block. +/// +/// Recognized at parse time so that any `proc` declarations inside +/// the braces register in the document's signature table under the +/// qualified name `::` (Tcl namespace semantics), and +/// the analyzer can offer the same hover / completion / signature +/// help / goto experience for namespaced procs as for top-level +/// ones. The body parses as a script just like a proc body, so +/// nested `namespace eval` blocks compose. +#[derive(Clone, Debug)] +pub struct NamespaceEval { + /// Bare-text namespace name when extractable (the common case), + /// `None` when the name word couldn't be reduced to literal text + /// (e.g. it contains substitutions). Multi-segment names like + /// `foo::bar` are preserved as-is and the analyzer uses them as + /// the full prefix. + pub name: Option, + pub name_span: Span, + pub body_span: Span, + /// The body parsed into statements. Spans are absolute (whole- + /// source) coordinates, same convention as [`Proc::body`]. + pub body: Vec, +} + +/// A `src ` import — load and evaluate another htcl module. +/// +/// The path's *form* is classified at load time, not here: leading +/// `@name/` resolves through the workspace's `vw.toml` dependencies, +/// a leading `/` is filesystem-absolute, anything else is relative to +/// the importing file's directory. `path` is `None` only when the +/// path word couldn't be reduced to literal text (e.g. it contains +/// `$var` / `[cmd]` substitutions); those imports are diagnosed +/// downstream rather than parsed structurally. +#[derive(Clone, Debug)] +pub struct SrcImport { + pub path: Option, + pub path_span: Span, +} + +/// A `proc` declaration. +/// +/// The outer shape (name, args span, body span) comes from the Phase 0 +/// parser. The structured args grammar (Phase 2) is reparsed from +/// `args_span` and stored in [`signature`](Self::signature). When +/// `signature` is `None` the args body couldn't be parsed at all +/// (e.g. mid-edit syntax error); diagnostics for that live in the +/// document's parse-error list. +#[derive(Clone, Debug)] +pub struct Proc { + /// Bare-text proc name when it could be extracted; `None` for + /// programmatically-named procs (e.g. names built from + /// substitution). + pub name: Option, + pub name_span: Span, + pub args_span: Span, + pub body_span: Span, + pub signature: Option, + /// Optional return-type annotation: the 4th word of a + /// `proc NAME { args } TYPE { body }` declaration, parsed as a + /// [`TypeExpr`]. `None` means "no annotation present" — the + /// proc still works, but downstream type-driven machinery + /// (REPL repr printer, hover, future call-site validation) + /// falls back to its untyped path. Bracketed forms like + /// `{dict}` are unwrapped before type-parsing. + pub return_type: Option, + /// Source span of the 4th-word type slot when present; `None` + /// when the proc has no annotation. The span covers the outer + /// word including any wrapping braces, so diagnostics can + /// underline the entire annotation. + pub return_type_span: Option, + /// The body parsed into statements, with spans in absolute + /// (whole-source) coordinates. Populated by a post-pass after the + /// outer parse; empty until then and for bodies that are pure + /// braced text with no commands. Lowering still ships the body + /// verbatim from [`body_span`](Self::body_span) — this field + /// exists so navigation, hover, and analysis can see *into* a + /// proc body. Nested procs declared here have their own `body` + /// populated recursively. + pub body: Vec, + /// Attributes attached to this proc declaration itself (as + /// opposed to [`ProcArg::attributes`], which live on individual + /// args). Populated by the parser from `@name(…)` items that + /// appear at statement position immediately BEFORE the `proc` + /// keyword. Mirrors the doc-comment attachment pattern the + /// parser already uses. Currently used by `vw test` to + /// recognize `@test`- and `@test(dedicated-eda)`-marked procs. + pub attributes: Vec, +} + +impl Proc { + /// Look up a proc-level attribute by name. Returns the first + /// match — attributes with duplicate names are unusual but not + /// rejected at the parse level. + pub fn attribute(&self, name: &str) -> Option<&Attribute> { + self.attributes.iter().find(|a| a.name == name) + } +} + +/// A `type NAME = UNDERLYING` declaration. +/// +/// Introduces a newtype wrapper around an existing type. The +/// validator requires the user to ALSO define three procs in the +/// `::` namespace: `repr` (rendering to a `string`), `from` +/// (lifting an underlying value, with optional validation), and +/// `to` (extracting the underlying value). See the validator for +/// the exact signature shapes enforced. +/// +/// Compile-time only — never lowered to Tcl. The newtype's runtime +/// representation is identical to the underlying type; the +/// distinction lives entirely in the analyzer / printer / future +/// type-checker. +#[derive(Clone, Debug)] +pub struct TypeDecl { + /// Bare-text type name when extractable; `None` for + /// programmatically-named declarations (vanishingly rare; + /// kept consistent with [`Proc::name`]'s convention). + pub name: Option, + pub name_span: Span, + /// The underlying type, parsed from the right-hand side of `=`. + /// `None` when the right-hand side couldn't be parsed as a + /// type expression (e.g. mid-edit). Diagnostics for that live + /// in the document's parse-error list. + pub underlying: Option, + /// Span of the underlying-type word (outer, including any + /// wrapping braces) so diagnostics can underline it. + pub underlying_span: Span, +} + +/// An `enum = { }` declaration. The body is +/// brace-wrapped and newline-separated; each variant is +/// `IDENT (':' TYPE)?`. +/// +/// Compile-time only — codegen lowers this to a `namespace eval +/// { … }` block containing auto-generated constructors, +/// `repr`/`from`/`to`, and `tag`/`payload` accessors. The +/// validator enforces variant-name uniqueness within an enum and +/// that variant payload types reference known type names. +#[derive(Clone, Debug)] +pub struct EnumDecl { + /// Bare-text enum name when extractable. + pub name: Option, + pub name_span: Span, + /// Declared variants, in source order. + pub variants: Vec, + /// Span of the brace-wrapped variants block (outer, including + /// the braces) so diagnostics can underline it. + pub body_span: Span, +} + +/// One variant inside an [`EnumDecl`]. `payload` is `None` for +/// empty-payload variants (e.g. `North` in `enum Direction = { +/// North; South: int }`). +#[derive(Clone, Debug)] +pub struct EnumVariant { + pub name: String, + pub name_span: Span, + pub payload: Option, + /// Span of the payload-type word, or zero-length at the + /// variant's end position for empty-payload variants. + pub payload_span: Span, + /// Span covering the full `NAME (':' TYPE)?` form. + pub span: Span, +} + +/// Side-table entry produced by the validator's overload-classifier +/// pass: for a public proc name that resolved to an enum-overload +/// set, records which enum drives dispatch and where each variant's +/// specialization lives. The codegen step uses this to synthesize +/// the public dispatcher proc; the analyzer uses it to render +/// overload information in hover / signature help. +#[derive(Clone, Debug)] +pub struct OverloadInfo { + /// The public, user-facing proc name (e.g. `handle_prop`). + pub public_name: String, + /// The enum this overload set dispatches on (e.g. `Property`). + pub enum_name: String, + /// Shared arg name across all overload arms (e.g. `v`). The + /// validator enforces every arm uses the same name so the + /// dispatcher can pass the payload via the kwargs protocol + /// (`- `) without per-arm gymnastics. + pub dispatch_arg_name: String, + /// One entry per variant, in declaration order on the enum. + pub variants: Vec, + /// Span on the first overload's name — used as the diagnostic + /// anchor for overload-set-wide errors. + pub anchor_span: Span, +} + +#[derive(Clone, Debug)] +pub struct OverloadVariant { + /// The variant short-name (e.g. `Scalar`, `Nested`). + pub variant_name: String, + /// The mangled internal proc name the specialization runs + /// under at runtime (e.g. `__handle_prop__Scalar`). + pub mangled_proc_name: String, + /// Span of the variant-arg annotation on the specialization's + /// first argument — diagnostic anchor when something's + /// specifically wrong with this arm. + pub dispatch_arg_span: Span, +} + +/// A type expression — the syntactic form of a type used in +/// `proc NAME { args } TYPE { body }` return annotations and on the +/// right-hand side of `type NAME = TYPE` declarations. +/// +/// Newtypes (`bd_cell`, `widget`, user inventions) and primitives +/// (`string`, `int`, `bool`, `unit`) share the [`Named`] variant — +/// the distinction lives in the validator's type table, not the +/// AST. Containers (`list`, `dict`, and any future shape +/// with the same `name` surface) are [`Generic`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TypeExpr { + Named { + name: String, + span: Span, + }, + Generic { + name: String, + name_span: Span, + args: Vec, + /// Full span including `<` … `>`. + span: Span, + }, + /// `Enum::Variant` — a qualified path naming a single variant + /// of a declared enum. Only legal as an arg-type annotation on + /// an overloaded handler proc (the dispatch indicator); the + /// validator rejects this variant anywhere else (return types, + /// generic args, nested positions). + Qualified { + namespace: String, + variant: String, + namespace_span: Span, + variant_span: Span, + /// Full span covering `namespace::variant`. + span: Span, + }, +} + +impl TypeExpr { + pub fn name(&self) -> &str { + match self { + TypeExpr::Named { name, .. } | TypeExpr::Generic { name, .. } => { + name + } + TypeExpr::Qualified { namespace, .. } => namespace, + } + } + + pub fn span(&self) -> Span { + match self { + TypeExpr::Named { span, .. } + | TypeExpr::Generic { span, .. } + | TypeExpr::Qualified { span, .. } => *span, + } + } +} + +/// Structured proc-argument signature. +/// +/// One entry per declared argument, in source order. The order is the +/// canonical positional order used when lowering keyword-arg call +/// sites to Tcl-positional calls for the EDA backend. +#[derive(Clone, Debug)] +pub struct ProcSignature { + pub args: Vec, + pub span: Span, + /// The declared return type, copied here from [`Proc::return_type`] + /// at parse time so the signature-table-based lookup paths + /// (REPL formatter, hover) don't have to re-walk back to the + /// Proc node. `None` for unannotated procs. + pub return_type: Option, +} + +impl ProcSignature { + pub fn find(&self, name: &str) -> Option<&ProcArg> { + self.args.iter().find(|a| a.name == name) + } +} + +#[derive(Clone, Debug)] +pub struct ProcArg { + pub name: String, + pub name_span: Span, + pub doc_comments: Vec, + /// Source span covering the whole `##` block that attaches to + /// this arg — `None` when the arg has no doc comments. See + /// [`Command::doc_comments_span`] for the analyzer-side rationale. + pub doc_comments_span: Option, + pub attributes: Vec, + /// Optional `: TYPE` annotation on the arg. `Some` when the + /// source carries `name: bd_cell` style; `None` when the arg + /// is untyped (the legacy form). Used by the validator (full + /// shape check on newtype repr/from/to procs) and by the + /// analyzer's hover / signature-help displays. + pub type_annotation: Option, + pub span: Span, +} + +impl ProcArg { + pub fn attribute(&self, name: &str) -> Option<&Attribute> { + self.attributes.iter().find(|a| a.name == name) + } +} + +/// Raw attribute as parsed: name plus zero or more comma-separated +/// values. Semantic interpretation (default, required, enum, range, +/// requires, conflicts, deprecated) lives in the validators, not +/// here — keeping the AST shape unopinionated lets new attribute +/// names land without a parser change. +#[derive(Clone, Debug)] +pub struct Attribute { + pub name: String, + pub name_span: Span, + pub values: Vec, + pub span: Span, +} + +#[derive(Clone, Debug)] +pub enum AttributeValue { + Integer { + value: i64, + span: Span, + }, + String { + value: String, + span: Span, + }, + Ident { + value: String, + span: Span, + }, + /// A `key=value` item in an attribute value list, e.g. + /// `@test(dedicated-eda part=xcvm3358-vsvh1747-2M-e-S)` where + /// `part=xcvm3358…` produces a `Keyed { key: "part", value: + /// Ident(…) }`. Positional items in the same list continue to + /// use the plain [`Integer`]/[`String`]/[`Ident`] variants, + /// so old consumers keep working. + Keyed { + key: String, + key_span: Span, + value: Box, + span: Span, + }, +} + +impl AttributeValue { + pub fn span(&self) -> Span { + match self { + AttributeValue::Integer { span, .. } + | AttributeValue::String { span, .. } + | AttributeValue::Ident { span, .. } + | AttributeValue::Keyed { span, .. } => *span, + } + } + + /// If this item is a `key=value` pair, return `(key, + /// inner_value)`. `None` for positional items. + pub fn as_keyed(&self) -> Option<(&str, &AttributeValue)> { + match self { + AttributeValue::Keyed { key, value, .. } => Some((key, value)), + _ => None, + } + } + + /// Render the value back to a Tcl-style literal, suitable for + /// comparison against a runtime arg or for emitting in lowered + /// Tcl. Integers and idents stringify as-is; strings get + /// double-quoted with naive escaping. + pub fn to_tcl_literal(&self) -> String { + match self { + AttributeValue::Integer { value, .. } => value.to_string(), + AttributeValue::Ident { value, .. } => value.clone(), + AttributeValue::String { value, .. } => { + let escaped = value.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") + } + AttributeValue::Keyed { key, value, .. } => { + format!("{key}={}", value.to_tcl_literal()) + } + } + } + + pub fn as_str(&self) -> &str { + match self { + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value, + AttributeValue::Integer { .. } => "", + AttributeValue::Keyed { .. } => "", + } + } +} + +#[derive(Clone, Debug)] +pub struct Comment { + /// Comment text with the leading `#` removed; for `##` doc + /// comments, both `#`s are removed. + pub text: String, + pub span: Span, + pub is_doc: bool, +} + +#[derive(Clone, Debug)] +pub struct ParseFailure { + pub message: String, + pub span: Span, +} + +#[derive(Clone, Debug)] +pub struct Word { + pub form: WordForm, + pub parts: Vec, + pub span: Span, + /// Populated at parse time for braced-word args that are + /// KNOWN to be script bodies: the third arg of `foreach`, the + /// fourth arg of `dict for`, both branches of `if`, etc. + /// `None` for everything else — including bare braced strings + /// and braced expressions. + /// + /// Interior spans are absolute (whole-source) coordinates, + /// same convention as [`Proc::body`]. Populated by + /// `parser::populate_procs` after the initial word parse. + /// + /// Downstream tools (validator, putr rewriter, LSP hover / + /// goto, syntax highlighter) descend into this when + /// non-`None` so behavior inside a control-flow body matches + /// behavior at the top level. + pub body: Option>, +} + +impl Word { + /// If this word is a single literal text part (no interpolation), + /// return its value. Useful for matching command names, fixed + /// keywords, and option flags without rebuilding the string. + pub fn as_text(&self) -> Option<&str> { + match self.parts.as_slice() { + [WordPart::Text { value, .. }] => Some(value), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WordForm { + Bare, + Quoted, + Braced, +} + +#[derive(Clone, Debug)] +pub enum WordPart { + Text { + value: String, + span: Span, + }, + VarRef { + name: String, + span: Span, + /// True when the source used the `${name}` bracketed form. + /// Preserved for lowering so `"${ip}_wrapper.vhd"` doesn't + /// re-emit as `"$ip_wrapper.vhd"` — Tcl's `$` reads a + /// greedy ident (`[A-Za-z0-9_:]+`) and would then try to + /// dereference the non-existent `ip_wrapper` variable. See + /// `lower_word_parts` for the emit side. + braced: bool, + }, + /// `[ cmd ... ]` command substitution. `source` is the raw interior + /// text (between the brackets) and `span` covers the whole + /// `[...]`. `body` is populated by a post-pass that recursively + /// parses the interior into statements with absolute spans, so + /// hover / goto / signature-help can descend in. + CmdSubst { + source: String, + span: Span, + body: Vec, + }, + Escape { + value: char, + span: Span, + }, +} + +impl WordPart { + pub fn span(&self) -> Span { + match self { + WordPart::Text { span, .. } + | WordPart::VarRef { span, .. } + | WordPart::CmdSubst { span, .. } + | WordPart::Escape { span, .. } => *span, + } + } +} diff --git a/vw-htcl/src/cmdline.rs b/vw-htcl/src/cmdline.rs new file mode 100644 index 0000000..cf37c2f --- /dev/null +++ b/vw-htcl/src/cmdline.rs @@ -0,0 +1,336 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Lightweight analysis of the partially-typed command at the cursor. +//! +//! Completion and signature help need to know, mid-edit, which command +//! the cursor sits in and which word is being typed. The full AST is +//! unreliable here *precisely because* the text is incomplete, so we +//! scan the raw source backward to the nearest command boundary +//! (newline, `;`, or the `[` that opens a command substitution) and +//! tokenize on whitespace. This is a deliberately shallow Tcl reader — +//! good enough to drive IDE affordances, not to execute. + +use crate::span::Span; + +#[derive(Clone, Debug)] +pub struct CmdLine<'a> { + /// Whitespace-separated complete words before the cursor. The + /// first, when present, is the command name. + pub words: Vec<&'a str>, + /// The word currently under the cursor: the trailing token when + /// the prefix doesn't end in whitespace, otherwise empty. + pub partial: &'a str, + /// Span of `partial` in the source — the range a completion should + /// replace. Zero-width (an insertion point) when `partial` is + /// empty. + pub partial_span: Span, +} + +impl CmdLine<'_> { + /// The command name (first complete word). `None` while the cursor + /// is still on the first word — i.e. command-name position. + pub fn command_name(&self) -> Option<&str> { + self.words.first().copied() + } + + /// True when the cursor is in command-name position (no complete + /// words precede it). + pub fn in_command_position(&self) -> bool { + self.words.is_empty() + } + + /// Flags (`-foo`) already supplied among the complete words after + /// the command name. + pub fn used_flags(&self) -> impl Iterator { + self.words + .iter() + .skip(1) + .copied() + .filter(|w| w.starts_with('-')) + } +} + +/// Analyze the command the cursor at `offset` is editing. +pub fn analyze(source: &str, offset: u32) -> CmdLine<'_> { + let off = (offset as usize).min(source.len()); + let bytes = source.as_bytes(); + + // Walk back to the start of the current command. The boundary + // depends on the cursor's *bracket nesting*: inside a `[ … ]`, + // newlines are whitespace (matching the parser), only `;` and the + // opening `[` terminate. Outside brackets, `\n` and `;` both + // terminate at the cursor's level. + // + // We track depth as we walk backward — each `]` going back means + // we're entering a deeper region, each `[` brings us back out. If + // we hit an unmatched `[` (the opening bracket of the substitution + // the cursor sits in), that's the command boundary. Otherwise the + // closest `\n`/`;` we passed at depth 0 wins. We have to scan past + // a candidate `\n`/`;` because an enclosing `[` further back would + // override it. + let mut depth: i32 = 0; + let mut nearest_top_sep: Option = None; + let mut bracket_open: Option = None; + let mut i = off; + while i > 0 { + i -= 1; + match bytes[i] { + b']' => depth += 1, + b'[' => { + if depth > 0 { + depth -= 1; + } else { + bracket_open = Some(i + 1); + break; + } + } + b'\n' | b';' + if depth == 0 + && nearest_top_sep.is_none() + && !escaped_by_backslash(bytes, i) + && !(bytes[i] == b'\n' + && next_line_is_flag_continuation(bytes, i)) => + { + // The `!escaped_by_backslash` guard above is Tcl's + // line-continuation rule: `\` and `\;` are + // escaped literals, not separators. Without it, a + // multi-line invocation like + // + // create_clk_wizard_clkout \ + // -cell $clk \ + // -req + // + // would have its completion fall off the cliff because + // the analyzer thought line 3 was a fresh command. + // + // The `next_line_is_flag_continuation` guard mirrors + // the parser's dash-line-continuation rule so the same + // shape without backslashes also completes correctly: + // + // create_clk_wizard_clkout + // -cell $clk + // -req + nearest_top_sep = Some(i + 1); + } + _ => {} + } + } + let start = bracket_open.or(nearest_top_sep).unwrap_or(0); + let prefix = &source[start..off]; + + // The partial word is the trailing run of non-whitespace, unless + // the prefix already ends in whitespace (then we're between words). + let partial_len: usize = prefix + .chars() + .rev() + .take_while(|c| !c.is_whitespace()) + .map(char::len_utf8) + .sum(); + let split = prefix.len() - partial_len; + let head = &prefix[..split]; + let partial = &prefix[split..]; + + CmdLine { + words: head.split_whitespace().collect(), + partial, + partial_span: Span::new((start + split) as u32, off as u32), + } +} + +/// Peek past `bytes[i]` (a `\n`) and any inline whitespace on the +/// next line: does the first non-whitespace byte look like a flag +/// (`-` followed by letter/digit/`-`)? Mirrors the parser's dash- +/// line-continuation rule so the analyzer's cursor-at-end +/// completion path agrees with the parser about whether an +/// unescaped newline ends a command or extends it. +fn next_line_is_flag_continuation(bytes: &[u8], i: usize) -> bool { + let mut j = i + 1; + while j < bytes.len() { + match bytes[j] { + b' ' | b'\t' | b'\r' => j += 1, + _ => break, + } + } + if j >= bytes.len() || bytes[j] != b'-' { + return false; + } + let next = bytes.get(j + 1).copied().unwrap_or(b'\0'); + // Flag-shaped: letter/digit/underscore/`-`. See the parser's + // `next_line_is_flag_continuation` for the rationale on the + // underscore case (real Vivado flags like `-_64bit`). + next.is_ascii_alphanumeric() || next == b'-' || next == b'_' +} + +/// True when the byte at `i` is preceded by an odd-length run of +/// backslashes — Tcl's escape rule. `bytes[i]` itself is not consulted. +fn escaped_by_backslash(bytes: &[u8], i: usize) -> bool { + let mut j = i; + let mut count = 0usize; + while j > 0 && bytes[j - 1] == b'\\' { + count += 1; + j -= 1; + } + count % 2 == 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at_end(src: &str) -> CmdLine<'_> { + analyze(src, src.len() as u32) + } + + #[test] + fn command_position_with_partial() { + let line = at_end("gr"); + assert!(line.in_command_position()); + assert_eq!(line.partial, "gr"); + assert_eq!(line.partial_span, Span::new(0, 2)); + } + + #[test] + fn argument_position_after_name() { + let line = at_end("greet "); + assert!(!line.in_command_position()); + assert_eq!(line.command_name(), Some("greet")); + assert_eq!(line.partial, ""); + assert_eq!(line.partial_span, Span::new(6, 6)); + } + + #[test] + fn partial_flag_after_name() { + let line = at_end("greet -na"); + assert_eq!(line.command_name(), Some("greet")); + assert_eq!(line.partial, "-na"); + assert_eq!(line.partial_span.slice("greet -na"), "-na"); + } + + #[test] + fn used_flags_are_reported() { + let line = at_end("f -a 1 -b "); + let used: Vec<&str> = line.used_flags().collect(); + assert_eq!(used, vec!["-a", "-b"]); + } + + #[test] + fn resets_at_command_substitution() { + // Only the text inside the `[...]` counts as the command. + let src = "puts [greet -na"; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("greet")); + assert_eq!(line.partial, "-na"); + } + + #[test] + fn resets_at_newline() { + let src = "set x 1\ngr"; + let line = analyze(src, src.len() as u32); + assert!(line.in_command_position()); + assert_eq!(line.partial, "gr"); + } + + #[test] + fn ignores_newlines_inside_brackets() { + // The cursor sits on `-cell ` in a multi-line `[ … ]`. The + // analyzer must skip the intervening newlines so it can still + // see `create_cpm5_cpm_pcie0` as the command name. + let src = "\ +set x [ + create_cpm5_cpm_pcie0 + -cell "; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("create_cpm5_cpm_pcie0")); + assert_eq!(line.partial, ""); + // The flag in the middle counts as already-used. + let used: Vec<&str> = line.used_flags().collect(); + assert_eq!(used, vec!["-cell"]); + } + + #[test] + fn active_partial_flag_across_lines() { + // Partial `-max_link_` typed on a fresh line of a multi-line + // bracket should still be recognized as the partial word, and + // the command name should still be the bracket's first word. + let src = "\ +set x [ + create_cpm5_cpm_pcie0 + -cell cpm5 + -max_link_"; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("create_cpm5_cpm_pcie0")); + assert_eq!(line.partial, "-max_link_"); + } + + /// Dash-line continuation without `\`: the completion path + /// must agree with the parser that a newline followed by a + /// `-flag` line extends the current command, so cursor-at-end + /// resolves to the command's flag position. + #[test] + fn dash_led_next_line_continuation_no_backslash() { + let src = "\ +create_clk_wizard_clkout + -cell $clk + -req"; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("create_clk_wizard_clkout")); + assert_eq!(line.partial, "-req"); + let used: Vec<&str> = line.used_flags().collect(); + assert_eq!(used, vec!["-cell"]); + } + + #[test] + fn backslash_newline_continuation_keeps_command_alive() { + // A `\` is Tcl's line continuation — the analyzer + // must look through it so the cursor at the end of line 3 is + // still "argument position of `create_clk_wizard_clkout`," + // not a fresh top-level command. + let src = "\ +create_clk_wizard_clkout \\ + -cell $clk \\ + -req"; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("create_clk_wizard_clkout")); + assert_eq!(line.partial, "-req"); + let used: Vec<&str> = line.used_flags().collect(); + assert_eq!(used, vec!["-cell"]); + } + + #[test] + fn escaped_double_backslash_newline_still_separates() { + // `\\` ends with a literal backslash followed by a + // real command boundary — the second-to-last command is a + // separate statement. + let src = "set x foo\\\\\nbar"; + let line = analyze(src, src.len() as u32); + // The trailing `\\` is a literal backslash, then `\n` + // separates, so `bar` is a fresh command. + assert!(line.in_command_position(), "{:?}", line.words); + assert_eq!(line.partial, "bar"); + } + + #[test] + fn skips_balanced_inner_brackets() { + // Walking back past a complete `[…]` shouldn't fool the + // analyzer into thinking the cursor is at top level when it's + // really inside another, *outer* bracket. + let src = "\ +set x [ + [a b] + outer "; + let line = analyze(src, src.len() as u32); + // The cursor's enclosing bracket is the outer one; its first + // word is the standalone `[a b]` substitution, not a simple + // identifier — so command_name is None, but partial is empty + // (we're between words on a continuation line). The point is + // that the *outer* bracket is what we recognized, not the + // inner one. + let used: Vec<&str> = line.used_flags().collect(); + assert!(used.is_empty(), "{used:?}"); + // `outer` is the second word inside the outer bracket; the + // first word was the `[…]` substitution itself. + assert!(line.words.contains(&"outer"), "{:?}", line.words); + } +} diff --git a/vw-htcl/src/complete.rs b/vw-htcl/src/complete.rs new file mode 100644 index 0000000..e2f0a0f --- /dev/null +++ b/vw-htcl/src/complete.rs @@ -0,0 +1,858 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Code completion for htcl. +//! +//! Two contexts, both keyed off [`cmdline::analyze`]: +//! +//! - **Command position** (typing the first word) → the names of +//! `proc`s declared in the document. +//! - **Argument position** (after a known proc's name) → that proc's +//! `-flag` arguments, minus any already supplied. +//! +//! Pure analysis: returns structured [`Completion`]s referencing the +//! document; the LSP backend maps them to `CompletionItem`s and the +//! REPL will render them its own way. Vivado builtins are not offered +//! yet — that needs the UG835 command database (project-plan Phase 8). + +use std::fmt::Write; + +use crate::ast::{ + Attribute, AttributeValue, CommandKind, Document, ProcArg, ProcSignature, + Stmt, +}; +use crate::cmdline::{self, CmdLine}; +use crate::span::Span; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CompletionKind { + /// A `proc` name in command position. + Proc, + /// A `-flag` keyword argument of a known proc. + Flag, + /// A value from a flag's `@enum(...)` constraint. + EnumValue, + /// A constructor call inferred from a `Construct with [path]` doc + /// hint on the target arg — inserts a multi-line command- + /// substitution block with the cursor positioned to start typing + /// the constructor's own flags. + Constructor, +} + +#[derive(Clone, Debug)] +pub struct Completion { + /// Text shown in the list (`greet`, `-name`, or the constructor + /// path for `Constructor` kind). Also used as the inserted text + /// unless `insert_text` overrides it. + pub label: String, + pub kind: CompletionKind, + /// Short, single-line annotation shown inline next to the label. + pub detail: Option, + /// Longer markdown shown in the item's documentation popup. + pub documentation: Option, + /// Source range the inserted text replaces (the partial word, or a + /// zero-width insertion point between words). + pub replace: Span, + /// Text to insert instead of `label` — used by snippet-shaped + /// completions (`Constructor`) whose visible label is a plain + /// name but whose insertion is a multi-line block. When `None` + /// the LSP layer inserts `label` verbatim. + pub insert_text: Option, + /// If true, `insert_text` uses LSP snippet syntax (`$0`, `${1:foo}`, + /// etc.) and the LSP layer sets `InsertTextFormat::SNIPPET`. + pub snippet: bool, +} + +struct ProcInfo<'a> { + /// Qualified name as it would be called — bare proc name for + /// top-level declarations, `::` for procs declared + /// inside `namespace eval` blocks. + name: String, + doc_comments: &'a [String], + signature: Option<&'a ProcSignature>, +} + +/// Completions available at `offset`. +pub fn complete_at( + document: &Document, + source: &str, + offset: u32, +) -> Vec { + complete_at_with_extras(document, source, offset, &[]) +} + +/// Same as [`complete_at`] but also considers a workspace-side +/// document as an extra source of proc definitions. +/// +/// The LSP calls this way so it can: +/// - use the **current** local text (`document` + `source`) for +/// cmdline-scan context — so what the user *just typed* is what +/// drives `in_command_position` / `partial` / `-flag` detection; +/// - AND still pick up cross-file proc names (`gtwiz_versal::configure`, +/// etc.) from a stale but recently-committed workspace parse +/// (`workspace_document`). +/// +/// Local procs shadow workspace ones on name collision. `workspace_ +/// document` should be a document parsed from the merged workspace +/// view (local + all transitive imports); the local prefix will +/// duplicate the local file's procs, so the shadowing rule dedupes +/// them without extra bookkeeping. +/// +/// Passing `None` is exactly equivalent to `complete_at` — no +/// workspace symbols get added. +pub fn complete_at_with_extras( + document: &Document, + source: &str, + offset: u32, + workspace_documents: &[&Document], +) -> Vec { + // Inside a proc's argument-declaration braces, command/flag + // completion is meaningless (attribute completion will live here + // later). Stay quiet rather than offer nonsense. + if in_proc_args(&document.stmts, offset) { + return Vec::new(); + } + + let line = cmdline::analyze(source, offset); + let mut procs = collect_procs(document); + // Merge in workspace-provided procs. Local procs already in + // `procs` shadow workspace ones — we only append a workspace + // proc when no local proc with the same qualified name exists. + for ws in workspace_documents { + let ws_procs = collect_procs(ws); + for wp in ws_procs { + if !procs.iter().any(|p| p.name == wp.name) { + procs.push(wp); + } + } + } + + if line.in_command_position() { + return complete_proc_names(&procs, &line); + } + + // If the previous complete word is a `-flag`, the cursor is in + // value position — even if the partial is empty (user just hit + // space after the flag). Offer the flag's `@enum(...)` choices + // when it has them; otherwise stay silent so the user can type a + // free-form value (string, int, etc.) without a flag list popping + // up in front of it. + // + // If the partial *starts with* `-` we step back into flag-typing + // mode regardless — the user is clearly typing a new flag. + let last_word_is_flag = line.words.len() >= 2 + && line.words.last().is_some_and(|w| w.starts_with('-')); + if last_word_is_flag && !line.partial.starts_with('-') { + // In value position, offer only value-shaped completions — + // enum choices and `Construct with [...]` snippets. Never + // fall through to flag completion: a free-form value slot + // has no reason to pop a flag list in front of what the user + // is typing. + let mut items = complete_enum_values(&procs, &line); + items.extend(complete_constructor(&procs, &line)); + return items; + } + + complete_flags(&procs, &line) +} + +/// If the flag currently in value position has a `Construct with +/// [path]` hint in its doc comment, emit a single snippet completion +/// that inserts a multi-line command-substitution block calling that +/// constructor. The cursor lands after the constructor name so the +/// user can immediately start typing its own `-flag value` pairs. +/// +/// Empty when the flag has no such hint (so the caller can fall +/// through to normal flag/value handling). +fn complete_constructor( + procs: &[ProcInfo<'_>], + line: &CmdLine<'_>, +) -> Vec { + let Some(name) = line.command_name() else { + return Vec::new(); + }; + let Some(proc) = procs.iter().find(|p| p.name == name) else { + return Vec::new(); + }; + let Some(sig) = proc.signature else { + return Vec::new(); + }; + let Some(last) = line.words.last() else { + return Vec::new(); + }; + let Some(flag) = last.strip_prefix('-') else { + return Vec::new(); + }; + let Some(arg) = sig.find(flag) else { + return Vec::new(); + }; + let Some(path) = extract_constructor_hint(&arg.doc_comments) else { + return Vec::new(); + }; + // Only offer when the partial (what the user has typed after the + // flag) is either empty or a prefix of the constructor path — a + // free-form value like `$my_var` shouldn't fight the constructor + // suggestion. + if !line.partial.is_empty() && !path.starts_with(line.partial) { + return Vec::new(); + } + // Constructor invocation, multi-line for readability, `$0` sets + // the LSP cursor after the constructor name so `-flag` completion + // picks up next. + let insert = format!("[\n {path} $0\n]"); + vec![Completion { + label: path.clone(), + kind: CompletionKind::Constructor, + detail: Some(format!("constructor for -{}", arg.name)), + documentation: Some(format!( + "Insert a `\\[{path} ...\\]` command-substitution block \ + for the `-{}` slot.", + arg.name + )), + replace: line.partial_span, + insert_text: Some(insert), + snippet: true, + }] +} + +/// Scan doc comments for the `Construct with [path::name]` idiom and +/// return the bracketed path if found. Matches on any line the doc +/// author put it on; case-insensitive on the `construct with` phrase +/// so `Construct` / `construct` / `CONSTRUCT` all work. +fn extract_constructor_hint(doc_comments: &[String]) -> Option { + for line in doc_comments { + let lower = line.to_ascii_lowercase(); + let mut cursor = 0; + while let Some(rel) = lower[cursor..].find("construct with [") { + let start = cursor + rel + "construct with [".len(); + if let Some(end_rel) = line[start..].find(']') { + let path = &line[start..start + end_rel]; + if is_valid_path(path) { + return Some(path.to_string()); + } + cursor = start + end_rel + 1; + } else { + break; + } + } + } + None +} + +fn is_valid_path(s: &str) -> bool { + !s.is_empty() + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == ':') + && s.chars().next().is_some_and(|c| !c.is_ascii_digit()) +} + +/// `@enum(…)` value completions when the cursor sits in value +/// position. Returns empty when the flag has no `@enum` (so the +/// caller can fall back to flag completion). +fn complete_enum_values( + procs: &[ProcInfo<'_>], + line: &CmdLine<'_>, +) -> Vec { + let Some(name) = line.command_name() else { + return Vec::new(); + }; + let Some(proc) = procs.iter().find(|p| p.name == name) else { + return Vec::new(); + }; + let Some(sig) = proc.signature else { + return Vec::new(); + }; + // The flag whose value we're completing is the last word on the + // line; if it isn't a `-flag`, the user is between options and + // there's nothing to enum-complete. + let Some(last) = line.words.last() else { + return Vec::new(); + }; + let Some(flag) = last.strip_prefix('-') else { + return Vec::new(); + }; + let Some(arg) = sig.find(flag) else { + return Vec::new(); + }; + let Some(enum_attr) = arg.attribute("enum") else { + return Vec::new(); + }; + + let needle = line.partial; + enum_attr + .values + .iter() + .filter_map(|v| { + let raw = enum_value_text(v); + // Filter by either the bare or quoted form so a user typing + // `Mas` matches the value `Master Mode` whose insert form + // is `"Master Mode"`. + if !raw.starts_with(needle) + && !quote_for_completion(&raw).starts_with(needle) + { + return None; + } + let insert = quote_for_completion(&raw); + Some(Completion { + label: insert.clone(), + kind: CompletionKind::EnumValue, + detail: Some(format!("value for -{}", arg.name)), + documentation: crate::doc::brief(&arg.doc_comments), + replace: line.partial_span, + insert_text: None, + snippet: false, + }) + }) + .collect() +} + +fn enum_value_text(v: &AttributeValue) -> String { + match v { + AttributeValue::Integer { value, .. } => value.to_string(), + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value.clone(), + // `key=value` in an @enum(…) doesn't make semantic sense — + // enum values are positional. Render the whole thing as + // the literal `key=value` string. + AttributeValue::Keyed { .. } => v.to_tcl_literal(), + } +} + +/// Quote `s` for use as a value on a call site if it can't ride as a +/// bare word. Mirrors the rule [`crate::emit::Word::lit`] uses: bare +/// when safe, double-quoted with `\`/`"` escapes otherwise. +fn quote_for_completion(s: &str) -> String { + let needs = s.is_empty() + || s.chars().any(|c| { + c.is_whitespace() + || matches!( + c, + ';' | '"' | '\\' | '[' | ']' | '{' | '}' | '$' | '#' + ) + }); + if needs { + let escaped = s.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") + } else { + s.to_string() + } +} + +fn complete_proc_names( + procs: &[ProcInfo<'_>], + line: &CmdLine<'_>, +) -> Vec { + procs + .iter() + .filter(|p| p.name.starts_with(line.partial)) + .map(|p| Completion { + label: p.name.to_string(), + kind: CompletionKind::Proc, + detail: first_doc_line(p.doc_comments), + documentation: proc_documentation(p), + replace: line.partial_span, + insert_text: None, + snippet: false, + }) + .collect() +} + +fn complete_flags( + procs: &[ProcInfo<'_>], + line: &CmdLine<'_>, +) -> Vec { + let Some(name) = line.command_name() else { + return Vec::new(); + }; + let Some(proc) = procs.iter().find(|p| p.name == name) else { + return Vec::new(); + }; + let Some(sig) = proc.signature else { + return Vec::new(); + }; + + let used: Vec<&str> = line.used_flags().collect(); + let needle = line.partial; + let bare_needle = needle.trim_start_matches('-'); + + sig.args + .iter() + .filter_map(|arg| { + let label = format!("-{}", arg.name); + // Don't re-offer a flag already on the line, unless it's + // the very word being typed. + if used.iter().any(|u| *u == label) && needle != label { + return None; + } + // Match either the dashed form (`-na`) or the bare name + // (`na`); an empty needle matches everything. + if !label.starts_with(needle) && !arg.name.starts_with(bare_needle) + { + return None; + } + Some(Completion { + label, + kind: CompletionKind::Flag, + detail: flag_detail(arg), + documentation: Some(arg_documentation(arg)), + replace: line.partial_span, + insert_text: None, + snippet: false, + }) + }) + .collect() +} + +fn collect_procs(document: &Document) -> Vec> { + let mut out = Vec::new(); + collect_procs_in(&document.stmts, "", &mut out); + out +} + +fn collect_procs_in<'a>( + stmts: &'a [Stmt], + prefix: &str, + out: &mut Vec>, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let qualified = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + out.push(ProcInfo { + name: qualified, + doc_comments: &cmd.doc_comments, + signature: proc.signature.as_ref(), + }); + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + let nested = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + collect_procs_in(&ns.body, &nested, out); + } + _ => {} + } + } +} + +/// True if `offset` is inside any proc's argument-declaration braces, +/// at any nesting depth. +fn in_proc_args(stmts: &[Stmt], offset: u32) -> bool { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Proc(proc) = &cmd.kind else { + continue; + }; + if proc.args_span.contains(offset) { + return true; + } + if in_proc_args(&proc.body, offset) { + return true; + } + } + false +} + +fn first_doc_line(docs: &[String]) -> Option { + crate::doc::brief(docs) +} + +fn proc_documentation(p: &ProcInfo<'_>) -> Option { + // Use `extended` (body only) here because the call site populates + // `CompletionItem::detail` with the brief sentence separately — + // shipping the full reflowed text would duplicate that sentence at + // the top of every popup. + let mut out = String::new(); + if let Some(ext) = crate::doc::extended(p.doc_comments) { + out.push_str(&ext); + } + if let Some(sig) = p.signature { + if !sig.args.is_empty() { + if !out.is_empty() { + out.push_str("\n\n"); + } + for arg in &sig.args { + write!(out, "- `-{}`", arg.name).unwrap(); + if let Some(d) = crate::doc::brief(&arg.doc_comments) { + write!(out, " — {d}").unwrap(); + } + out.push('\n'); + } + } + } + (!out.is_empty()).then_some(out) +} + +fn arg_documentation(arg: &ProcArg) -> String { + // `extended` only — the brief sentence is handled by the caller's + // `detail` field; see `proc_documentation` for the rationale. + let mut out = String::new(); + if let Some(ext) = crate::doc::extended(&arg.doc_comments) { + out.push_str(&ext); + } + for attr in &arg.attributes { + if !out.is_empty() { + out.push('\n'); + } + write!(out, "- `{}`", render_attribute(attr)).unwrap(); + } + out +} + +/// Single-line label shown inline next to the flag in the completion +/// popup. We prepend any `@enum(...)` / `@default(...)` constraint — +/// rendered with its actual values rather than the bare attribute +/// name — so the user sees *what's allowed* without having to expand +/// the documentation pane. The doc-brief, if any, follows after a +/// dash. +fn flag_detail(arg: &ProcArg) -> Option { + let mut parts: Vec = Vec::new(); + for attr in &arg.attributes { + parts.push(render_attribute(attr)); + } + let brief = crate::doc::brief(&arg.doc_comments); + match (parts.is_empty(), brief) { + (true, b) => b, + (false, None) => Some(parts.join(" ")), + (false, Some(b)) => Some(format!("{} — {b}", parts.join(" "))), + } +} + +/// `@name(value, value, ...)` if the attribute carries values, +/// `@name` otherwise. Values are rendered via +/// [`AttributeValue::to_tcl_literal`] so strings get quoted and +/// integers/idents render as-is — same convention `proc_args` uses +/// when echoing back a defaulted call site. +fn render_attribute(attr: &Attribute) -> String { + if attr.values.is_empty() { + format!("@{}", attr.name) + } else { + let vals: Vec = attr + .values + .iter() + .map(AttributeValue::to_tcl_literal) + .collect(); + format!("@{}({})", attr.name, vals.join(", ")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + /// Build `src` plus a cursor at the `|` marker, returning the + /// marker-free source and the byte offset of the cursor. + fn cursor(src_with_marker: &str) -> (String, u32) { + let offset = src_with_marker.find('|').expect("no cursor marker"); + let src = src_with_marker.replacen('|', "", 1); + (src, offset as u32) + } + + fn labels(src_with_marker: &str) -> Vec { + let (src, off) = cursor(src_with_marker); + let parsed = parse(&src); + complete_at(&parsed.document, &src, off) + .into_iter() + .map(|c| c.label) + .collect() + } + + #[test] + fn proc_names_in_command_position() { + let src = "\ +proc greet {} { }\n\ +proc grumble {} { }\n\ +gr|\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["greet", "grumble"]); + } + + #[test] + fn proc_names_filtered_by_prefix() { + let src = "\ +proc greet {} { }\n\ +proc grumble {} { }\n\ +gree|\n"; + assert_eq!(labels(src), vec!["greet"]); + } + + #[test] + fn flags_in_argument_position() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg |\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["-depth", "-width"]); + } + + #[test] + fn flags_filtered_by_partial() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -w|\n"; + assert_eq!(labels(src), vec!["-width"]); + } + + #[test] + fn already_used_flag_is_not_reoffered() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -width 8 |\n"; + assert_eq!(labels(src), vec!["-depth"]); + } + + #[test] + fn completes_call_inside_proc_body() { + let src = "\ +proc helper {} { }\n\ +proc outer {} {\n hel|\n}\n"; + assert_eq!(labels(src), vec!["helper"]); + } + + #[test] + fn no_completion_inside_arg_decls() { + let src = "\ +proc greet {} { }\n\ +proc cfg {\n wi|\n} { }\n"; + assert!(labels(src).is_empty()); + } + + #[test] + fn enum_value_position_offers_choices() { + // `2.5_GT/s` etc. aren't valid `attribute_value_ident`s, so + // the IP generator quotes them in `@enum(…)` — the proc-args + // grammar parses them as strings. The completion labels come + // back bare here because no whitespace requires re-quoting. + let src = "\ +proc cfg {\n @enum(\"2.5_GT/s\", \"5.0_GT/s\", \"8.0_GT/s\") max_link_speed\n} { }\n\ +cfg -max_link_speed |\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["2.5_GT/s", "5.0_GT/s", "8.0_GT/s"]); + } + + #[test] + fn enum_values_filter_by_partial() { + let src = "\ +proc cfg {\n @enum(\"2.5_GT/s\", \"5.0_GT/s\", \"8.0_GT/s\") max_link_speed\n} { }\n\ +cfg -max_link_speed 5|\n"; + assert_eq!(labels(src), vec!["5.0_GT/s"]); + } + + #[test] + fn enum_completion_kind_marks_items() { + let src = "\ +proc cfg {\n @enum(target, controller) kind\n} { }\n\ +cfg -kind |\n"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + assert!(items.iter().all(|c| c.kind == CompletionKind::EnumValue)); + } + + #[test] + fn enum_value_with_spaces_gets_quoted() { + let src = "\ +proc cfg {\n @enum(\"Master Mode\", \"Slave Mode\") role\n} { }\n\ +cfg -role |\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["\"Master Mode\"", "\"Slave Mode\""]); + } + + #[test] + fn flag_without_enum_offers_no_completions_at_value_position() { + // For a flag with no `@enum` the user is expected to type a + // free-form value. Popping a flag list there is wrong; it + // gets in the way of the actual value the user is typing. + let src = "\ +proc cfg {\n @default(0) width\n @default(0) depth\n} { }\n\ +cfg -width |\n"; + assert!(labels(src).is_empty(), "{:?}", labels(src)); + } + + #[test] + fn flag_completion_returns_after_value_is_typed() { + // After the value is typed, the cursor is between args again + // — show the next flags. + let src = "\ +proc cfg {\n @default(0) width\n @default(0) depth\n} { }\n\ +cfg -width 8 |\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["-depth"]); + } + + #[test] + fn dash_partial_keeps_flag_completion() { + // Typing `-` after a complete flag should still mean "new + // flag," not "enum value." + let src = "\ +proc cfg {\n @enum(a, b) mode\n @default(0) width\n} { }\n\ +cfg -mode -|\n"; + let got = labels(src); + assert!(got.contains(&"-width".to_string()), "{got:?}"); + assert!(!got.contains(&"a".to_string()), "{got:?}"); + } + + #[test] + fn unknown_command_offers_no_flags() { + let src = "puts |\n"; + assert!(labels(src).is_empty()); + } + + #[test] + fn flag_completion_carries_doc_and_detail() { + // Multi-sentence doc: the brief sentence joins the attribute + // summary in `detail`, the rest goes in `documentation`. They + // must NOT overlap — an LSP client renders both, and a + // repeated leading sentence reads as a duplicate. + let src = "\ +proc cfg { + ## Bus width in bits. Must be a power of two. + @default(8) width +} { } +cfg | +"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + let item = items.iter().find(|c| c.label == "-width").unwrap(); + assert_eq!(item.kind, CompletionKind::Flag); + assert_eq!( + item.detail.as_deref(), + Some("@default(8) — Bus width in bits.") + ); + let doc = item.documentation.as_deref().unwrap(); + assert!(doc.contains("Must be a power of two."), "{doc}"); + assert!( + !doc.contains("Bus width in bits."), + "documentation should not repeat the brief: {doc}" + ); + assert!(doc.contains("@default(8)"), "{doc}"); + } + + #[test] + fn flag_detail_renders_enum_alternatives_and_default() { + // The whole point: a user scanning the completion list sees + // exactly which values are allowed (`@enum(...)`) and what + // ships by default (`@default(...)`) without having to expand + // the doc pane. + let src = "\ +proc cfg { + @enum(LOW, HIGH, OPTIMIZED) @default(OPTIMIZED) bandwidth +} { } +cfg | +"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + let item = items.iter().find(|c| c.label == "-bandwidth").unwrap(); + assert_eq!( + item.detail.as_deref(), + Some("@enum(LOW, HIGH, OPTIMIZED) @default(OPTIMIZED)") + ); + let doc = item.documentation.as_deref().unwrap(); + assert!(doc.contains("@enum(LOW, HIGH, OPTIMIZED)"), "{doc}"); + assert!(doc.contains("@default(OPTIMIZED)"), "{doc}"); + } + + #[test] + fn constructor_hint_offers_snippet_in_value_position() { + // A doc line of the form `Construct with [namespaced::path]` + // on the target arg turns into a snippet completion in value + // position — the label is the constructor path, the insert + // text is a `[\n path $0\n]` block with cursor placed + // after the constructor name. + let src = "\ +namespace eval demo {}\n\ +proc demo::child { -x: int } {}\n\ +proc parent {\n ## Construct with [demo::child].\n child: any\n} { }\n\ +parent -child |\n"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + let item = items + .iter() + .find(|c| c.kind == CompletionKind::Constructor) + .unwrap_or_else(|| { + panic!( + "no constructor completion; got {:?}", + items + .iter() + .map(|c| (&c.label, c.kind)) + .collect::>() + ) + }); + assert_eq!(item.label, "demo::child"); + assert!(item.snippet); + let insert = item.insert_text.as_deref().unwrap(); + assert!(insert.contains("demo::child"), "insert={insert:?}"); + assert!(insert.contains("$0"), "insert={insert:?}"); + assert!(insert.starts_with('['), "insert={insert:?}"); + assert!(insert.trim_end().ends_with(']'), "insert={insert:?}"); + } + + #[test] + fn constructor_hint_absent_produces_no_extra_completion() { + // No `Construct with ...` doc → no constructor completion. + // Value position on a non-enum, non-hinted flag returns empty. + let src = "\ +proc parent {\n ## An arbitrary slot.\n child: any\n} { }\n\ +parent -child |\n"; + assert!(labels(src).is_empty()); + } + + #[test] + fn constructor_hint_case_insensitive() { + let src = "\ +proc parent {\n ## construct with [foo::bar]\n slot: any\n} { }\n\ +parent -slot |\n"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + assert!(items + .iter() + .any(|c| c.kind == CompletionKind::Constructor + && c.label == "foo::bar")); + } + + #[test] + fn flag_detail_quotes_string_enum_values() { + // Values that started life as a quoted string in the source + // must stay quoted in the detail line — `"Master Mode"` not + // `Master Mode` — so the displayed text is what the user + // would type back as the value. + let src = "\ +proc cfg { + @enum(\"Master Mode\", \"Slave Mode\") role +} { } +cfg | +"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + let item = items.iter().find(|c| c.label == "-role").unwrap(); + assert_eq!( + item.detail.as_deref(), + Some("@enum(\"Master Mode\", \"Slave Mode\")") + ); + } +} diff --git a/vw-htcl/src/doc.rs b/vw-htcl/src/doc.rs new file mode 100644 index 0000000..4e7f749 --- /dev/null +++ b/vw-htcl/src/doc.rs @@ -0,0 +1,358 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Render helpers for doc-comment blocks. +//! +//! `##` doc comments in source are typically wrapped at a comfortable +//! editing column (~80 chars). When a display surface — an LSP hover, +//! signature help, completion documentation — joins those lines +//! verbatim, the source wrap survives into the rendered markdown. +//! Most LSP clients then treat the first wrapped fragment as a "brief +//! summary," which is almost always a mid-sentence truncation. +//! +//! [`reflow_doc_comments`] converts a slice of doc-comment lines into +//! markdown-clean text: consecutive non-empty lines collapse into one +//! paragraph (joined with a single space), and a blank line becomes a +//! paragraph break (`\n\n`). The first paragraph then reads as a +//! complete unit — usually one or more whole sentences — instead of +//! the editor-wrap fragment that surfaces today. + +/// One-line summary suitable for an inline annotation (LSP +/// `CompletionItem::detail`, a parameter list's `— brief` suffix, +/// etc.). Takes the first reflowed paragraph and trims to its first +/// sentence — the convention rustdoc, godoc, and most doc generators +/// follow for "short description vs full body." +/// +/// Returns `None` when `lines` has no non-blank content. Falls back +/// to the whole first paragraph when no sentence terminator (`.`, +/// `!`, `?` followed by whitespace or end-of-string) is found. +pub fn brief(lines: &[String]) -> Option { + let reflowed = reflow_doc_comments(lines); + if reflowed.is_empty() { + return None; + } + let first_paragraph = reflowed.split("\n\n").next().unwrap(); + let bytes = first_paragraph.as_bytes(); + for (i, &b) in bytes.iter().enumerate() { + if !matches!(b, b'.' | b'!' | b'?') { + continue; + } + let next = bytes.get(i + 1).copied(); + if next.is_none() || matches!(next, Some(b' ' | b'\t' | b'\n')) { + return Some(first_paragraph[..=i].to_string()); + } + } + Some(first_paragraph.to_string()) +} + +/// Extended description — everything **after** the first sentence, +/// reflowed into markdown. +/// +/// Pairs with [`brief`]: an LSP-facing renderer puts `brief` in +/// `CompletionItem::detail` (the inline summary next to the label) +/// and `extended` in `documentation` (the body popup). Splitting +/// this way avoids the duplication that occurs when both fields +/// start with the same sentence. +/// +/// Returns `None` when there is no content after the first sentence +/// — e.g. when the doc is a single-sentence summary with no body. +pub fn extended(lines: &[String]) -> Option { + let reflowed = reflow_doc_comments(lines); + if reflowed.is_empty() { + return None; + } + let bytes = reflowed.as_bytes(); + let mut split_at = None; + for (i, &b) in bytes.iter().enumerate() { + if !matches!(b, b'.' | b'!' | b'?') { + continue; + } + let next = bytes.get(i + 1).copied(); + if next.is_none() || matches!(next, Some(b' ' | b'\t' | b'\n')) { + split_at = Some(i + 1); + break; + } + } + // No sentence terminator means the whole reflow IS the brief — + // nothing to put in the body. + let after = reflowed[split_at?..].trim_start(); + (!after.is_empty()).then(|| after.to_string()) +} + +/// Word-wrap `text` into lines no wider than `width` chars. Used by +/// doc-comment generators that want source files with paragraphs +/// re-flowed to a comfortable editing width (the LSP reflows again +/// for display, but a wrapped source is easier for humans to read +/// and diff). +/// +/// A single word longer than `width` is left on a line by itself +/// rather than truncated. +pub fn wrap_paragraph(text: &str, width: usize) -> Vec { + let mut out: Vec = Vec::new(); + let mut current = String::new(); + for word in text.split_whitespace() { + if current.is_empty() { + current.push_str(word); + } else if current.len() + 1 + word.len() <= width { + current.push(' '); + current.push_str(word); + } else { + out.push(std::mem::take(&mut current)); + current.push_str(word); + } + } + if !current.is_empty() { + out.push(current); + } + out +} + +/// Reflow doc-comment lines into a markdown string. See module docs. +pub fn reflow_doc_comments(lines: &[String]) -> String { + let mut out = String::new(); + let mut paragraph = String::new(); + let flush = |paragraph: &mut String, out: &mut String| { + if paragraph.is_empty() { + return; + } + if !out.is_empty() { + out.push_str("\n\n"); + } + out.push_str(paragraph); + paragraph.clear(); + }; + for line in lines { + let trimmed = line.trim(); + if trimmed.is_empty() { + flush(&mut paragraph, &mut out); + } else { + if !paragraph.is_empty() { + paragraph.push(' '); + } + paragraph.push_str(&render_refs(trimmed)); + } + } + flush(&mut paragraph, &mut out); + out +} + +/// Rewrite `[NAME]` tokens as `` `NAME` `` so hover-popup markdown +/// renders them as inline code — visually distinct from prose, and +/// (in editors that honor code-span click handlers) discoverable as +/// something a reader can act on. The analyzer's goto/hover paths +/// already resolve the cursor to the same reference; this is the +/// display side of the same feature. +/// +/// Interior chars accepted: letters, digits, `_`, and `:` (for +/// namespace qualification). Anything else is left as-is — a +/// prose sentence like "see [1]" or "[TODO: refactor]" isn't a +/// reference. +fn render_refs(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'[' { + let content_start = i + 1; + // First char of a Tcl-style ident must be a letter or + // underscore; digits and `:`-prefixed forms don't count + // (they're prose or footnote-style refs, not identifiers). + if content_start < bytes.len() + && (bytes[content_start].is_ascii_alphabetic() + || bytes[content_start] == b'_') + { + let mut j = content_start; + while j < bytes.len() && bytes[j] != b']' { + let b = bytes[j]; + let ok = + b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + if !ok { + break; + } + j += 1; + } + if j < bytes.len() && bytes[j] == b']' && j > content_start { + out.push('`'); + out.push_str(&s[content_start..j]); + out.push('`'); + i = j + 1; + continue; + } + } + } + out.push(bytes[i] as char); + i += 1; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lines(arr: [&str; N]) -> Vec { + arr.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn single_wrapped_paragraph_becomes_one_line() { + let out = reflow_doc_comments(&lines([ + "Create an external port in the current block design and connect that to the", + "selected block pin.", + ])); + assert_eq!( + out, + "Create an external port in the current block design and connect that to the selected block pin." + ); + } + + #[test] + fn blank_line_becomes_paragraph_break() { + let out = reflow_doc_comments(&lines([ + "Summary line one.", + "", + "Body line two,", + "wrapped.", + ])); + assert_eq!(out, "Summary line one.\n\nBody line two, wrapped."); + } + + #[test] + fn leading_and_trailing_blanks_are_dropped() { + let out = reflow_doc_comments(&lines(["", "Hello.", "", ""])); + assert_eq!(out, "Hello."); + } + + #[test] + fn empty_input_returns_empty_string() { + assert_eq!(reflow_doc_comments(&[]), ""); + } + + #[test] + fn brief_extracts_first_sentence_from_wrapped_lines() { + let out = brief(&lines([ + "Create an external port in the current block design and connect that to the", + "selected block pin. If a bd_cell is specified, all pins are made external.", + ])); + assert_eq!( + out.as_deref(), + Some( + "Create an external port in the current block design and connect that to the selected block pin." + ) + ); + } + + #[test] + fn brief_handles_single_sentence_proc() { + let out = brief(&lines(["Width of the data bus in bits."])); + assert_eq!(out.as_deref(), Some("Width of the data bus in bits.")); + } + + #[test] + fn brief_falls_back_to_paragraph_when_no_terminator() { + let out = brief(&lines(["just a phrase", "with no period"])); + assert_eq!(out.as_deref(), Some("just a phrase with no period")); + } + + #[test] + fn brief_returns_none_for_empty_input() { + assert!(brief(&[]).is_none()); + assert!(brief(&lines([""])).is_none()); + } + + #[test] + fn extended_skips_the_summary_sentence() { + let out = extended(&lines([ + "Summary. Body sentence in same paragraph.", + "", + "Second paragraph here.", + ])); + assert_eq!( + out.as_deref(), + Some("Body sentence in same paragraph.\n\nSecond paragraph here.") + ); + } + + #[test] + fn extended_returns_none_for_single_sentence_doc() { + assert!(extended(&lines(["Width of the data bus in bits."])).is_none()); + } + + #[test] + fn extended_brief_round_trip_covers_full_text() { + // Together, `brief` and `extended` should reproduce every + // visible character of the reflowed input (modulo a single + // separator between them). + let input = lines([ + "First sentence.", + "Continued first paragraph.", + "", + "Second paragraph.", + ]); + let b = brief(&input).unwrap(); + let e = extended(&input).unwrap(); + let full = reflow_doc_comments(&input); + // The recombined text should equal the reflow (with a space + // between b and e since the brief is part of paragraph 1). + assert!(full.starts_with(&b)); + assert!(full.ends_with(&e)); + } + + #[test] + fn brief_does_not_trip_on_decimal_or_versal_dots() { + // `3.4` shouldn't end the sentence — terminator must be + // followed by whitespace or end-of-string. + let out = + brief(&lines(["Source IP-XACT: xilinx.com:ip:versal_cips:3.4"])); + assert_eq!( + out.as_deref(), + Some("Source IP-XACT: xilinx.com:ip:versal_cips:3.4") + ); + } + + #[test] + fn wrap_paragraph_breaks_at_word_boundaries() { + let out = wrap_paragraph("one two three four five six", 12); + assert_eq!(out, vec!["one two", "three four", "five six"]); + } + + #[test] + fn wrap_paragraph_keeps_oversize_words_on_their_own_line() { + let out = wrap_paragraph("short superlongword end", 8); + assert_eq!(out, vec!["short", "superlongword", "end"]); + } + + #[test] + fn single_leading_space_is_trimmed_per_line() { + // `##` doc comments may include a leading space after the + // `##` marker that gets preserved in the parsed string; we + // trim each line so the leading space doesn't become a + // double-space inside the joined paragraph. + let out = reflow_doc_comments(&lines([" word one", " word two"])); + assert_eq!(out, "word one word two"); + } + + /// `[NAME]` refs in doc-comment text render as inline code + /// spans so hover popups distinguish them from prose. + #[test] + fn ref_tokens_render_as_inline_code() { + let out = reflow_doc_comments(&lines([ + "Construct with [dcmac::mac_port] before calling [dcmac::create].", + ])); + assert_eq!( + out, + "Construct with `dcmac::mac_port` before calling `dcmac::create`." + ); + } + + /// Not every `[…]` in prose is a reference. Only accept alnum + + /// `_` + `:` interiors; anything else stays as-is. + #[test] + fn non_ref_brackets_left_alone() { + let out = reflow_doc_comments(&lines([ + "See [1] and [TODO: refactor] for details.", + ])); + assert_eq!(out, "See [1] and [TODO: refactor] for details."); + } +} diff --git a/vw-htcl/src/emit.rs b/vw-htcl/src/emit.rs new file mode 100644 index 0000000..c21e264 --- /dev/null +++ b/vw-htcl/src/emit.rs @@ -0,0 +1,553 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Build and emit htcl source code. +//! +//! Distinct from [`crate::ast`], which is the parser's CST and carries +//! spans, doc comments as raw text, and a structure optimized for +//! analysis. `emit` is the dual: for code generation. No spans, +//! ergonomic constructors, and a [`Display`](std::fmt::Display) impl +//! that produces well-formed, indented htcl text. +//! +//! The model is small on purpose. The [`Word`] variants line up with +//! the parser's [`crate::ast::WordForm`] / [`crate::ast::WordPart`] +//! distinctions (bare / quoted / braced / `$var` / `[cmd]`), and +//! [`Word::lit`] picks the safest word form for a runtime string. The +//! [`ToHtcl`] trait is the interpolation interface used by `vw-quote`'s +//! `quote_htcl!` macro and by hand-written generators. + +use std::fmt; + +/// A complete htcl document being built. +#[derive(Clone, Debug, Default)] +pub struct Doc { + pub items: Vec, +} + +impl Doc { + pub fn new() -> Self { + Self::default() + } + + pub fn push(&mut self, item: impl Into) -> &mut Self { + self.items.push(item.into()); + self + } + + pub fn cmd(&mut self, cmd: Command) -> &mut Self { + self.items.push(Item::Command(cmd)); + self + } + + pub fn comment(&mut self, text: impl Into) -> &mut Self { + self.items.push(Item::Comment(text.into())); + self + } + + pub fn doc(&mut self, text: impl Into) -> &mut Self { + self.items.push(Item::DocComment(text.into())); + self + } + + pub fn blank(&mut self) -> &mut Self { + self.items.push(Item::Blank); + self + } +} + +#[derive(Clone, Debug)] +pub enum Item { + Command(Command), + /// Regular `# ...` comment (one line, no leading `#`). + Comment(String), + /// Doc `## ...` comment (one line, no leading `##`). Doc comments + /// attached to a specific command live on [`Command::doc_comments`]. + DocComment(String), + /// Emit a blank line. + Blank, +} + +impl From for Item { + fn from(c: Command) -> Self { + Item::Command(c) + } +} + +/// A single htcl command (one logical line, possibly with a body +/// block). +#[derive(Clone, Debug, Default)] +pub struct Command { + /// `##` doc comments emitted immediately above the command. + pub doc_comments: Vec, + /// The command name and its arguments, in order. + pub words: Vec, + /// Optional braced body emitted as `{ … }` after the words, with + /// its contents indented. Used by `proc`, `if`, `while`, etc. + pub body: Option, +} + +impl Command { + /// `name arg1 arg2 …` with no body. Most generic command shape. + pub fn call(name: impl Into, args: I) -> Self + where + I: IntoIterator, + W: Into, + { + let mut words = vec![name.into()]; + words.extend(args.into_iter().map(Into::into)); + Self { + words, + ..Self::default() + } + } + + pub fn with_doc(mut self, doc: impl Into) -> Self { + self.doc_comments.push(doc.into()); + self + } + + pub fn with_body(mut self, body: Doc) -> Self { + self.body = Some(body); + self + } +} + +/// One word of an htcl command. +/// +/// The variants correspond to the parser's word forms. Prefer +/// [`Word::lit`] when you have a runtime string and want the safest +/// form chosen for you; the named constructors are for when you know +/// the form (e.g. you're producing a `$var` reference deliberately). +#[derive(Clone, Debug)] +pub enum Word { + /// A bare unquoted word. Caller is responsible for ensuring `s` + /// contains no whitespace or shell-special characters; prefer + /// [`Word::lit`] when in doubt. + Bare(String), + /// A double-quoted word (`"…"`). Tcl substitution applies inside; + /// the content is escaped during emit so embedded `"` and `\` + /// are safe. + Quoted(String), + /// A braced word (`{…}`). No substitution; embedded `{`/`}` are + /// the caller's responsibility (typically rare). + Braced(String), + /// A `$name` variable reference. + Var(String), + /// A `[ cmd ]` command substitution; `s` is the interior text, + /// emitted verbatim. + CmdSubst(String), + /// Pre-formatted text inserted as-is. Caller is responsible for + /// it being a valid single word. Useful when composing fragments + /// produced elsewhere. + Raw(String), +} + +impl Word { + /// Choose the smallest safe word form for `s`: bare when it + /// contains only word-safe ASCII characters, double-quoted with + /// escapes otherwise. Empty strings become `""`. + pub fn lit(s: impl Into) -> Word { + let s = s.into(); + if needs_quoting(&s) { + Word::Quoted(s) + } else { + Word::Bare(s) + } + } + + /// `$name` reference. The name is not validated. + pub fn var(name: impl Into) -> Word { + Word::Var(name.into()) + } +} + +fn needs_quoting(s: &str) -> bool { + if s.is_empty() { + return true; + } + s.chars().any(|c| { + c.is_whitespace() + || matches!(c, ';' | '"' | '\\' | '[' | ']' | '{' | '}' | '$' | '#') + }) +} + +impl From<&str> for Word { + fn from(s: &str) -> Self { + Word::lit(s) + } +} + +impl From for Word { + fn from(s: String) -> Self { + Word::lit(s) + } +} + +// --------------------------------------------------------------------------- +// ToHtcl — the interpolation interface for `quote_htcl!`. +// --------------------------------------------------------------------------- + +/// Produce a [`Word`] for interpolation into emitted htcl. +/// +/// Implemented for the common Rust value types. Pass any `T: ToHtcl` +/// to `#expr` slots in `quote_htcl!`; the macro calls +/// `(&expr).to_htcl()` to get the inserted word. +pub trait ToHtcl { + fn to_htcl(&self) -> Word; +} + +impl ToHtcl for Word { + fn to_htcl(&self) -> Word { + self.clone() + } +} +impl ToHtcl for str { + fn to_htcl(&self) -> Word { + Word::lit(self) + } +} +impl ToHtcl for String { + fn to_htcl(&self) -> Word { + Word::lit(self.clone()) + } +} +impl ToHtcl for &T { + fn to_htcl(&self) -> Word { + (*self).to_htcl() + } +} +impl ToHtcl for bool { + fn to_htcl(&self) -> Word { + Word::Bare(if *self { "1".into() } else { "0".into() }) + } +} + +macro_rules! impl_to_htcl_display { + ($($t:ty),* $(,)?) => { + $( + impl ToHtcl for $t { + fn to_htcl(&self) -> Word { + Word::Bare(self.to_string()) + } + } + )* + }; +} +impl_to_htcl_display!(i8, i16, i32, i64, i128, isize); +impl_to_htcl_display!(u8, u16, u32, u64, u128, usize); +impl_to_htcl_display!(f32, f64); + +// --------------------------------------------------------------------------- +// ToTcl — interpolation interface for `quote_tcl!`. +// --------------------------------------------------------------------------- + +/// Produce a [`Word`] for interpolation into emitted *pure Tcl*. +/// +/// Distinct from [`ToHtcl`] so that compiler intrinsics (the `repr` +/// codegen module, `kwargs` shim helpers, future ones) which emit +/// Tcl bodies — not htcl — can carry an independent vocabulary if +/// they grow it. For now the surface is intentionally identical: +/// the same Rust value types yield the same [`Word`] under both +/// traits. The split exists so future Tcl-only forms (typed +/// `Tcl_Obj` handle quoting, namespaced-proc-name formatting, +/// etc.) can land on `ToTcl` without changing `ToHtcl`'s contract. +pub trait ToTcl { + fn to_tcl(&self) -> Word; +} + +impl ToTcl for Word { + fn to_tcl(&self) -> Word { + self.clone() + } +} +impl ToTcl for str { + fn to_tcl(&self) -> Word { + Word::lit(self) + } +} +impl ToTcl for String { + fn to_tcl(&self) -> Word { + Word::lit(self.clone()) + } +} +impl ToTcl for &T { + fn to_tcl(&self) -> Word { + (*self).to_tcl() + } +} +impl ToTcl for bool { + fn to_tcl(&self) -> Word { + Word::Bare(if *self { "1".into() } else { "0".into() }) + } +} + +macro_rules! impl_to_tcl_display { + ($($t:ty),* $(,)?) => { + $( + impl ToTcl for $t { + fn to_tcl(&self) -> Word { + Word::Bare(self.to_string()) + } + } + )* + }; +} +impl_to_tcl_display!(i8, i16, i32, i64, i128, isize); +impl_to_tcl_display!(u8, u16, u32, u64, u128, usize); +impl_to_tcl_display!(f32, f64); + +// --------------------------------------------------------------------------- +// Emit — Display impls produce well-formed htcl text. +// --------------------------------------------------------------------------- + +impl fmt::Display for Doc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + emit_doc(f, self, 0) + } +} + +impl fmt::Display for Item { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + emit_item(f, self, 0) + } +} + +impl fmt::Display for Command { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + emit_command(f, self, 0) + } +} + +impl fmt::Display for Word { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + emit_word(f, self) + } +} + +const INDENT: &str = " "; + +fn emit_indent(f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { + for _ in 0..level { + f.write_str(INDENT)?; + } + Ok(()) +} + +fn emit_doc( + f: &mut fmt::Formatter<'_>, + doc: &Doc, + level: usize, +) -> fmt::Result { + for item in &doc.items { + emit_item(f, item, level)?; + } + Ok(()) +} + +fn emit_item( + f: &mut fmt::Formatter<'_>, + item: &Item, + level: usize, +) -> fmt::Result { + match item { + Item::Command(c) => emit_command(f, c, level), + Item::Comment(text) => { + emit_indent(f, level)?; + writeln!(f, "# {text}") + } + Item::DocComment(text) => { + emit_indent(f, level)?; + writeln!(f, "## {text}") + } + Item::Blank => writeln!(f), + } +} + +fn emit_command( + f: &mut fmt::Formatter<'_>, + cmd: &Command, + level: usize, +) -> fmt::Result { + for doc in &cmd.doc_comments { + emit_indent(f, level)?; + writeln!(f, "## {doc}")?; + } + emit_indent(f, level)?; + let mut first = true; + for w in &cmd.words { + if !first { + f.write_str(" ")?; + } + emit_word(f, w)?; + first = false; + } + if let Some(body) = &cmd.body { + if body.items.is_empty() { + f.write_str(" {}\n")?; + } else { + f.write_str(" {\n")?; + emit_doc(f, body, level + 1)?; + emit_indent(f, level)?; + f.write_str("}\n")?; + } + } else { + f.write_str("\n")?; + } + Ok(()) +} + +fn emit_word(f: &mut fmt::Formatter<'_>, w: &Word) -> fmt::Result { + match w { + Word::Bare(s) => f.write_str(s), + Word::Quoted(s) => { + f.write_str("\"")?; + for c in s.chars() { + match c { + '\\' => f.write_str("\\\\")?, + '"' => f.write_str("\\\"")?, + '$' => f.write_str("\\$")?, + '[' => f.write_str("\\[")?, + ']' => f.write_str("\\]")?, + other => f.write_fmt(format_args!("{other}"))?, + } + } + f.write_str("\"") + } + Word::Braced(s) => { + f.write_str("{")?; + f.write_str(s)?; + f.write_str("}") + } + Word::Var(name) => { + f.write_str("$")?; + f.write_str(name) + } + Word::CmdSubst(s) => { + f.write_str("[")?; + f.write_str(s)?; + f.write_str("]") + } + Word::Raw(s) => f.write_str(s), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn word_lit_picks_bare_when_safe() { + assert!( + matches!(Word::lit("hello"), Word::Bare(ref s) if s == "hello") + ); + assert!(matches!(Word::lit("32.0_GT/s"), Word::Bare(_))); + } + + #[test] + fn word_lit_quotes_when_special() { + let cases = ["with space", "has\"quote", "has$dollar", "has;semi", ""]; + for c in cases { + assert!( + matches!(Word::lit(c), Word::Quoted(_)), + "expected quoted for {c:?}" + ); + } + } + + #[test] + fn emit_command_simple() { + let cmd = Command::call("puts", ["hi"]); + assert_eq!(format!("{cmd}"), "puts hi\n"); + } + + #[test] + fn emit_command_quotes_when_needed() { + let cmd = Command::call("puts", ["hello world"]); + assert_eq!(format!("{cmd}"), "puts \"hello world\"\n"); + } + + #[test] + fn emit_doc_full_proc() { + // proc greet {name} { puts "hi $name" } + let inner = Command::call("puts", [Word::Quoted("hi $name".into())]); + let body = { + let mut d = Doc::new(); + d.cmd(inner); + d + }; + let proc = Command { + doc_comments: vec!["Say hi.".into()], + words: vec![ + Word::Bare("proc".into()), + Word::Bare("greet".into()), + Word::Braced("name".into()), + ], + body: Some(body), + }; + let mut doc = Doc::new(); + doc.cmd(proc); + let out = format!("{doc}"); + let expected = "\ +## Say hi. +proc greet {name} { + puts \"hi \\$name\" +} +"; + assert_eq!(out, expected); + } + + #[test] + fn empty_body_emits_braces() { + let cmd = Command { + words: vec![ + Word::Bare("proc".into()), + Word::Bare("f".into()), + Word::Braced("".into()), + ], + body: Some(Doc::new()), + ..Default::default() + }; + assert_eq!(format!("{cmd}"), "proc f {} {}\n"); + } + + #[test] + fn to_htcl_basic_types() { + assert!(matches!("hi".to_htcl(), Word::Bare(ref s) if s == "hi")); + assert!(matches!(42i64.to_htcl(), Word::Bare(ref s) if s == "42")); + assert!(matches!(true.to_htcl(), Word::Bare(ref s) if s == "1")); + } + + #[test] + fn emitted_output_round_trips_through_parser() { + // Build a doc, emit it, re-parse, and check we get a structurally + // similar document — proves the emitter is producing well-formed + // htcl that the parser accepts. + use crate::parser::parse; + let mut body = Doc::new(); + body.cmd(Command::call("puts", [Word::Quoted("hi $name".into())])); + let proc = Command { + words: vec![ + Word::Bare("proc".into()), + Word::Bare("greet".into()), + Word::Braced("name".into()), + ], + body: Some(body), + ..Default::default() + }; + let mut doc = Doc::new(); + doc.cmd(proc); + let text = doc.to_string(); + let parsed = parse(&text); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + // First (and only) statement should be the proc. + let stmt = &parsed.document.stmts[0]; + let crate::ast::Stmt::Command(cmd) = stmt else { + panic!("expected command, got {stmt:?}"); + }; + let crate::ast::CommandKind::Proc(p) = &cmd.kind else { + panic!("expected proc"); + }; + assert_eq!(p.name.as_deref(), Some("greet")); + } +} diff --git a/vw-htcl/src/enum_parse.rs b/vw-htcl/src/enum_parse.rs new file mode 100644 index 0000000..5938c04 --- /dev/null +++ b/vw-htcl/src/enum_parse.rs @@ -0,0 +1,318 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Mini-parser for the variants block of an `enum NAME = { ... }` +//! declaration. +//! +//! Grammar (operates on the text INSIDE the body braces, not +//! including the braces themselves): +//! +//! ```text +//! Variants ::= Sep* (Variant Sep+ Variant)* Sep* +//! Variant ::= Ident (':' Type)? +//! Sep ::= '\n' | comment | doc_comment | whitespace +//! ``` +//! +//! Variants are newline-separated (mirroring `proc {a; b}` arg-list +//! style); blank lines and `##` doc comments are ignored. The payload +//! type, when present, is parsed via [`crate::type_parse`] verbatim +//! — so anything that grammar accepts (primitives, newtypes, +//! generics, qualified) is legal here too. A future tightening could +//! reject `Qualified` payloads as nonsensical; v1 keeps it permissive +//! and lets the validator decide. + +use crate::ast::EnumVariant; +use crate::span::Span; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnumParseError { + pub message: String, + pub span: Span, +} + +/// Parse the body of an enum declaration. `text` is the contents +/// INSIDE the body braces (not including the braces). `base_offset` +/// is the absolute byte position of `text[0]` in the original +/// source so returned spans land in the right place. +pub fn parse( + text: &str, + base_offset: u32, +) -> Result, EnumParseError> { + let mut p = Parser::new(text, base_offset); + let mut variants = Vec::new(); + loop { + p.skip_separators(); + if p.eof() { + break; + } + variants.push(p.parse_variant()?); + } + Ok(variants) +} + +struct Parser<'a> { + text: &'a str, + bytes: &'a [u8], + pos: usize, + base: u32, +} + +impl<'a> Parser<'a> { + fn new(text: &'a str, base: u32) -> Self { + Self { + text, + bytes: text.as_bytes(), + pos: 0, + base, + } + } + + fn eof(&self) -> bool { + self.pos >= self.bytes.len() + } + + fn here(&self) -> u32 { + self.base + self.pos as u32 + } + + fn here_span(&self) -> Span { + let h = self.here(); + Span::new(h, h) + } + + fn span_from(&self, start: usize) -> Span { + Span::new(self.base + start as u32, self.base + self.pos as u32) + } + + fn skip_horizontal_ws(&mut self) { + while self.pos < self.bytes.len() { + let c = self.bytes[self.pos]; + if c == b' ' || c == b'\t' || c == b'\r' { + self.pos += 1; + } else { + break; + } + } + } + + /// Skip newlines, whitespace, regular `#` comments, and `##` + /// doc comments. Variants are separated by at least one newline + /// (or are at the start of the body). + fn skip_separators(&mut self) { + loop { + // Any whitespace, including newlines. + while self.pos < self.bytes.len() + && self.bytes[self.pos].is_ascii_whitespace() + { + self.pos += 1; + } + if self.eof() { + break; + } + // Comment line — consume to next newline. `##` doc + // comments are dropped here; if a future revision needs + // to attach docs to variants, this is the spot. + if self.bytes[self.pos] == b'#' { + while self.pos < self.bytes.len() + && self.bytes[self.pos] != b'\n' + { + self.pos += 1; + } + continue; + } + break; + } + } + + /// Variant := IDENT (':' TYPE)? + fn parse_variant(&mut self) -> Result { + let start = self.pos; + let (name, name_span) = self.parse_ident()?; + self.skip_horizontal_ws(); + let payload_pos = self.pos; + let (payload, payload_span) = if self.pos < self.bytes.len() + && self.bytes[self.pos] == b':' + { + self.pos += 1; // ':' + self.skip_horizontal_ws(); + let type_start = self.pos; + // Consume up to end-of-line or end-of-input. The + // type-text-extraction window stops at newline so a + // bad payload doesn't bleed into the next variant. + while self.pos < self.bytes.len() && self.bytes[self.pos] != b'\n' { + self.pos += 1; + } + // Trim trailing horizontal whitespace from the + // payload text so spans are tight. + let mut end = self.pos; + while end > type_start + && matches!(self.bytes[end - 1], b' ' | b'\t' | b'\r') + { + end -= 1; + } + let payload_text = &self.text[type_start..end]; + let span = Span::new( + self.base + type_start as u32, + self.base + end as u32, + ); + let ty = crate::type_parse::parse( + payload_text, + self.base + type_start as u32, + ) + .map_err(|e| EnumParseError { + message: e.message, + span: e.span, + })?; + (Some(ty), span) + } else { + // Empty-payload variant. Span is a zero-width point + // right after the name. + let here = Span::new( + self.base + payload_pos as u32, + self.base + payload_pos as u32, + ); + (None, here) + }; + Ok(EnumVariant { + name, + name_span, + payload, + payload_span, + span: self.span_from(start), + }) + } + + fn parse_ident(&mut self) -> Result<(String, Span), EnumParseError> { + self.skip_horizontal_ws(); + let start = self.pos; + if self.eof() { + return Err(EnumParseError { + message: "expected variant name, found end of body".into(), + span: self.here_span(), + }); + } + let first = self.bytes[self.pos]; + if !(first.is_ascii_alphabetic() || first == b'_') { + return Err(EnumParseError { + message: format!( + "expected variant name, found `{}`", + first as char + ), + span: self.here_span(), + }); + } + self.pos += 1; + while self.pos < self.bytes.len() { + let c = self.bytes[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' { + self.pos += 1; + } else { + break; + } + } + let name = self.text[start..self.pos].to_string(); + Ok((name, self.span_from(start))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::TypeExpr; + + fn p(s: &str) -> Vec { + parse(s, 0).unwrap_or_else(|e| panic!("parse failed: {e:?}")) + } + + #[test] + fn empty_body() { + let v = p(""); + assert!(v.is_empty()); + let v = p(" \n\n "); + assert!(v.is_empty()); + } + + #[test] + fn single_variant_with_payload() { + let v = p("Scalar: string"); + assert_eq!(v.len(), 1); + assert_eq!(v[0].name, "Scalar"); + let ty = v[0].payload.as_ref().unwrap(); + assert_eq!(ty.name(), "string"); + } + + #[test] + fn single_empty_payload_variant() { + let v = p("North"); + assert_eq!(v.len(), 1); + assert_eq!(v[0].name, "North"); + assert!(v[0].payload.is_none()); + } + + #[test] + fn mixed_payload_and_empty() { + let v = p("\n North\n South: int\n East\n West\n"); + assert_eq!(v.len(), 4); + assert_eq!(v[0].name, "North"); + assert!(v[0].payload.is_none()); + assert_eq!(v[1].name, "South"); + assert_eq!(v[1].payload.as_ref().unwrap().name(), "int"); + assert_eq!(v[2].name, "East"); + assert!(v[2].payload.is_none()); + assert_eq!(v[3].name, "West"); + assert!(v[3].payload.is_none()); + } + + #[test] + fn generic_payload() { + let v = p("\n Scalar: string\n Nested: dict\n"); + assert_eq!(v.len(), 2); + let TypeExpr::Generic { name, args, .. } = + v[1].payload.as_ref().unwrap() + else { + panic!() + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + assert_eq!(args[1].name(), "Property"); + } + + #[test] + fn comments_skipped_between_variants() { + let v = p( + "\n# leading comment\n## doc comment\nScalar: string\n# trailing\nNested: int\n", + ); + assert_eq!(v.len(), 2); + assert_eq!(v[0].name, "Scalar"); + assert_eq!(v[1].name, "Nested"); + } + + #[test] + fn err_invalid_first_char() { + let e = parse("123Foo: int", 0).unwrap_err(); + assert!(e.message.contains("variant name")); + } + + #[test] + fn err_bad_payload_type() { + let e = parse("Scalar: Option { + let table = signature_table(document); + // Try a doc-comment `[NAME]` reference first — it's cheap and + // rules out any structural resolution when the cursor is inside + // a `##` block. Otherwise fall through to the structural paths. + if let Some(span) = + definition_in_doc_comment(document, source, offset, &table) + { + return Some(span); + } + definition_in_stmts(&document.stmts, None, document, &table, source, offset) + // Fallback: a `$var` the structured tree keeps opaque — inside + // a command substitution or an `if`/`while` condition. Found by + // scanning the source and resolving against the enclosing + // proc's scope. + .or_else(|| definition_of_scanned_var(document, source, offset)) + // Fallback: cursor on a type-name annotation (arg type, + // return type, `type … = TYPE` underlying, generic arg). + .or_else(|| definition_of_type(document, offset)) +} + +/// Cursor on a type name in a proc signature or a `type` decl's +/// underlying → return the matching type declaration's name span. +/// Handles qualified names (`dcmac::MacPortProps`) via the parser's +/// `Qualified` variant plus the type-table lookup helper. +fn definition_of_type(document: &Document, offset: u32) -> Option { + let ty = type_expr_at(document, offset)?; + let name = type_expr_lookup_name(ty); + let decl = find_type_decl(document, &name)?; + Some(decl.name_span) +} + +/// Resolve a `[NAME]` reference embedded in a `##` doc-comment +/// block. Returns `None` when the cursor isn't inside any command's +/// or arg's doc-comment span, or when the `[…]` at the cursor +/// doesn't name a proc declared in this document. +/// +/// Cross-file references (e.g. `[Properties::from]` where +/// `Properties::from` lives in `types.htcl`) resolve when the +/// document has already sourced that file; unresolved names simply +/// return `None`, which the LSP client renders as "no definition +/// available" without disrupting the fallback paths. +fn definition_in_doc_comment( + document: &Document, + source: &str, + offset: u32, + _table: &SignatureTable<'_>, +) -> Option { + let block = enclosing_doc_block(&document.stmts, offset)?; + let name = extract_ref_at(source, block, offset)?; + let proc = find_proc_decl(document, &name)?; + Some(proc.name_span) +} + +/// Return the doc-comment span that contains `offset`, if any. Walks +/// commands, proc signatures, nested proc bodies, and namespace-eval +/// bodies. +fn enclosing_doc_block(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if let Some(span) = cmd.doc_comments_span { + if span.contains(offset) { + return Some(span); + } + } + // ProcArg doc-comment blocks sit inside the proc's args span. + if let CommandKind::Proc(proc) = &cmd.kind { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(span) = arg.doc_comments_span { + if span.contains(offset) { + return Some(span); + } + } + } + } + if let Some(span) = enclosing_doc_block(&proc.body, offset) { + return Some(span); + } + } + if let CommandKind::NamespaceEval(ns) = &cmd.kind { + if let Some(span) = enclosing_doc_block(&ns.body, offset) { + return Some(span); + } + } + } + None +} + +/// Given a doc-comment block-span and a cursor offset inside it, +/// find a `[NAME]` reference whose interior span contains `offset` +/// and return `NAME`. Interior must be a valid ident (letters, +/// digits, `_`, `:` for namespace qualification). `[` and `]` are +/// the disambiguators from surrounding prose. +fn extract_ref_at(source: &str, block: Span, offset: u32) -> Option { + let bytes = source.as_bytes(); + let start = block.start as usize; + let end = (block.end as usize).min(bytes.len()); + let off = offset as usize; + if off < start || off > end { + return None; + } + // Scan for `[…]` pairs. `[` and `]` are cheap to find; the + // interior is validated once we have both delimiters. + let mut i = start; + while i < end { + if bytes[i] == b'[' { + let content_start = i + 1; + // Ident-start rule: first char must be a letter or `_`. + // Same as `render_refs` in doc.rs — keeps the analyzer + // and the renderer in agreement on what counts as a ref. + if content_start < end + && (bytes[content_start].is_ascii_alphabetic() + || bytes[content_start] == b'_') + { + let mut j = content_start; + while j < end && bytes[j] != b']' { + let b = bytes[j]; + let ok = + b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + if !ok { + break; + } + j += 1; + } + if j < end && bytes[j] == b']' && j > content_start { + // Reference spans `[NAME]` inclusive. The cursor + // hits a ref if it's on any of `[`, `NAME`, or `]`. + if off >= i && off <= j { + let name = + std::str::from_utf8(&bytes[content_start..j]) + .ok()? + .to_string(); + return Some(name); + } + i = j + 1; + continue; + } + } + } + i += 1; + } + None +} + +fn definition_of_scanned_var( + document: &Document, + source: &str, + offset: u32, +) -> Option { + let (name, _) = scan_var_ref(source, offset)?; + let (stmts, enclosing) = innermost_scope(document, offset); + resolve_var_def(&name, stmts, enclosing, offset).map(|d| d.def_span()) +} + +/// Resolve the definition at `offset` within `stmts`, descending into +/// proc bodies. `enclosing` is the proc whose body `stmts` belongs to +/// (`None` at the top level), used to resolve variables to parameters. +/// `document` is the whole document so call sites — at any nesting +/// depth — can find their declaring proc, which always lives at the +/// top level. +fn definition_in_stmts<'a>( + stmts: &'a [Stmt], + enclosing: Option<&'a Proc>, + document: &'a Document, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + + // Inside a proc declaration, attribute ident values can + // reference sibling args by name. Resolve those to the arg's + // declaration site. + if let CommandKind::Proc(proc) = &cmd.kind { + if let Some(span) = definition_in_proc_decl(proc, offset) { + return Some(span); + } + // Cursor on the proc's own name — "goto def" of the def + // itself is the same span. Not super useful but + // consistent. + if proc.name_span.contains(offset) { + return Some(proc.name_span); + } + // Otherwise the cursor is somewhere in the body: recurse, + // making this proc the enclosing scope. + return definition_in_stmts( + &proc.body, + Some(proc), + document, + table, + source, + offset, + ); + } + + // `namespace eval { … }` — descend into the populated + // body directly. Without this arm we'd fall through to the + // brace-body reparse path, which works but discards the + // pre-populated proc bodies and re-triggers a full parse. + if let CommandKind::NamespaceEval(ns) = &cmd.kind { + if let Some(span) = definition_in_stmts( + &ns.body, enclosing, document, table, source, offset, + ) { + return Some(span); + } + } + + // Cursor on a `$var` reference → its definition in scope. + if let Some(span) = definition_of_var(cmd, stmts, enclosing, offset) { + return Some(span); + } + + // Generic call site. Two flavors: + // 1. Cursor on the call name → proc declaration. + // 2. Cursor on a `-flag` arg → that arg's decl in the proc. + if let Some(span) = definition_in_call(cmd, document, table, offset) { + return Some(span); + } + + // Cursor inside a `[ … ]` command substitution → recurse into + // its parsed body so goto works on calls written inline. + if let Some(span) = + definition_in_cmd_substs(cmd, document, table, source, offset) + { + return Some(span); + } + + // Cursor inside a `{ … }` control-flow body (the second word + // of `if`, the third of `while`, the body of `foreach`, + // etc.). The parser leaves those as opaque `Braced` words — + // they're semantically Tcl scripts, but there's no + // eager-parse pass like there is for `[ … ]`. So we reparse + // on demand when goto lands the cursor inside one. + if let Some(span) = + definition_in_braced_bodies(cmd, document, table, source, offset) + { + return Some(span); + } + } + None +} + +/// When the cursor sits inside a `{ … }` word of a control-flow +/// command, reparse that word's interior as a htcl fragment, shift +/// all spans back into whole-source coordinates, and recurse into +/// [`definition_in_stmts`]. Without this, `if { … … }`, +/// `while { … … }`, `foreach x $xs { … … }`, and +/// friends never resolve their body calls — goto-def returns +/// "no definition found" from inside every generated +/// `if {[llength …]} { }` scaffold. +/// +/// Only `Braced` words are considered; the enclosing command's +/// head has to be one of a small list of body-hosts so we don't +/// waste a parse on `list {a b c}`-style data braces (their content +/// is a Tcl list, not a script). +fn definition_in_braced_bodies<'a>( + cmd: &'a Command, + document: &'a Document, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option { + let head = cmd.words.first().and_then(|w| w.as_text())?; + if !is_body_host(head) { + return None; + } + // The interior text lives in the source we're parsing against + // — we don't have that here directly, but we do have span + // access to the whole-source Command tree. Each `Braced` word's + // interior is `span.start+1 .. span.end-1` in the outer + // source. We recover it via the WordPart::Text that the parser + // populates for braced words. + for word in cmd.words.iter().skip(1) { + if !matches!(word.form, WordForm::Braced) { + continue; + } + if !word.span.contains(offset) { + continue; + } + // First-part Text carries the interior string with a span + // starting at word.span.start + 1 (the byte after `{`). + let Some(WordPart::Text { + value, + span: text_span, + }) = word.parts.first() + else { + continue; + }; + // Reparse the body as a bracket-body-mode fragment so + // newlines are whitespace and multi-line control flow + // parses cleanly. + // Reparse the brace-body against `source` — the WHOLE + // outer document. That lets us pass `source` back into + // `populate_procs` below so nested `[ … ]` CmdSubst + // bodies get filled in against the same coordinate space + // the outer AST uses. Without the populate pass, a call + // like `set cell [vivado_cmd::create_bd_cell …]` inside + // `if {$bd} { … }` still has an empty CmdSubst.body and + // the recursion bottoms out at `set`. + // + // Order matters: shift first (spans become absolute), then + // populate — the shifted CmdSubst.span carries the + // absolute offset `populate_cmd_subst_parts` needs to shift + // its reparsed interior into place. + let (mut stmts, mut errs) = crate::parser::parse_fragment( + value.as_str(), + crate::parser::Mode::BracketBody, + ); + let delta = text_span.start; + for s in &mut stmts { + crate::parser::shift_stmt(s, delta); + } + crate::parser::populate_procs(&mut stmts, source, &mut errs); + // Recurse into the reparsed stmts. Passing `None` as the + // enclosing proc mirrors what `definition_in_cmd_substs` + // does — variable resolution across a control-flow body + // is out of scope for this fix; the call-site → proc-decl + // path is what matters. + return definition_in_stmts( + &stmts, None, document, table, source, offset, + ); + } + None +} + +/// Command names whose brace-args hold Tcl scripts rather than +/// data. Restricting the lookup to these keeps us from waking the +/// parser on data braces (`list {a b c}`, `dict {k v}`, etc.). +fn is_body_host(head: &str) -> bool { + matches!( + head, + "if" | "elseif" + | "else" + | "while" + | "for" + | "foreach" + | "catch" + | "try" + | "finally" + | "eval" + | "uplevel" + | "namespace" + | "on" + | "apply" + // `dict for` — same rationale as `hover::is_body_host`. + | "dict" + ) +} + +fn definition_in_cmd_substs<'a>( + cmd: &'a Command, + document: &'a Document, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option { + for word in &cmd.words { + if !word.span.contains(offset) { + continue; + } + for part in &word.parts { + if let crate::ast::WordPart::CmdSubst { span, body, .. } = part { + if span.contains(offset) { + return definition_in_stmts( + body, None, document, table, source, offset, + ); + } + } + } + } + None +} + +/// If the cursor is on a `$var` reference (a real [`WordPart::VarRef`]) +/// in `cmd`, resolve it to its definition within `scope_stmts` or a +/// parameter of `enclosing`. +fn definition_of_var<'a>( + cmd: &'a Command, + scope_stmts: &'a [Stmt], + enclosing: Option<&'a Proc>, + offset: u32, +) -> Option { + let name = var_ref_at(cmd, offset)?; + resolve_var_def(name, scope_stmts, enclosing, offset).map(|d| d.def_span()) +} + +/// The name of the `$var` reference under the cursor, if any. Walks +/// word parts so it also fires inside quoted words (`"hi $name"`) and +/// array syntax (`$arr($idx)`). +fn var_ref_at(cmd: &Command, offset: u32) -> Option<&str> { + for word in &cmd.words { + if !word.span.contains(offset) { + continue; + } + for part in &word.parts { + if let WordPart::VarRef { name, span, .. } = part { + if span.contains(offset) { + return Some(name.as_str()); + } + } + } + } + None +} + +fn definition_in_call<'a>( + cmd: &'a Command, + document: &'a Document, + table: &SignatureTable<'a>, + offset: u32, +) -> Option { + let first = cmd.words.first()?; + let name = first.as_text()?; + + // Cursor on the call name. + if first.span.contains(offset) { + let proc = find_proc_decl(document, name)?; + return Some(proc.name_span); + } + + // Cursor on one of the `-flag` words. Look the flag up in the + // called proc's signature and return that arg's name_span. + let sig = *table.get(name)?; + for word in cmd.words.iter().skip(1) { + if !word.span.contains(offset) { + continue; + } + let text = word.as_text()?; + let flag = text.strip_prefix('-')?; + let arg = sig.find(flag)?; + return Some(arg.name_span); + } + + None +} + +/// Find the `proc` declaration that registers under `name` in the +/// document's signature table. Walks `namespace eval` bodies +/// recursively so a call to `project::set_target_language` resolves +/// to the inner `proc set_target_language` inside +/// `namespace eval project { … }`. +fn find_proc_decl<'a>(document: &'a Document, name: &str) -> Option<&'a Proc> { + find_proc_decl_in(&document.stmts, "", name) +} + +fn find_proc_decl_in<'a>( + stmts: &'a [Stmt], + prefix: &str, + name: &str, +) -> Option<&'a Proc> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(decl_name) = proc.name.as_deref() else { + continue; + }; + let qualified = if prefix.is_empty() { + decl_name.to_string() + } else { + format!("{prefix}::{decl_name}") + }; + if qualified == name { + return Some(proc); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(ns_name) = ns.name.as_deref() else { + continue; + }; + let nested = if prefix.is_empty() { + ns_name.to_string() + } else { + format!("{prefix}::{ns_name}") + }; + if let Some(found) = find_proc_decl_in(&ns.body, &nested, name) + { + return Some(found); + } + } + _ => {} + } + } + None +} + +fn definition_in_proc_decl(proc: &Proc, offset: u32) -> Option { + let sig = proc.signature.as_ref()?; + for arg in &sig.args { + for attr in &arg.attributes { + for value in &attr.values { + let AttributeValue::Ident { value: name, span } = value else { + continue; + }; + if !span.contains(offset) { + continue; + } + if let Some(target) = find_sibling_arg(sig, name) { + return Some(target.name_span); + } + // Ident value naming an unknown arg — no definition. + return None; + } + } + } + None +} + +fn find_sibling_arg<'a>( + sig: &'a ProcSignature, + name: &str, +) -> Option<&'a ProcArg> { + sig.args.iter().find(|a| a.name == name) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn first(src: &str, needle: &str) -> u32 { + src.find(needle).expect("needle not found") as u32 + } + + fn nth(src: &str, needle: &str, n: usize) -> u32 { + let mut start = 0; + for i in 0..=n { + let pos = src[start..] + .find(needle) + .map(|p| start + p) + .expect("needle not found enough times"); + if i == n { + return pos as u32; + } + start = pos + needle.len(); + } + unreachable!() + } + + #[test] + fn call_to_proc_decl() { + let src = "\ +proc greet {\n name\n} { puts hi }\n\ +greet -name there\n"; + let parsed = parse(src); + // Cursor on the `g` of the call-site `greet`. + let pos = first(src, "greet -"); + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Should point at the `greet` in the proc declaration (first + // occurrence after `proc `). + let decl_span = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::Proc(p) + if p.name.as_deref() == Some("greet") => + { + Some(p.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, decl_span); + } + + #[test] + fn attribute_ident_to_sibling_arg() { + let src = "\ +proc f {\n has_a\n @requires(has_a) has_b\n} { }\n"; + let parsed = parse(src); + // Cursor on `has_a` inside `@requires(has_a)`. + // First occurrence is the declaration; second is the + // attribute argument. + let pos = nth(src, "has_a", 1); + let target = definition_at(&parsed.document, src, pos).unwrap(); + let decl_pos = first(src, "has_a"); + assert_eq!(target.start, decl_pos); + } + + #[test] + fn call_to_unknown_proc_returns_none() { + let src = "puts hello\n"; + let parsed = parse(src); + assert!( + definition_at(&parsed.document, src, first(src, "puts")).is_none() + ); + } + + #[test] + fn attribute_ident_to_unknown_arg_returns_none() { + let src = "proc f {\n @requires(typo) only\n} { }\n"; + let parsed = parse(src); + let pos = first(src, "typo"); + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + #[test] + fn call_flag_to_arg_decl() { + let src = "\ +proc show {\n flag_a\n width\n} { }\n\ +show -width 16\n"; + let parsed = parse(src); + // Cursor on `-width` at the call site. + let pos = first(src, "-width"); + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Decl `width` arg name is the second `width` in the source. + let decl_pos = nth(src, "width", 0); + assert_eq!(target.start, decl_pos); + } + + #[test] + fn call_to_namespaced_proc_resolves_to_inner_decl() { + // `project::set_target_language` at the call site should + // resolve to `proc set_target_language` declared inside the + // matching `namespace eval project { ... }` block. + let src = "\ +namespace eval project { + proc set_target_language { + proj + language + } { } +} +project::set_target_language -proj p -language VHDL +"; + let parsed = parse(src); + let pos = first(src, "project::set_target_language"); + let target = definition_at(&parsed.document, src, pos).unwrap(); + // The decl's name span covers just `set_target_language` + // (without the namespace prefix), which appears as the + // first occurrence of that bare token in the source. + let decl_pos = first(src, "set_target_language"); + assert_eq!(target.start, decl_pos); + } + + #[test] + fn call_inside_proc_body_to_proc_decl() { + // Mirrors interface.htcl: a call to a top-level proc from + // inside another proc's body. + let src = "\ +proc if_tport {\n type\n name\n} { }\n\ +proc axis {\n width\n} {\n if_tport\n}\n"; + let parsed = parse(src); + // Cursor on the `if_tport` call inside `axis`'s body — the + // second occurrence of `if_tport`. + let pos = nth(src, "if_tport", 1); + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Resolves to the `if_tport` name in the declaration (first + // occurrence). + assert_eq!(target.start, first(src, "if_tport")); + } + + #[test] + fn var_ref_to_set_in_same_body() { + // Mirrors interface.htcl: `$mode` resolves to `set mode ...`. + let src = "\ +proc axis_if {\n kind\n} {\n\ + set mode hello\n\ + use_it $mode\n}\n"; + let parsed = parse(src); + let pos = first(src, "$mode") + 1; // on the `m` of `$mode` + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Should point at the `mode` in `set mode`. + assert_eq!(target.start, first(src, "mode")); + } + + #[test] + fn var_ref_to_proc_parameter() { + // `$name` has no `set`, so it resolves to the proc parameter. + let src = "\ +proc axis_if {\n kind\n name\n} {\n\ + use_it $name\n}\n"; + let parsed = parse(src); + let pos = first(src, "$name") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Parameter `name` decl is the first occurrence of `name`. + assert_eq!(target.start, first(src, "name")); + } + + #[test] + fn var_ref_to_variable_declaration() { + let src = "\ +proc p {} {\n\ + variable vlnv\n\ + use_it $vlnv\n}\n"; + let parsed = parse(src); + let pos = first(src, "$vlnv") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + assert_eq!(target.start, first(src, "vlnv")); + } + + #[test] + fn var_ref_inside_dict_for_body_goes_to_kv_binder() { + let src = "\ +set deps [some_proc] +dict for {lib srcs} $deps { + puts $lib +} +"; + let parsed = parse(src); + let pos = first(src, "$lib\n") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Target should be the `{lib srcs}` braced word. + assert_eq!(target.start, first(src, "{lib srcs}")); + } + + #[test] + fn var_ref_inside_foreach_body_goes_to_binder() { + let src = "foreach x {a b c} {\n puts $x\n}\n"; + let parsed = parse(src); + let pos = first(src, "$x\n") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + assert_eq!(target.start, first(src, "x {a")); + } + + #[test] + fn unknown_var_ref_returns_none() { + let src = "proc p {} {\n use_it $nope\n}\n"; + let parsed = parse(src); + let pos = first(src, "$nope") + 1; + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + #[test] + fn var_ref_inside_opaque_condition_resolves_to_param() { + // `$kind` lives inside an `if` condition that sits inside a + // command substitution — both opaque to the structured tree. + // The source-scan fallback still resolves it to the parameter. + let src = "\ +proc axis_if {\n kind\n} {\n\ + set mode [\n\ + if {$kind == controller} { Master }\n\ + ]\n}\n"; + let parsed = parse(src); + let pos = nth(src, "$kind", 0) + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + assert_eq!(target.start, first(src, "kind")); + } + + #[test] + fn call_inside_command_substitution_resolves_to_decl() { + let src = "\ +proc create_cpm5 {\n name\n} { puts hi }\n\ +set cell [create_cpm5 -name x]\n"; + let parsed = parse(src); + // The second occurrence of `create_cpm5` (the call inside `[…]`). + let pos = nth(src, "create_cpm5", 1); + let target = definition_at(&parsed.document, src, pos).unwrap(); + assert_eq!(target.start, first(src, "create_cpm5")); + } + + #[test] + fn call_flag_to_unknown_arg_returns_none() { + let src = "\ +proc show {\n width\n} { }\n\ +show -widthz 16\n"; + let parsed = parse(src); + let pos = first(src, "-widthz"); + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + /// `[NAME]` in a proc's leading `##` doc block resolves to that + /// proc's declaration when NAME is defined in the same document. + #[test] + fn doc_ref_in_proc_block_resolves_to_proc() { + let src = "\ +proc target {} { puts hi } +## See [target] for related config. +proc other {} { return 1 } +"; + let parsed = parse(src); + // Cursor on the `t` inside `[target]`. + let pos = first(src, "[target]") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + let expected = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::Proc(p) + if p.name.as_deref() == Some("target") => + { + Some(p.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, expected); + } + + /// `[NAME]` in a proc arg's `##` block resolves too — this is + /// exactly the shape the generator emits for `-port0`-style args. + #[test] + fn doc_ref_in_proc_arg_block_resolves() { + let src = "\ +proc mac_port {} { puts ok } +proc create { + ## Configuration for MAC port 0. Construct with [mac_port]. + port0 +} { return 1 } +"; + let parsed = parse(src); + let pos = first(src, "[mac_port]") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + let expected = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::Proc(p) + if p.name.as_deref() == Some("mac_port") => + { + Some(p.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, expected); + } + + /// Unresolved `[NAME]` — target doesn't exist — falls through to + /// None without touching the structural paths. + #[test] + fn doc_ref_to_unknown_returns_none() { + let src = "\ +## See [nonexistent] for details. +proc foo {} { puts hi } +"; + let parsed = parse(src); + let pos = first(src, "[nonexistent]") + 1; + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + /// Cursor on prose inside a doc comment (not on a `[…]` token) + /// falls through to structural resolution. Sanity check that the + /// doc-ref path doesn't intercept every cursor position in a `##`. + #[test] + fn doc_prose_falls_through() { + let src = "\ +## Just prose, no refs here at all. +proc target {} { puts hi } +"; + let parsed = parse(src); + // Cursor on the `p` of "prose" — inside the block but not + // inside a `[…]`. + let pos = first(src, "prose"); + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + /// Goto on a return-type annotation resolves to the `type` decl. + #[test] + fn goto_on_return_type_finds_type_decl() { + let src = "\ +type MyProps = string +proc use {name} MyProps { return $name } +"; + let parsed = parse(src); + // Cursor on `MyProps` in the return-type slot (2nd occurrence). + let pos = nth(src, "MyProps", 1); + let target = definition_at(&parsed.document, src, pos).unwrap(); + let expected = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::TypeDecl(td) + if td.name.as_deref() == Some("MyProps") => + { + Some(td.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, expected); + } + + /// Goto on a qualified type name (`dcmac::MacPortProps`) resolves + /// to the declaration when the type_table key matches. + #[test] + fn goto_on_qualified_type_finds_decl() { + let src = "\ +namespace eval dcmac {} +namespace eval dcmac::T {} +type dcmac::T = string +proc dcmac::T::from {v: string} dcmac::T { return $v } +proc dcmac::T::to {v: dcmac::T} string { return $v } +proc dcmac::T::repr {v: dcmac::T} string { return $v } +proc use {port0: dcmac::T} string { return $port0 } +"; + let parsed = parse(src); + // Cursor on `T` inside `port0: dcmac::T` (the last occurrence). + let pos = src.rfind("dcmac::T").unwrap() as u32 + 7; // land on `T` + let target = definition_at(&parsed.document, src, pos).unwrap(); + let expected = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::TypeDecl(td) + if td.name.as_deref() == Some("dcmac::T") => + { + Some(td.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, expected); + } +} diff --git a/vw-htcl/src/hover.rs b/vw-htcl/src/hover.rs new file mode 100644 index 0000000..e8c7910 --- /dev/null +++ b/vw-htcl/src/hover.rs @@ -0,0 +1,1193 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Find the htcl construct at a given byte offset. +//! +//! Used by `vw analyzer` for `textDocument/hover` and (later) by the +//! REPL for inline hover-style popups. Pure analysis — returns a +//! structured [`HoverTarget`] referencing into the document; the +//! caller formats it (markdown for LSP, a ratatui widget for the +//! REPL, etc). + +use crate::ast::{ + Command, CommandKind, Document, Proc, ProcArg, ProcSignature, Stmt, Word, + WordForm, WordPart, +}; +use crate::lower::{signature_table, SignatureTable}; +use crate::scope::{ + find_type_decl, innermost_scope, resolve_var_def, scan_var_ref, + type_expr_at, type_expr_lookup_name, VarDef, +}; +use crate::span::Span; + +/// A construct the cursor is on, plus the data needed to render +/// hover content. Lifetime-tied to the [`Document`] passed into +/// [`hover_at`]. +#[derive(Clone, Debug)] +pub enum HoverTarget<'a> { + /// Cursor is on the name of a `proc` declaration. The proc's own + /// signature contains the docs. + ProcDef { proc: &'a Proc, span: Span }, + /// Cursor is on the name of an argument inside a `proc` + /// declaration's args braces. + ProcArgDef { + proc_name: String, + arg: &'a ProcArg, + span: Span, + }, + /// Cursor is on the first word of a command that resolves to a + /// known structured proc — i.e. a call to a documented proc. + CallSite { + proc_name: String, + signature: &'a ProcSignature, + span: Span, + }, + /// Cursor is on a `-flag` word in a call to a known proc. + CallArg { + proc_name: String, + arg: &'a ProcArg, + span: Span, + }, + /// Cursor is on a `$var` reference that resolves to a local + /// (`set`/`variable`) rather than a parameter. The span is the + /// reference itself. `ty` carries the type inferred from the + /// binding's RHS when the shape is knowable (a `[typed_proc ...]` + /// call substitution, a `$other_typed_var` copy, or a + /// `true`/`false` literal); `None` when the RHS is opaque. + LocalVar { + name: String, + span: Span, + ty: Option, + }, + /// Cursor is on the name of an `enum` declaration. Shows the + /// variants block as a hover popup. + EnumDef { + decl: &'a crate::ast::EnumDecl, + span: Span, + }, + /// Cursor is on a type name in a proc signature or type-decl + /// underlying — e.g. `MyNewtype` in `proc f {} MyNewtype { … }` + /// or `dcmac::MacPortProps` in `-port0: dcmac::MacPortProps`. + /// Resolves to a declared `type` in the document. + TypeDef { + decl: &'a crate::ast::TypeDecl, + span: Span, + }, +} + +impl HoverTarget<'_> { + pub fn span(&self) -> Span { + match self { + HoverTarget::ProcDef { span, .. } + | HoverTarget::ProcArgDef { span, .. } + | HoverTarget::CallSite { span, .. } + | HoverTarget::CallArg { span, .. } + | HoverTarget::LocalVar { span, .. } + | HoverTarget::EnumDef { span, .. } + | HoverTarget::TypeDef { span, .. } => *span, + } + } +} + +pub fn hover_at<'a>( + document: &'a Document, + source: &'a str, + offset: u32, +) -> Option> { + let table = signature_table(document); + // Doc-comment `[NAME]` reference — cheap to detect and rules + // out any structural resolution when the cursor is inside a + // `##` block. Renders the target proc's signature as the hover + // content, same as if the cursor were on a call site. + if let Some(t) = hover_in_doc_comment(document, source, offset, &table) { + return Some(t); + } + // Cursor on the name-word of a `set VAR X` binding → treat it + // as the local's definition site, showing the same + // name-and-inferred-type hover that a `$VAR` reference would. + // Runs before the general stmt walker because that walker + // resolves `set` as a generic call and returns nothing useful + // for the name-word position. + if let Some(t) = hover_in_set_binding(document, source, offset, &table) { + return Some(t); + } + // Cursor on a binder-name inside a control-flow varname-list + // (`foreach {a b}`, `dict for {k v}`, `catch BODY resvar`). + // Same LocalVar hover shape as the `set NAME` binding site so + // the two look identical to the reader. + if let Some(t) = hover_in_control_flow_binding(document, source, offset) { + return Some(t); + } + hover_in_stmts(&document.stmts, &table, source, offset) + // Fallback: a `$var` reference — including one buried in opaque + // text (a command substitution or `if`/`while` condition). + .or_else(|| hover_scanned_var(document, source, offset, &table)) + // Fallback: cursor on a type-name annotation (arg type, + // return type, `type … = TYPE` underlying, generic arg). + .or_else(|| hover_of_type(document, offset)) +} + +/// Cursor on the name-word of a `set NAME VALUE` command (the +/// binding site, no `$` prefix). Reuses the same `infer_local_type` +/// walker as the `$var` fallback so the hover shows `$NAME: T` in +/// both places consistently. +fn hover_in_set_binding<'a>( + document: &'a Document, + _source: &'a str, + offset: u32, + sig_table: &crate::lower::SignatureTable<'a>, +) -> Option> { + use crate::ast::CommandKind; + let (stmts, enclosing) = innermost_scope(document, offset); + let cmd = find_set_command_at(stmts, offset)?; + // Skip the containing proc when scanning nested control-flow — + // for now `find_set_command_at` only looks at top-level stmts of + // the innermost scope, which covers the common case. + let CommandKind::Set = cmd.kind else { + return None; + }; + let name_word = cmd.words.get(1)?; + if !name_word.span.contains(offset) { + return None; + } + let name = name_word.as_text()?.to_string(); + let ty = infer_local_type( + stmts, + enclosing, + sig_table, + document, + &name, + name_word.span, + ); + Some(HoverTarget::LocalVar { + name, + span: name_word.span, + ty, + }) +} + +/// Cursor sits on a bare identifier inside a control-flow varname +/// list — the `lib` / `srcs` inside `dict for {lib srcs} …`, an +/// `a` / `b` inside `foreach {a b} …`, or the result-var of +/// `catch { … } err`. Returns a `LocalVar` target pointing at the +/// whole braced word (or the bare word), so hover shows the same +/// `$name` shape it would show on a `$name` reference lower in +/// the body. +/// +/// Sub-token spans inside the braced list aren't currently +/// tracked by the parser, so the returned span covers the whole +/// braced group — the click target still lands on `lib` (or +/// wherever the cursor is), just with a slightly wider highlight. +fn hover_in_control_flow_binding<'a>( + document: &'a Document, + source: &'a str, + offset: u32, +) -> Option> { + use crate::ast::{CommandKind, WordForm, WordPart}; + let (stmts, _enclosing) = innermost_scope(document, offset); + // Walk statements looking for a Generic-command whose head is + // a body-host and whose varname arg contains `offset`. + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + if !matches!(cmd.kind, CommandKind::Generic) { + continue; + } + let Some(head) = cmd.words.first().and_then(|w| w.as_text()) else { + continue; + }; + // Collect (word-idx, is-varname-list) tuples per head. + let varname_word_indices: Vec = match head { + "foreach" => { + // Word 1, 3, 5, … up to body_idx (last word). + let body_idx = cmd.words.len().saturating_sub(1); + (1..body_idx).step_by(2).collect() + } + "dict" => { + if cmd.words.get(1).and_then(|w| w.as_text()) == Some("for") { + vec![2] + } else { + continue; + } + } + "catch" => vec![2, 3], + _ => continue, + }; + for idx in varname_word_indices { + let Some(word) = cmd.words.get(idx) else { + continue; + }; + if !word.span.contains(offset) { + continue; + } + // Figure out which sub-name the cursor is on. + let target_name = match word.form { + WordForm::Bare => word.as_text()?.to_string(), + WordForm::Braced => { + let WordPart::Text { + value, + span: text_span, + } = word.parts.first()? + else { + continue; + }; + // Find the whitespace-delimited token at the + // cursor offset within the braced interior. + let rel = offset.saturating_sub(text_span.start) as usize; + let bytes = value.as_bytes(); + let mut start = rel.min(bytes.len()); + while start > 0 + && !bytes[start - 1].is_ascii_whitespace() + && bytes[start - 1] != b'{' + { + start -= 1; + } + let mut end = rel.min(bytes.len()); + while end < bytes.len() + && !bytes[end].is_ascii_whitespace() + && bytes[end] != b'}' + { + end += 1; + } + if start >= end { + continue; + } + value[start..end].to_string() + } + _ => continue, + }; + let _ = source; // reserved for future sub-token spans + return Some(HoverTarget::LocalVar { + name: target_name, + span: word.span, + ty: None, + }); + } + } + None +} + +fn find_set_command_at(stmts: &[Stmt], offset: u32) -> Option<&Command> { + use crate::ast::{CommandKind, Stmt}; + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + if matches!(cmd.kind, CommandKind::Set) { + return Some(cmd); + } + } + None +} + +/// Cursor on a type name → return a `TypeDef` hover target for the +/// matching declaration. Mirror of [`crate::goto::definition_of_type`]. +fn hover_of_type(document: &Document, offset: u32) -> Option> { + let ty = type_expr_at(document, offset)?; + let name = type_expr_lookup_name(ty); + let decl = find_type_decl(document, &name)?; + Some(HoverTarget::TypeDef { + decl, + span: ty.span(), + }) +} + +/// Hover for a `[NAME]` reference inside a `##` doc-comment block. +/// The result mirrors `HoverTarget::CallSite` for the referenced +/// proc, so the LSP formatter renders the target's signature with +/// its own docs — exactly what a reader following the reference +/// wants to see. +fn hover_in_doc_comment<'a>( + document: &'a Document, + source: &str, + offset: u32, + table: &SignatureTable<'a>, +) -> Option> { + let block = enclosing_doc_block(&document.stmts, offset)?; + let name = extract_ref_at(source, block, offset)?; + // Anchor the hover span on the `[NAME]` reference itself so the + // editor highlights just that token. + let ref_span = ref_span_at(source, block, offset)?; + let sig = *table.get(&name)?; + Some(HoverTarget::CallSite { + proc_name: name, + signature: sig, + span: ref_span, + }) +} + +/// Return the doc-comment span containing `offset`. Mirror of +/// [`crate::goto::enclosing_doc_block`] — kept crate-local because +/// both consumers want the same rule. +fn enclosing_doc_block(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if let Some(span) = cmd.doc_comments_span { + if span.contains(offset) { + return Some(span); + } + } + if let CommandKind::Proc(proc) = &cmd.kind { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(span) = arg.doc_comments_span { + if span.contains(offset) { + return Some(span); + } + } + } + } + if let Some(span) = enclosing_doc_block(&proc.body, offset) { + return Some(span); + } + } + if let CommandKind::NamespaceEval(ns) = &cmd.kind { + if let Some(span) = enclosing_doc_block(&ns.body, offset) { + return Some(span); + } + } + } + None +} + +/// Extract the identifier inside a `[NAME]` reference at `offset`. +/// See goto.rs's `extract_ref_at` for the disambiguation rules. +fn extract_ref_at(source: &str, block: Span, offset: u32) -> Option { + let (_, name) = find_ref(source, block, offset)?; + Some(name) +} + +/// Return the inclusive span of the `[NAME]` token at `offset` in +/// the block, so the hover popup anchors on the reference (not the +/// entire doc block). +fn ref_span_at(source: &str, block: Span, offset: u32) -> Option { + let (span, _) = find_ref(source, block, offset)?; + Some(span) +} + +fn find_ref(source: &str, block: Span, offset: u32) -> Option<(Span, String)> { + let bytes = source.as_bytes(); + let start = block.start as usize; + let end = (block.end as usize).min(bytes.len()); + let off = offset as usize; + if off < start || off > end { + return None; + } + let mut i = start; + while i < end { + if bytes[i] == b'[' { + let content_start = i + 1; + // Ident-start rule mirrors doc.rs / goto.rs. + if content_start < end + && (bytes[content_start].is_ascii_alphabetic() + || bytes[content_start] == b'_') + { + let mut j = content_start; + while j < end && bytes[j] != b']' { + let b = bytes[j]; + let ok = + b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + if !ok { + break; + } + j += 1; + } + if j < end && bytes[j] == b']' && j > content_start { + if off >= i && off <= j { + let name = + std::str::from_utf8(&bytes[content_start..j]) + .ok()? + .to_string(); + return Some(( + Span::new(i as u32, (j + 1) as u32), + name, + )); + } + i = j + 1; + continue; + } + } + } + i += 1; + } + None +} + +/// Hover for a `$var` reference found by scanning the source. Resolves +/// to a parameter (rendered like an arg) or a local (`set`/`variable`). +fn hover_scanned_var<'a>( + document: &'a Document, + source: &str, + offset: u32, + sig_table: &crate::lower::SignatureTable<'a>, +) -> Option> { + let (name, span) = scan_var_ref(source, offset)?; + let (stmts, enclosing) = innermost_scope(document, offset); + match resolve_var_def(&name, stmts, enclosing, offset)? { + VarDef::Param(arg) => Some(HoverTarget::ProcArgDef { + proc_name: enclosing + .and_then(|p| p.name.clone()) + .unwrap_or_default(), + arg, + // Anchor the hover on the reference, not the declaration. + span, + }), + VarDef::Local(def_span) => { + let ty = infer_local_type( + stmts, enclosing, sig_table, document, &name, def_span, + ); + Some(HoverTarget::LocalVar { name, span, ty }) + } + } +} + +/// Infer the type of the local variable `name` whose defining `set` +/// command's name-word lives at `def_span`. Walks `scope_stmts` in +/// order to seed `VarTypeTable` with any typed `set`s that come +/// before the target — so a chain like `set a [typed]; set b $a` +/// still types `b`. Seeds parameter types too, in case `set b $arg` +/// forwards a parameter through a local. Returns `None` when the +/// RHS is opaque (the same policy the validator uses). +fn infer_local_type<'a>( + scope_stmts: &'a [Stmt], + enclosing: Option<&'a crate::ast::Proc>, + sig_table: &crate::lower::SignatureTable<'a>, + document: &'a Document, + name: &str, + def_span: Span, +) -> Option { + use crate::ast::{CommandKind, Stmt}; + let mut var_table = crate::validate::VarTypeTable::new(); + // Parameter types are visible to `set` RHS inference — a forward + // like `set out $arg` propagates the arg's type through. + if let Some(proc) = enclosing { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + var_table.insert(arg.name.clone(), ty.clone()); + } + } + } + } + // Full document-wide proc table so `[user_proc]` return-type + // inference kicks in for user procs without an annotated + // return type — same walker the REPL / putr chain uses. + let proc_table = crate::validate::build_proc_table(document); + for stmt in scope_stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !matches!(cmd.kind, CommandKind::Set) { + continue; + } + let Some(name_word) = cmd.words.get(1) else { + continue; + }; + let Some(value_word) = cmd.words.get(2) else { + continue; + }; + let Some(binding_name) = name_word.as_text() else { + continue; + }; + // Type the RHS in the pre-target var-table (chain support). + let ty = crate::validate::value_type_with_procs( + value_word, + sig_table, + &var_table, + Some(&proc_table), + ); + if let Some(ref t) = ty { + var_table.insert(binding_name.to_string(), t.clone()); + } + // Return the type of the specific binding the hover is on — + // matched by name-word span so shadowed bindings before it + // don't overwrite the answer. + if name_word.span == def_span && binding_name == name { + return ty; + } + } + None +} + +/// Find the hover target at `offset` within `stmts`, descending into +/// proc bodies. The signature table is the document-wide (top-level) +/// one, so a call inside a body still resolves to the proc it names. +fn hover_in_stmts<'a>( + stmts: &'a [Stmt], + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + if let Some(target) = hover_in_command(cmd, table, source, offset) { + return Some(target); + } + } + None +} + +fn hover_in_command<'a>( + cmd: &'a Command, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + let primary = match &cmd.kind { + CommandKind::Proc(proc) => hover_in_proc_decl(proc, offset) + // Cursor isn't on the proc's name or an arg — look inside + // the body. + .or_else(|| hover_in_stmts(&proc.body, table, source, offset)), + CommandKind::NamespaceEval(ns) => { + // `namespace eval { … }` — descend into the + // populated body. The parser's post-pass already + // reparsed the block into `ns.body`, so we walk the + // structured AST rather than re-triggering the on-demand + // brace-body reparse. + hover_in_stmts(&ns.body, table, source, offset) + } + CommandKind::EnumDecl(decl) => { + // Cursor on the enum's name → show the variants. + if decl.name_span.contains(offset) { + Some(HoverTarget::EnumDef { + decl, + span: decl.name_span, + }) + } else { + None + } + } + _ => hover_in_call(cmd, table, offset), + }; + primary + .or_else(|| hover_in_cmd_substs(&cmd.words, table, source, offset)) + .or_else(|| hover_in_braced_bodies(cmd, table, source, offset)) +} + +/// Cursor inside a `{ … }` control-flow body (the second word of +/// `if`, the third of `while`, the body of `foreach`, etc.). The +/// parser leaves those as opaque `Braced` words — semantically +/// Tcl scripts, but no eager-parse pass like `[ … ]` gets. So we +/// reparse the body on demand when hover lands the cursor inside +/// one. Mirrors `goto::definition_in_braced_bodies` (same fix, same +/// motivating case — IP-wrapper `set_property` calls sit inside +/// `if {[llength $_vw_d] > 0} { … }` scaffolds). +fn hover_in_braced_bodies<'a>( + cmd: &'a Command, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + let head = cmd.words.first().and_then(|w| w.as_text())?; + if !is_body_host(head) { + return None; + } + for word in cmd.words.iter().skip(1) { + if !matches!(word.form, WordForm::Braced) { + continue; + } + if !word.span.contains(offset) { + continue; + } + let Some(WordPart::Text { + value, + span: text_span, + }) = word.parts.first() + else { + continue; + }; + // Reparse against the fragment text; shift spans up to + // whole-source coordinates; THEN run `populate_procs` so + // nested `[ … ]` CmdSubst bodies inside the brace-body + // get their own recursive parse. Without the populate + // pass, a call like + // `set cell [vivado_cmd::create_bd_cell …]` inside an + // `if {$bd} { … }` branch has an empty CmdSubst body and + // the transient walker can't reach the call. + let (mut stmts, mut errs) = crate::parser::parse_fragment( + value.as_str(), + crate::parser::Mode::BracketBody, + ); + let delta = text_span.start; + for s in &mut stmts { + crate::parser::shift_stmt(s, delta); + } + crate::parser::populate_procs(&mut stmts, source, &mut errs); + // NOTE: the reparsed stmts are owned by this call; but + // `HoverTarget` variants only borrow from `Command` / + // `Proc` / `ProcArg` / `ProcSignature` values that we've + // been threading via `&'a` from the outer document. + // Anything produced from the *reparsed* fragment would need + // its own owned storage — and there's nowhere to put it in + // the current HoverTarget shape. Rather than restructure + // that lifetime, we recurse only to look up calls that + // resolve through `table` (which lives at the top level + // and is already `'a`). + return hover_in_stmts_transient(&stmts, table, source, offset); + } + None +} + +/// Sig-table-only pass over transient (locally-owned) stmts: +/// only the `hover_in_call` branch that resolves through the +/// document-level `SignatureTable<'a>` is reachable. See the +/// comment in [`hover_in_braced_bodies`] for the lifetime story. +fn hover_in_stmts_transient<'a>( + stmts: &[Stmt], + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + // Cursor on a call name → hover its proc via `table`. + if let Some(target) = hover_call_via_table(cmd, table, offset) { + return Some(target); + } + // Nested `[ … ]` inside the reparsed body — recurse via + // the same transient walker so an `if { if { … [call] } }` + // chain works. + for word in &cmd.words { + if !word.span.contains(offset) { + continue; + } + for part in &word.parts { + if let WordPart::CmdSubst { span, body, .. } = part { + if span.contains(offset) { + return hover_in_stmts_transient( + body, table, source, offset, + ); + } + } + } + } + // Nested braced body inside the reparsed body — same idea. + let head = cmd.words.first().and_then(|w| w.as_text()); + if let Some(head) = head { + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if !matches!(word.form, WordForm::Braced) { + continue; + } + if !word.span.contains(offset) { + continue; + } + let Some(WordPart::Text { + value, + span: text_span, + }) = word.parts.first() + else { + continue; + }; + let (mut inner, mut inner_errs) = + crate::parser::parse_fragment( + value.as_str(), + crate::parser::Mode::BracketBody, + ); + let delta = text_span.start; + for s in &mut inner { + crate::parser::shift_stmt(s, delta); + } + crate::parser::populate_procs( + &mut inner, + source, + &mut inner_errs, + ); + return hover_in_stmts_transient( + &inner, table, source, offset, + ); + } + } + } + } + None +} + +/// Sig-table-only variant of [`hover_in_call`] that avoids taking +/// any borrow from `cmd` — returned data references only `'a` +/// values that live in the outer document via the sig table. +fn hover_call_via_table<'a>( + cmd: &Command, + table: &SignatureTable<'a>, + offset: u32, +) -> Option> { + let first = cmd.words.first()?; + let name_text = first.as_text()?; + let sig = *table.get(name_text)?; + // Cursor on the call name → the callee's signature. + if first.span.contains(offset) { + return Some(HoverTarget::CallSite { + proc_name: name_text.to_string(), + signature: sig, + span: first.span, + }); + } + // Cursor on a `-flag` word → the corresponding arg's docs. + for word in cmd.words.iter().skip(1) { + if !word.span.contains(offset) { + continue; + } + let text = word.as_text()?; + let flag = text.strip_prefix('-')?; + let arg = sig.find(flag)?; + return Some(HoverTarget::CallArg { + proc_name: name_text.to_string(), + arg, + span: word.span, + }); + } + None +} + +/// Command names whose brace-args hold Tcl scripts rather than +/// data. Same list as [`crate::goto`]'s counterpart. Exposed to +/// the crate so the unused-var pass (`crate::unused`) can reuse +/// it without a third copy. +pub(crate) fn is_body_host(head: &str) -> bool { + matches!( + head, + "if" | "elseif" + | "else" + | "while" + | "for" + | "foreach" + | "catch" + | "try" + | "finally" + | "eval" + | "uplevel" + | "namespace" + | "on" + | "apply" + // `dict for` — head word alone doesn't disambiguate + // (`dict get`/`dict set`/… have no script body); we + // include `dict` here and let the per-word-form loop + // skip non-braced args. False positives cost a + // reparse but never produce spurious hover results. + | "dict" + ) +} + +/// Descend into any `[ … ]` command substitutions on this command's +/// words so hover works on calls written inline, e.g. +/// `set cell [create_cpm5 -name x]`. +fn hover_in_cmd_substs<'a>( + words: &'a [Word], + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + for word in words { + if !word.span.contains(offset) { + continue; + } + for part in &word.parts { + if let crate::ast::WordPart::CmdSubst { span, body, .. } = part { + if span.contains(offset) { + return hover_in_stmts(body, table, source, offset); + } + } + } + } + None +} + +fn hover_in_proc_decl<'a>( + proc: &'a Proc, + offset: u32, +) -> Option> { + if proc.name_span.contains(offset) { + return Some(HoverTarget::ProcDef { + proc, + span: proc.name_span, + }); + } + if let Some(sig) = proc.signature.as_ref() { + for arg in &sig.args { + if arg.name_span.contains(offset) { + let proc_name = proc.name.clone().unwrap_or_default(); + return Some(HoverTarget::ProcArgDef { + proc_name, + arg, + span: arg.name_span, + }); + } + } + } + None +} + +fn hover_in_call<'a>( + cmd: &'a Command, + table: &SignatureTable<'a>, + offset: u32, +) -> Option> { + let first = cmd.words.first()?; + let name = first.as_text()?; + let sig = *table.get(name)?; + + if first.span.contains(offset) { + return Some(HoverTarget::CallSite { + proc_name: name.to_string(), + signature: sig, + span: first.span, + }); + } + + // Walk remaining words looking for the `-flag` under the cursor. + // Value words (the token after a flag) don't trigger hover — + // they could be anything from a literal to a [cmd subst], and + // there's no general definition to point at. + for word in cmd.words.iter().skip(1) { + if !word.span.contains(offset) { + continue; + } + let text = word.as_text()?; + let flag = text.strip_prefix('-')?; + let arg = sig.find(flag)?; + return Some(HoverTarget::CallArg { + proc_name: name.to_string(), + arg, + span: word.span, + }); + } + None +} + +// Helpers retained for symmetric use from formatters that want to +// pretty-print attributes etc. without re-walking from raw AST. +#[allow(dead_code)] +fn _word_text(word: &Word) -> Option<&str> { + word.as_text() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn at(src: &str, needle: &str, occurrence: usize) -> u32 { + let mut start = 0; + for i in 0..=occurrence { + let pos = src[start..] + .find(needle) + .map(|p| start + p) + .expect("needle not found"); + if i == occurrence { + return pos as u32; + } + start = pos + needle.len(); + } + unreachable!() + } + + fn first(src: &str, needle: &str) -> u32 { + at(src, needle, 0) + } + + #[test] + fn hover_on_call_name() { + let src = "\ +proc greet {\n @default(\"world\") name\n} { puts $name }\n\ +greet -name there\n"; + let parsed = parse(src); + let target = + hover_at(&parsed.document, src, first(src, "greet -")).unwrap(); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "greet"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } + + #[test] + fn hover_on_call_arg_flag() { + let src = "\ +proc greet {\n @default(\"world\") name\n} { puts $name }\n\ +greet -name there\n"; + let parsed = parse(src); + let pos = first(src, "-name there"); + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::CallArg { arg, proc_name, .. } => { + assert_eq!(proc_name, "greet"); + assert_eq!(arg.name, "name"); + } + other => panic!("expected CallArg, got {other:?}"), + } + } + + #[test] + fn hover_on_value_word_returns_none() { + let src = "\ +proc greet {\n @default(\"world\") name\n} { puts $name }\n\ +greet -name there\n"; + let parsed = parse(src); + let pos = first(src, "there"); + assert!(hover_at(&parsed.document, src, pos).is_none()); + } + + #[test] + fn hover_on_proc_decl_name() { + let src = "proc greet {\n name\n} { puts $name }\n"; + let parsed = parse(src); + let pos = first(src, "greet"); + let target = hover_at(&parsed.document, src, pos).unwrap(); + assert!(matches!(target, HoverTarget::ProcDef { .. })); + } + + #[test] + fn hover_on_proc_arg_decl() { + let src = "proc greet {\n @default(\"x\") name\n} { puts hi }\n"; + let parsed = parse(src); + let pos = first(src, "name"); // first "name" is in args + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::ProcArgDef { arg, .. } => { + assert_eq!(arg.name, "name"); + assert_eq!(arg.attributes[0].name, "default"); + } + other => panic!("expected ProcArgDef, got {other:?}"), + } + } + + #[test] + fn hover_on_call_inside_proc_body() { + // A call to a documented proc from within another proc's body + // should hover, just like a top-level call. + let src = "\ +proc if_tport {\n type\n name\n} { }\n\ +proc axis {\n width\n} {\n if_tport\n}\n"; + let parsed = parse(src); + let pos = at(src, "if_tport", 1); + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "if_tport"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } + + #[test] + fn hover_on_call_inside_command_substitution() { + // The interior of `[ … ]` is now parsed; hover on the inner + // call's name should report the proc the same way it does at + // the top level. + let src = "\ +proc create_cpm5 {\n @default(0) name\n} { puts hi }\n\ +set cell [create_cpm5 -name x]\n"; + let parsed = parse(src); + let pos = at(src, "create_cpm5", 1); // the call inside [ ] + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "create_cpm5"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } + + #[test] + fn hover_on_unknown_call_returns_none() { + let src = "puts hello\n"; + let parsed = parse(src); + let pos = first(src, "puts"); + assert!(hover_at(&parsed.document, src, pos).is_none()); + } + + #[test] + fn hover_on_var_in_condition_shows_param() { + // `$kind` inside an opaque `if` condition resolves, via the + // source scan, to the proc parameter — rendered like an arg. + let src = "\ +proc axis_if {\n @enum(target, controller) kind\n} {\n\ + set m [ if {$kind == controller} { a } ]\n}\n"; + let parsed = parse(src); + let pos = first(src, "$kind") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::ProcArgDef { arg, .. } => { + assert_eq!(arg.name, "kind"); + assert_eq!(arg.attributes[0].name, "enum"); + } + other => panic!("expected ProcArgDef, got {other:?}"), + } + } + + #[test] + fn hover_on_local_var_reports_local() { + let src = "\ +proc p {} {\n set count 0\n use $count\n}\n"; + let parsed = parse(src); + let pos = first(src, "$count") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, .. } => assert_eq!(name, "count"), + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_var_ref_inside_dict_for_body_resolves_to_kv_binder() { + // `dict for {lib srcs} $deps { … $lib … }` — hovering the + // `$lib` reference in the body should resolve to the + // binder at the `{lib srcs}` list. + let src = "\ +set deps [some_proc] +dict for {lib srcs} $deps { + puts $lib +} +"; + let parsed = parse(src); + let pos = first(src, "$lib\n") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, .. } => assert_eq!(name, "lib"), + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_bare_name_inside_dict_for_kv_list_resolves() { + // Cursor on `lib` INSIDE `{lib srcs}` — the binding site + // itself. Same LocalVar hover shape as if the cursor were + // on `$lib` later in the body. + let src = "dict for {lib srcs} $deps { puts $lib }\n"; + let parsed = parse(src); + let pos = first(src, "lib srcs"); + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, .. } => assert_eq!(name, "lib"), + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_foreach_kv_list_binder_resolves() { + // Same story for `foreach {a b} $pairs { … }`. + let src = "foreach {a b} $pairs { puts $a }\n"; + let parsed = parse(src); + let pos = first(src, "a b}"); + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, .. } => assert_eq!(name, "a"), + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_set_binding_name_infers_type() { + // Cursor on the LHS of `set x [typed]` (no `$`) — should + // resolve as a LocalVar with the RHS's type, same as + // hovering `$x` later would. + let src = "\ +proc make_it {} string { return hi } +proc p {} { + set x [make_it] +} +"; + let parsed = parse(src); + // Cursor lands on the `x` of `set x` (not `$x`). + let pos = first(src, "set x ") + "set ".len() as u32; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, ty, .. } => { + assert_eq!(name, "x"); + let ty = ty.expect("expected inferred type"); + assert!( + matches!( + ty, + crate::ast::TypeExpr::Named { ref name, .. } + if name == "string" + ), + "got {ty:?}", + ); + } + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_local_var_infers_type_from_typed_call() { + // `set x [typed_proc ...]` where `typed_proc` has an + // annotated return type → hovering `$x` should carry that + // type. + let src = "\ +proc make_it {} string { return hi } +proc p {} { + set x [make_it] + use $x +} +"; + let parsed = parse(src); + let pos = first(src, "$x") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, ty, .. } => { + assert_eq!(name, "x"); + let ty = ty.expect("expected inferred type"); + assert!( + matches!( + ty, + crate::ast::TypeExpr::Named { ref name, .. } + if name == "string" + ), + "got {ty:?}", + ); + } + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_local_var_untyped_rhs_reports_no_type() { + let src = "\ +proc p {} { + set x [some_untyped] + use $x +} +"; + let parsed = parse(src); + let pos = first(src, "$x") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { ty, .. } => { + assert!(ty.is_none(), "expected no type, got {ty:?}"); + } + other => panic!("expected LocalVar, got {other:?}"), + } + } + + /// `[NAME]` inside a `##` block renders as a CallSite hover on + /// the referenced proc — same as if the cursor were on a live + /// call. Lets the reader hover the reference and see the + /// target's signature. + #[test] + fn doc_ref_hovers_as_target_call_site() { + let src = "\ +## Documented target proc. +proc target {} { puts hi } +## See [target] for more. +proc caller {} { return 1 } +"; + let parsed = crate::parser::parse(src); + let pos = src.find("[target]").unwrap() as u32 + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "target"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } +} diff --git a/vw-htcl/src/lib.rs b/vw-htcl/src/lib.rs new file mode 100644 index 0000000..ec89d50 --- /dev/null +++ b/vw-htcl/src/lib.rs @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! htcl language layer. +//! +//! Provides the parser, concrete syntax tree, and analysis passes that +//! every htcl-consuming subcommand of `vw` shares. The same code drives +//! `vw run`, `vw check`, the LSP (`vw analyzer`), and (eventually) the +//! REPL (`vw repl`). Keeping a single source of truth for parsing and +//! analysis is the durable fix for the "compiler vs. IDE drift" failure +//! mode of language tooling. +//! +//! This v0 covers the Phase 0 subset from the project plan: literals, +//! variables, command substitution, `set`, `proc` (vanilla form), +//! generic command invocations, and comments. Control flow, structured +//! proc grammar, modules, and dependency-aware imports come in later +//! phases. + +// `quote_tcl!` (and `quote_htcl!`) generate code that names this +// crate as `::vw_htcl::…`. Within the crate itself that path +// doesn't resolve by default; this directive aliases the current +// crate as `vw_htcl` so the macros work uniformly inside and +// outside of vw-htcl. Standard Rust idiom for self-targeting +// proc-macros. +extern crate self as vw_htcl; + +pub mod ast; +pub mod cmdline; +pub mod complete; +pub mod doc; +pub mod emit; +pub mod enum_parse; +pub mod goto; +pub mod hover; +pub mod line_index; +pub mod loader; +pub mod lower; +pub mod overload; +pub mod parser; +pub mod proc_args; +pub mod putr; +pub mod references; +pub mod rename; +pub mod repr; +pub mod scope; +pub mod signature_help; +pub mod span; +pub mod src_path; +pub mod type_parse; +pub mod undefined; +pub use undefined::{top_level_var_names, top_level_var_types}; +pub mod unused; +pub mod validate; + +pub use complete::{ + complete_at, complete_at_with_extras, Completion, CompletionKind, +}; +pub use goto::definition_at; +pub use hover::{hover_at, HoverTarget}; +pub use loader::{ + load as load_program, load_with_observer as load_program_with_observer, + load_with_preloaded as load_program_with_preloaded, ImportEdge, LoadError, + LoadObserver, LoadedFile, LoadedProgram, SourceRegion, +}; +pub use lower::{ + extern_rename_prelude, is_extern_call, lower_command, + lower_command_with_putr, lower_command_with_putr_and_index, + lower_proc_decl_with_name, lower_proc_decl_with_name_and_index, + rewrite_externs, signature_table, ExternRewrite, SignatureTable, + EXTERN_PREFIX, +}; +pub use overload::emit_dispatcher; +pub use references::{find_references_in, identify_at, ReferenceTarget}; +pub use rename::{rename_at, RenameEdit}; +pub use repr::{ + emit_enum_prelude, emit_primitive_prelude, emit_repr, emit_repr_with_types, +}; +pub use signature_help::{signature_help_at, SignatureHelp}; +pub use src_path::{ + classify as classify_src_path, PathKind, ResolveError, Resolver, +}; +pub use validate::{ + build_enum_decl_table, build_signature_table_with_overloads, + build_type_decl_table, mangle_specialization, validate, + validate_with_all_extras, validate_with_all_extras_and_vars, + validate_with_extras, validate_with_signatures, + Diagnostic as ValidatorDiagnostic, OverloadTable, Severity, +}; + +pub use ast::{ + Attribute, AttributeValue, Command, CommandKind, Document, EnumDecl, + EnumVariant, OverloadInfo, OverloadVariant, Proc, ProcArg, ProcSignature, + SrcImport, Stmt, TypeDecl, TypeExpr, Word, WordPart, +}; +pub use line_index::{LineCol, LineIndex}; +pub use parser::{parse, ParseError, ParseOutput}; +pub use span::Span; diff --git a/vw-htcl/src/line_index.rs b/vw-htcl/src/line_index.rs new file mode 100644 index 0000000..4375eca --- /dev/null +++ b/vw-htcl/src/line_index.rs @@ -0,0 +1,163 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Byte-offset ↔ line/column conversion. +//! +//! LSP positions are 0-indexed `(line, character)` where `character` +//! counts UTF-16 code units, not bytes. We honor that here so the +//! editor's cursor lands where the user expects on non-ASCII source. + +use crate::span::Span; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LineCol { + pub line: u32, + pub character: u32, +} + +#[derive(Clone, Debug)] +pub struct LineIndex { + /// Byte offset of the start of each line. `line_starts[0] == 0`. + line_starts: Vec, + /// Full source text; needed for the UTF-8 → UTF-16 column + /// conversion. + text: String, +} + +impl LineIndex { + pub fn new(text: &str) -> Self { + let mut line_starts = vec![0u32]; + for (i, b) in text.bytes().enumerate() { + if b == b'\n' { + line_starts.push((i + 1) as u32); + } + } + Self { + line_starts, + text: text.to_string(), + } + } + + pub fn position(&self, byte_offset: u32) -> LineCol { + let offset = byte_offset.min(self.text.len() as u32); + let line_idx = match self.line_starts.binary_search(&offset) { + Ok(i) => i, + Err(i) => i - 1, + }; + let line_start = self.line_starts[line_idx]; + let line_text = &self.text[line_start as usize..offset as usize]; + let character = line_text.encode_utf16().count() as u32; + LineCol { + line: line_idx as u32, + character, + } + } + + pub fn range(&self, span: Span) -> (LineCol, LineCol) { + (self.position(span.start), self.position(span.end)) + } + + /// Convert a UTF-16 line/character position back to a byte + /// offset. Inverse of [`position`](Self::position); used to map + /// LSP positions from clients (which speak UTF-16) into byte + /// offsets the rest of the analysis uses. + /// + /// Clamps gracefully: a line past EOF returns the source length; + /// a character past EOL returns the offset of the line ending. + pub fn offset_of(&self, lc: LineCol) -> u32 { + let Some(&line_start) = self.line_starts.get(lc.line as usize) else { + return self.text.len() as u32; + }; + let line_end = self + .line_starts + .get(lc.line as usize + 1) + .copied() + .map(|n| n.saturating_sub(1)) + .unwrap_or(self.text.len() as u32); + let line_text = &self.text[line_start as usize..line_end as usize]; + let mut byte_offset = line_start; + let mut char_count: u32 = 0; + for ch in line_text.chars() { + if char_count >= lc.character { + break; + } + char_count += ch.len_utf16() as u32; + byte_offset += ch.len_utf8() as u32; + } + byte_offset + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic_positions() { + let idx = LineIndex::new("abc\nde\nf"); + assert_eq!( + idx.position(0), + LineCol { + line: 0, + character: 0 + } + ); + assert_eq!( + idx.position(3), + LineCol { + line: 0, + character: 3 + } + ); + assert_eq!( + idx.position(4), + LineCol { + line: 1, + character: 0 + } + ); + assert_eq!( + idx.position(7), + LineCol { + line: 2, + character: 0 + } + ); + } + + #[test] + fn utf16_character_count() { + // `é` is one UTF-16 code unit, two UTF-8 bytes. + let idx = LineIndex::new("é\nx"); + let pos = idx.position(2); // byte after `é` + assert_eq!( + pos, + LineCol { + line: 0, + character: 1 + } + ); + } + + #[test] + fn offset_of_round_trips() { + let idx = LineIndex::new("abc\nde\nf"); + for &b in &[0u32, 1, 3, 4, 6, 7] { + assert_eq!(idx.offset_of(idx.position(b)), b); + } + } + + #[test] + fn offset_of_clamps_past_line_end() { + let idx = LineIndex::new("abc\nde"); + // line 0, character 100 → end of line 0 (byte 3) + assert_eq!( + idx.offset_of(LineCol { + line: 0, + character: 100 + }), + 3 + ); + } +} diff --git a/vw-htcl/src/loader.rs b/vw-htcl/src/loader.rs new file mode 100644 index 0000000..8d8caca --- /dev/null +++ b/vw-htcl/src/loader.rs @@ -0,0 +1,661 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Recursive `src` import resolution. +//! +//! Reads an entry-point .htcl file, parses it, resolves every `src` +//! statement via [`crate::src_path::Resolver`], and recursively pulls +//! in each imported module's contents. Idempotent on canonical +//! (realpath'd) file paths — a file imported by N callers loads +//! exactly once. +//! +//! The output is a single flat [`LoadedProgram`] carrying: +//! +//! - the concatenated source text (imports first, in topological +//! order, then the entry file's non-`src` content), which downstream +//! stages (lower, the analyzer, `vw run`) consume as if it were one +//! document; +//! - the set of canonical paths that were loaded, for cache +//! invalidation and tooling. + +use std::collections::HashSet; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::ast::{CommandKind, Stmt}; +use crate::parser::parse; +use crate::src_path::{ResolveError, Resolver}; + +#[derive(Debug, thiserror::Error)] +pub enum LoadError { + #[error("reading {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("resolving `src {raw}` from {importer}: {source}")] + Resolve { + importer: PathBuf, + raw: String, + #[source] + source: ResolveError, + }, + #[error( + "`src` import at {importer}:{line} has a non-literal path (it \ + contains `$var` or `[cmd]` substitution); module paths must \ + be a plain string" + )] + DynamicPath { importer: PathBuf, line: u32 }, + #[error("parse errors in {path}")] + Parse { + path: PathBuf, + errors: Vec, + }, +} + +/// Hooks called as the loader makes progress. Lets the CLI surface +/// real-time `Sourcing …` / `Checking …` lines without baking display +/// concerns into the loader. +/// +/// Events fire in dependency-first order, which matches Cargo's +/// "compile deps before the top crate" convention: for each `src` +/// import we hit, [`on_source`](Self::on_source) fires immediately, +/// the import is loaded (recursing through *its* dependencies first), +/// and only then does [`on_parsed`](Self::on_parsed) fire for that +/// file. The entry file's `on_parsed` fires last. +pub trait LoadObserver { + /// A `src ` statement is about to be resolved and loaded. + /// `resolved` is the on-disk path the loader is about to read, + /// so the observer can render dep-relative labels like + /// `@cpm/cpm_pcie1_axibar2pcie` even when `raw` itself is a + /// relative path (`./cpm_pcie1_axibar2pcie`) inside a dep. + fn on_source(&mut self, _raw: &str, _resolved: &Path) {} + /// `file` finished parsing. `raw` is the original `src` text when + /// this file was reached through an import (so callers can render + /// `amd-htcl/cpm5` rather than the full filesystem path); `None` + /// for the entry file. + fn on_parsed(&mut self, _file: &Path, _raw: Option<&str>) {} +} + +struct NoopObserver; +impl LoadObserver for NoopObserver {} + +#[derive(Debug, Default)] +pub struct LoadedProgram { + /// Flattened htcl source — every loaded file's non-`src` content, + /// concatenated. Downstream stages (lower, the analyzer in CLI + /// mode, `vw run`) consume this as if it were one document. + pub source: String, + /// Files seen, in the order [`load_file`] first visits them + /// (importer-first, depth-first). Each entry carries the file's + /// canonical path and its original on-disk text so callers can + /// map a span in [`source`](Self::source) back to a line/column + /// in the file it actually came from. + pub files: Vec, + /// Per-region map from byte ranges in [`source`](Self::source) + /// to `(file_index, file_offset)`. Regions are emitted in order + /// as content is concatenated, so the slice is sorted by + /// `flat_start` and non-overlapping — `locate` does a binary + /// search. + pub regions: Vec, +} + +#[derive(Debug, Clone)] +pub struct LoadedFile { + pub path: PathBuf, + pub source: String, + /// The `src` import that pulled this file in, or `None` for + /// the entry file the loader was started against. Captured at + /// load time so analyzers and the REPL can render call chains + /// like `failing.htcl:12 ← importer.htcl:4 (src @dep/foo) + /// ← entry.htcl:1 (src ip/cips)` — which is the htcl-level + /// equivalent of a stack trace. + pub imported_via: Option, + /// Modification time of the file at read time. Populated when + /// the fs metadata call succeeds; `None` when it doesn't + /// (unlikely on a file we just read, but the API allows it). + /// Used by the REPL's cross-batch loader cache: if the + /// current mtime differs from this stored one, the file + /// gets re-read on the next `src` — this is what makes + /// live-editing an already-sourced `.htcl` file work + /// interactively without a full restart. + pub mtime: Option, +} + +/// An edge in the import graph: this file was loaded because the +/// file at index [`Self::importer_file`] executed a `src` statement +/// covering [`Self::src_span`] in the importer's source. +#[derive(Debug, Clone, Copy)] +pub struct ImportEdge { + pub importer_file: usize, + /// Span of the `src` statement in the **importer's file-local + /// source** (i.e. an offset into the importer's + /// [`LoadedFile::source`], *not* into the flattened + /// [`LoadedProgram::source`]). + pub src_span: crate::span::Span, +} + +#[derive(Debug, Clone, Copy)] +pub struct SourceRegion { + /// Inclusive byte start in the flattened source. + pub flat_start: u32, + /// Exclusive byte end in the flattened source. + pub flat_end: u32, + /// Index into [`LoadedProgram::files`]. + pub file_index: u32, + /// Byte offset of the start of this region in the originating + /// file's source. + pub file_offset: u32, +} + +impl LoadedProgram { + /// Map a byte offset in [`source`](Self::source) back to its + /// originating file's index and the byte offset within that file. + pub fn locate(&self, offset: u32) -> Option<(usize, u32)> { + // `regions` is sorted by `flat_start`; find the last region + // whose start is at or before `offset` and verify the offset + // falls inside it. + let idx = self.regions.partition_point(|r| r.flat_start <= offset); + if idx == 0 { + return None; + } + let region = &self.regions[idx - 1]; + if offset >= region.flat_end { + return None; + } + Some(( + region.file_index as usize, + region.file_offset + (offset - region.flat_start), + )) + } + + /// Map a span in the flattened source to `(file_index, + /// file_local_span)`. Assumes the span lies within a single + /// originating file's contribution — true for diagnostics emitted + /// against a single word/command, which is the use case we care + /// about. + pub fn locate_span( + &self, + span: crate::span::Span, + ) -> Option<(usize, crate::span::Span)> { + let (file_index, file_start) = self.locate(span.start)?; + let length = span.end.saturating_sub(span.start); + Some(( + file_index, + crate::span::Span::new(file_start, file_start + length), + )) + } + + /// Walk the import chain from `file_index` toward the entry, + /// yielding each [`ImportEdge`] in order (nearest first). The + /// entry file has no edge and so produces no items. + pub fn ancestry( + &self, + file_index: usize, + ) -> impl Iterator + '_ { + let mut cur = self.files.get(file_index).and_then(|f| f.imported_via); + std::iter::from_fn(move || { + let edge = cur?; + cur = self + .files + .get(edge.importer_file) + .and_then(|f| f.imported_via); + Some(edge) + }) + } +} + +/// Read `entry` and recursively resolve its imports. Each file is +/// loaded at most once; circular imports (a → b → a) short-circuit on +/// the second visit. +pub fn load( + entry: &Path, + resolver: &Resolver, +) -> Result { + let mut noop = NoopObserver; + load_with_observer(entry, resolver, &mut noop) +} + +/// Like [`load`], but reports progress through `observer` so the CLI +/// can print `Sourcing …` and `Checking …` lines. +pub fn load_with_observer( + entry: &Path, + resolver: &Resolver, + observer: &mut dyn LoadObserver, +) -> Result { + load_with_preloaded( + entry, + resolver, + observer, + &std::collections::HashMap::new(), + ) +} + +/// Same as [`load_with_observer`] but seeds the "already loaded" +/// map with paths → mtime-at-load-time. +/// +/// The REPL uses this to skip re-parsing files a prior batch has +/// already sourced. Without it, `src ip/gtm` in a REPL session +/// that just `--load prime.htcl`-ed the same file re-parses +/// **every** transitive import from scratch — the loader's own +/// `self.loaded` cache is per-load-call, so cross-batch +/// redundancy explodes into O(minutes) for a large tree. +/// +/// A preloaded path short-circuits ONLY when the file's current +/// on-disk mtime matches the stored one. That's what lets a user +/// edit `ip/gtm.htcl`, then re-run `src ip/gtm` at the REPL and +/// actually see the change — the loader stats the target, sees +/// the mtime bump, and re-reads + re-parses instead of taking +/// the cached-empty path. A stat per hit is negligible next to +/// what the re-parse would cost. +/// +/// A missing mtime (`None`) on either side downgrades to +/// "unknown → always reload" — safer than silently reusing +/// possibly-stale content. +pub fn load_with_preloaded( + entry: &Path, + resolver: &Resolver, + observer: &mut dyn LoadObserver, + preloaded: &std::collections::HashMap, +) -> Result { + let entry = entry.canonicalize().unwrap_or_else(|_| entry.to_path_buf()); + let mut state = State { + program: LoadedProgram::default(), + // Seed `preloaded_mtimes` — the actual short-circuit + // check is inside `load_file` and gates on the mtime + // being unchanged since the prior batch stored it. The + // entry path itself is intentionally omitted even if the + // caller included it; that path is the batch's own + // scratch or the user's typed input, which the caller + // wants walked regardless. + preloaded_mtimes: preloaded + .iter() + .filter(|(p, _)| *p != &entry) + .map(|(p, t)| (p.clone(), *t)) + .collect(), + loaded: HashSet::new(), + in_progress: HashSet::new(), + resolver, + observer, + }; + state.load_file(&entry, None, None)?; + Ok(state.program) +} + +struct State<'r, 'o> { + program: LoadedProgram, + /// Files the loader has already read this call (its own dedup + /// tracking). Grows as `load_file` recurses. + loaded: HashSet, + /// Cross-call preloaded set: path → mtime-when-prior-batch- + /// loaded-it. `load_file` short-circuits only when the entry + /// is here AND the current mtime matches. Never mutated + /// during a load call. + preloaded_mtimes: std::collections::HashMap, + in_progress: HashSet, + resolver: &'r Resolver, + observer: &'o mut dyn LoadObserver, +} + +impl State<'_, '_> { + fn load_file( + &mut self, + path: &Path, + reached_via: Option<&str>, + imported_via: Option, + ) -> Result<(), LoadError> { + if self.loaded.contains(path) || self.in_progress.contains(path) { + return Ok(()); + } + // Cross-batch cache: skip re-parse only when this file + // was preloaded by a prior batch AND its on-disk mtime + // hasn't budged. Any mismatch (unknown current mtime, + // different mtime, no stored mtime) forces a fresh + // read so live edits show up on the next `src`. + if let Some(stored_mtime) = self.preloaded_mtimes.get(path) { + let current_mtime = + fs::metadata(path).ok().and_then(|m| m.modified().ok()); + if current_mtime == Some(*stored_mtime) { + return Ok(()); + } + } + self.in_progress.insert(path.to_path_buf()); + + let source = fs::read_to_string(path).map_err(|e| LoadError::Io { + path: path.to_path_buf(), + source: e, + })?; + // Stat right after the read so the recorded mtime matches + // the source we just captured. Failing to stat (fs + // permissions, deleted between read and stat, etc.) + // downgrades to `None`, which the preloaded-cache path + // treats as "unknown → always reload". + let mtime = fs::metadata(path).ok().and_then(|m| m.modified().ok()); + let parsed = parse(&source); + if !parsed.errors.is_empty() { + return Err(LoadError::Parse { + path: path.to_path_buf(), + errors: parsed.errors, + }); + } + + // Register the file up front so we have a stable index for + // every chunk we emit on its behalf. + let file_index = self.program.files.len() as u32; + self.program.files.push(LoadedFile { + path: path.to_path_buf(), + source: source.clone(), + imported_via, + mtime, + }); + + // Walk the parsed document, copying text in span order. Any + // `src` statement triggers a recursion so the imported content + // lands in the flat source before we continue the importer's + // remaining text. Each pushed slice gets a `SourceRegion` + // entry so locations in the flat source can be mapped back. + let mut cursor = 0usize; + let parent_dir = path.parent().unwrap_or_else(|| Path::new(".")); + for stmt in &parsed.document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(import) = &cmd.kind else { + continue; + }; + self.emit_chunk( + &source, + cursor, + cmd.span.start as usize, + file_index, + ); + cursor = cmd.span.end as usize; + // Skip the trailing newline that terminated the `src` + // command so we don't leave a stray blank line behind. + if source.as_bytes().get(cursor) == Some(&b'\n') { + cursor += 1; + } + + let Some(raw) = import.path.as_deref() else { + let line = line_of(&source, cmd.span.start) + 1; + return Err(LoadError::DynamicPath { + importer: path.to_path_buf(), + line, + }); + }; + let resolved = + self.resolver.resolve(parent_dir, raw).map_err(|source| { + LoadError::Resolve { + importer: path.to_path_buf(), + raw: raw.to_string(), + source, + } + })?; + if !self.loaded.contains(&resolved) + && !self.in_progress.contains(&resolved) + { + self.observer.on_source(raw, &resolved); + } + self.load_file( + &resolved, + Some(raw), + Some(ImportEdge { + importer_file: file_index as usize, + src_span: cmd.span, + }), + )?; + } + // Tail after the last `src`. + self.emit_chunk(&source, cursor, source.len(), file_index); + if !self.program.source.ends_with('\n') { + // Synthetic newline so subsequent files don't run on; no + // region for it — it didn't come from any input file. + self.program.source.push('\n'); + } + + self.in_progress.remove(path); + self.loaded.insert(path.to_path_buf()); + self.observer.on_parsed(path, reached_via); + Ok(()) + } + + /// Push `source[start..end]` onto the flat source and record a + /// region mapping that byte range back to the file it came from. + fn emit_chunk( + &mut self, + source: &str, + start: usize, + end: usize, + file_index: u32, + ) { + if start >= end { + return; + } + let flat_start = self.program.source.len() as u32; + self.program.source.push_str(&source[start..end]); + let flat_end = self.program.source.len() as u32; + self.program.regions.push(SourceRegion { + flat_start, + flat_end, + file_index, + file_offset: start as u32, + }); + } +} + +fn line_of(source: &str, byte: u32) -> u32 { + source[..(byte as usize).min(source.len())] + .bytes() + .filter(|b| *b == b'\n') + .count() as u32 +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn workspace() -> tempfile::TempDir { + tempfile::tempdir().unwrap() + } + + #[test] + fn loads_a_single_file_unchanged() { + let dir = workspace(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "puts hi\n").unwrap(); + let prog = load(&entry, &Resolver::new()).unwrap(); + assert_eq!(prog.source.trim(), "puts hi"); + assert_eq!(prog.files.len(), 1); + } + + #[test] + fn imports_local_file_and_drops_src_statement() { + let dir = workspace(); + fs::write(dir.path().join("lib.htcl"), "proc f {} { puts hi }\n") + .unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src lib\nf\n").unwrap(); + + let prog = load(&entry, &Resolver::new()).unwrap(); + // Imported content first, no `src` statement, then the importer. + assert_eq!( + prog.source, "proc f {} { puts hi }\nf\n", + "actual: {:?}", + prog.source + ); + assert_eq!(prog.files.len(), 2); + } + + #[test] + fn idempotent_across_diamond_imports() { + // main → a, b ; a → c ; b → c — c must load exactly once. + let dir = workspace(); + fs::write(dir.path().join("c.htcl"), "proc c {} {}\n").unwrap(); + fs::write(dir.path().join("a.htcl"), "src c\nproc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "src c\nproc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\nsrc b\n").unwrap(); + + let prog = load(&entry, &Resolver::new()).unwrap(); + let occurrences = prog.source.matches("proc c {}").count(); + assert_eq!(occurrences, 1, "c loaded multiple times: {}", prog.source); + } + + #[test] + fn cycle_does_not_loop_forever() { + let dir = workspace(); + fs::write(dir.path().join("a.htcl"), "src b\nproc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "src a\nproc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\n").unwrap(); + let prog = load(&entry, &Resolver::new()).unwrap(); + assert!(prog.source.contains("proc a")); + assert!(prog.source.contains("proc b")); + } + + #[test] + fn named_dependency_resolves_through_the_cache() { + let dir = workspace(); + let dep_root = dir.path().join("cache").join("xilinx-ip-deadbeef"); + fs::create_dir_all(&dep_root).unwrap(); + fs::write(dep_root.join("cpm5.htcl"), "proc create_cpm5 {} {}\n") + .unwrap(); + let resolver = Resolver::new().with_dep("xilinx-ip", dep_root); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src @xilinx-ip/cpm5\ncreate_cpm5\n").unwrap(); + let prog = load(&entry, &resolver).unwrap(); + assert!(prog.source.contains("proc create_cpm5")); + assert!(prog.source.contains("\ncreate_cpm5\n")); + } + + #[test] + fn observer_fires_in_dependency_order() { + // entry → a → c ; entry → b + // Expect: source a, parse a (after source c, parse c), + // source b, parse b, parse entry. + let dir = workspace(); + fs::write(dir.path().join("c.htcl"), "proc c {} {}\n").unwrap(); + fs::write(dir.path().join("a.htcl"), "src c\nproc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "proc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\nsrc b\n").unwrap(); + + #[derive(Default)] + struct Recorder { + events: Vec, + } + impl LoadObserver for Recorder { + fn on_source(&mut self, raw: &str, _resolved: &Path) { + self.events.push(format!("source {raw}")); + } + fn on_parsed(&mut self, file: &Path, raw: Option<&str>) { + let label = match raw { + Some(r) => r.to_string(), + None => file + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("?") + .to_string(), + }; + self.events.push(format!("parse {label}")); + } + } + + let mut rec = Recorder::default(); + load_with_observer(&entry, &Resolver::new(), &mut rec).unwrap(); + assert_eq!( + rec.events, + vec![ + "source a", + "source c", + "parse c", + "parse a", + "source b", + "parse b", + "parse main", + ] + ); + } + + #[test] + fn observer_suppresses_source_for_already_loaded_imports() { + // Diamond: main → a → c ; main → b → c. `c` is encountered + // twice via `src` but only loaded once, so "Sourcing c" + // should fire exactly once. + let dir = workspace(); + fs::write(dir.path().join("c.htcl"), "proc c {} {}\n").unwrap(); + fs::write(dir.path().join("a.htcl"), "src c\nproc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "src c\nproc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\nsrc b\n").unwrap(); + + #[derive(Default)] + struct Counter { + source_c: usize, + parse_c: usize, + } + impl LoadObserver for Counter { + fn on_source(&mut self, raw: &str, _resolved: &Path) { + if raw == "c" { + self.source_c += 1; + } + } + fn on_parsed(&mut self, file: &Path, _raw: Option<&str>) { + if file.file_stem().and_then(|s| s.to_str()) == Some("c") { + self.parse_c += 1; + } + } + } + let mut counter = Counter::default(); + load_with_observer(&entry, &Resolver::new(), &mut counter).unwrap(); + assert_eq!(counter.source_c, 1); + assert_eq!(counter.parse_c, 1); + } + + #[test] + fn regions_map_each_byte_back_to_its_originating_file() { + // entry uses `set` from one local file and `puts` from another. + let dir = workspace(); + fs::write(dir.path().join("a.htcl"), "proc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "proc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\nputs hello\nsrc b\nputs done\n").unwrap(); + let prog = load(&entry, &Resolver::new()).unwrap(); + + // Pick a byte in the middle of `puts hello` — should map back + // to the entry file (main.htcl). + let puts_hello_at_flat = + prog.source.find("puts hello").expect("puts hello in flat") as u32; + let (idx, file_offset) = + prog.locate(puts_hello_at_flat).expect("locate puts hello"); + assert_eq!( + prog.files[idx].path.file_name().and_then(|s| s.to_str()), + Some("main.htcl") + ); + // In main.htcl the line `puts hello` sits right after `src a\n`, + // so file_offset is at byte 6 (`s`=0,1,2,r=3,c=4,a=5,\n=6). + // Actually 'src a\n' = 6 bytes (s,r,c,space,a,\n), so puts starts at 6. + assert_eq!(file_offset, 6); + + // Pick a byte in the middle of `proc a` — should map to a.htcl. + let proc_a_at_flat = + prog.source.find("proc a").expect("proc a in flat") as u32; + let (idx_a, _) = prog.locate(proc_a_at_flat).expect("locate proc a"); + assert_eq!( + prog.files[idx_a].path.file_name().and_then(|s| s.to_str()), + Some("a.htcl") + ); + } + + #[test] + fn unknown_dep_surfaces_helpful_error() { + let dir = workspace(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src @nope/cpm5\n").unwrap(); + let err = load(&entry, &Resolver::new()).unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("unknown dependency"), "{msg}"); + } +} diff --git a/vw-htcl/src/lower.rs b/vw-htcl/src/lower.rs new file mode 100644 index 0000000..3d7edf4 --- /dev/null +++ b/vw-htcl/src/lower.rs @@ -0,0 +1,929 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Lower htcl to plain Tcl for the EDA backend. +//! +//! Phase 2 lowering: +//! +//! - Structured `proc` declarations emit `proc name {arg1 arg2 ...} +//! body`, where the arg list is the declared canonical order with +//! no attributes (Vivado's Tcl doesn't understand `@default` etc.). +//! - Call sites to a known structured proc rewrite their `-flag +//! value` form to a positional list in the canonical order, with +//! defaults filled in for omitted args. +//! - Everything else (comments, unknown commands, calls to commands +//! without a structured signature) passes through verbatim. +//! +//! Limitation: only top-level proc declarations and top-level call +//! sites are lowered. Calls *inside* a proc body are not rewritten — +//! the body text is shipped as-is. Phase 3+ will recursively lower +//! nested commands once we have static analysis of proc bodies. + +use std::collections::HashMap; + +use crate::ast::{ + Command, CommandKind, Document, NamespaceEval, Proc, ProcSignature, Stmt, + Word, WordForm, WordPart, +}; +use crate::line_index::LineIndex; + +pub type SignatureTable<'a> = HashMap; + +/// Empty putr rewrite map used as the default when a caller +/// doesn't have one. See [`lower_command_with_putr`] for the +/// keyed-lookup semantics. +fn empty_putr_map() -> &'static crate::putr::RewriteMap { + use std::sync::OnceLock; + static EMPTY: OnceLock = OnceLock::new(); + EMPTY.get_or_init(crate::putr::RewriteMap::new) +} + +/// Walk `doc` and collect every proc's signature — top-level and +/// nested inside `namespace eval` blocks. Namespaced procs register +/// under their qualified name (`::`), matching the +/// signature table the validator builds so call-site lowering works +/// uniformly for both shapes. +pub fn signature_table(doc: &Document) -> SignatureTable<'_> { + let mut table = HashMap::new(); + collect_into(&doc.stmts, "", &mut table); + table +} + +fn collect_into<'a>( + stmts: &'a [Stmt], + prefix: &str, + table: &mut SignatureTable<'a>, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let Some(sig) = proc.signature.as_ref() else { + continue; + }; + let qualified = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + table.insert(qualified, sig); + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + let nested = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + collect_into(&ns.body, &nested, table); + } + _ => {} + } + } +} + +/// Lower one top-level command into its Tcl equivalent for the EDA +/// backend. See [`lower_command_with_putr`] for the variant that +/// takes a `putr` rewrite map; callers with no putr calls (or who +/// don't care) can invoke this simpler form. +/// +/// Builds a fresh [`LineIndex`] every call — fine for one-off +/// uses (tests, single-statement lowering) but pathological when +/// looped over a document with thousands of top-level procs. Bulk +/// callers should build the index once and use +/// [`lower_command_with_putr_and_index`] to skip the per-call +/// rebuild — a 19MB flat document has 1000× the newline-scan +/// cost of a single-statement fragment. +pub fn lower_command( + cmd: &Command, + source: &str, + table: &SignatureTable<'_>, +) -> String { + let line_index = LineIndex::new(source); + lower_command_with_putr_and_index( + cmd, + source, + table, + empty_putr_map(), + &line_index, + ) +} + +/// Lower one top-level command, consulting `putr_map` first: when +/// `cmd.span` is a key in the map, the map's replacement Tcl is +/// used verbatim in place of the standard lowering. This is how +/// `putr $x` becomes `puts [T::repr -v $x]` at emit time without +/// mutating the source string. +/// +/// Recurses into proc bodies / namespace-eval bodies / cmd-subst +/// bodies with the same `putr_map`, so `putr` calls buried inside +/// any of those still get the rewrite. +pub fn lower_command_with_putr( + cmd: &Command, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, +) -> String { + let line_index = LineIndex::new(source); + lower_command_with_putr_and_index(cmd, source, table, putr_map, &line_index) +} + +/// Bulk-friendly variant of [`lower_command_with_putr`] that +/// accepts a pre-built [`LineIndex`] instead of constructing one +/// per call. Callers looping over thousands of top-level +/// statements MUST use this form — every `lower_proc_decl` +/// consult needs line-of-offset lookups, and rebuilding the +/// index over a 19MB flat source per proc was the O(procs × +/// source_len) accidental quadratic that made auto-loading a +/// large `.htcl` take minutes. +pub fn lower_command_with_putr_and_index( + cmd: &Command, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + // Fast path: if this command IS a putr rewrite target, emit + // the replacement verbatim. No further descent needed — the + // replacement is a complete Tcl expression. + if let Some(replacement) = putr_map.get(&cmd.span) { + return replacement.clone(); + } + match &cmd.kind { + CommandKind::Proc(proc) => { + lower_proc_decl(proc, source, table, putr_map, line_index) + } + CommandKind::NamespaceEval(ns) => { + lower_namespace_eval(ns, source, table, putr_map, line_index) + } + // `src` is a module import; by the time we lower we expect the + // [`crate::loader`] flatten pass to have already inlined every + // import's contents and dropped the `src` statements. Anything + // that slips through here we render as a no-op comment so the + // emitted Tcl is still well-formed. + CommandKind::Src(import) => { + let path = import.path.as_deref().unwrap_or(""); + format!("# vw: unresolved `src {path}` — loader bypass") + } + // Newtype declarations are compile-time only — they feed the + // analyzer / printer machinery but ship nothing to Vivado. + // Drop entirely (empty Tcl, no whitespace). + CommandKind::TypeDecl(_) => String::new(), + // Enum declarations are also compile-time-only at this + // layer — the codegen path (vw-htcl/src/repr.rs) emits the + // auto-generated `namespace eval ` block separately + // through the same wrap-with-repr pipeline used for the + // primitive prelude. The decl itself ships nothing. + CommandKind::EnumDecl(_) => String::new(), + _ => { + // Verbatim, but reconstructed word-by-word so that any + // `[ … ]` substitution inside the command gets its own + // commands lowered through the same pipeline (extern + // rewrites, multi-line bracket flattening). + // + // No keyword→positional rewrite here: htcl is keyword- + // only at the call site, and the rewrite from `-flag + // value` pairs to local variables happens at runtime + // in the wrapper's `::vw::kwargs $args { ... }` prelude + // (emitted by `lower_proc_decl`). That lets call sites + // anywhere — top-level, inside a proc body, inside a + // `[ ... ]`, inside an `eval` — work uniformly without + // the lowerer needing to see every call site. + lower_words(&cmd.words, source, table, putr_map, line_index) + } + } +} + +/// Lower a `namespace eval` block: recurse on each inner statement +/// (so inner proc declarations get their htcl attributes stripped +/// and gain the `::vw::kwargs` runtime prelude) and wrap the +/// result in `namespace eval { ... }`. Output is a single +/// Tcl-valid string the EDA backend can `eval` directly. +fn lower_namespace_eval( + ns: &NamespaceEval, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + let name = ns.name.as_deref().unwrap_or(""); + let mut body = String::new(); + for stmt in &ns.body { + let Stmt::Command(cmd) = stmt else { continue }; + let line = lower_command_with_putr_and_index( + cmd, source, table, putr_map, line_index, + ); + if !line.is_empty() { + body.push_str(&line); + body.push('\n'); + } + } + format!("namespace eval {name} {{\n{body}}}") +} + +/// Lower a `proc` declaration into a Tcl proc whose runtime +/// signature is `args` (variadic). The first line of the body is a +/// generated `::vw::kwargs $args { name default ... }` call that +/// parses the caller's `-flag value` pairs into local variables +/// matching the declared parameter names — defaults applied where +/// the caller didn't supply a flag. After the prelude the original +/// body runs unchanged, using `$name`, `$dir`, etc. just as if +/// they were standard Tcl parameters. +/// +/// Why this shape: htcl is keyword-only at the call site. Doing +/// the parse at runtime (in the wrapper) means every call site +/// works the same — top-level, inside a proc body, inside a +/// `[ ... ]` substitution, inside an `eval`. The previous +/// architecture rewrote `-flag value` → positional at compile +/// time, but only for top-level calls the lowerer could see; calls +/// inside proc bodies stayed verbatim and broke at runtime against +/// a positional-only wrapper proc. +/// +/// Procs without a parsed signature (parser couldn't extract one +/// from the args list, e.g. mid-edit syntax error) pass through as +/// plain Tcl: `proc name { } { }`. The +/// `::vw::kwargs` prelude is only emitted when we know what +/// parameters to declare. +fn lower_proc_decl( + proc: &Proc, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + lower_proc_decl_with_name_and_index( + proc, source, table, None, putr_map, line_index, + ) +} + +/// Like [`lower_proc_decl`] but uses `name_override` as the emitted +/// proc name instead of `proc.name`. Used by the REPL when lowering +/// an enum-overload specialization under its mangled name +/// (`____`) — the source name on the parsed proc +/// is the user-visible public name (`handle_prop`), but the +/// dispatcher needs the specialization to live under its mangled +/// alias so the runtime switch can find it. +pub fn lower_proc_decl_with_name( + proc: &Proc, + source: &str, + table: &SignatureTable<'_>, + name_override: Option<&str>, + putr_map: &crate::putr::RewriteMap, +) -> String { + let line_index = LineIndex::new(source); + lower_proc_decl_with_name_and_index( + proc, + source, + table, + name_override, + putr_map, + &line_index, + ) +} + +/// Bulk-friendly variant: takes a pre-built [`LineIndex`] instead +/// of constructing one over the entire source per call. The old +/// unindexed form was the O(procs × source_len) hotspot that made +/// `prepare` for a 19MB flat document take ~85s. +pub fn lower_proc_decl_with_name_and_index( + proc: &Proc, + source: &str, + table: &SignatureTable<'_>, + name_override: Option<&str>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + let name = name_override.or(proc.name.as_deref()).unwrap_or(""); + // Re-emit the body by walking its parsed statements rather + // than slicing raw text. This is what gives htcl's "newlines + // inside `[ … ]` are whitespace" semantics inside proc bodies + // too — verbatim slicing leaves Tcl to interpret the + // newlines as command separators, which silently splits a + // multi-line `set x [ foo \n -a 1 \n -b 2 \n]` into four + // separate calls and drops every flag arg. + // + // Critically, we pad the emitted body with blank lines so + // each lowered statement lands on the SAME line it occupied + // in the source. Tcl's `info frame` reports body lines + // relative to the script text it was given — without padding, + // collapsing a 5-line `[ ... ]` to one line shifts every + // subsequent statement upward and the stack trace's + // "line N in proc X" ends up pointing at unrelated source + // lines. With padding, Tcl's body line N == source body + // line N == `body_start_file_line + N - 1`, which is what + // the REPL's `ProcLocation::resolve_body_line` already + // assumes. + let body_open_line = line_index.position(proc.body_span.start).line; // 0-based + let body = if proc.body.is_empty() { + proc.body_span.slice(source).to_string() + } else { + let mut out = String::new(); + // First emitted body line corresponds to one line after + // the line containing `{`. We track 0-based file lines + // throughout. + let mut cur_line = body_open_line + 1; + for stmt in &proc.body { + let Stmt::Command(cmd) = stmt else { continue }; + let stmt_line = line_index.position(cmd.span.start).line; + while cur_line < stmt_line { + out.push('\n'); + cur_line += 1; + } + let line = lower_command_with_putr_and_index( + cmd, source, table, putr_map, line_index, + ); + if line.is_empty() { + continue; + } + out.push_str(&line); + out.push('\n'); + cur_line += 1 + line.matches('\n').count() as u32; + } + out + }; + let Some(sig) = proc.signature.as_ref() else { + // Couldn't parse a structured signature — emit the proc + // verbatim. Tcl will accept it if the raw arg text is + // valid Tcl; otherwise the user already has a parse-error + // diagnostic from the upstream parser. + let args_list = proc.args_span.slice(source); + return format!("proc {name} {{{args_list}}} {{{body}}}"); + }; + let sig_dict = build_kwargs_sig_dict(sig); + // Put `::vw::kwargs` on the SAME line as the opening `{` so + // it doesn't eat the first source line of the body and shift + // subsequent statements. Tcl treats "the line containing `{`" + // as body line 1 — putting the kwargs preamble there means + // body line 2 onward maps 1:1 to source lines, matching + // what the padding loop above produced. + format!( + "proc {name} {{args}} {{ ::vw::kwargs $args {{{sig_dict}}}\n{body}}}" + ) +} + +/// Render the parameter list as a flat `name default name default +/// ...` Tcl dict for [`::vw::kwargs`] to consume. The default for +/// an arg without `@default` is the empty string `""` — at which +/// point the validator has already complained at compile time +/// about missing required args. Quote each default through +/// [`AttributeValue::to_tcl_literal`] so integers, idents, and +/// strings all round-trip correctly. +fn build_kwargs_sig_dict(sig: &ProcSignature) -> String { + let mut out = String::new(); + for (i, arg) in sig.args.iter().enumerate() { + if i > 0 { + out.push(' '); + } + out.push_str(&arg.name); + out.push(' '); + let default = arg + .attribute("default") + .and_then(|attr| attr.values.first()) + .map(|v| v.to_tcl_literal()) + .unwrap_or_else(|| "\"\"".to_string()); + out.push_str(&default); + } + out +} + +/// The syntactic prefix that marks a call to a runtime-Tcl proc +/// (an "extern") rather than an htcl-defined proc. Anywhere in +/// lowered text, `extern::name` rewrites to a mangled Tcl symbol +/// the lowering's prelude has aliased to the underlying proc. +pub const EXTERN_PREFIX: &str = "extern::"; + +/// Result of [`rewrite_externs`]: the lowered text with every +/// `extern::name` reference replaced by its mangled Tcl form, plus +/// the deduplicated set of external names that were referenced. +/// Callers feed `names` to [`extern_rename_prelude`] to build the +/// one-time setup that exposes each extern at its mangled name. +#[derive(Clone, Debug)] +pub struct ExternRewrite { + pub text: String, + pub names: Vec, +} + +/// Rewrite every `extern::` in `text` to `::` — the +/// Tcl-absolute form that anchors the lookup at the global +/// namespace. Returns the rewritten text plus the unique, sorted +/// set of names seen. +/// +/// Anchoring at `::` matters because htcl wrappers live inside +/// `namespace eval vivado { … }`. Inside that namespace a bare +/// `create_project` resolution searches the *current* namespace +/// first and finds `vivado::create_project` (the wrapper itself!) — +/// infinite recursion. The leading `::` skips the current- +/// namespace search and goes straight to the global, where the +/// unshadowed Vivado native lives. +/// +/// The rewrite is text-level, not AST-level. Proc bodies lower +/// as raw text, and a textual pass cleanly catches calls at any +/// nesting depth — inside `[ … ]`, inside multi-arm +/// `if {…} { … extern::foo … } else { … }`, etc. Word-boundary +/// detection on the leading side prevents `not_extern::foo` from +/// triggering; the trailing identifier is parsed greedily so +/// `extern::a::b::c` rewrites as one unit (→ `::a::b::c`). +pub fn rewrite_externs(text: &str) -> ExternRewrite { + let mut out = String::with_capacity(text.len()); + let mut names: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if i + EXTERN_PREFIX.len() <= bytes.len() + && &bytes[i..i + EXTERN_PREFIX.len()] == EXTERN_PREFIX.as_bytes() + && (i == 0 || !is_extern_ident_byte(bytes[i - 1])) + { + let name_start = i + EXTERN_PREFIX.len(); + let name_end = scan_extern_name_end(bytes, name_start); + if name_end > name_start { + let name = &text[name_start..name_end]; + // Leading `::` makes the lookup absolute (global + // namespace) — necessary inside `namespace eval + // vivado { … }` so the wrapper body doesn't + // recurse on itself. + out.push_str("::"); + out.push_str(name); + names.insert(name.to_string()); + i = name_end; + continue; + } + } + let ch_end = next_char_boundary(text, i); + out.push_str(&text[i..ch_end]); + i = ch_end; + } + ExternRewrite { + text: out, + names: names.into_iter().collect(), + } +} + +fn is_extern_ident_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || b == b'_' +} + +fn scan_extern_name_end(bytes: &[u8], start: usize) -> usize { + let mut i = start; + while i < bytes.len() { + if is_extern_ident_byte(bytes[i]) { + i += 1; + } else if bytes[i] == b':' + && bytes.get(i + 1).copied() == Some(b':') + && bytes.get(i + 2).copied().is_some_and(is_extern_ident_byte) + { + i += 2; + } else { + break; + } + } + i +} + +fn next_char_boundary(s: &str, start: usize) -> usize { + let mut end = start + 1; + while end < s.len() && !s.is_char_boundary(end) { + end += 1; + } + end +} + +/// Historically emitted a rename prelude that aliased each Vivado +/// native to a mangled name so wrappers could forward to the +/// underlying proc without recursing on themselves. With wrappers +/// now living in the `vivado::` namespace and no longer shadowing +/// the globals they wrap, no rename is needed — `extern::foo` +/// just rewrites to bare `foo`, which Tcl resolves to the global +/// native. Kept as a public symbol so callers don't have to track +/// the layering change; returns the empty string. +pub fn extern_rename_prelude(_names: &[String]) -> String { + String::new() +} + +/// True when `call_name` is the explicit `extern::…` form — used +/// by the validator to skip the unknown-call check for these +/// deliberately-external invocations. +pub fn is_extern_call(call_name: &str) -> bool { + call_name.starts_with(EXTERN_PREFIX) +} + +/// Reconstruct a command's words as lowered Tcl text. Splits the +/// problem along the AST's natural boundaries so each piece is +/// handled by the right rules: +/// +/// - Bare and quoted words are rebuilt part-by-part. Plain text, +/// `$var` references, and `\x` escapes go through verbatim; +/// `[ … ]` substitutions recurse into the lowering pipeline so +/// keyword → positional rewriting applies to calls *inside* a +/// `set proj [ create_project … ]`, and multi-line bracket +/// bodies collapse to one Tcl statement by construction. +/// - Braced words are literal text — Tcl never substitutes inside +/// `{ … }`, so the parser doesn't even surface `CmdSubst` parts +/// for them; we ship them as raw source. +fn lower_words( + words: &[Word], + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + // Preserve source-level adjacency between consecutive words. + // The parser splits `{*}$var` into two AST words ({*} as a + // braced "*", $var as a bare word) but their source spans + // touch — Tcl reads them as the expand-prefix operator. + // Joining with a literal space would force `{*} $var`, which + // Tcl reinterprets as a literal-`*`-arg followed by `$var`. + // Checking adjacency keeps the no-space form for `{*}$var` + // while still spacing genuinely-whitespace-separated words. + let mut out = String::new(); + for (i, w) in words.iter().enumerate() { + if i > 0 { + let prev_end = words[i - 1].span.end; + if w.span.start > prev_end { + out.push(' '); + } + } + out.push_str(&lower_word(w, source, table, putr_map, line_index)); + } + out +} + +fn lower_word( + word: &Word, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + match word.form { + WordForm::Bare => { + lower_word_parts(&word.parts, source, table, putr_map, line_index) + } + WordForm::Quoted => { + let inner = lower_word_parts( + &word.parts, + source, + table, + putr_map, + line_index, + ); + format!("\"{inner}\"") + } + WordForm::Braced => word.span.slice(source).to_string(), + } +} + +fn lower_word_parts( + parts: &[WordPart], + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + let mut out = String::new(); + for part in parts { + match part { + WordPart::Text { value, .. } => out.push_str(value), + WordPart::VarRef { name, braced, .. } => { + // Preserve the source's braced form. Emitting a + // plain `$name` where the source had `${name}` + // breaks interpolations like `"${ip}_wrapper.vhd"`: + // Tcl reads `$ip_wrapper` as one greedy ident + // and errors with "no such variable ip_wrapper". + // Preserving the braces also happens to be a + // no-op for typical `$var` refs — we only wrap + // when the source did. + if *braced { + out.push_str("${"); + out.push_str(name); + out.push('}'); + } else { + out.push('$'); + out.push_str(name); + } + } + WordPart::Escape { value, .. } => { + out.push('\\'); + out.push(*value); + } + WordPart::CmdSubst { body, .. } => { + let lowered: Vec = body + .iter() + .filter_map(|s| match s { + Stmt::Command(c) => { + Some(lower_command_with_putr_and_index( + c, source, table, putr_map, line_index, + )) + } + _ => None, + }) + .filter(|s| !s.trim().is_empty()) + .collect(); + out.push('['); + out.push_str(&lowered.join("; ")); + out.push(']'); + } + } + } + out +} + +/// Helper retained for symmetry with future analyzers that want to +/// inspect a word's literal form without re-walking its parts. +#[allow(dead_code)] +fn word_text(word: &Word) -> Option { + let mut out = String::new(); + for part in &word.parts { + match part { + WordPart::Text { value, .. } => out.push_str(value), + WordPart::Escape { value, .. } => out.push(*value), + _ => return None, + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn lowered(src: &str) -> Vec { + let parsed = parse(src); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + let table = signature_table(&parsed.document); + parsed + .document + .stmts + .iter() + .filter_map(|s| match s { + Stmt::Command(c) => Some(lower_command(c, src, &table)), + _ => None, + }) + .collect() + } + + #[test] + fn proc_decl_emits_kwargs_prelude() { + // Every htcl proc lowers to `proc name {args} { ::vw::kwargs + // ... ; body }` — the runtime helper parses the caller's + // `-flag value` pairs into local variables matching the + // declared param names, with defaults applied where the + // caller didn't supply a flag. Body text passes through + // unchanged. + let src = "proc f {\n @default(0) a\n @default(1) b\n} { puts hi }\n"; + let out = lowered(src); + assert!( + out[0].starts_with("proc f {args} {"), + "wrong arg-list form: {}", + out[0] + ); + assert!( + out[0].contains("::vw::kwargs $args {a 0 b 1}"), + "missing or wrong kwargs prelude: {}", + out[0] + ); + assert!(out[0].contains("puts hi"), "lost body: {}", out[0]); + } + + #[test] + fn call_with_flags_ships_verbatim() { + // No more compile-time keyword→positional rewrite. The call + // ships as the user typed it; the wrapper proc parses the + // keywords at runtime via its kwargs prelude. + let src = "proc f {\n a\n b\n} { puts hi }\nf -b 22 -a 11\n"; + let out = lowered(src); + assert_eq!(out[1], "f -b 22 -a 11"); + } + + #[test] + fn call_with_omitted_arg_ships_verbatim() { + // The wrapper's default is wired in at runtime by + // ::vw::kwargs; the call site doesn't need to fill it. + let src = "proc f {\n @default(7) a\n b\n} { puts hi }\nf -b 22\n"; + let out = lowered(src); + assert_eq!(out[1], "f -b 22"); + } + + #[test] + fn inner_call_inside_brackets_ships_verbatim() { + // What this test used to assert (`[make xc foo]` — + // keyword→positional rewrite for the inner call) is no + // longer the architecture. The inner call ships verbatim; + // the wrapper parses `-part`/`-name` at runtime. The only + // transformation we still apply is multi-line bracket + // flattening. + let src = "proc make { + @default(\"\") part + name +} { puts ok } +set proj [ + make + -part xc + -name foo +] +"; + let out = lowered(src); + assert_eq!(out.len(), 2, "{:?}", out); + let set_line = &out[1]; + // Inner call stays keyword-form: `make -part xc -name foo`. + assert!( + set_line.contains("[make -part xc -name foo]"), + "inner call should ship verbatim; got: {set_line}" + ); + // Multi-line bracket body still collapses to one line. + assert!( + !set_line.contains('\n'), + "expected single line; got: {set_line:?}" + ); + } + + #[test] + fn call_inside_proc_body_ships_verbatim() { + // Regression guard for the create_bd_design bug: a + // keyword-form call to a known wrapper, nested inside + // another proc's body, must NOT be rewritten. In the old + // architecture the lowerer only saw top-level call sites + // and silently failed to translate this one, so at runtime + // Tcl handed `-name cips` to a positional-only wrapper + // proc and errored "wrong # args". Now the wrapper parses + // keywords at runtime, so we just ship the call as-is. + let src = "proc create_bd_design { @default(\"\") name } { puts ok }\n\ + proc configure_cips {} {\n \ + create_bd_design -name cips\n\ + }\n"; + let out = lowered(src); + // The configure_cips proc decl is the second statement. + // Its body should still contain the keyword-form call — + // we don't touch it at compile time. + assert!( + out[1].contains("create_bd_design -name cips"), + "call inside proc body should ship verbatim; got:\n{}", + out[1] + ); + } + + #[test] + fn proc_with_no_default_emits_empty_string_default() { + // An htcl arg without `@default` is implicitly required — + // the validator catches a missing-flag call at compile + // time. At runtime we still need a placeholder default so + // `::vw::kwargs` doesn't blow up when the variable is + // referenced before the (missing) `-flag` would have set + // it; we use `""` (empty string). + let src = "proc f {\n required_arg\n} { puts hi }\n"; + let out = lowered(src); + assert!( + out[0].contains("::vw::kwargs $args {required_arg \"\"}"), + "wrong default for required arg: {}", + out[0] + ); + } + + #[test] + fn multiline_bracket_substitution_collapses_to_one_line() { + // The exact shape that broke the REPL: an outer call whose + // sole arg is a `[ … ]` substitution spanning multiple + // source lines. Tcl would parse the bracket body as N + // separate commands; we have to flatten the newlines. + let src = "set proj [\n create_project\n -in_memory 1\n -name foo\n]\n"; + let out = lowered(src); + assert_eq!(out.len(), 1); + // No literal newline inside the brackets after lowering. + let between = out[0] + .split_once('[') + .and_then(|(_, rest)| rest.rsplit_once(']')) + .map(|(inner, _)| inner) + .unwrap(); + assert!(!between.contains('\n'), "lowered: {:?}", out[0]); + // The full call must still parse as `set proj [ ... ]`. + assert!(out[0].starts_with("set proj [")); + assert!(out[0].trim_end().ends_with(']')); + } + + #[test] + fn nested_multiline_brackets_all_collapse() { + // `[outer [inner ...] ...]` — newlines inside both layers + // become spaces; the parser sees nested CmdSubst so the + // recursive collection covers both. + let src = + "set x [\n foo\n -a [\n bar\n -b 1\n ]\n]\n"; + let out = lowered(src); + assert!(!out[0].contains('\n'), "lowered: {:?}", out[0]); + } + + #[test] + fn newlines_inside_braced_groups_stay_intact() { + // Inside `{ … }` the brackets are literal, not a + // substitution. The parser doesn't emit a `CmdSubst` for + // them so we must not strip newlines from braced bodies. + let src = "proc f {} {\n puts a\n puts b\n}\n"; + let out = lowered(src); + // The proc-decl lowering builds its own output (not the + // verbatim path), so it preserves body newlines. + assert!(out[0].contains('\n'), "lowered: {:?}", out[0]); + } + + #[test] + fn rewrite_externs_anchors_at_global_namespace() { + let r = rewrite_externs( + "set cmd [list extern::set_property]\n\ + extern::create_project -name foo\n", + ); + // Leading `::` anchors the lookup at Tcl's global + // namespace, which is where unshadowed Vivado natives + // live — necessary so wrapper bodies inside `namespace + // eval vivado { … }` don't recurse on themselves. + assert!(r.text.contains("[list ::set_property]"), "{}", r.text); + assert!(r.text.contains("::create_project -name foo"), "{}", r.text); + assert!(!r.text.contains("extern::"), "{}", r.text); + assert_eq!(r.names, vec!["create_project", "set_property"]); + } + + #[test] + fn rewrite_externs_preserves_namespaced_names() { + let r = rewrite_externs("extern::common::send_msg_id A B C\n"); + // Same anchoring for multi-segment names — + // `::common::send_msg_id` resolves the leading namespace + // search from the global root. + assert!(r.text.contains("::common::send_msg_id A B C"), "{}", r.text); + assert_eq!(r.names, vec!["common::send_msg_id"]); + } + + #[test] + fn rewrite_externs_respects_word_boundary() { + let r = rewrite_externs("set x not_extern::foo\n"); + assert_eq!(r.text, "set x not_extern::foo\n"); + assert!(r.names.is_empty()); + } + + #[test] + fn extern_rename_prelude_is_empty() { + // Wrappers no longer shadow globals (they live in the + // `vivado::` namespace), so the historical rename plumbing + // is unnecessary. The helper still exists for API stability + // but always returns empty. + let p = extern_rename_prelude(&["set_property".to_string()]); + assert!(p.is_empty(), "{p}"); + } + + #[test] + fn is_extern_call_recognizes_prefix() { + assert!(is_extern_call("extern::set_property")); + assert!(is_extern_call("extern::common::send_msg_id")); + assert!(!is_extern_call("set_property")); + assert!(!is_extern_call("not_extern::foo")); + } + + #[test] + fn braced_var_ref_preserves_braces_in_output() { + // Regression: `"${ip}_wrapper.vhd"` in htcl source used to + // lower to `"$ip_wrapper.vhd"`, which Tcl reads as + // `$ip_wrapper` (one greedy identifier) — dereferencing a + // non-existent variable. Preserving the braces is the fix. + let src = "puts \"${ip}_wrapper.vhd\"\n"; + let out = lowered(src); + assert_eq!( + out[0], "puts \"${ip}_wrapper.vhd\"", + "braced form must round-trip", + ); + } + + #[test] + fn bare_var_ref_still_uses_bare_form() { + // The braces are a source-level distinction; unadorned + // `$var` should still emit as `$var`, not `${var}` (Tcl + // handles both but the bare form is the idiomatic one). + let src = "puts \"$ip.bd\"\n"; + let out = lowered(src); + assert_eq!(out[0], "puts \"$ip.bd\""); + } + + #[test] + fn unknown_command_passes_through() { + let src = "puts \"hello $x\"\n"; + let out = lowered(src); + assert_eq!(out[0], "puts \"hello $x\""); + } + + #[test] + fn string_default_quotes_correctly_in_kwargs_sig() { + // Defaults are stamped into the proc's kwargs-prelude sig + // dict, not into the call site. A `@default("hi")` becomes + // the literal `"hi"` (quoted) in the dict — `::vw::kwargs` + // sets `$greeting` to it when the caller omits the flag. + let src = "proc f {\n @default(\"hi\") greeting\n} { puts hi }\n"; + let out = lowered(src); + assert!( + out[0].contains("::vw::kwargs $args {greeting \"hi\"}"), + "default should appear quoted in the sig dict: {}", + out[0] + ); + } +} diff --git a/vw-htcl/src/overload.rs b/vw-htcl/src/overload.rs new file mode 100644 index 0000000..9f5027a --- /dev/null +++ b/vw-htcl/src/overload.rs @@ -0,0 +1,162 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Codegen for the enum-overload dispatcher. +//! +//! When the validator classifies a set of procs sharing a name as +//! a valid enum-overload (see +//! [`crate::validate::build_signature_table_with_overloads`]), the +//! lowerer rewrites each specialization under its mangled name +//! (`____`) and emits a single public dispatcher +//! proc that switches on the tagged value's variant tag and calls +//! the right specialization with the unwrapped payload. +//! +//! Runtime model: +//! +//! ```tcl +//! proc handle_prop {v args} { +//! switch -- [lindex $v 0] { +//! Scalar { return [__handle_prop__Scalar [lindex $v 1] {*}$args] } +//! Nested { return [__handle_prop__Nested [lindex $v 1] {*}$args] } +//! default { error "handle_prop: unknown variant '[lindex $v 0]'" } +//! } +//! } +//! ``` +//! +//! The payload is unwrapped before the specialization runs — the +//! body of `proc handle_prop {v: Property::Scalar} ...` sees `$v` +//! as the bare payload (a `string`), matching Haskell `case` +//! semantics. +//! +//! Empty-payload variants still get `[lindex $v 1]` passed through +//! (it's the empty string for a single-element list) — for those +//! the specialization's body shouldn't reference `$v` and the +//! lowering should ideally drop the arg, but for v1 we pass +//! uniformly for simplicity. + +use crate::ast::OverloadInfo; + +/// Generate the public-name dispatcher proc for an overload set. +/// +/// `tail_arg_names` is the list of tail arg names (after the +/// dispatched first arg). They thread through via `{*}$args` so +/// the public signature stays `{v args}` regardless of arity — +/// keeping the dispatcher uniform across overload sets with +/// different tail shapes. Specializations always receive the +/// payload as their first positional arg, then the tail by +/// position. +pub fn emit_dispatcher(info: &OverloadInfo) -> String { + let mut out = String::new(); + // The dispatcher takes the same kwargs envelope every other + // proc takes (so calls can pass `- `). It + // extracts the dispatched-arg value from the kwargs args list + // and switches on its tag. Specializations receive the + // payload via the same kwargs protocol — `- ` + // — so their bodies bind `$` to the unwrapped payload + // naturally. + let pub_name = &info.public_name; + let arg = &info.dispatch_arg_name; + out.push_str(&format!("proc {pub_name} {{args}} {{\n")); + // Grab the dispatched arg's value from the kwargs args list. + // We look for `- ` pairwise; if the user passes + // positional, well, that's not a supported form for overloaded + // procs in v1 (kwargs-only). + out.push_str(&format!( + " set __vw_disp \"\"\n \ + foreach {{__vw_k __vw_v}} $args {{\n \ + if {{$__vw_k eq \"-{arg}\"}} {{ set __vw_disp $__vw_v; break }}\n \ + }}\n" + )); + out.push_str(" switch -- [lindex $__vw_disp 0] {\n"); + for v in &info.variants { + // Build a new args list with the dispatched-arg's value + // replaced by the unwrapped payload, then forward the full + // args list (including any tail args we pass through for + // future multi-arg overload support) to the specialization. + out.push_str(&format!( + " {variant} {{\n \ + set __vw_new [list]\n \ + foreach {{__vw_k __vw_v}} $args {{\n \ + if {{$__vw_k eq \"-{arg}\"}} {{\n \ + lappend __vw_new $__vw_k [lindex $__vw_v 1]\n \ + }} else {{\n \ + lappend __vw_new $__vw_k $__vw_v\n \ + }}\n \ + }}\n \ + return [{mangled} {{*}}$__vw_new]\n \ + }}\n", + variant = v.variant_name, + mangled = v.mangled_proc_name, + )); + } + out.push_str(&format!( + " default {{ error \"{pub_name}: unknown variant '[lindex $__vw_disp 0]'\" }}\n" + )); + out.push_str(" }\n"); + out.push_str("}\n"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::{OverloadInfo, OverloadVariant}; + use crate::span::Span; + use crate::validate::mangle_specialization; + + fn info(public: &str, variants: &[&str]) -> OverloadInfo { + OverloadInfo { + public_name: public.into(), + enum_name: "E".into(), + dispatch_arg_name: "v".into(), + variants: variants + .iter() + .map(|v| OverloadVariant { + variant_name: (*v).into(), + mangled_proc_name: mangle_specialization(public, v), + dispatch_arg_span: Span::new(0, 0), + }) + .collect(), + anchor_span: Span::new(0, 0), + } + } + + #[test] + fn dispatcher_two_arms() { + let i = info("handle_prop", &["Scalar", "Nested"]); + let d = emit_dispatcher(&i); + // Dispatcher takes the standard `{args}` kwargs envelope. + assert!(d.contains("proc handle_prop {args}")); + // Walks kwargs to find the `-v ` pair. + assert!(d.contains("if {$__vw_k eq \"-v\"}")); + assert!(d.contains("switch -- [lindex $__vw_disp 0]")); + // Each arm forwards to its mangled specialization with the + // unwrapped payload spliced back into the kwargs list. + assert!(d.contains("Scalar {")); + assert!(d.contains("__handle_prop__Scalar")); + assert!(d.contains("Nested {")); + assert!(d.contains("__handle_prop__Nested")); + assert!(d.contains("default {")); + assert!(d.contains("unknown variant")); + } + + #[test] + fn dispatcher_single_arm() { + let i = info("only_one", &["Solo"]); + let d = emit_dispatcher(&i); + assert!(d.contains("proc only_one {args}")); + assert!(d.contains("__only_one__Solo")); + } + + #[test] + fn dispatcher_includes_default_arm() { + // Future-proof against runtime corruption / unanticipated + // tag values. The validator's exhaustiveness check guards + // the source side; the default arm guards the runtime + // side. + let i = info("foo", &["A", "B"]); + let d = emit_dispatcher(&i); + assert!(d.contains("default { error")); + } +} diff --git a/vw-htcl/src/parser.rs b/vw-htcl/src/parser.rs new file mode 100644 index 0000000..2235430 --- /dev/null +++ b/vw-htcl/src/parser.rs @@ -0,0 +1,2472 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! htcl source parser. +//! +//! Builds a [`Document`] CST plus a list of [`ParseError`]s. The parser +//! is error-tolerant: when a command can't be parsed it is recorded as +//! a [`Stmt::Error`] and the parser resyncs at the next statement +//! boundary (newline or semicolon). This is what makes the same parser +//! usable from the LSP, where input is incomplete by definition. +//! +//! The outer statement loop is hand-rolled (it owns recovery and +//! collects doc comments); inner pieces — words, parts, escapes — +//! drive [`winnow::LocatingSlice`] for position tracking. As the grammar +//! grows past Phase 0 the inner pieces will lean on winnow combinators +//! more heavily. + +use winnow::stream::{Location, Stream}; +use winnow::LocatingSlice; + +use crate::ast::*; +use crate::span::Span; + +type Input<'i> = LocatingSlice<&'i str>; + +#[derive(Clone, Debug)] +pub struct ParseError { + pub message: String, + pub span: Span, +} + +#[derive(Clone, Debug)] +pub struct ParseOutput { + pub document: Document, + pub errors: Vec, +} + +pub fn parse(source: &str) -> ParseOutput { + let mut input = LocatingSlice::new(source); + let mut errors = Vec::new(); + let mut document = + parse_document(&mut input, source, &mut errors, Mode::Toplevel); + populate_procs(&mut document.stmts, source, &mut errors); + ParseOutput { document, errors } +} + +/// Statement-termination mode for the parser. +/// +/// At the top level (and inside proc bodies, which are themselves +/// scripts) a newline ends a command — the historical Tcl rule. Inside +/// a `[ … ]` command substitution we relax that: newlines are +/// whitespace and only `;` (or the closing bracket, which is EOF for +/// the interior parser) terminates a command. That lets a single call +/// span lines without `\` continuations, e.g. +/// +/// ```htcl +/// set x [ +/// create_cpm5_cpm_pcie0 +/// -cell cpm5 +/// -max_link_speed 32.0_GT/s +/// ] +/// ``` +/// +/// Multi-command `[…]` (rare in practice — only the last command's +/// value flows out) still works via explicit `;`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Mode { + Toplevel, + BracketBody, +} + +/// Post-pass over every `proc` — top-level *and* nested — that fills +/// in the structured args [`signature`](crate::ast::Proc::signature) +/// and parses the proc [`body`](crate::ast::Proc::body) into real +/// statements. +/// +/// The body is parsed as a standalone fragment (its braces are +/// already stripped by [`inner_text_span`]); the resulting spans are +/// relative to the fragment, so they're shifted by the body's start +/// offset back into whole-source coordinates before being stored. +/// After shifting they're absolute, which lets the recursion process +/// nested procs against the original `source` uniformly. +pub(crate) fn populate_procs( + stmts: &mut [crate::ast::Stmt], + source: &str, + errors: &mut Vec, +) { + use crate::ast::{CommandKind, Stmt}; + use crate::proc_args::parse_proc_args; + for stmt in stmts.iter_mut() { + let Stmt::Command(cmd) = stmt else { continue }; + + // Walk every word's parts and parse `[…]` command substitution + // interiors into statements. Spans inside the parsed body get + // shifted into whole-source coordinates so downstream analyses + // can navigate them uniformly with top-level commands. + for word in &mut cmd.words { + populate_cmd_subst_parts(&mut word.parts, source, errors); + } + + match &mut cmd.kind { + CommandKind::Proc(proc) => { + let (sig, errs) = parse_proc_args(source, proc.args_span); + errors.extend(errs); + proc.signature = Some(sig); + + // Parse the return-type annotation if the outer + // parse recorded one. We have `source` and the + // error sink here, so this is the right place to + // do it — `classify_command` only recorded the + // (inner, brace-stripped) span. + if let Some(rt_span) = proc.return_type_span { + let text = rt_span.slice(source); + match crate::type_parse::parse(text, rt_span.start) { + Ok(ty) => proc.return_type = Some(ty), + Err(e) => errors.push(ParseError { + message: e.message, + span: e.span, + }), + } + } + // Mirror the return type onto the signature so the + // signature-table-based lookup paths (REPL repr + // formatter, hover) see it without re-walking + // back to the Proc node. + if let Some(sig) = proc.signature.as_mut() { + sig.return_type = proc.return_type.clone(); + } + + let delta = proc.body_span.start; + let body_text = proc.body_span.slice(source); + // Proc bodies are scripts — newlines still terminate + // statements there. + let (mut body_stmts, body_errs) = + parse_fragment(body_text, Mode::Toplevel); + for stmt in &mut body_stmts { + shift_stmt(stmt, delta); + } + for mut err in body_errs { + err.span = err.span.shifted(delta); + errors.push(err); + } + proc.body = body_stmts; + + // Spans are now absolute, so nested procs can be processed + // against the same `source`. + populate_procs(&mut proc.body, source, errors); + } + CommandKind::TypeDecl(td) => { + let text = td.underlying_span.slice(source); + match crate::type_parse::parse(text, td.underlying_span.start) { + Ok(ty) => td.underlying = Some(ty), + Err(e) => errors.push(ParseError { + message: e.message, + span: e.span, + }), + } + } + CommandKind::EnumDecl(ed) => { + let text = ed.body_span.slice(source); + match crate::enum_parse::parse(text, ed.body_span.start) { + Ok(vs) => ed.variants = vs, + Err(e) => errors.push(ParseError { + message: e.message, + span: e.span, + }), + } + } + CommandKind::NamespaceEval(ns) => { + // Same body-recursion as `proc` — the braced body is + // a script fragment, parsed in toplevel mode so + // newlines terminate statements normally. + let delta = ns.body_span.start; + let body_text = ns.body_span.slice(source); + let (mut body_stmts, body_errs) = + parse_fragment(body_text, Mode::Toplevel); + for stmt in &mut body_stmts { + shift_stmt(stmt, delta); + } + for mut err in body_errs { + err.span = err.span.shifted(delta); + errors.push(err); + } + ns.body = body_stmts; + populate_procs(&mut ns.body, source, errors); + } + _ => { + // Generic (unrecognized) command: check the head + // word against the Tcl-builtin control-flow set + // and, when it's one of `foreach` / `for` / + // `while` / `if` / `catch` / `dict for`, + // recursively parse the arg-word(s) that are + // KNOWN to be script bodies. Populates + // `Word::body`; downstream tools (validator, + // putr rewrite, hover, goto, syntax highlight) + // key off that field to descend uniformly with + // top-level statements. + populate_control_flow_bodies(cmd, source, errors); + } + } + } +} + +/// Body-arg positions per builtin. Word-index 0 is the head, so +/// `foreach var list body` puts the body at word 3. `dict for +/// {kv} dict body` is the two-word head form. +fn populate_control_flow_bodies( + cmd: &mut crate::ast::Command, + source: &str, + errors: &mut Vec, +) { + // Word 0: head. Everything below tests it against the known + // set. Non-literal heads (interpolation, subst) fall through + // — we can't statically match them and the analyzer treats + // them as generic. + let Some(head) = cmd.words.first().and_then(|w| w.as_text()) else { + return; + }; + // Collect word-indices whose braced form should be parsed as + // a script. Multiple positions handle `for INIT COND STEP + // BODY` and `if COND BODY [elseif COND BODY]* [else BODY]`. + let body_positions: Vec = match head { + "foreach" => vec![cmd.words.len().saturating_sub(1)], + "while" => vec![2], + "for" => vec![1, 3, 4], // init, step, body — cond is an expr + "catch" => vec![1], + "dict" + // `dict for {kv} DICT BODY` — head is the two-word + // sub-command form. Only recognize the `for` variant + // for body descent; other `dict` sub-commands have no + // script args. + if cmd.words.get(1).and_then(|w| w.as_text()) == Some("for") => { + vec![cmd.words.len().saturating_sub(1)] + } + "if" => { + // `if COND BODY [elseif COND BODY]* [else BODY]`. + // Scan word-by-word: after `if`/`elseif` we skip the + // condition and mark the next word as a body; after + // `else` the very next word is a body. + let mut out = Vec::new(); + let mut i = 1usize; + while i < cmd.words.len() { + let w = cmd.words.get(i).and_then(|w| w.as_text()); + if w == Some("elseif") || i == 1 { + // condition at i (or i+1 for elseif), body at i+1 (or i+2) + let (cond_idx, body_idx) = if w == Some("elseif") { + (i + 1, i + 2) + } else { + (i, i + 1) + }; + let _ = cond_idx; + if body_idx < cmd.words.len() { + out.push(body_idx); + } + i = body_idx + 1; + } else if w == Some("else") { + if i + 1 < cmd.words.len() { + out.push(i + 1); + } + i += 2; + } else { + i += 1; + } + } + out + } + _ => return, + }; + for idx in body_positions { + let Some(word) = cmd.words.get_mut(idx) else { + continue; + }; + if word.form != crate::ast::WordForm::Braced { + // Only descend when the arg is a braced literal — + // an interpolated body (`$body_var`) can't be + // statically parsed. + continue; + } + // Interior text: `span` covers `{...}`; strip the two + // brace bytes. + let interior_start = word.span.start + 1; + let interior_end = word.span.end.saturating_sub(1); + if interior_end <= interior_start { + continue; + } + let interior = &source[interior_start as usize..interior_end as usize]; + let (mut body_stmts, body_errs) = + parse_fragment(interior, Mode::Toplevel); + for stmt in &mut body_stmts { + shift_stmt(stmt, interior_start); + } + for mut err in body_errs { + err.span = err.span.shifted(interior_start); + errors.push(err); + } + populate_procs(&mut body_stmts, source, errors); + word.body = Some(body_stmts); + } +} + +fn populate_cmd_subst_parts( + parts: &mut [WordPart], + source: &str, + errors: &mut Vec, +) { + for part in parts { + let WordPart::CmdSubst { + source: text, + span, + body, + } = part + else { + continue; + }; + // `span` covers the whole `[...]` including the brackets, and + // `text` is the interior; the first interior byte sits at + // `span.start + 1`. + let delta = span.start + 1; + // Bracket-body mode: newlines are whitespace, so multi-line + // calls inside `[ … ]` parse as one command without `\`. + let (mut body_stmts, body_errs) = + parse_fragment(text, Mode::BracketBody); + for s in &mut body_stmts { + shift_stmt(s, delta); + } + for mut err in body_errs { + err.span = err.span.shifted(delta); + errors.push(err); + } + *body = body_stmts; + populate_procs(body, source, errors); + } +} + +/// Parse a fragment of htcl (e.g. a proc body) into statements. Spans +/// are relative to `text`; the caller shifts them into whole-source +/// coordinates. +pub(crate) fn parse_fragment( + text: &str, + mode: Mode, +) -> (Vec, Vec) { + let mut input = LocatingSlice::new(text); + let mut errors = Vec::new(); + let document = parse_document(&mut input, text, &mut errors, mode); + (document.stmts, errors) +} + +pub(crate) fn shift_stmt(stmt: &mut crate::ast::Stmt, delta: u32) { + use crate::ast::Stmt; + match stmt { + Stmt::Command(cmd) => shift_command(cmd, delta), + Stmt::Comment(c) => c.span = c.span.shifted(delta), + Stmt::Error(e) => e.span = e.span.shifted(delta), + } +} + +fn shift_command(cmd: &mut Command, delta: u32) { + cmd.span = cmd.span.shifted(delta); + for word in &mut cmd.words { + shift_word(word, delta); + } + // At this stage nested procs carry only the spans produced by + // `parse_document`; `signature` is still `None` and `body` empty, + // both filled later by the caller's `populate_procs` recursion. + match &mut cmd.kind { + CommandKind::Proc(proc) => { + proc.name_span = proc.name_span.shifted(delta); + proc.args_span = proc.args_span.shifted(delta); + proc.body_span = proc.body_span.shifted(delta); + if let Some(ref mut s) = proc.return_type_span { + *s = s.shifted(delta); + } + // `return_type` is None at this point (parsed later in + // `populate_procs` using the now-absolute span), so + // there's nothing to shift inside it. + } + CommandKind::NamespaceEval(ns) => { + ns.name_span = ns.name_span.shifted(delta); + ns.body_span = ns.body_span.shifted(delta); + } + CommandKind::TypeDecl(td) => { + td.name_span = td.name_span.shifted(delta); + td.underlying_span = td.underlying_span.shifted(delta); + } + CommandKind::EnumDecl(ed) => { + ed.name_span = ed.name_span.shifted(delta); + ed.body_span = ed.body_span.shifted(delta); + // Variants are filled later by `populate_procs` using + // the now-absolute body_span, so there's nothing to + // shift inside them yet. + } + _ => {} + } +} + +fn shift_word(word: &mut Word, delta: u32) { + word.span = word.span.shifted(delta); + for part in &mut word.parts { + let span = match part { + WordPart::Text { span, .. } + | WordPart::VarRef { span, .. } + | WordPart::CmdSubst { span, .. } + | WordPart::Escape { span, .. } => span, + }; + *span = span.shifted(delta); + } +} + +#[derive(Clone, Debug)] +struct InnerError { + message: String, + #[allow(dead_code)] + span: Span, +} + +fn parse_document( + input: &mut Input<'_>, + source: &str, + errors: &mut Vec, + mode: Mode, +) -> Document { + let start = input.location(); + let mut stmts = Vec::new(); + let mut pending_docs: Vec = Vec::new(); + // Span covering all currently-pending `##` lines from the first + // `#` byte to the last line's end. Grows with each new `##` + // encountered; cleared alongside `pending_docs`. Used to seed + // the attached command's `doc_comments_span` so the analyzer + // can answer "is the cursor inside this doc block?" + let mut pending_docs_span: Option = None; + // Attributes at statement position (`@test`, `@test(dedicated-eda)`, + // etc.) attach to the next `proc` command, mirroring + // `pending_docs`. Cleared with a warning if the next command + // isn't a proc. + let mut pending_attrs: Vec = Vec::new(); + + loop { + skip_inline_ws(input, source, mode); + if at_eof(input, source) { + break; + } + let c = current_char(input, source); + // In `BracketBody`, `\n` is whitespace consumed by + // `skip_inline_ws`, so it never reaches this match. + let is_separator = match mode { + Mode::Toplevel => c == '\n' || c == ';', + Mode::BracketBody => c == ';', + }; + match c { + _ if is_separator => { + advance_char(input); + // A statement separator drops any orphan doc comments; + // doc comments only attach to the immediately + // following command. + if matches!(c, ';') { + // semicolons don't break doc attachment within a line + } else { + // Blank line breaks the doc-comment run only if + // the next non-whitespace is itself a blank line. + // For v0 we keep this simple: any `\n` between a + // doc comment and the next command keeps the + // attachment so long as nothing else intervenes. + } + continue; + } + '#' => { + let comment = parse_comment(input, source); + if comment.is_doc { + pending_docs.push(comment.text.clone()); + // Extend the block-span to cover this line. The + // comment's own span starts at its `#` and ends + // at the line's end. + pending_docs_span = Some(match pending_docs_span { + Some(prev) => Span::new(prev.start, comment.span.end), + None => comment.span, + }); + } else { + pending_docs.clear(); + pending_docs_span = None; + } + stmts.push(Stmt::Comment(comment)); + } + '@' => { + // Statement-position attribute — `@name` or + // `@name(v1, v2)`. Accumulates into + // `pending_attrs` and attaches to the next `proc` + // command via the drain in the `Ok(cmd)` arm below. + match parse_top_level_attribute(input, source, errors) { + Some(attr) => pending_attrs.push(attr), + None => { + // Parser already recorded the error and + // resynced. Drop any accumulated attrs so + // a garbage line doesn't attach half a + // block to the next proc. + pending_attrs.clear(); + } + } + } + _ => { + let cmd_start = input.location(); + match parse_command(input, source, mode) { + Ok(mut cmd) => { + cmd.doc_comments = std::mem::take(&mut pending_docs); + cmd.doc_comments_span = pending_docs_span.take(); + // Drain pending attributes into the command: + // procs get them via `Proc.attributes`; any + // other command shape drops them with a + // warning. + if !pending_attrs.is_empty() { + match &mut cmd.kind { + CommandKind::Proc(proc) => { + proc.attributes = + std::mem::take(&mut pending_attrs); + } + _ => { + let first = + pending_attrs.first().unwrap().span; + let last = + pending_attrs.last().unwrap().span; + errors.push(ParseError { + message: "attribute attached to \ + non-proc statement — \ + only `proc` declarations \ + accept attributes here" + .into(), + span: Span::new(first.start, last.end), + }); + pending_attrs.clear(); + } + } + } + stmts.push(Stmt::Command(cmd)); + } + Err(err) => { + pending_docs.clear(); + pending_docs_span = None; + // Resync to the next statement boundary. In + // `BracketBody` only `;` breaks; the surrounding + // `]` is EOF for the interior parser. + while !at_eof(input, source) { + let c = current_char(input, source); + let stop = match mode { + Mode::Toplevel => c == '\n' || c == ';', + Mode::BracketBody => c == ';', + }; + if stop { + break; + } + advance_char(input); + } + let span = Span::new( + cmd_start as u32, + input.location() as u32, + ); + errors.push(ParseError { + message: err.message.clone(), + span, + }); + stmts.push(Stmt::Error(ParseFailure { + message: err.message, + span, + })); + } + } + } + } + } + + Document { + stmts, + span: Span::new(start as u32, input.location() as u32), + } +} + +/// Parse a statement-position attribute — `@name` or +/// `@name(v1, v2, …)`. Attribute values accept the same shapes +/// [`crate::proc_args`]' parser does (int, string, ident), plus +/// kebab-case idents (`dedicated-eda`) for readable multi-word +/// tokens like `@test(dedicated-eda)`. +/// +/// Returns `None` when the leading identifier is missing or the +/// argument list is malformed — errors are appended to `errors` +/// verbatim; the caller drops any accumulated attributes to avoid +/// half-parsed items sticking to the next proc. +fn parse_top_level_attribute( + input: &mut Input<'_>, + source: &str, + errors: &mut Vec, +) -> Option { + let start = input.location() as u32; + advance_char(input); // '@' + let name_start = input.location() as u32; + let name = consume_ident(input, source); + let name_end = input.location() as u32; + if name.is_empty() { + errors.push(ParseError { + message: "expected attribute name after `@`".into(), + span: Span::new(start, input.location() as u32), + }); + return None; + } + let name_span = Span::new(name_start, name_end); + let mut values: Vec = Vec::new(); + if !at_eof(input, source) && current_char(input, source) == '(' { + advance_char(input); + loop { + skip_attr_ws(input, source); + if at_eof(input, source) { + errors.push(ParseError { + message: "unterminated attribute argument list".into(), + span: Span::new(start, input.location() as u32), + }); + break; + } + if current_char(input, source) == ')' { + advance_char(input); + break; + } + match parse_attribute_item(input, source) { + Some(v) => values.push(v), + None => { + errors.push(ParseError { + message: "expected attribute value".into(), + span: Span::new( + input.location() as u32, + input.location() as u32 + 1, + ), + }); + // Resync to comma/whitespace/`)` so the rest + // of the list still parses. + while !at_eof(input, source) { + let c = current_char(input, source); + if c == ',' + || c == ')' + || c == '\n' + || c == ' ' + || c == '\t' + { + break; + } + advance_char(input); + } + } + } + skip_attr_ws(input, source); + if at_eof(input, source) { + continue; + } + // Item separator: optional comma. Whitespace between + // items (already consumed by `skip_attr_ws`) is also + // a valid separator — matches the shape + // `@test(dedicated-eda part=xcvm3358…)` where items + // are space-separated. + if current_char(input, source) == ',' { + advance_char(input); + } + } + } + Some(Attribute { + name, + name_span, + values, + span: Span::new(start, input.location() as u32), + }) +} + +/// Consume `[A-Za-z_][A-Za-z0-9_]*` at the input cursor. +fn consume_ident(input: &mut Input<'_>, source: &str) -> String { + let mut out = String::new(); + let mut first = true; + while !at_eof(input, source) { + let c = current_char(input, source); + let ok = if first { + c.is_alphabetic() || c == '_' + } else { + c.is_alphanumeric() || c == '_' + }; + if !ok { + break; + } + out.push(c); + advance_char(input); + first = false; + } + out +} + +/// Ident with support for internal hyphens (`dedicated-eda`). +/// Consumes `[A-Za-z_]([A-Za-z0-9_-]*[A-Za-z0-9_])?` — a leading +/// alphabetic/underscore, followed by any mix of alphanumeric, +/// underscore, or hyphen, but disallowing a trailing hyphen. Used +/// only for attribute value idents; keeps proc names and +/// everything else on the stricter `consume_ident` rule. +fn consume_kebab_ident(input: &mut Input<'_>, source: &str) -> String { + let mut out = String::new(); + let mut first = true; + while !at_eof(input, source) { + let c = current_char(input, source); + let ok = if first { + c.is_alphabetic() || c == '_' + } else { + c.is_alphanumeric() || c == '_' || c == '-' + }; + if !ok { + break; + } + out.push(c); + advance_char(input); + first = false; + } + // Trim a trailing hyphen — `foo-` isn't a valid identifier and + // rolling it back lets the surrounding parser see the `-` as + // its own token if it wants to. + while out.ends_with('-') { + out.pop(); + // We can't easily un-advance the winnow cursor here, so + // trailing hyphens are consumed but not part of the name. + // In attribute-value context the `-` would then be + // followed by `,` or `)`, and neither position accepts a + // dangling hyphen — the caller's resync handles it. + } + out +} + +/// Skip horizontal whitespace + newlines inside an attribute +/// argument list. Attribute lists can wrap across lines, so we +/// consume `\n` as freely as space/tab. +fn skip_attr_ws(input: &mut Input<'_>, source: &str) { + while !at_eof(input, source) { + let c = current_char(input, source); + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + advance_char(input); + } else { + break; + } + } +} + +/// Parse one item in an attribute argument list — either a plain +/// positional value (int, string, ident, kebab-ident) or a +/// `key=value` pair. Wraps `key=value` in [`AttributeValue::Keyed`] +/// so downstream can distinguish. +fn parse_attribute_item( + input: &mut Input<'_>, + source: &str, +) -> Option { + let start = input.location() as u32; + // Peek: if the current position parses as `ident=`, it's a + // keyed item. Otherwise, delegate to positional + // `parse_attribute_value`. We do the peek without committing: + // save the cursor, tentatively consume an ident, check for + // `=`, and either commit or rewind. + // + // Rewind story: `input` here is a winnow `LocatingSlice`; + // its clone captures the internal cursor so the tentative + // parse is undoable. + let saved = *input; + let key_start = input.location() as u32; + let key = consume_ident(input, source); + let is_keyed = !key.is_empty() + && !at_eof(input, source) + && current_char(input, source) == '='; + if !is_keyed { + // Not a `key=value` item — rewind and parse as positional. + *input = saved; + return parse_attribute_value(input, source); + } + let key_span = Span::new(key_start, input.location() as u32); + advance_char(input); // consume '=' + // Value can be any regular attribute value shape. + let value = parse_attribute_value(input, source)?; + let end = input.location() as u32; + Some(AttributeValue::Keyed { + key, + key_span, + value: Box::new(value), + span: Span::new(start, end), + }) +} + +fn parse_attribute_value( + input: &mut Input<'_>, + source: &str, +) -> Option { + let start = input.location() as u32; + if at_eof(input, source) { + return None; + } + let c = current_char(input, source); + if c == '"' { + advance_char(input); + let mut buf = String::new(); + while !at_eof(input, source) && current_char(input, source) != '"' { + if current_char(input, source) == '\\' { + advance_char(input); + if !at_eof(input, source) { + buf.push(current_char(input, source)); + advance_char(input); + } + } else { + buf.push(current_char(input, source)); + advance_char(input); + } + } + if !at_eof(input, source) { + advance_char(input); // closing " + } + Some(AttributeValue::String { + value: buf, + span: Span::new(start, input.location() as u32), + }) + } else if c == '-' || c.is_ascii_digit() { + let mut buf = String::new(); + if c == '-' { + buf.push('-'); + advance_char(input); + } + while !at_eof(input, source) + && current_char(input, source).is_ascii_digit() + { + buf.push(current_char(input, source)); + advance_char(input); + } + buf.parse::() + .ok() + .map(|value| AttributeValue::Integer { + value, + span: Span::new(start, input.location() as u32), + }) + } else if c.is_alphabetic() || c == '_' { + let value = consume_kebab_ident(input, source); + Some(AttributeValue::Ident { + value, + span: Span::new(start, input.location() as u32), + }) + } else { + None + } +} + +fn parse_comment(input: &mut Input<'_>, source: &str) -> Comment { + let start = input.location(); + advance_char(input); // leading `#` + let mut is_doc = false; + if !at_eof(input, source) && current_char(input, source) == '#' { + is_doc = true; + advance_char(input); + } + // Leading single space after `#` / `##` is conventionally part of + // the marker; trim it so callers see the raw comment text. + if !at_eof(input, source) && current_char(input, source) == ' ' { + advance_char(input); + } + let text_start = input.location(); + while !at_eof(input, source) { + let c = current_char(input, source); + if c == '\n' { + break; + } + advance_char(input); + } + let text_end = input.location(); + Comment { + text: source[text_start..text_end].to_string(), + span: Span::new(start as u32, text_end as u32), + is_doc, + } +} + +fn parse_command( + input: &mut Input<'_>, + source: &str, + mode: Mode, +) -> Result { + let start = input.location(); + let mut words = Vec::new(); + loop { + skip_inline_ws(input, source, mode); + if at_eof(input, source) { + break; + } + let c = current_char(input, source); + // Line-continuation on a leading-dash next line: the + // configurator shape `cmd\n -flag val\n -flag val\n` + // parses as one command without needing `\` at every EOL. + // Only triggers mid-command (`!words.is_empty()`) — a `-` + // at the start of a fresh statement stays a new statement, + // even if it doesn't lex as a command name. + if mode == Mode::Toplevel + && c == '\n' + && !words.is_empty() + && next_line_is_flag_continuation(input, source) + { + advance_char(input); + continue; + } + let terminate = match mode { + Mode::Toplevel => c == '\n' || c == ';', + // In bracket-body, only `;` terminates a command — `\n` + // is whitespace consumed by `skip_inline_ws`. + Mode::BracketBody => c == ';', + }; + if terminate { + break; + } + // Inline comment at word-start position (mid-command). The + // configurator idiom for commenting out an arg line — + // + // set cfg [ + // versal_cips::configure + // -enable_reg_interface 1 + // #-intf_parent_pin_list 0 + // ] + // + // — needs the parser to eat the `#-intf_parent_pin_list 0` + // as a comment, otherwise it lands as a word and the + // analyzer flags `expected keyword argument`. Only fires + // MID-command (`!words.is_empty()`) so `#` at + // command-start on the top-level (a real Tcl comment) still + // reaches the outer `parse_document` handler; inside + // brackets `words.is_empty()` at line-start is normal + // because bracket-body has no prior context, but the + // enclosing `[…]` was already parsed as a CmdSubst so any + // `#` INSIDE the subst body's first command *does* have + // words already (the command name). + if c == '#' && !words.is_empty() { + skip_to_end_of_line(input, source); + continue; + } + words.push(parse_word(input, source)?); + } + if words.is_empty() { + return Err(InnerError { + message: "expected command".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + let span = Span::new(start as u32, input.location() as u32); + let kind = classify_command(&words); + Ok(Command { + words, + span, + kind, + doc_comments: Vec::new(), + doc_comments_span: None, + }) +} + +fn classify_command(words: &[Word]) -> CommandKind { + let Some(first) = words.first() else { + return CommandKind::Generic; + }; + match first.as_text() { + Some("set") => CommandKind::Set, + Some("src") if words.len() == 2 => { + let path_word = &words[1]; + CommandKind::Src(SrcImport { + path: path_word.as_text().map(String::from), + path_span: path_word.span, + }) + } + Some("proc") if words.len() >= 4 => { + let name_word = &words[1]; + let args_word = &words[2]; + // 5 words = return-type slot present: + // proc NAME { args } TYPE { body } + // 4 words = no return type: + // proc NAME { args } { body } + // (>5 words is treated as 5+ junk; the body is taken + // from words[4] and the rest is silently ignored. + // The return-type slot is parsed in `populate_procs` + // where we have `source` and an error sink — we only + // record the span here.) + let (return_type_span, body_word) = if words.len() >= 5 { + (Some(inner_text_span(&words[3])), &words[4]) + } else { + (None, &words[3]) + }; + let name = name_word.as_text().map(|s| s.to_string()); + CommandKind::Proc(Proc { + name, + name_span: name_word.span, + args_span: inner_text_span(args_word), + body_span: inner_text_span(body_word), + signature: None, + return_type: None, + return_type_span, + body: Vec::new(), + attributes: Vec::new(), + }) + } + // `type NAME = UNDERLYING` newtype declaration. The `=` may + // be its own word (`type T = U`) or fused (`type T=U`) — + // Tcl word splitting is whitespace-driven, so we accept + // either by checking the third word. The underlying type + // is parsed in `populate_procs`'s second pass (same + // rationale as `proc`'s return type). + Some("type") if words.len() >= 3 => { + let name_word = &words[1]; + let underlying_word = + if words.len() >= 4 && words[2].as_text() == Some("=") { + &words[3] + } else { + &words[2] + }; + CommandKind::TypeDecl(crate::ast::TypeDecl { + name: name_word.as_text().map(String::from), + name_span: name_word.span, + underlying: None, + underlying_span: inner_text_span(underlying_word), + }) + } + // `enum NAME = { …variants… }` sum-type declaration. + // Same `=`-may-or-may-not-be-its-own-word convention as + // `type`. The body word is brace-wrapped; its contents + // (the variant list) are parsed in `populate_procs`'s + // second pass, when we have the source + error sink. + Some("enum") if words.len() >= 3 => { + let name_word = &words[1]; + let body_word = + if words.len() >= 4 && words[2].as_text() == Some("=") { + &words[3] + } else { + &words[2] + }; + CommandKind::EnumDecl(crate::ast::EnumDecl { + name: name_word.as_text().map(String::from), + name_span: name_word.span, + variants: Vec::new(), + body_span: inner_text_span(body_word), + }) + } + Some("namespace") + if words.len() >= 4 + && words.get(1).and_then(Word::as_text) == Some("eval") => + { + let name_word = &words[2]; + let body_word = &words[3]; + CommandKind::NamespaceEval(crate::ast::NamespaceEval { + name: name_word.as_text().map(String::from), + name_span: name_word.span, + body_span: inner_text_span(body_word), + body: Vec::new(), + }) + } + _ => CommandKind::Generic, + } +} + +/// For a braced word, return the span of the brace contents (without +/// the braces themselves). For any other word, return its full span. +/// Used so Phase 2's structured-proc reparse and the LSP can point at +/// the parseable interior directly. +fn inner_text_span(word: &Word) -> Span { + if word.form == WordForm::Braced { + if let [WordPart::Text { span, .. }] = word.parts.as_slice() { + return *span; + } + } + word.span +} + +fn parse_word(input: &mut Input<'_>, source: &str) -> Result { + let start = input.location(); + let c = current_char(input, source); + let (form, parts) = match c { + '{' => parse_braced_word(input, source)?, + '"' => parse_quoted_word(input, source)?, + _ => parse_bare_word(input, source)?, + }; + let end = input.location(); + Ok(Word { + form, + parts, + span: Span::new(start as u32, end as u32), + // `body` is set by `populate_procs` for known control-flow + // builtins whose Nth arg is a script body. Initial parse + // leaves it None. + body: None, + }) +} + +fn parse_braced_word( + input: &mut Input<'_>, + source: &str, +) -> Result<(WordForm, Vec), InnerError> { + let open_cp = input.checkpoint(); + let open = input.location(); + advance_char(input); // { + let inner_start = input.location(); + let mut depth = 1usize; + while !at_eof(input, source) { + let c = current_char(input, source); + match c { + '\\' => { + advance_char(input); + if !at_eof(input, source) { + advance_char(input); + } + } + '{' => { + depth += 1; + advance_char(input); + } + '}' => { + depth -= 1; + if depth == 0 { + let inner_end = input.location(); + advance_char(input); + let text = source[inner_start..inner_end].to_string(); + return Ok(( + WordForm::Braced, + vec![WordPart::Text { + value: text, + span: Span::new( + inner_start as u32, + inner_end as u32, + ), + }], + )); + } + advance_char(input); + } + _ => advance_char(input), + } + } + // Unterminated: rewind to just past the open brace so the outer + // loop's resync can find the next statement boundary instead of + // being stuck at EOF. + input.reset(&open_cp); + advance_char(input); + Err(InnerError { + message: "unterminated brace group".into(), + span: Span::new(open as u32, (open + 1) as u32), + }) +} + +fn parse_quoted_word( + input: &mut Input<'_>, + source: &str, +) -> Result<(WordForm, Vec), InnerError> { + let open = input.location(); + advance_char(input); // " + let parts = collect_parts(input, source, Some('"'))?; + if at_eof(input, source) || current_char(input, source) != '"' { + return Err(InnerError { + message: "unterminated string".into(), + span: Span::new(open as u32, input.location() as u32), + }); + } + advance_char(input); // closing " + Ok((WordForm::Quoted, parts)) +} + +fn parse_bare_word( + input: &mut Input<'_>, + source: &str, +) -> Result<(WordForm, Vec), InnerError> { + let start = input.location(); + let parts = collect_parts(input, source, None)?; + if parts.is_empty() { + return Err(InnerError { + message: "expected word".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + Ok((WordForm::Bare, parts)) +} + +/// Accumulate [`WordPart`]s. +/// +/// `terminator` controls the stop condition: `Some('"')` for +/// double-quoted words (stops at the closing quote, newlines are +/// content), `None` for bare words (stops at whitespace, `;`, `\n`, +/// EOF). +fn collect_parts( + input: &mut Input<'_>, + source: &str, + terminator: Option, +) -> Result, InnerError> { + let mut parts = Vec::new(); + let mut text_buf = String::new(); + let mut text_start: Option = None; + + let flush = |parts: &mut Vec, + buf: &mut String, + start: &mut Option, + end: u32| { + if let Some(s) = start.take() { + if !buf.is_empty() { + parts.push(WordPart::Text { + value: std::mem::take(buf), + span: Span::new(s, end), + }); + } + buf.clear(); + } + }; + + loop { + if at_eof(input, source) { + break; + } + let c = current_char(input, source); + if Some(c) == terminator { + break; + } + if terminator.is_none() { + match c { + ' ' | '\t' | '\r' | '\n' | ';' => break, + _ => {} + } + } + match c { + '$' => { + flush( + &mut parts, + &mut text_buf, + &mut text_start, + input.location() as u32, + ); + parts.push(parse_var_ref(input, source)?); + } + '[' => { + flush( + &mut parts, + &mut text_buf, + &mut text_start, + input.location() as u32, + ); + parts.push(parse_cmd_subst(input, source)?); + } + '\\' => { + flush( + &mut parts, + &mut text_buf, + &mut text_start, + input.location() as u32, + ); + parts.push(parse_escape(input, source)?); + } + _ => { + if text_start.is_none() { + text_start = Some(input.location() as u32); + } + text_buf.push(c); + advance_char(input); + } + } + } + flush( + &mut parts, + &mut text_buf, + &mut text_start, + input.location() as u32, + ); + Ok(parts) +} + +fn parse_var_ref( + input: &mut Input<'_>, + source: &str, +) -> Result { + let start = input.location(); + advance_char(input); // $ + if at_eof(input, source) { + return Err(InnerError { + message: "expected variable name after `$`".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + let mut name = String::new(); + if current_char(input, source) == '{' { + advance_char(input); + while !at_eof(input, source) { + let c = current_char(input, source); + if c == '}' { + advance_char(input); + return Ok(WordPart::VarRef { + name, + span: Span::new(start as u32, input.location() as u32), + braced: true, + }); + } + name.push(c); + advance_char(input); + } + return Err(InnerError { + message: "unterminated `${...}`".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + while !at_eof(input, source) { + let c = current_char(input, source); + if c.is_alphanumeric() || c == '_' || c == ':' { + name.push(c); + advance_char(input); + } else { + break; + } + } + Ok(WordPart::VarRef { + name, + span: Span::new(start as u32, input.location() as u32), + braced: false, + }) +} + +fn parse_cmd_subst( + input: &mut Input<'_>, + source: &str, +) -> Result { + let start = input.location(); + advance_char(input); // [ + let inner_start = input.location(); + let mut depth = 1usize; + while !at_eof(input, source) { + let c = current_char(input, source); + match c { + '\\' => { + advance_char(input); + if !at_eof(input, source) { + advance_char(input); + } + } + '[' => { + depth += 1; + advance_char(input); + } + ']' => { + depth -= 1; + if depth == 0 { + let inner_end = input.location(); + advance_char(input); + let span = Span::new(start as u32, input.location() as u32); + let text = source[inner_start..inner_end].to_string(); + return Ok(WordPart::CmdSubst { + source: text, + span, + body: Vec::new(), + }); + } + advance_char(input); + } + _ => advance_char(input), + } + } + Err(InnerError { + message: "unterminated `[...]` command substitution".into(), + span: Span::new(start as u32, input.location() as u32), + }) +} + +fn parse_escape( + input: &mut Input<'_>, + source: &str, +) -> Result { + let start = input.location(); + advance_char(input); // backslash + if at_eof(input, source) { + return Err(InnerError { + message: "trailing `\\` at end of input".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + let c = current_char(input, source); + advance_char(input); + let value = match c { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + '\\' => '\\', + '"' => '"', + '[' => '[', + ']' => ']', + '{' => '{', + '}' => '}', + '$' => '$', + other => other, + }; + Ok(WordPart::Escape { + value, + span: Span::new(start as u32, input.location() as u32), + }) +} + +/// Peek past a `\n` and any inline whitespace on the immediately- +/// following line: does that line's first non-whitespace byte +/// begin a flag-shaped token (`-` followed by a letter, digit, +/// or another `-`)? Called by [`parse_command`] to decide whether +/// a newline should be treated as command continuation. +/// +/// Deliberately does NOT peek across a second `\n` — a blank line +/// terminates the continuation. Callers rely on this to model +/// "paragraph breaks" naturally, matching a reader's intuition. +/// +/// Doesn't consume input; only inspects `source` bytes. +fn next_line_is_flag_continuation(input: &Input<'_>, source: &str) -> bool { + let bytes = source.as_bytes(); + // Cursor sits on `\n`; look ahead starting at the byte after. + let mut i = input.location() + 1; + while i < bytes.len() { + match bytes[i] { + b' ' | b'\t' | b'\r' => i += 1, + _ => break, + } + } + if i >= bytes.len() || bytes[i] != b'-' { + return false; + } + // Look at what follows the `-`. Flag-shaped: letter, digit, + // underscore, or a second `-` (for `--end-of-options` idiom). + // Anything else (whitespace, EOF, punctuation) declines to + // continue. Underscore is included because real Vivado flag + // names like `-_64bit` (create_bd_cell's 64-bit BAR flag) are + // valid — without `_` in the set, the continuation rule + // breaks on them and Tcl tries to execute `-_64bit` as a + // command. + let next = bytes.get(i + 1).copied().unwrap_or(b'\0'); + next.is_ascii_alphanumeric() || next == b'-' || next == b'_' +} + +/// Consume input up to the next `\n` (not consuming the `\n` +/// itself). Used to treat `#`-prefixed lines mid-command as +/// inline comments — see the callsite in `parse_command`. +fn skip_to_end_of_line(input: &mut Input<'_>, source: &str) { + while !at_eof(input, source) { + if current_char(input, source) == '\n' { + break; + } + advance_char(input); + } +} + +fn skip_inline_ws(input: &mut Input<'_>, source: &str, mode: Mode) { + while !at_eof(input, source) { + let c = current_char(input, source); + if c == ' ' || c == '\t' || c == '\r' { + advance_char(input); + } else if mode == Mode::BracketBody && c == '\n' { + // Inside `[ … ]` the newline isn't a statement terminator; + // it's just whitespace. + advance_char(input); + } else if c == '\\' { + let pos = input.location(); + if pos + 1 < source.len() && source.as_bytes()[pos + 1] == b'\n' { + advance_char(input); + advance_char(input); + } else { + break; + } + } else { + break; + } + } +} + +fn at_eof(input: &Input<'_>, source: &str) -> bool { + input.location() >= source.len() +} + +fn current_char(input: &Input<'_>, source: &str) -> char { + source[input.location()..].chars().next().unwrap_or('\0') +} + +fn advance_char(input: &mut Input<'_>) { + let _ = input.next_token(); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_empty() { + let out = parse(""); + assert!(out.document.stmts.is_empty()); + assert!(out.errors.is_empty()); + } + + #[test] + fn control_flow_bodies_get_parsed_into_word_body() { + // `dict for {kv} DICT BODY` — head is 2-word `dict for`, + // body is the LAST arg. The braced body should get its + // interior parsed and populated on `Word::body`, with + // absolute spans, so hover/goto/putr can descend. + let src = "dict for {lib srcs} $deps {\n puts $lib\n putr $srcs\n}"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("expected command"); + }; + // Last word is the body braced-word. + let body_word = cmd.words.last().unwrap(); + assert_eq!(body_word.form, crate::ast::WordForm::Braced); + let body = body_word + .body + .as_ref() + .expect("dict-for body should have parsed statements"); + // Body has three top-level commands: puts, putr, and a + // synthesized command boundary if parsing picked one up. + // Expect at least two commands (puts + putr). + assert!(body.len() >= 2, "body: {body:?}"); + let Stmt::Command(first) = &body[0] else { + panic!("expected first body stmt to be a command"); + }; + assert_eq!(first.words[0].as_text(), Some("puts")); + // Spans should be absolute (whole-source). + assert!( + first.span.start > 20, + "expected absolute span past outer command head, got {:?}", + first.span, + ); + } + + #[test] + fn foreach_body_populated() { + let src = "foreach x {a b c} {\n puts $x\n}"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("expected command"); + }; + // Body is the last (third-index) word. + let body_word = cmd.words.last().unwrap(); + assert!( + body_word.body.is_some(), + "foreach body should have parsed statements", + ); + } + + #[test] + fn if_bodies_populated() { + let src = "if {$x > 0} {\n puts big\n} else {\n puts small\n}"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("expected command"); + }; + // Both bodies (word 2 for the `if` body, word 4 for the + // else body) should be populated. + assert!(cmd.words[2].body.is_some(), "if-body should be parsed"); + assert!(cmd.words[4].body.is_some(), "else-body should be parsed"); + } + + #[test] + fn parse_comment_and_doc() { + let src = "# regular\n## doc text\nputs hi\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + assert_eq!(out.document.stmts.len(), 3); + let Stmt::Command(cmd) = &out.document.stmts[2] else { + panic!("expected command, got {:?}", out.document.stmts[2]); + }; + assert_eq!(cmd.doc_comments, vec!["doc text".to_string()]); + assert_eq!(cmd.words[0].as_text(), Some("puts")); + assert_eq!(cmd.words[1].as_text(), Some("hi")); + } + + #[test] + fn parse_set_command() { + let out = parse("set x 42"); + assert!(out.errors.is_empty()); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert!(matches!(cmd.kind, CommandKind::Set)); + assert_eq!(cmd.words.len(), 3); + } + + #[test] + fn parse_proc_braced() { + let src = "proc greet {name} { puts \"hi $name\" }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!("expected proc, got {:?}", cmd.kind); + }; + assert_eq!(proc.name.as_deref(), Some("greet")); + let args = proc.args_span.slice(src); + let body = proc.body_span.slice(src); + assert_eq!(args, "name"); + assert!(body.contains("puts")); + // No return type slot. + assert!(proc.return_type.is_none()); + assert!(proc.return_type_span.is_none()); + } + + #[test] + fn parse_proc_with_return_type_named() { + let src = "proc f {} string { return foo }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + assert_eq!(proc.name.as_deref(), Some("f")); + let body = proc.body_span.slice(src); + assert!(body.contains("return foo")); + let ty = proc.return_type.as_ref().expect("return type set"); + assert_eq!(ty.name(), "string"); + match ty { + crate::ast::TypeExpr::Named { .. } => {} + _ => panic!("expected Named"), + } + } + + #[test] + fn parse_proc_with_return_type_generic_no_whitespace() { + let src = "proc f {} list { return {} }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + let ty = proc.return_type.as_ref().unwrap(); + let crate::ast::TypeExpr::Generic { name, args, .. } = ty else { + panic!("expected Generic") + }; + assert_eq!(name, "list"); + assert_eq!(args.len(), 1); + assert_eq!(args[0].name(), "bd_cell"); + } + + #[test] + fn parse_proc_with_return_type_nested_generic() { + let src = "proc f {} list> { return {} }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + let ty = proc.return_type.as_ref().unwrap(); + let crate::ast::TypeExpr::Generic { args, .. } = ty else { + panic!() + }; + let crate::ast::TypeExpr::Generic { + name: inner_name, + args: inner_args, + .. + } = &args[0] + else { + panic!("expected nested Generic") + }; + assert_eq!(inner_name, "dict"); + assert_eq!(inner_args.len(), 2); + } + + #[test] + fn parse_proc_with_return_type_bracketed_whitespace() { + let src = "proc f {} {dict} { return {} }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + let ty = proc.return_type.as_ref().unwrap(); + let crate::ast::TypeExpr::Generic { name, args, .. } = ty else { + panic!() + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + assert_eq!(args[1].name(), "int"); + } + + #[test] + fn parse_proc_with_invalid_return_type_emits_diagnostic() { + let src = "proc f {} list< { return {} }\n"; + let out = parse(src); + assert!( + !out.errors.is_empty(), + "expected a parse-error diagnostic for bad type" + ); + assert!(out.errors.iter().any(|e| e.message.contains("expected") + || e.message.contains("unterminated"))); + } + + #[test] + fn parse_type_decl_named() { + let src = "type bd_cell = string\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::TypeDecl(td) = &cmd.kind else { + panic!("expected TypeDecl, got {:?}", cmd.kind) + }; + assert_eq!(td.name.as_deref(), Some("bd_cell")); + let underlying = td.underlying.as_ref().unwrap(); + assert_eq!(underlying.name(), "string"); + } + + #[test] + fn parse_type_decl_generic_underlying() { + let src = "type fancy_dict = {dict}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::TypeDecl(td) = &cmd.kind else { + panic!() + }; + let crate::ast::TypeExpr::Generic { name, args, .. } = + td.underlying.as_ref().unwrap() + else { + panic!() + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + } + + #[test] + fn parse_type_decl_without_equals_works() { + // `type T U` (no `=`) is also accepted — the `=` is sugar. + let src = "type widget string\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::TypeDecl(td) = &cmd.kind else { + panic!() + }; + assert_eq!(td.name.as_deref(), Some("widget")); + assert_eq!(td.underlying.as_ref().unwrap().name(), "string"); + } + + #[test] + fn parse_type_decl_with_bad_underlying_emits_diagnostic() { + let src = "type foo = \n"; + let out = parse(src); + assert!( + !out.errors.is_empty(), + "expected diagnostic for malformed underlying type" + ); + } + + // --- enum declarations ----------------------------------------- + + #[test] + fn parse_enum_decl_simple() { + let src = "enum Direction = {\n North\n South\n East\n West\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::EnumDecl(ed) = &cmd.kind else { + panic!("expected EnumDecl, got {:?}", cmd.kind); + }; + assert_eq!(ed.name.as_deref(), Some("Direction")); + assert_eq!(ed.variants.len(), 4); + for v in &ed.variants { + assert!(v.payload.is_none(), "expected empty-payload variant"); + } + let names: Vec<&str> = + ed.variants.iter().map(|v| v.name.as_str()).collect(); + assert_eq!(names, vec!["North", "South", "East", "West"]); + } + + #[test] + fn parse_enum_decl_with_payloads() { + let src = "enum Property = {\n Scalar: string\n Nested: dict\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::EnumDecl(ed) = &cmd.kind else { + panic!() + }; + assert_eq!(ed.name.as_deref(), Some("Property")); + assert_eq!(ed.variants.len(), 2); + assert_eq!(ed.variants[0].name, "Scalar"); + assert_eq!(ed.variants[0].payload.as_ref().unwrap().name(), "string"); + assert_eq!(ed.variants[1].name, "Nested"); + let crate::ast::TypeExpr::Generic { name, args, .. } = + ed.variants[1].payload.as_ref().unwrap() + else { + panic!(); + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + assert_eq!(args[1].name(), "Property"); + } + + #[test] + fn parse_enum_decl_mixed_payload_and_empty() { + let src = + "enum Mix = {\n Empty\n WithInt: int\n Other\n WithList: list\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::EnumDecl(ed) = &cmd.kind else { + panic!() + }; + assert_eq!(ed.variants.len(), 4); + assert!(ed.variants[0].payload.is_none()); + assert_eq!(ed.variants[1].payload.as_ref().unwrap().name(), "int"); + assert!(ed.variants[2].payload.is_none()); + assert_eq!(ed.variants[3].payload.as_ref().unwrap().name(), "list"); + } + + #[test] + fn parse_enum_decl_without_equals() { + let src = "enum Color {\n Red\n Green\n Blue\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::EnumDecl(ed) = &cmd.kind else { + panic!() + }; + assert_eq!(ed.name.as_deref(), Some("Color")); + assert_eq!(ed.variants.len(), 3); + } + + #[test] + fn parse_enum_decl_with_bad_variant_emits_diagnostic() { + // 123Foo is not a valid identifier — should diagnose. + let src = "enum Bad = {\n 123Foo: int\n}\n"; + let out = parse(src); + assert!( + !out.errors.is_empty(), + "expected diagnostic for malformed variant" + ); + } + + #[test] + fn parse_proc_with_qualified_arg_type() { + // The `E::V` qualified syntax for overloaded handler args. + let src = + "proc handle_prop {v: Property::Scalar} string { return $v }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + let sig = proc.signature.as_ref().unwrap(); + assert_eq!(sig.args.len(), 1); + let arg = &sig.args[0]; + assert_eq!(arg.name, "v"); + let crate::ast::TypeExpr::Qualified { + namespace, variant, .. + } = arg.type_annotation.as_ref().unwrap() + else { + panic!( + "expected Qualified type annotation, got {:?}", + arg.type_annotation + ); + }; + assert_eq!(namespace, "Property"); + assert_eq!(variant, "Scalar"); + } + + #[test] + fn parse_variable_and_subst() { + let src = "puts $x [foo bar]"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert_eq!(cmd.words.len(), 3); + let WordPart::VarRef { name, .. } = &cmd.words[1].parts[0] else { + panic!("expected var ref"); + }; + assert_eq!(name, "x"); + let WordPart::CmdSubst { source: src, .. } = &cmd.words[2].parts[0] + else { + panic!("expected cmd subst"); + }; + assert_eq!(src, "foo bar"); + } + + #[test] + fn parse_quoted_with_subst() { + let src = r#"puts "hello $name""#; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert_eq!(cmd.words[1].form, WordForm::Quoted); + assert_eq!(cmd.words[1].parts.len(), 2); + let WordPart::Text { value, .. } = &cmd.words[1].parts[0] else { + panic!(); + }; + assert_eq!(value, "hello "); + let WordPart::VarRef { name, .. } = &cmd.words[1].parts[1] else { + panic!(); + }; + assert_eq!(name, "name"); + } + + #[test] + fn recovers_from_unterminated_brace() { + let src = "puts {oops\nputs ok\n"; + let out = parse(src); + assert!(!out.errors.is_empty()); + assert!(out.errors[0].message.contains("brace group")); + // After the error we should still see the second `puts ok`. + let ok_cmd = out.document.stmts.iter().find_map(|s| match s { + Stmt::Command(c) + if c.words.first().and_then(|w| w.as_text()) + == Some("puts") + && c.words.get(1).and_then(|w| w.as_text()) + == Some("ok") => + { + Some(c) + } + _ => None, + }); + assert!( + ok_cmd.is_some(), + "expected recovery: {:?}", + out.document.stmts + ); + } + + #[test] + fn proc_body_parses_into_statements_with_absolute_spans() { + let src = "proc outer {\n a\n} {\n inner_call foo\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!("expected proc"); + }; + assert_eq!(proc.body.len(), 1, "{:?}", proc.body); + let Stmt::Command(body_cmd) = &proc.body[0] else { + panic!("expected command in body"); + }; + // Span is absolute: it slices back to the original source. + assert_eq!(body_cmd.words[0].span.slice(src), "inner_call"); + assert_eq!( + body_cmd.span.start as usize, + src.find("inner_call").unwrap() + ); + } + + #[test] + fn nested_proc_body_is_parsed_recursively() { + let src = + "proc outer {\n a\n} {\n proc inner {\n b\n} {\n deep\n}\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(outer) = &cmd.kind else { + panic!("expected outer proc"); + }; + let Stmt::Command(inner_cmd) = &outer.body[0] else { + panic!("expected inner proc command"); + }; + let CommandKind::Proc(inner) = &inner_cmd.kind else { + panic!("expected inner proc"); + }; + assert_eq!(inner.name.as_deref(), Some("inner")); + // Inner proc got its signature and body populated too. + assert!(inner.signature.is_some()); + let Stmt::Command(deep) = &inner.body[0] else { + panic!("expected deep command"); + }; + assert_eq!(deep.words[0].span.slice(src), "deep"); + } + + #[test] + fn parses_src_statement() { + let src = "src common/log\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Src(import) = &cmd.kind else { + panic!("expected Src, got {:?}", cmd.kind); + }; + assert_eq!(import.path.as_deref(), Some("common/log")); + assert_eq!(import.path_span.slice(src), "common/log"); + } + + #[test] + fn parses_src_with_named_dep_prefix() { + let out = parse("src @xilinx-ip/cpm5\n"); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Src(import) = &cmd.kind else { + panic!("expected Src"); + }; + assert_eq!(import.path.as_deref(), Some("@xilinx-ip/cpm5")); + } + + #[test] + fn src_with_extra_words_is_generic() { + // `src a b` isn't a valid import — it falls back to generic so + // the validator can report it as an unknown command rather than + // the parser silently accepting it. + let out = parse("src a b\n"); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert!(matches!(cmd.kind, CommandKind::Generic), "{:?}", cmd.kind); + } + + #[test] + fn bracket_body_treats_newlines_as_whitespace() { + // Multi-line call inside `[ … ]` parses as a *single* command, + // no backslash continuations needed. + let src = "\ +set cell [ + create_cpm5_cpm_pcie0 + -cell cpm5 + -max_link_speed 32.0_GT/s +] +"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(set_cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert!(matches!(set_cmd.kind, CommandKind::Set)); + // The `set`'s value word is the cmd-subst; its body should be + // a single command with five words. + let WordPart::CmdSubst { body, .. } = &set_cmd.words[2].parts[0] else { + panic!("expected CmdSubst"); + }; + assert_eq!(body.len(), 1, "{body:#?}"); + let Stmt::Command(inner) = &body[0] else { + panic!(); + }; + let word_texts: Vec<&str> = + inner.words.iter().filter_map(|w| w.as_text()).collect(); + assert_eq!( + word_texts, + vec![ + "create_cpm5_cpm_pcie0", + "-cell", + "cpm5", + "-max_link_speed", + "32.0_GT/s", + ] + ); + } + + #[test] + fn bracket_body_still_separates_on_semicolon() { + // Explicit `;` keeps the multi-command form available inside + // brackets for users who want it. + let src = "set x [a 1 ; b 2]\n"; + let out = parse(src); + assert!(out.errors.is_empty()); + let Stmt::Command(set_cmd) = &out.document.stmts[0] else { + panic!(); + }; + let WordPart::CmdSubst { body, .. } = &set_cmd.words[2].parts[0] else { + panic!(); + }; + assert_eq!(body.len(), 2, "{body:#?}"); + } + + #[test] + fn toplevel_newlines_still_terminate() { + // The bracket-body relaxation does not leak to the top level. + let src = "puts a\nputs b\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 2); + } + + /// Line continuation via a leading `-` on the next line: the + /// common flag-per-line configurator shape (`create_foo`, + /// newline, indented `-bar val`, newline, `-baz val`, …) + /// parses as one command without needing a trailing `\`. + #[test] + fn dash_leading_next_line_continues_command() { + let src = "create_foo\n -bar 1\n -baz 2\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 1, "{cmds:#?}"); + let words: Vec<&str> = + cmds[0].words.iter().filter_map(Word::as_text).collect(); + assert_eq!(words, ["create_foo", "-bar", "1", "-baz", "2"]); + } + + /// A `--` (end-of-options) continuation also chains — same + /// leading-dash shape. + #[test] + fn double_dash_next_line_continues_command() { + let src = "cmd -a 1\n -- rest\n"; + let out = parse(src); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("{:#?}", out.document.stmts); + }; + assert_eq!(cmd.words.len(), 5); + } + + /// Non-dash next line still terminates. A regression here + /// would break every existing top-level script. + #[test] + fn non_dash_next_line_terminates_as_before() { + let src = "puts a\nputs b\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 2); + } + + /// A blank line between the header and a dash-led line breaks + /// the continuation — the header stands alone and the dash- + /// led line becomes a new (probably weird) command. This + /// matches how a reader intuits paragraph breaks: an empty + /// line is a stronger separator than a newline. + /// Vivado flags like `-_64bit` start with `-_` — the underscore + /// must be recognized as flag-shaped so the continuation rule + /// keeps the line inside the enclosing command instead of + /// starting a new (invalid) command. + #[test] + fn dash_underscore_line_continues_command() { + let src = "create_bar\n -cell foo\n -_64bit 1\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 1, "{cmds:#?}"); + let words: Vec<&str> = + cmds[0].words.iter().filter_map(Word::as_text).collect(); + assert_eq!(words, ["create_bar", "-cell", "foo", "-_64bit", "1"]); + } + + /// Trailing whitespace on the previous line must not defeat the + /// dash-continuation rule — real-world files often carry a stray + /// space at end of line, and we want the multi-line command to + /// still parse as one command. + #[test] + fn dash_continuation_survives_trailing_ws_on_previous_line() { + let src = "cmd\n -foo 1 \n -bar 2\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 1, "{cmds:#?}"); + assert_eq!(cmds[0].words.len(), 5); + } + + #[test] + fn blank_line_before_dash_breaks_continuation() { + let src = "cmd\n\n -a 1\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 2, "{cmds:#?}"); + } + + #[test] + fn proc_body_newlines_still_terminate() { + // Proc bodies are scripts; the relaxation is bracket-only. + let src = "proc f {} {\n puts a\n puts b\n}\n"; + let out = parse(src); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!(); + }; + assert_eq!(proc.body.len(), 2); + } + + #[test] + fn semicolon_separates_commands() { + let src = "set a 1; set b 2"; + let out = parse(src); + assert!(out.errors.is_empty()); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 2); + } + + #[test] + fn inline_comment_arg_line_is_stripped() { + // The configurator idiom for commenting out an arg line. + // Parser should eat `#-intf_parent_pin_list 0` and leave a + // clean word list `[configure, -enable_reg_interface, 1]`. + let src = "\ +set cfg [ + configure + -enable_reg_interface 1 + #-intf_parent_pin_list 0 +]\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "parse errors: {:?}", out.errors); + // The set command has three words: `set`, `cfg`, `[…]`. + let set = out + .document + .stmts + .iter() + .find_map(|s| match s { + crate::ast::Stmt::Command(c) + if c.words.first().and_then(|w| w.as_text()) + == Some("set") => + { + Some(c) + } + _ => None, + }) + .expect("set command"); + // The CmdSubst body should contain one command with 3 words + // (the `#-intf_parent_pin_list 0` line is eaten). + let bracket = &set.words[2]; + let crate::ast::WordPart::CmdSubst { body, .. } = &bracket.parts[0] + else { + panic!("expected CmdSubst"); + }; + let inner_cmd = body + .iter() + .find_map(|s| match s { + crate::ast::Stmt::Command(c) => Some(c), + _ => None, + }) + .expect("configure command"); + let word_texts: Vec<&str> = + inner_cmd.words.iter().filter_map(|w| w.as_text()).collect(); + assert_eq!( + word_texts, + vec!["configure", "-enable_reg_interface", "1"], + "expected the commented arg line to be gone", + ); + } + + #[test] + fn hash_mid_command_line_is_still_comment() { + // `#` mid-command at word-start, even without a newline + // in-between, is treated as a comment. Matches the + // configurator ergonomics — inside `[cmd -a x #-b y]` + // the `#-b y` gets eaten. + let src = "[configure -a x #-b y]\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "parse errors: {:?}", out.errors); + } + + #[test] + fn top_level_attribute_attaches_to_proc() { + // Regression: `@test` above `proc foo {} { ... }` should + // populate `proc.attributes` with a single attribute named + // `test`. Consumed by `vw test`. + let src = "@test\nproc foo {} { puts hi }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("expected command"); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!("expected proc"); + }; + assert_eq!(proc.attributes.len(), 1); + assert_eq!(proc.attributes[0].name, "test"); + assert!(proc.attributes[0].values.is_empty()); + } + + #[test] + fn attribute_with_kebab_value_parses() { + // `@test(dedicated-eda)` — kebab-case ident allowed in + // attribute value position only. This shape drives the + // `vw test` runner's shared-vs-dedicated bucket choice. + let src = "@test(dedicated-eda)\nproc foo {} { }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!(); + }; + assert_eq!(proc.attributes.len(), 1); + let attr = &proc.attributes[0]; + assert_eq!(attr.name, "test"); + assert_eq!(attr.values.len(), 1); + match &attr.values[0] { + AttributeValue::Ident { value, .. } => { + assert_eq!(value, "dedicated-eda") + } + other => panic!("expected Ident, got {other:?}"), + } + } + + #[test] + fn multiple_attributes_stack_on_one_proc() { + let src = "@test\n@another\nproc foo {} { }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!(); + }; + assert_eq!(proc.attributes.len(), 2); + assert_eq!(proc.attributes[0].name, "test"); + assert_eq!(proc.attributes[1].name, "another"); + } + + #[test] + fn attribute_with_whitespace_separated_items() { + // `@test(dedicated-eda part=xcvm3358-vsvh1747-2M-e-S)` — + // no commas between items, one positional + one keyed. + let src = "\ +@test(dedicated-eda part=xcvm3358-vsvh1747-2M-e-S) +proc foo {} { } +"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!(); + }; + assert_eq!(proc.attributes.len(), 1); + let attr = &proc.attributes[0]; + assert_eq!(attr.name, "test"); + assert_eq!(attr.values.len(), 2); + // Positional item. + match &attr.values[0] { + AttributeValue::Ident { value, .. } => { + assert_eq!(value, "dedicated-eda") + } + other => panic!("expected Ident, got {other:?}"), + } + // Keyed item. + match &attr.values[1] { + AttributeValue::Keyed { key, value, .. } => { + assert_eq!(key, "part"); + match value.as_ref() { + AttributeValue::Ident { value, .. } => { + assert_eq!(value, "xcvm3358-vsvh1747-2M-e-S") + } + other => panic!("expected Ident, got {other:?}"), + } + } + other => panic!("expected Keyed, got {other:?}"), + } + } + + #[test] + fn attribute_comma_separated_still_works() { + // Backward-compat: `@enum(a, b, c)` still parses as three + // positional idents. + let src = "\ +proc foo { + @enum(a, b, c) x +} { } +"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + } + + #[test] + fn attribute_on_non_proc_command_reports_error() { + // `@test` above a `set` command doesn't make sense — the + // parser records a diagnostic and drops the attrs so they + // don't leak onto whatever the NEXT command is. + let src = "@test\nset x 1\n"; + let out = parse(src); + assert!( + out.errors.iter().any(|e| e + .message + .contains("attribute attached to non-proc statement")), + "expected non-proc-attachment diagnostic, got {:?}", + out.errors, + ); + } +} diff --git a/vw-htcl/src/proc_args.rs b/vw-htcl/src/proc_args.rs new file mode 100644 index 0000000..60d207c --- /dev/null +++ b/vw-htcl/src/proc_args.rs @@ -0,0 +1,599 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Parser for the structured proc-arg grammar (Phase 2). +//! +//! Operates on the inner contents of a `proc`'s args braces — the +//! span passed in must point at the text between the braces, not +//! including the braces themselves. All spans on the returned AST +//! nodes are absolute file offsets, so they slot directly into the +//! main document's diagnostics without rebasing. +//! +//! Grammar: +//! +//! ```text +//! args := arg_item* +//! arg_item := doc_comment* attribute* IDENT ( ':' TYPE )? +//! attribute := '@' IDENT ( '(' value ( ',' value )* ')' )? +//! value := integer | string | ident +//! TYPE := IDENT ( '<' TYPE ( ',' TYPE )* '>' )? +//! ``` +//! +//! The optional `: TYPE` slot turns an arg from an opaque +//! identifier into a typed one. Adoption is gradual — existing +//! libraries without annotations still parse identically. +//! +//! Whitespace, blank lines, and non-doc comments are skippable +//! between items. + +use crate::ast::{Attribute, AttributeValue, ProcArg, ProcSignature}; +use crate::parser::ParseError; +use crate::span::Span; + +pub fn parse_proc_args( + full_source: &str, + args_span: Span, +) -> (ProcSignature, Vec) { + let inner = args_span.slice(full_source); + let mut state = State { + inner, + base: args_span.start, + pos: 0, + errors: Vec::new(), + }; + let mut args = Vec::new(); + state.parse_args(&mut args); + let State { errors, .. } = state; + ( + ProcSignature { + args, + span: args_span, + // Filled in later by the parser's `populate_procs` + // pass once the return-type annotation has been parsed. + return_type: None, + }, + errors, + ) +} + +struct State<'a> { + inner: &'a str, + /// Absolute file offset where `inner` starts. + base: u32, + /// Byte offset into `inner`. + pos: usize, + errors: Vec, +} + +impl<'a> State<'a> { + fn at_eof(&self) -> bool { + self.pos >= self.inner.len() + } + + fn current(&self) -> char { + self.inner[self.pos..].chars().next().unwrap_or('\0') + } + + fn peek_at(&self, offset: usize) -> char { + let target = self.pos + offset; + if target >= self.inner.len() { + '\0' + } else { + self.inner[target..].chars().next().unwrap_or('\0') + } + } + + fn abs(&self) -> u32 { + self.base + self.pos as u32 + } + + fn bump(&mut self) { + if let Some(c) = self.inner[self.pos..].chars().next() { + self.pos += c.len_utf8(); + } + } + + fn skip_horizontal_ws(&mut self) { + while !self.at_eof() { + let c = self.current(); + if c == ' ' || c == '\t' || c == '\r' { + self.bump(); + } else { + break; + } + } + } + + /// Consume blank lines, comments, and whitespace; doc comments + /// (`##`) are collected and returned so they can attach to the + /// next arg item. The returned span (when non-empty) covers the + /// whole `##` block from its first `#` byte to the newline after + /// its last line, matching [`Command::doc_comments_span`]'s + /// convention. + fn skip_separators(&mut self) -> (Vec, Option) { + let mut docs = Vec::new(); + let mut docs_span: Option = None; + loop { + // Whitespace including newlines + while !self.at_eof() { + let c = self.current(); + if c.is_whitespace() { + self.bump(); + } else { + break; + } + } + if self.at_eof() { + break; + } + if self.current() == '#' { + let line_start = self.abs(); + let is_doc = self.peek_at(1) == '#'; + self.bump(); + if is_doc { + self.bump(); + } + if !self.at_eof() && self.current() == ' ' { + self.bump(); + } + let text_start = self.pos; + while !self.at_eof() && self.current() != '\n' { + self.bump(); + } + let text = self.inner[text_start..self.pos].to_string(); + if is_doc { + let line_end = self.abs(); + docs.push(text); + docs_span = Some(match docs_span { + Some(prev) => Span::new(prev.start, line_end), + None => Span::new(line_start, line_end), + }); + } + continue; + } + break; + } + (docs, docs_span) + } + + fn parse_args(&mut self, out: &mut Vec) { + loop { + let (docs, docs_span) = self.skip_separators(); + if self.at_eof() { + if !docs.is_empty() { + // Doc comments with nothing to attach to. Warn so + // the user knows they're unused. + self.errors.push(ParseError { + message: "doc comment with no following argument" + .into(), + span: Span::new(self.abs(), self.abs()), + }); + } + break; + } + let item_start = self.abs(); + let mut attributes = Vec::new(); + // Attributes can be interleaved with whitespace and + // doc comments themselves can't appear between attrs, + // only at the head — that's the convention from the + // project plan ("doc comments first, then attributes in + // any order, then the argument name"). + while !self.at_eof() && self.current() == '@' { + if let Some(attr) = self.parse_attribute() { + attributes.push(attr); + } + self.skip_horizontal_ws(); + // Allow newlines between attributes. + while !self.at_eof() && self.current() == '\n' { + self.bump(); + self.skip_horizontal_ws(); + } + } + // Identifier. + self.skip_horizontal_ws(); + if self.at_eof() { + self.errors.push(ParseError { + message: "expected argument name".into(), + span: Span::new(item_start, self.abs()), + }); + break; + } + let name_start = self.abs(); + let name = self.consume_ident(); + if name.is_empty() { + let c = self.current(); + self.errors.push(ParseError { + message: format!("expected argument name, found {c}"), + span: Span::new(self.abs(), self.abs() + 1), + }); + // Resync: drop whatever non-whitespace junk is here + // so we can try the next item. + while !self.at_eof() && !self.current().is_whitespace() { + self.bump(); + } + continue; + } + let name_span = Span::new(name_start, self.abs()); + // Optional `: TYPE` annotation. Tcl strings allow `:` + // in bare words, but we're in the structured-args + // sub-grammar — distinct rules apply here. A `:` + // immediately after the arg name (with optional + // horizontal whitespace) opens the annotation slot. + self.skip_horizontal_ws(); + let type_annotation = if !self.at_eof() && self.current() == ':' { + self.bump(); // ':' + self.skip_horizontal_ws(); + let ty_start = self.abs(); + // Consume up to whitespace or end of arg. The type + // mini-parser handles its own internal grammar + // (idents, '<', ',', '>'); we just need to slice + // the right text out of the source. + while !self.at_eof() { + let c = self.current(); + if c.is_whitespace() || c == '#' { + break; + } + self.bump(); + } + let ty_end = self.abs(); + let text = &self.inner[(ty_start - self.base) as usize + ..(ty_end - self.base) as usize]; + match crate::type_parse::parse(text, ty_start) { + Ok(ty) => Some(ty), + Err(e) => { + self.errors.push(ParseError { + message: e.message, + span: e.span, + }); + None + } + } + } else { + None + }; + let span = Span::new(item_start, self.abs()); + out.push(ProcArg { + name, + name_span, + doc_comments: docs, + doc_comments_span: docs_span, + attributes, + type_annotation, + span, + }); + } + } + + fn parse_attribute(&mut self) -> Option { + let start = self.abs(); + self.bump(); // '@' + let name_start = self.abs(); + let name = self.consume_ident(); + if name.is_empty() { + self.errors.push(ParseError { + message: "expected attribute name after @".into(), + span: Span::new(start, self.abs()), + }); + return None; + } + let name_span = Span::new(name_start, self.abs()); + let mut values = Vec::new(); + if !self.at_eof() && self.current() == '(' { + self.bump(); + loop { + self.skip_horizontal_ws(); + // Allow newlines inside the value list + while !self.at_eof() && self.current() == '\n' { + self.bump(); + self.skip_horizontal_ws(); + } + if self.at_eof() { + self.errors.push(ParseError { + message: "unterminated attribute argument list".into(), + span: Span::new(start, self.abs()), + }); + break; + } + if self.current() == ')' { + self.bump(); + break; + } + match self.parse_value() { + Some(v) => values.push(v), + None => { + // Drop characters up to `,` or `)` to resync. + while !self.at_eof() { + let c = self.current(); + if c == ',' || c == ')' || c == '\n' { + break; + } + self.bump(); + } + } + } + self.skip_horizontal_ws(); + while !self.at_eof() && self.current() == '\n' { + self.bump(); + self.skip_horizontal_ws(); + } + if self.at_eof() { + continue; + } + if self.current() == ',' { + self.bump(); + } + } + } + Some(Attribute { + name, + name_span, + values, + span: Span::new(start, self.abs()), + }) + } + + fn parse_value(&mut self) -> Option { + let start = self.abs(); + let c = self.current(); + if c == '"' { + self.bump(); + let text_start = self.pos; + let mut buf = String::new(); + while !self.at_eof() && self.current() != '"' { + if self.current() == '\\' { + self.bump(); + if !self.at_eof() { + buf.push(self.current()); + self.bump(); + } + } else { + buf.push(self.current()); + self.bump(); + } + } + let _ = text_start; + if self.at_eof() { + self.errors.push(ParseError { + message: "unterminated string".into(), + span: Span::new(start, self.abs()), + }); + } else { + self.bump(); // closing " + } + Some(AttributeValue::String { + value: buf, + span: Span::new(start, self.abs()), + }) + } else if c == '-' || c.is_ascii_digit() { + let mut buf = String::new(); + if c == '-' { + buf.push('-'); + self.bump(); + } + while !self.at_eof() && self.current().is_ascii_digit() { + buf.push(self.current()); + self.bump(); + } + match buf.parse::() { + Ok(value) => Some(AttributeValue::Integer { + value, + span: Span::new(start, self.abs()), + }), + Err(_) => { + self.errors.push(ParseError { + message: format!("invalid integer: {buf}"), + span: Span::new(start, self.abs()), + }); + None + } + } + } else if is_ident_start(c) { + let value = self.consume_ident(); + Some(AttributeValue::Ident { + value, + span: Span::new(start, self.abs()), + }) + } else { + self.errors.push(ParseError { + message: format!("expected attribute value, found {c}"), + span: Span::new(start, self.abs() + 1), + }); + None + } + } + + fn consume_ident(&mut self) -> String { + let mut out = String::new(); + let mut first = true; + while !self.at_eof() { + let c = self.current(); + let ok = if first { + is_ident_start(c) + } else { + is_ident_continue(c) + }; + if !ok { + break; + } + out.push(c); + self.bump(); + first = false; + } + out + } +} + +fn is_ident_start(c: char) -> bool { + c.is_alphabetic() || c == '_' +} + +fn is_ident_continue(c: char) -> bool { + c.is_alphanumeric() || c == '_' +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(input: &str) -> (ProcSignature, Vec) { + // Pretend the inner args text starts at offset 0 in a virtual + // source identical to `input`. + let span = Span::new(0, input.len() as u32); + parse_proc_args(input, span) + } + + #[test] + fn empty_signature() { + let (sig, errs) = parse(""); + assert!(errs.is_empty()); + assert!(sig.args.is_empty()); + } + + #[test] + fn plain_arg_names() { + let (sig, errs) = parse("a b c"); + assert!(errs.is_empty(), "{:?}", errs); + let names: Vec<&str> = + sig.args.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(names, vec!["a", "b", "c"]); + } + + #[test] + fn arg_with_named_type_annotation() { + let (sig, errs) = parse("object: bd_cell"); + assert!(errs.is_empty(), "{:?}", errs); + assert_eq!(sig.args.len(), 1); + let a = &sig.args[0]; + assert_eq!(a.name, "object"); + let ty = a.type_annotation.as_ref().expect("type set"); + assert_eq!(ty.name(), "bd_cell"); + } + + #[test] + fn arg_with_generic_type_annotation() { + let (sig, errs) = parse("cells: list"); + assert!(errs.is_empty(), "{:?}", errs); + let a = &sig.args[0]; + let crate::ast::TypeExpr::Generic { name, args, .. } = + a.type_annotation.as_ref().unwrap() + else { + panic!() + }; + assert_eq!(name, "list"); + assert_eq!(args[0].name(), "bd_cell"); + } + + #[test] + fn typed_and_untyped_args_mix() { + let (sig, errs) = parse("a b: string c"); + assert!(errs.is_empty(), "{:?}", errs); + assert_eq!(sig.args.len(), 3); + assert!(sig.args[0].type_annotation.is_none()); + assert_eq!( + sig.args[1].type_annotation.as_ref().unwrap().name(), + "string" + ); + assert!(sig.args[2].type_annotation.is_none()); + } + + #[test] + fn arg_with_attrs_and_type() { + let (sig, errs) = parse("@default(0) count: int"); + assert!(errs.is_empty(), "{:?}", errs); + let a = &sig.args[0]; + assert_eq!(a.name, "count"); + assert_eq!(a.attributes.len(), 1); + assert_eq!(a.attributes[0].name, "default"); + assert_eq!(a.type_annotation.as_ref().unwrap().name(), "int"); + } + + #[test] + fn arg_with_invalid_type_emits_diagnostic() { + let (_sig, errs) = parse("v: ` call sites into +//! `puts [::repr -v ]` (typed) or `puts ` +//! (untyped fallback). +//! +//! `putr` is a compile-time-only shim — by the time source reaches +//! Tcl, every occurrence has been rewritten in place. This lets one +//! syntactic form cover the ergonomic case (`putr $cpm5_cfg` at the +//! REPL prompt to dump a typed value's `repr` output) without +//! needing a runtime `putr` proc, and without depending on Tcl-level +//! shape detection. +//! +//! ## Walker +//! +//! The rewrite pass walks the FULL parsed AST — top-level +//! statements, proc bodies (populated by +//! [`crate::parser::populate_procs`]), namespace-eval bodies, and +//! command-substitution interiors. It mirrors the scope discipline +//! in `validate::validate_stmts`: +//! +//! - Proc bodies push a fresh [`VarTypeTable`] frame seeded with +//! the proc's typed parameters (from `ProcArg.type_annotation`). +//! - Namespace-eval bodies and command-substitution bodies share +//! the enclosing frame (mirrors Tcl semantics — `namespace eval` +//! creates a namespace but not a fresh local-variable scope, and +//! `[…]` runs in the caller's frame). +//! - `set VAR ` records the value's inferred type in the +//! current frame so downstream `putr $VAR` sees it. +//! +//! ## Rewriting +//! +//! Rewrites are exposed as a `HashMap` keyed by the +//! putr command's source span. `crate::lower::lower_command` +//! consults the map at emit time — when the current command's +//! span matches a key, it emits the replacement Tcl instead of +//! lowering the original. This avoids mutating the source +//! string, which would shift byte offsets and break +//! `LoadedProgram::locate_span` for anything after the rewrite +//! site. +//! +//! ## Fallbacks +//! +//! - `putr $x` where `x`'s type is unknown → `puts $x` (plain +//! fallback; no worse than what the user would type directly). +//! - `putr` with zero args or more than one → left untouched. The +//! analyzer's builtin recognition still accepts them; if the +//! caller wanted something else the diagnostic layer flags it +//! through the normal path. +//! - `putr ` → falls to the untyped path. Literal +//! strings don't carry `T::repr` targets; `puts "hello"` is what +//! the user gets and what they probably want. + +use std::collections::HashMap; + +use crate::ast::{ + Command, CommandKind, Document, Proc, ProcSignature, Stmt, WordPart, +}; +use crate::span::Span; +use crate::validate::{ + build_proc_table, build_signature_table, value_type_with_procs, + VarTypeTable, +}; + +/// A map from putr command span → replacement Tcl. `crate::lower` +/// consults this at emit time so the lowered proc body and the +/// lowered top-level statements both pick up the rewrite. Empty +/// when the input contained no `putr` calls; safe to build and +/// pass into lowering unconditionally. +pub type RewriteMap = HashMap; + +/// Build the rewrite map for every `putr ` command in +/// `document`, dispatching through the argument's type's `repr` +/// proc when the type is statically knowable and falling back to +/// plain `puts` when it isn't. Equivalent to +/// [`rewrite_with_extras`] with an empty extras map — most +/// callers that don't have prior-batch state use this. +pub fn rewrite(source: &str, document: &Document) -> RewriteMap { + rewrite_with_extras(source, document, &HashMap::new(), &HashMap::new()) +} + +/// Same as [`rewrite`] but accepts prior-batch context so the +/// REPL can see variable types that came from earlier commits. +/// +/// - `extra_sigs`: signatures from prior batches, merged into the +/// local signature table so `putr [prior_batch_proc]` resolves. +/// - `extra_var_types`: prior-batch top-level variable bindings +/// (from `Session::top_level_var_types()`). Seeded into the +/// walker's initial `VarTypeTable` frame so `putr $prior_var` +/// dispatches through the right `T::repr`. Later `set` bindings +/// in the current document shadow. +/// +/// `source` must be the same source `document` was parsed from — +/// the walker uses AST spans as byte offsets into it. +pub fn rewrite_with_extras( + source: &str, + document: &Document, + extra_sigs: &HashMap, + extra_var_types: &HashMap, +) -> RewriteMap { + let mut sig_diags = Vec::new(); + let mut table = build_signature_table(document, &mut sig_diags); + // Prior-batch signatures fill in the gaps; the current + // document's entries win (entry().or_insert is a no-op on + // present keys). + for (name, sig) in extra_sigs { + table.entry(name.clone()).or_insert(*sig); + } + // Proc table for return-type inference on unannotated procs + // — lets `putr [some_proc]` and `set x [some_proc]; putr $x` + // both dispatch through the right `T::repr` even when + // `some_proc` has no `-> T` annotation. Built from the + // current document only; prior-batch procs aren't inferrable + // here (their bodies aren't in `document`), but their + // annotated returns already flow through `extra_sigs`. + let proc_table = build_proc_table(document); + let mut rewrites: RewriteMap = HashMap::new(); + let mut top_var_table: VarTypeTable = extra_var_types.clone(); + walk_stmts( + source, + &document.stmts, + &table, + &proc_table, + &mut top_var_table, + &mut rewrites, + ); + rewrites +} + +fn walk_stmts( + source: &str, + stmts: &[Stmt], + sig_table: &HashMap, + proc_table: &HashMap, + var_table: &mut VarTypeTable, + rewrites: &mut RewriteMap, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // `set VAR ` binding — seed the var-type table + // before recursing so downstream `putr $VAR` sees the + // type. Same shape as validate.rs's set-binding hook. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type_with_procs( + value_word, + sig_table, + var_table, + Some(proc_table), + ) { + var_table.insert(name.to_string(), ty); + } + } + } + } + // `putr ` — the actual work. + if let Some(replacement) = + try_rewrite_putr(source, cmd, sig_table, proc_table, var_table) + { + rewrites.insert(cmd.span, replacement); + } + // Recurse into structured commands. + match &cmd.kind { + CommandKind::Proc(proc) => { + // Fresh scope per proc body, seeded with typed + // parameters. Same pattern as + // `validate::validate_stmts`'s proc handling. + let mut proc_scope = VarTypeTable::new(); + if let Some(sig) = &proc.signature { + for a in &sig.args { + if let Some(ty) = &a.type_annotation { + proc_scope.insert(a.name.clone(), ty.clone()); + } + } + } + walk_stmts( + source, + &proc.body, + sig_table, + proc_table, + &mut proc_scope, + rewrites, + ); + } + CommandKind::NamespaceEval(ns) => { + walk_stmts( + source, &ns.body, sig_table, proc_table, var_table, + rewrites, + ); + } + _ => {} + } + // Descend into `[ … ]` command substitution bodies on any + // word — matches how validate.rs walks these. `putr` + // buried inside a `[…]` still gets rewritten. + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + walk_stmts( + source, body, sig_table, proc_table, var_table, + rewrites, + ); + } + } + // Descend into control-flow braced bodies (foreach, + // dict for, for, while, if, catch — populated by + // `parser::populate_control_flow_bodies`). Without + // this, `putr` inside a `foreach { … }` body would + // reach Tcl as a literal command call. + if let Some(body) = &word.body { + walk_stmts( + source, body, sig_table, proc_table, var_table, rewrites, + ); + } + } + } +} + +/// If `cmd` is a `putr ` call, return the replacement Tcl +/// source. `None` for non-putr commands and for `putr` with the +/// wrong arg count (which we leave untouched — the analyzer will +/// flag the arity issue through its normal path). +fn try_rewrite_putr( + source: &str, + cmd: &Command, + sig_table: &HashMap, + proc_table: &HashMap, + var_table: &VarTypeTable, +) -> Option { + let head = cmd.words.first()?; + if head.as_text() != Some("putr") { + return None; + } + // Exactly one argument. `putr` matches `puts`'s single-value + // shape; multi-word invocations get left as-is. + if cmd.words.len() != 2 { + return None; + } + let arg = &cmd.words[1]; + // `value_type_with_procs` covers `$var`, `[proc-call]` (with + // return-type inference for unannotated procs via proc_table), + // and bare `true`/`false`. Everything else returns None and + // lands in the plain-puts fallback. + let inferred = + value_type_with_procs(arg, sig_table, var_table, Some(proc_table)); + let arg_source = arg.span.slice(source); + Some(match inferred { + Some(ty) => { + let dispatch = crate::repr::dispatch_name(&ty); + format!("puts [{dispatch} -v {arg_source}]") + } + None => { + // Fallback: `puts `. Same behavior the + // caller would get from typing `puts $x` directly — + // no worse than that, and consistent with what happens + // for `putr `. + format!("puts {arg_source}") + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + /// Test helper: apply the rewrite map to the source top-down + /// so tests can assert on the fully-substituted result. The + /// real emit path doesn't do this — it consults the map + /// per-command during lowering — but for unit tests it's the + /// clearest way to see what came out. + fn rewrite_str(input: &str) -> String { + let parsed = parse(input); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors, + ); + let map = rewrite(input, &parsed.document); + // Apply in reverse span order to preserve earlier byte + // offsets. + let mut entries: Vec<_> = map.into_iter().collect(); + entries.sort_by_key(|(s, _)| std::cmp::Reverse(s.start)); + let mut out = input.to_string(); + for (span, replacement) in entries { + out.replace_range( + span.start as usize..span.end as usize, + &replacement, + ); + } + out + } + + #[test] + fn typed_var_dispatches_through_repr() { + // `configure_x` is annotated → return type flows to `x` + // via `set x [configure_x]` → `putr $x` sees x's type. + let src = "\ +proc configure_x {} MyType { return foo } +set x [configure_x] +putr $x +"; + let out = rewrite_str(src); + // MyType::repr with the -v envelope. + assert!( + out.contains("puts [MyType::repr -v $x]"), + "expected MyType::repr dispatch, got:\n{out}", + ); + // Original putr line is gone. + assert!(!out.contains("putr $x"), "putr $x still present:\n{out}"); + } + + #[test] + fn untyped_var_falls_back_to_plain_puts() { + // No proc annotation → var type unknown → plain `puts $x`. + let src = "\ +proc make_something {} { return foo } +set x [make_something] +putr $x +"; + let out = rewrite_str(src); + assert!(out.contains("puts $x"), "expected plain puts, got:\n{out}",); + // The rewrite should NOT have introduced a repr dispatch. + assert!( + !out.contains("::repr -v $x"), + "unexpected repr dispatch for untyped var:\n{out}", + ); + } + + #[test] + fn inline_proc_call_uses_return_type() { + // `putr [make]` — no intermediate binding; the arg is a + // direct `[proc-call]`, `value_type` sees the return type. + let src = "\ +proc make {} MyType { return foo } +putr [make] +"; + let out = rewrite_str(src); + assert!( + out.contains("puts [MyType::repr -v [make]]"), + "expected inline dispatch, got:\n{out}", + ); + } + + #[test] + fn inside_proc_body_uses_arg_type() { + // Proc parameter with a type annotation → visible to the + // walker's per-proc scope frame. + let src = "\ +proc show { v: MyType } { + putr $v +} +"; + let out = rewrite_str(src); + assert!( + out.contains("puts [MyType::repr -v $v]"), + "expected in-body dispatch, got:\n{out}", + ); + } + + #[test] + fn wrong_arity_left_alone() { + // `putr` with zero args — outside the rewrite's target + // shape, we leave the source untouched. Analyzer will + // handle the arity complaint through its normal path. + let src = "putr\n"; + let out = rewrite_str(src); + assert_eq!(out, src); + } + + #[test] + fn unannotated_proc_return_type_inferred_from_body() { + // `wrapper` has no return-type annotation, but its body + // ends with `return $inner` where `inner` was set from a + // typed proc. The rewrite should still resolve + // `wrapper`'s return type via body inference, then flow + // it into `$x`, then dispatch `putr $x` through the + // right repr — the specific pattern that made + // `putr $_gtm` fall to plain puts before this fix. + let src = "\ +proc typed_ctor {} MyType { return foo } +proc wrapper {} { + set inner [typed_ctor] + return $inner +} +set x [wrapper] +putr $x +"; + let out = rewrite_str(src); + assert!( + out.contains("puts [MyType::repr -v $x]"), + "expected inferred MyType dispatch, got:\n{out}", + ); + } +} diff --git a/vw-htcl/src/references.rs b/vw-htcl/src/references.rs new file mode 100644 index 0000000..a005bf2 --- /dev/null +++ b/vw-htcl/src/references.rs @@ -0,0 +1,1324 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Symbol-references core. +//! +//! The vw-htcl side of `textDocument/references` and +//! `textDocument/rename`. Splits the problem into two orthogonal +//! pieces: +//! +//! 1. **[`identify_at`]** — given a cursor `offset`, figure out what +//! the user is pointing at. Returns a [`ReferenceTarget`] that +//! names the symbol in a way independent of the source file it +//! was found in. +//! 2. **[`find_references_in`]** — given a target, walk a document +//! and return every source span that is a use or a decl of that +//! target. +//! +//! Both procs and types cross file boundaries in real workspaces, +//! so the LSP layer typically calls (1) on the file the cursor is +//! in and (2) on every `.htcl` file under the workspace root. The +//! two functions never talk to each other — the target flows in as +//! a plain value. +//! +//! Locals and proc args have file-local scope by construction, so +//! `find_references_in` returns the empty set for them when passed +//! a document that doesn't contain the declaration. The LSP layer +//! uses this to skip the cross-file scan for local kinds. + +use crate::ast::{ + AttributeValue, Command, CommandKind, Document, Proc, ProcSignature, Stmt, + TypeExpr, Word, WordForm, WordPart, +}; +use crate::hover::is_body_host; +use crate::scope::{resolve_var_def, scan_var_ref, VarDef}; +use crate::span::Span; + +/// A symbol whose references we want to find. Kinds carry the +/// identifying data needed to match uses across files (procs and +/// types by qualified name) or to bound the scope to a single +/// declaration (locals and proc args). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ReferenceTarget { + /// A proc named exactly `name` — either bare (`configure_gtm`) or + /// namespaced (`vivado_cmd::create_bd_cell`). Matching is + /// literal on the head-word text of every command, plus the + /// name-word of every `proc` decl. Cross-file. + Proc { name: String }, + /// A `type NAME = …` declaration and every `: NAME` / + /// `Generic` reference. `name` is the declared (possibly + /// qualified) form. Cross-file. + Type { name: String }, + /// A single variant of an enum. Referred to as + /// `::` in qualified type annotations and as + /// `::` at construction sites. Cross-file. + EnumVariant { enum_name: String, variant: String }, + /// A `set VAR …` / `variable VAR` / `foreach VAR …` / `upvar + /// … LOCAL` inside a specific scope. File-local by + /// construction. `decl_scope_span` is the span that bounds the + /// scope (a proc body's span, or the whole document for + /// top-level locals) so that scans in OTHER files return the + /// empty set instead of accidentally matching a same-name + /// local elsewhere. + Local { name: String, decl_scope_span: Span }, + /// A proc-arg parameter. Emitted when the cursor is on the arg + /// itself, on a body-level `$name` reference, or on an + /// attribute-ident value that names the arg. File-local by + /// the same reasoning as [`Local`]. + ProcArg { + proc_name: Option, + arg_name: String, + /// Span of the enclosing proc body, used to bound the ref + /// scan the same way `Local` does. + decl_scope_span: Span, + }, +} + +/// If `offset` lands on something we can identify as a reference +/// target, return it. Order of tries is narrowest-first so the +/// less-specific fallbacks don't misclassify. +/// +/// Returns `None` when the cursor isn't on an identifier we know +/// how to track (whitespace, comment interior, keyword, arbitrary +/// argument text, etc.). +pub fn identify_at( + document: &Document, + source: &str, + offset: u32, +) -> Option { + // 1. Proc-arg identification. Narrowest by construction — + // only fires when the cursor is on an arg-name-shaped word + // inside a proc signature, an attr ident referring to a + // sibling arg, or a `$name` in a body resolving to an arg. + if let Some((proc, arg_name)) = + find_proc_arg_at(&document.stmts, source, offset) + { + return Some(ReferenceTarget::ProcArg { + proc_name: proc.name.clone(), + arg_name, + decl_scope_span: proc.body_span, + }); + } + // 2. Enum-variant reference (`Enum::Variant` in a type + // annotation, or `Enum::Variant` in a construction call + // like `Enum::Variant -payload $x`). Narrower than plain + // Proc because a Qualified type annotation is only + // parsed that way when the AST tagged it as such. + if let Some(t) = identify_enum_variant_at(document, offset) { + return Some(t); + } + // 3. Type reference or type decl. Fires on `type NAME = …` + // name spans and on any Named/Generic type-expression + // name-span in a decl or annotation. + if let Some(t) = identify_type_at(document, offset) { + return Some(t); + } + // 4. Proc reference or proc decl. Fires on the name span of + // a `proc` decl and on the head word of any Command. + if let Some(t) = identify_proc_at(document, offset) { + return Some(t); + } + // 5. Local variable — `set`/`variable`/`foreach`/`upvar` decl + // spans, and `$name` references in the enclosing scope. + if let Some(t) = identify_local_at(document, source, offset) { + return Some(t); + } + None +} + +/// Walk `document` collecting every span that references +/// `target`. Includes both use sites and (for cross-file kinds) +/// the declaration span itself so the rename pipeline gets a +/// single unified list. +/// +/// For file-local kinds (`Local`, `ProcArg`), returns the empty +/// set when the document doesn't contain the declaration — +/// callers use this to skip the cross-file scan. +pub fn find_references_in( + document: &Document, + source: &str, + target: &ReferenceTarget, +) -> Vec { + let mut out = Vec::new(); + match target { + ReferenceTarget::Proc { name } => { + find_proc_refs(&document.stmts, name, &mut out); + } + ReferenceTarget::Type { name } => { + find_type_refs(&document.stmts, name, &mut out); + } + ReferenceTarget::EnumVariant { enum_name, variant } => { + find_enum_variant_refs( + &document.stmts, + enum_name, + variant, + &mut out, + ); + } + ReferenceTarget::Local { + name, + decl_scope_span, + } => { + // Only scan if the scope is present in THIS document. + // `decl_scope_span` is an absolute-source offset from + // whatever document defined the local; a different + // file would have different byte offsets and this + // check would fail — which is the intent (file-local + // kinds don't leak across files). + if let Some(scope_stmts) = + scope_stmts_by_span(document, *decl_scope_span) + { + collect_local_decl_spans(scope_stmts, name, &mut out); + collect_var_ref_spans(scope_stmts, source, name, &mut out); + } + } + ReferenceTarget::ProcArg { + arg_name, + decl_scope_span, + .. + } => { + if let Some(proc) = + find_proc_by_body_span(&document.stmts, *decl_scope_span) + { + if let Some(sig) = &proc.signature { + if let Some(arg) = + sig.args.iter().find(|a| &a.name == arg_name) + { + out.push(arg.name_span); + } + for attr_span in attribute_ident_ref_spans(sig, arg_name) { + out.push(attr_span); + } + } + collect_var_ref_spans(&proc.body, source, arg_name, &mut out); + } + } + } + out.sort_by_key(|s| (s.start, s.end)); + out.dedup(); + out +} + +// ─── identify_at helpers ──────────────────────────────────────────── + +fn identify_proc_at( + document: &Document, + offset: u32, +) -> Option { + // Cursor on a `proc NAME { … }` decl? + if let Some(name) = proc_decl_name_at(&document.stmts, offset) { + return Some(ReferenceTarget::Proc { name }); + } + // Cursor on a command's head word (any call)? + if let Some(name) = command_head_name_at(&document.stmts, offset) { + return Some(ReferenceTarget::Proc { name }); + } + None +} + +fn proc_decl_name_at(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(name) = &proc.name { + if proc.name_span.contains(offset) { + return Some(name.clone()); + } + } + if let Some(name) = proc_decl_name_at(&proc.body, offset) { + return Some(name); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(name) = proc_decl_name_at(&ns.body, offset) { + return Some(name); + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + if let Some(name) = proc_decl_name_at(body, offset) { + return Some(name); + } + } + } + } + } + None +} + +fn command_head_name_at(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if let CommandKind::Generic = cmd.kind { + if let Some(head) = cmd.words.first() { + if head.span.contains(offset) { + if let Some(t) = head.as_text() { + return Some(t.to_string()); + } + } + } + } + // Recurse for nested calls / bodies. + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(n) = command_head_name_at(&proc.body, offset) { + return Some(n); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(n) = command_head_name_at(&ns.body, offset) { + return Some(n); + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + if let Some(n) = command_head_name_at(body, offset) { + return Some(n); + } + } + } + } + } + None +} + +fn identify_type_at( + document: &Document, + offset: u32, +) -> Option { + // Cursor on a `type NAME = …` decl name? + if let Some(name) = type_decl_name_at(&document.stmts, offset) { + return Some(ReferenceTarget::Type { name }); + } + // Cursor on a type-expression name (annotation or nested)? + if let Some(name) = type_expr_name_at(&document.stmts, offset) { + return Some(ReferenceTarget::Type { name }); + } + None +} + +fn type_decl_name_at(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::TypeDecl(td) => { + if let Some(name) = &td.name { + if td.name_span.contains(offset) { + return Some(name.clone()); + } + } + } + CommandKind::Proc(proc) => { + if let Some(n) = type_decl_name_at(&proc.body, offset) { + return Some(n); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(n) = type_decl_name_at(&ns.body, offset) { + return Some(n); + } + } + _ => {} + } + } + None +} + +fn type_expr_name_at(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + if let Some(n) = type_expr_name_span_hit(ty, offset) + { + return Some(n); + } + } + } + } + if let Some(ty) = &proc.return_type { + if let Some(n) = type_expr_name_span_hit(ty, offset) { + return Some(n); + } + } + if let Some(n) = type_expr_name_at(&proc.body, offset) { + return Some(n); + } + } + CommandKind::TypeDecl(td) => { + if let Some(ty) = &td.underlying { + if let Some(n) = type_expr_name_span_hit(ty, offset) { + return Some(n); + } + } + } + CommandKind::EnumDecl(ed) => { + for variant in &ed.variants { + if let Some(ty) = &variant.payload { + if let Some(n) = type_expr_name_span_hit(ty, offset) { + return Some(n); + } + } + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(n) = type_expr_name_at(&ns.body, offset) { + return Some(n); + } + } + _ => {} + } + } + None +} + +/// If `offset` lands on the name portion of `ty`, return that +/// name. Recurses through generic args. For `Qualified` we +/// deliberately return `None` — those are handled by the +/// enum-variant identifier, which is more specific. +fn type_expr_name_span_hit(ty: &TypeExpr, offset: u32) -> Option { + match ty { + TypeExpr::Named { name, span } => { + if span.contains(offset) { + Some(name.clone()) + } else { + None + } + } + TypeExpr::Generic { + name, + name_span, + args, + .. + } => { + if name_span.contains(offset) { + return Some(name.clone()); + } + for a in args { + if let Some(n) = type_expr_name_span_hit(a, offset) { + return Some(n); + } + } + None + } + TypeExpr::Qualified { .. } => None, + } +} + +fn identify_enum_variant_at( + document: &Document, + offset: u32, +) -> Option { + // Cursor on an enum-decl variant name? + if let Some(t) = enum_decl_variant_at(&document.stmts, offset) { + return Some(t); + } + // Cursor on a `Qualified{ns, variant}` type annotation in a + // proc arg (overload-arm shape)? + if let Some(t) = qualified_type_variant_at(&document.stmts, offset) { + return Some(t); + } + // Cursor on a construction call `Enum::Variant -payload …` + // — matched purely by name shape (`NAME::NAME`) in a command + // head. Only recognized when there's a matching enum decl + // anywhere in the document. + if let Some(t) = enum_construct_head_at(document, offset) { + return Some(t); + } + None +} + +fn enum_decl_variant_at( + stmts: &[Stmt], + offset: u32, +) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::EnumDecl(ed) => { + let Some(enum_name) = &ed.name else { continue }; + for variant in &ed.variants { + if variant.name_span.contains(offset) { + return Some(ReferenceTarget::EnumVariant { + enum_name: enum_name.clone(), + variant: variant.name.clone(), + }); + } + } + } + CommandKind::Proc(proc) => { + if let Some(t) = enum_decl_variant_at(&proc.body, offset) { + return Some(t); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(t) = enum_decl_variant_at(&ns.body, offset) { + return Some(t); + } + } + _ => {} + } + } + None +} + +fn qualified_type_variant_at( + stmts: &[Stmt], + offset: u32, +) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(TypeExpr::Qualified { + namespace, + variant, + span, + .. + }) = &arg.type_annotation + { + if span.contains(offset) { + return Some(ReferenceTarget::EnumVariant { + enum_name: namespace.clone(), + variant: variant.clone(), + }); + } + } + } + } + if let Some(t) = qualified_type_variant_at(&proc.body, offset) { + return Some(t); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(t) = qualified_type_variant_at(&ns.body, offset) { + return Some(t); + } + } + _ => {} + } + } + None +} + +fn enum_construct_head_at( + document: &Document, + offset: u32, +) -> Option { + let (name, span) = command_head_qualified_at(&document.stmts, offset)?; + // Parse as `Enum::Variant` — first-level namespace only. + let (ns, variant) = split_first_scope(&name)?; + // Only accept if there's a matching enum decl in the doc. + if !enum_decl_exists(&document.stmts, ns) { + return None; + } + // `span` was the whole head-word; the cursor lands on the + // string covered by it, so ownership by variant vs namespace + // is captured by the "any part of the compound" rule the + // caller wanted. Return the variant identity either way. + let _ = span; + Some(ReferenceTarget::EnumVariant { + enum_name: ns.to_string(), + variant: variant.to_string(), + }) +} + +fn command_head_qualified_at( + stmts: &[Stmt], + offset: u32, +) -> Option<(String, Span)> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if let CommandKind::Generic = cmd.kind { + if let Some(head) = cmd.words.first() { + if head.span.contains(offset) { + if let Some(t) = head.as_text() { + if t.contains("::") { + return Some((t.to_string(), head.span)); + } + } + } + } + } + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(x) = command_head_qualified_at(&proc.body, offset) { + return Some(x); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(x) = command_head_qualified_at(&ns.body, offset) { + return Some(x); + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + if let Some(x) = command_head_qualified_at(body, offset) { + return Some(x); + } + } + } + } + } + None +} + +fn split_first_scope(name: &str) -> Option<(&str, &str)> { + let (ns, rest) = name.split_once("::")?; + // Only match single-segment variants: `Enum::Variant`, not + // `Enum::Nested::Something`. + if rest.contains("::") { + return None; + } + Some((ns, rest)) +} + +fn enum_decl_exists(stmts: &[Stmt], name: &str) -> bool { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::EnumDecl(ed) if ed.name.as_deref() == Some(name) => { + return true; + } + CommandKind::Proc(proc) if enum_decl_exists(&proc.body, name) => { + return true; + } + CommandKind::NamespaceEval(ns) + if enum_decl_exists(&ns.body, name) => + { + return true; + } + _ => {} + } + } + false +} + +fn identify_local_at( + document: &Document, + source: &str, + offset: u32, +) -> Option { + let (scope_stmts, scope_span, enclosing) = + innermost_scope(document, offset)?; + // On a decl target? + if let Some(name) = local_decl_name_at(scope_stmts, offset) { + return Some(ReferenceTarget::Local { + name: name.to_string(), + decl_scope_span: scope_span, + }); + } + // On a `$var` that resolves to a local? + let name = var_ref_name_in_scope(scope_stmts, source, offset)?; + match resolve_var_def(&name, scope_stmts, enclosing, offset)? { + VarDef::Local(_) => Some(ReferenceTarget::Local { + name, + decl_scope_span: scope_span, + }), + VarDef::Param(_) => None, + } +} + +// ─── find_references_in helpers ───────────────────────────────────── + +fn find_proc_refs(stmts: &[Stmt], name: &str, out: &mut Vec) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if proc.name.as_deref() == Some(name) { + out.push(proc.name_span); + } + find_proc_refs(&proc.body, name, out); + } + CommandKind::NamespaceEval(ns) => { + find_proc_refs(&ns.body, name, out); + } + CommandKind::Generic => { + if let Some(head) = cmd.words.first() { + if head.as_text() == Some(name) { + out.push(head.span); + } + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + find_proc_refs(body, name, out); + } + } + } + } +} + +fn find_type_refs(stmts: &[Stmt], name: &str, out: &mut Vec) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::TypeDecl(td) => { + if td.name.as_deref() == Some(name) { + out.push(td.name_span); + } + if let Some(ty) = &td.underlying { + collect_type_expr_name_spans(ty, name, out); + } + } + CommandKind::EnumDecl(ed) => { + for variant in &ed.variants { + if let Some(ty) = &variant.payload { + collect_type_expr_name_spans(ty, name, out); + } + } + } + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + collect_type_expr_name_spans(ty, name, out); + } + } + } + if let Some(ty) = &proc.return_type { + collect_type_expr_name_spans(ty, name, out); + } + find_type_refs(&proc.body, name, out); + } + CommandKind::NamespaceEval(ns) => { + find_type_refs(&ns.body, name, out); + } + _ => {} + } + } +} + +fn collect_type_expr_name_spans( + ty: &TypeExpr, + name: &str, + out: &mut Vec, +) { + match ty { + TypeExpr::Named { name: n, span } => { + if n == name { + out.push(*span); + } + } + TypeExpr::Generic { + name: n, + name_span, + args, + .. + } => { + if n == name { + out.push(*name_span); + } + for a in args { + collect_type_expr_name_spans(a, name, out); + } + } + TypeExpr::Qualified { .. } => {} + } +} + +fn find_enum_variant_refs( + stmts: &[Stmt], + enum_name: &str, + variant: &str, + out: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::EnumDecl(ed) + if ed.name.as_deref() == Some(enum_name) => + { + for v in &ed.variants { + if v.name == variant { + out.push(v.name_span); + } + } + } + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(TypeExpr::Qualified { + namespace, + variant: v2, + span, + .. + }) = &arg.type_annotation + { + if namespace == enum_name && v2 == variant { + out.push(*span); + } + } + } + } + find_enum_variant_refs(&proc.body, enum_name, variant, out); + } + CommandKind::NamespaceEval(ns) => { + find_enum_variant_refs(&ns.body, enum_name, variant, out); + } + CommandKind::Generic => { + if let Some(head) = cmd.words.first() { + let expected = format!("{enum_name}::{variant}"); + if head.as_text() == Some(&expected) { + out.push(head.span); + } + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + find_enum_variant_refs(body, enum_name, variant, out); + } + } + } + } +} + +// ─── local + proc-arg helpers, adapted from rename.rs ────────────── + +/// Walk the AST looking for a scope whose enclosing span +/// contains `offset`. Returns the statements list, the scope's +/// bounding span, and the enclosing proc when nested inside one +/// (for arg-name resolution). +fn innermost_scope( + document: &Document, + offset: u32, +) -> Option<(&[Stmt], Span, Option<&Proc>)> { + // Try each proc body's stmts, deepest-first. Fall back to + // the document top level. + fn inner<'a>( + stmts: &'a [Stmt], + _top_span: Span, + offset: u32, + enclosing: Option<&'a Proc>, + ) -> Option<(&'a [Stmt], Span, Option<&'a Proc>)> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) if proc.body_span.contains(offset) => { + return inner( + &proc.body, + proc.body_span, + offset, + Some(proc), + ) + .or(Some(( + &proc.body, + proc.body_span, + Some(proc), + ))); + } + CommandKind::NamespaceEval(ns) + if ns.body_span.contains(offset) => + { + return inner(&ns.body, ns.body_span, offset, enclosing) + .or(Some((&ns.body, ns.body_span, enclosing))); + } + _ => {} + } + } + None + } + let doc_span = Span::new(0, u32::MAX); + inner(&document.stmts, doc_span, offset, None).or(Some(( + &document.stmts, + doc_span, + None, + ))) +} + +/// Return the stmts of the scope whose bounding span equals +/// `scope_span`. `Span::new(0, u32::MAX)` means "the document +/// top level." +fn scope_stmts_by_span( + document: &Document, + scope_span: Span, +) -> Option<&[Stmt]> { + if scope_span.start == 0 && scope_span.end == u32::MAX { + return Some(&document.stmts); + } + scope_stmts_by_span_in(&document.stmts, scope_span) +} + +fn scope_stmts_by_span_in(stmts: &[Stmt], scope_span: Span) -> Option<&[Stmt]> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if proc.body_span == scope_span { + return Some(&proc.body); + } + if let Some(s) = scope_stmts_by_span_in(&proc.body, scope_span) + { + return Some(s); + } + } + CommandKind::NamespaceEval(ns) => { + if ns.body_span == scope_span { + return Some(&ns.body); + } + if let Some(s) = scope_stmts_by_span_in(&ns.body, scope_span) { + return Some(s); + } + } + _ => {} + } + } + None +} + +fn find_proc_by_body_span(stmts: &[Stmt], body_span: Span) -> Option<&Proc> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if proc.body_span == body_span { + return Some(proc); + } + if let Some(p) = find_proc_by_body_span(&proc.body, body_span) { + return Some(p); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(p) = find_proc_by_body_span(&ns.body, body_span) { + return Some(p); + } + } + _ => {} + } + } + None +} + +/// Reuse of [`crate::rename`]'s proc-arg identification without +/// borrowing its private helpers. +fn find_proc_arg_at<'a>( + stmts: &'a [Stmt], + source: &str, + offset: u32, +) -> Option<(&'a Proc, String)> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + if proc.args_span.contains(offset) { + for arg in &sig.args { + if arg.name_span.contains(offset) { + return Some((proc, arg.name.clone())); + } + for a in &arg.attributes { + for v in &a.values { + if let AttributeValue::Ident { + value, + span, + } = v + { + if span.contains(offset) + && sig + .args + .iter() + .any(|x| &x.name == value) + { + return Some((proc, value.clone())); + } + } + } + } + } + } + if proc.body_span.contains(offset) { + if let Some(name) = + var_ref_name_in(&proc.body, source, offset) + { + if sig.args.iter().any(|a| a.name == name) { + return Some((proc, name)); + } + } + } + } + if let Some(x) = find_proc_arg_at(&proc.body, source, offset) { + return Some(x); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(x) = find_proc_arg_at(&ns.body, source, offset) { + return Some(x); + } + } + _ => {} + } + } + None +} + +fn var_ref_name_in( + _stmts: &[Stmt], + source: &str, + offset: u32, +) -> Option { + scan_var_ref(source, offset).map(|(n, _)| n) +} + +fn var_ref_name_in_scope( + _stmts: &[Stmt], + source: &str, + offset: u32, +) -> Option { + scan_var_ref(source, offset).map(|(n, _)| n) +} + +fn local_decl_name_at(stmts: &[Stmt], offset: u32) -> Option<&str> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let words = &cmd.words; + let Some(head) = words.first().and_then(|w| w.as_text()) else { + continue; + }; + // set / variable / foreach / upvar — same rules as rename.rs. + match head { + "set" | "variable" | "foreach" => { + if let Some(w) = words.get(1) { + if w.form == WordForm::Bare && w.span.contains(offset) { + if let Some(t) = w.as_text() { + return Some(t); + } + } + } + } + "upvar" => { + // `upvar [LEVEL] remote local [remote local]…` + // Skip an optional leading level (`#N` or digits), + // then walk pairs — cursor on the LOCAL of any + // pair identifies that local as the target. + let mut idx = 1; + if let Some(w) = words.get(idx) { + if let Some(t) = w.as_text() { + if t.starts_with('#') + || t.chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) + { + idx += 1; + } + } + } + while idx + 1 < words.len() { + let local_word = &words[idx + 1]; + if local_word.span.contains(offset) { + if let Some(t) = local_word.as_text() { + return Some(t); + } + } + idx += 2; + } + } + _ => {} + } + if let CommandKind::Proc(proc) = &cmd.kind { + if let Some(n) = local_decl_name_at(&proc.body, offset) { + return Some(n); + } + } + if is_body_host(head) { + for word in words.iter().skip(1) { + if let WordForm::Braced = word.form { + if word.span.contains(offset) { + // We don't re-parse braced bodies here — + // rename.rs's local pass already covers + // that via its own reparse. Skip. + } + } + } + } + } + None +} + +fn collect_local_decl_spans(stmts: &[Stmt], name: &str, out: &mut Vec) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // Skip nested scopes — locals don't cross. + if matches!( + &cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) { + continue; + } + let Some(head) = cmd.words.first().and_then(|w| w.as_text()) else { + continue; + }; + match head { + "set" | "variable" | "foreach" => { + if let Some(w) = cmd.words.get(1) { + if w.form == WordForm::Bare && w.as_text() == Some(name) { + out.push(w.span); + } + } + } + "upvar" => { + let mut idx = 1; + if let Some(w) = cmd.words.get(idx) { + if let Some(t) = w.as_text() { + if t.starts_with('#') + || t.chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) + { + idx += 1; + } + } + } + while idx + 1 < cmd.words.len() { + let local_word = &cmd.words[idx + 1]; + if local_word.as_text() == Some(name) { + out.push(local_word.span); + } + idx += 2; + } + } + _ => {} + } + // Body-host commands (if/while/foreach/…) — descend into + // their braced bodies. They run in the SAME frame, so a + // `set foo` inside an `if` body is the enclosing scope's + // local, not a separate scope. + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if let Some(inner_stmts) = + crate::unused::reparse_braced_body(word, "") + { + collect_local_decl_spans(&inner_stmts, name, out); + } + } + } + } +} + +fn collect_var_ref_spans( + stmts: &[Stmt], + source: &str, + name: &str, + out: &mut Vec, +) { + walk_var_ref_spans(stmts, source, name, out); +} + +fn walk_var_ref_spans( + stmts: &[Stmt], + source: &str, + name: &str, + out: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // Skip nested scopes. + if matches!( + &cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) { + continue; + } + for word in &cmd.words { + walk_var_ref_spans_in_word(word, source, name, out); + } + // Descend into body-host braced bodies (if/while/foreach/…). + if let Some(head) = cmd.words.first().and_then(|w| w.as_text()) { + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if let Some(inner_stmts) = + crate::unused::reparse_braced_body(word, source) + { + walk_var_ref_spans(&inner_stmts, source, name, out); + } + } + } + } + } +} + +fn walk_var_ref_spans_in_word( + word: &Word, + source: &str, + name: &str, + out: &mut Vec, +) { + for part in &word.parts { + match part { + WordPart::VarRef { name: n, span, .. } => { + if n == name { + // Span covers `$name`; the target is the + // identifier portion (skip the leading `$`). + out.push(Span::new(span.start + 1, span.end)); + } + } + WordPart::CmdSubst { body, .. } => { + walk_var_ref_spans(body, source, name, out); + } + WordPart::Text { .. } | WordPart::Escape { .. } => {} + } + } +} + +fn attribute_ident_ref_spans(sig: &ProcSignature, arg_name: &str) -> Vec { + let mut out = Vec::new(); + for arg in &sig.args { + for a in &arg.attributes { + for v in &a.values { + if let AttributeValue::Ident { value, span } = v { + if value == arg_name { + out.push(*span); + } + } + } + } + } + out +} + +// Sanity: the AST re-exports we need for the pattern matches above. +#[allow(dead_code)] +fn _unused_shape_check(w: &Word, _s: &str, _c: &Command) { + let _ = w.form; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn parsed(src: &str) -> crate::ast::Document { + parse(src).document + } + + fn find_offset(src: &str, needle: &str) -> u32 { + src.find(needle).unwrap() as u32 + } + + #[test] + fn identify_proc_on_decl_name() { + let src = "proc configure_gtm {} { }\n"; + let d = parsed(src); + let offset = find_offset(src, "configure_gtm"); + let target = identify_at(&d, src, offset).expect("identified"); + assert_eq!( + target, + ReferenceTarget::Proc { + name: "configure_gtm".into() + } + ); + } + + #[test] + fn identify_proc_on_call_site() { + let src = "proc configure_gtm {} { }\nconfigure_gtm\n"; + let d = parsed(src); + // Cursor on the CALL of configure_gtm (after the decl). + let offset = src.rfind("configure_gtm").unwrap() as u32; + let target = identify_at(&d, src, offset).expect("identified"); + assert_eq!( + target, + ReferenceTarget::Proc { + name: "configure_gtm".into() + } + ); + } + + #[test] + fn find_proc_refs_covers_decl_and_calls() { + let src = "\ +proc configure_gtm {} { } +configure_gtm +proc other {} { configure_gtm } +"; + let d = parsed(src); + let target = ReferenceTarget::Proc { + name: "configure_gtm".into(), + }; + let refs = find_references_in(&d, src, &target); + // 3 hits: decl name + top-level call + nested call in `other`. + assert_eq!(refs.len(), 3, "spans: {refs:?}"); + } + + #[test] + fn identify_type_on_decl_and_annotation() { + let src = "\ +type MyThing = string +proc use_it {v: MyThing} { } +"; + let d = parsed(src); + // Cursor on the decl name. + let offset_decl = find_offset(src, "MyThing"); + assert_eq!( + identify_at(&d, src, offset_decl), + Some(ReferenceTarget::Type { + name: "MyThing".into() + }) + ); + // Cursor on the annotation. + let offset_ann = src.rfind("MyThing").unwrap() as u32; + assert_eq!( + identify_at(&d, src, offset_ann), + Some(ReferenceTarget::Type { + name: "MyThing".into() + }) + ); + } + + #[test] + fn find_type_refs_covers_decl_and_annotations() { + let src = "\ +type MyThing = string +proc a {v: MyThing} MyThing { } +proc b {v: MyThing} { } +"; + let d = parsed(src); + let target = ReferenceTarget::Type { + name: "MyThing".into(), + }; + let refs = find_references_in(&d, src, &target); + // decl + a's arg-type + a's return-type + b's arg-type = 4. + assert_eq!(refs.len(), 4, "spans: {refs:?}"); + } + + #[test] + fn identify_enum_variant_on_decl_variant() { + let src = "\ +enum Color = { + Red + Green +} +"; + let d = parsed(src); + let offset = find_offset(src, "Red"); + assert_eq!( + identify_at(&d, src, offset), + Some(ReferenceTarget::EnumVariant { + enum_name: "Color".into(), + variant: "Red".into(), + }) + ); + } +} diff --git a/vw-htcl/src/rename.rs b/vw-htcl/src/rename.rs new file mode 100644 index 0000000..f42d423 --- /dev/null +++ b/vw-htcl/src/rename.rs @@ -0,0 +1,441 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Local-scope rename. +//! +//! `textDocument/rename` for identifiers whose scope is confined to +//! the current file. Explicitly in scope: +//! +//! - proc parameters (rename the decl in the signature, every `$name` +//! in the body, and any attribute-ident value that names the arg — +//! e.g. `@requires(name)`) +//! - `set NAME value` locals +//! - `variable NAME` locals +//! - `foreach ITER $list { … }` iterators +//! - `upvar [LEVEL] remote LOCAL` locals +//! +//! Explicitly OUT of scope (returns `None`): +//! +//! - proc names, type names, enum names — renaming these would break +//! call sites we can't see from a single file, so refuse rather +//! than emit an incomplete edit set. +//! - proc-arg **flag references** at call sites (`caller -oldname …`) — +//! same reason: cross-file. The user renaming a proc arg only gets +//! the body-local rename; call sites keep the old flag name until +//! they're touched manually. +//! +//! The cursor is allowed on any of: the decl itself, a `$name` +//! reference to it, or an attribute-ident value referring to a +//! sibling arg. Each maps to the same rename operation. + +use crate::ast::Document; +use crate::span::Span; + +/// A single text-substitution edit to apply. Spans are absolute in +/// the source we were given. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RenameEdit { + pub span: Span, + pub new_text: String, +} + +/// Compute the edits needed to rename the identifier at `offset` to +/// `new_name`. Returns `None` when: +/// +/// - `new_name` isn't a valid Tcl identifier +/// - The cursor isn't on something we know how to rename locally +/// - The cursor is on a construct whose rename would leak beyond the +/// current file (proc names, types, etc.) +/// +/// The returned edits are sorted by span start and deduplicated so +/// clients can apply them as-is. +pub fn rename_at( + document: &Document, + source: &str, + offset: u32, + new_name: &str, +) -> Option> { + if !is_valid_tcl_ident_or_qualified(new_name) { + return None; + } + // Route through the [`crate::references`] core so proc / + // type / enum-variant renames flow through the same + // identify + collect pipeline as `textDocument/references`. + // Locals + proc args are still file-local; the LSP layer + // decides whether to also scan sibling files. + let target = crate::references::identify_at(document, source, offset)?; + let spans = + crate::references::find_references_in(document, source, &target); + if spans.is_empty() { + return None; + } + let replacement = replacement_for(&target, new_name); + let edits = spans + .into_iter() + .map(|span| RenameEdit { + span, + new_text: replacement.clone(), + }) + .collect(); + Some(finalize_edits(edits)) +} + +/// Pick the exact text to substitute at each ref span for a +/// given target. Straightforward for locals / proc args / type +/// names (the new bare name goes in verbatim). Procs preserve +/// their namespace prefix so a rename of a call site like +/// `vivado_cmd::create_bd_cell` targets the LAST segment only +/// when the user typed a bare name. +fn replacement_for( + target: &crate::references::ReferenceTarget, + new_name: &str, +) -> String { + use crate::references::ReferenceTarget; + match target { + ReferenceTarget::Proc { name } + if name.contains("::") && !new_name.contains("::") => + { + // Preserve the namespace prefix from the original. + let ns_prefix = + name.rsplit_once("::").map(|(ns, _)| ns).unwrap_or(""); + format!("{ns_prefix}::{new_name}") + } + ReferenceTarget::EnumVariant { enum_name, .. } => { + // Enum variant refs span the whole `Enum::Variant` + // form in call-head/qualified positions, so we + // preserve the enum prefix. + format!("{enum_name}::{new_name}") + } + _ => new_name.to_string(), + } +} + +/// A rename target's new text is either a plain identifier or a +/// fully qualified path (`ns::name`). Reject anything that +/// doesn't parse as one of those so the substituted text stays +/// valid htcl. +fn is_valid_tcl_ident_or_qualified(s: &str) -> bool { + if s.is_empty() { + return false; + } + for seg in s.split("::") { + if seg.is_empty() { + return false; + } + if !is_valid_tcl_ident(seg) { + return false; + } + } + true +} + +/// Tcl identifiers accept letters, digits, underscore, and `::` +/// (namespace separator). For rename we only allow the first three: +/// renaming across a namespace boundary changes visibility rules, +/// which is outside "local rename" semantics. +fn is_valid_tcl_ident(s: &str) -> bool { + if s.is_empty() { + return false; + } + let mut bytes = s.bytes(); + let first = bytes.next().unwrap(); + if !(first.is_ascii_alphabetic() || first == b'_') { + return false; + } + bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') +} + +fn finalize_edits(mut edits: Vec) -> Vec { + edits.sort_by_key(|e| (e.span.start, e.span.end)); + edits.dedup_by(|a, b| a.span == b.span && a.new_text == b.new_text); + edits +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn at(src: &str, needle: &str, occurrence: usize) -> u32 { + let mut start = 0; + for i in 0..=occurrence { + let pos = src[start..] + .find(needle) + .map(|p| start + p) + .expect("needle not found enough times"); + if i == occurrence { + return pos as u32; + } + start = pos + needle.len(); + } + unreachable!() + } + + /// Apply edits and return the resulting source. + fn apply(src: &str, edits: &[RenameEdit]) -> String { + let mut out = src.to_string(); + for edit in edits.iter().rev() { + let s = edit.span.start as usize; + let e = edit.span.end as usize; + out.replace_range(s..e, &edit.new_text); + } + out + } + + fn edits_of(src: &str, pos: u32, new_name: &str) -> Vec { + let parsed = parse(src); + rename_at(&parsed.document, src, pos, new_name).unwrap_or_default() + } + + #[test] + fn rename_set_local_from_decl_position() { + let src = "\ +proc f {} { + set mode fast + puts $mode + return $mode +} +"; + let pos = at(src, "mode", 0); // the `mode` in `set mode` + let edits = edits_of(src, pos, "kind"); + assert!(!edits.is_empty(), "no edits produced"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set kind fast"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + assert!(renamed.contains("return $kind"), "{renamed}"); + assert!(!renamed.contains("mode"), "{renamed}"); + } + + #[test] + fn rename_set_local_from_var_ref_position() { + let src = "\ +proc f {} { + set mode fast + puts $mode +} +"; + // Cursor on the `m` inside `$mode`. + let pos = at(src, "$mode", 0) + 1; + let edits = edits_of(src, pos, "kind"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set kind fast"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + } + + #[test] + fn rename_proc_arg_from_decl() { + let src = "\ +proc f { + mode +} { + puts $mode + return $mode +} +"; + let pos = at(src, "mode", 0); // arg decl + let edits = edits_of(src, pos, "kind"); + let renamed = apply(src, &edits); + // Arg decl updated + body refs updated. + assert!(renamed.contains(" kind"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + assert!(renamed.contains("return $kind"), "{renamed}"); + } + + #[test] + fn rename_proc_arg_from_body_var_ref() { + let src = "\ +proc f { + mode +} { + puts $mode +} +"; + let pos = at(src, "$mode", 0) + 1; + let edits = edits_of(src, pos, "kind"); + let renamed = apply(src, &edits); + assert!(renamed.contains(" kind"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + } + + #[test] + fn rename_proc_arg_updates_attribute_ident() { + // `@requires(has_a)` in the sig references the sibling arg + // by name; renaming `has_a` should update that reference. + let src = "\ +proc f { + has_a + @requires(has_a) has_b +} { + puts $has_a +} +"; + let pos = at(src, "has_a", 0); // decl of has_a + let edits = edits_of(src, pos, "has_x"); + let renamed = apply(src, &edits); + assert!(renamed.contains(" has_x\n"), "{renamed}"); + assert!(renamed.contains("@requires(has_x)"), "{renamed}"); + assert!(renamed.contains("puts $has_x"), "{renamed}"); + } + + #[test] + fn rename_foreach_iterator() { + let src = "\ +proc f {} { + foreach item [list 1 2 3] { + puts $item + } +} +"; + let pos = at(src, "item", 0); // cursor on iter decl + let edits = edits_of(src, pos, "elem"); + let renamed = apply(src, &edits); + assert!(renamed.contains("foreach elem "), "{renamed}"); + assert!(renamed.contains("puts $elem"), "{renamed}"); + } + + #[test] + fn rename_upvar_local() { + let src = "\ +proc f {} { + upvar 1 remote local + puts $local +} +"; + let pos = at(src, "local", 0); // upvar's local half + let edits = edits_of(src, pos, "here"); + let renamed = apply(src, &edits); + assert!(renamed.contains("upvar 1 remote here"), "{renamed}"); + assert!(renamed.contains("puts $here"), "{renamed}"); + } + + #[test] + fn rename_covers_uses_inside_if_body() { + let src = "\ +proc f {} { + set mode fast + if {$mode == fast} { + puts $mode + } +} +"; + let pos = at(src, "mode", 0); + let edits = edits_of(src, pos, "kind"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set kind fast"), "{renamed}"); + // Both the condition and the body refs get updated via the + // brace-body reparse. + assert!(renamed.contains("if {$kind == fast}"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + } + + #[test] + fn rename_does_not_leak_into_nested_proc_scope() { + // Outer `set foo` and inner `proc g { foo }` share a name + // but are unrelated scopes. Renaming the outer must not + // touch the inner. + let src = "\ +proc outer {} { + set foo 1 + proc g {foo} { + puts $foo + } + puts $foo +} +"; + let pos = at(src, "foo", 0); // outer set decl + let edits = edits_of(src, pos, "bar"); + let renamed = apply(src, &edits); + // Outer decl + outer use renamed. + assert!(renamed.contains("set bar 1"), "{renamed}"); + // Inner proc's arg and its body ref stay `foo`. + assert!(renamed.contains("proc g {foo}"), "{renamed}"); + assert!(renamed.contains("puts $foo\n }"), "{renamed}"); + } + + #[test] + fn refuse_invalid_new_name() { + // Bad syntax for a Tcl identifier still gets refused. The + // qualified-name `ns::var` form is now accepted (proc / + // enum-variant renames need to write it). + let src = "proc f {} { set x 1; puts $x }\n"; + let pos = at(src, "set x", 0) + 4; + let parsed = parse(src); + assert!(rename_at(&parsed.document, src, pos, "").is_none()); + assert!(rename_at(&parsed.document, src, pos, "1foo").is_none()); + assert!(rename_at(&parsed.document, src, pos, "foo-bar").is_none()); + } + + #[test] + fn rename_proc_from_decl_covers_call_sites() { + // Cursor on the proc name at its decl → both the decl and + // every call to `greet` in the same file get rewritten. + let src = "\ +proc greet {} { puts hi } +greet +proc other {} { greet } +"; + let pos = at(src, "greet", 0); + let edits = edits_of(src, pos, "hello"); + let renamed = apply(src, &edits); + assert!(renamed.contains("proc hello {}"), "{renamed}"); + assert_eq!(renamed.matches("hello").count(), 3, "{renamed}"); + } + + #[test] + fn rename_proc_from_call_site() { + // Cursor on a call site → same set of edits as from the + // decl; the identify pass just picks the same target. + let src = "\ +proc greet {} { puts hi } +greet +"; + let pos = at(src, "greet", 1); + let edits = edits_of(src, pos, "hello"); + let renamed = apply(src, &edits); + assert!(renamed.contains("proc hello {}"), "{renamed}"); + assert!(!renamed.contains("greet"), "{renamed}"); + } + + #[test] + fn rename_top_level_set() { + let src = "\ +set root /tmp +puts $root +"; + let pos = at(src, "root", 0); + let edits = edits_of(src, pos, "dir"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set dir /tmp"), "{renamed}"); + assert!(renamed.contains("puts $dir"), "{renamed}"); + } + + #[test] + fn rename_from_cursor_on_dollar_sign() { + // Cursor on the `$` itself, not the letter after. + let src = "\ +proc f {} { + set x 1 + puts $x +} +"; + let pos = at(src, "$x", 0); + let edits = edits_of(src, pos, "y"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set y 1"), "{renamed}"); + assert!(renamed.contains("puts $y"), "{renamed}"); + } + + #[test] + fn is_valid_ident_smoke() { + assert!(is_valid_tcl_ident("foo")); + assert!(is_valid_tcl_ident("_foo")); + assert!(is_valid_tcl_ident("foo_bar")); + assert!(is_valid_tcl_ident("f1")); + assert!(!is_valid_tcl_ident("")); + assert!(!is_valid_tcl_ident("1foo")); + assert!(!is_valid_tcl_ident("foo-bar")); + assert!(!is_valid_tcl_ident("foo bar")); + assert!(!is_valid_tcl_ident("foo::bar")); + } +} diff --git a/vw-htcl/src/repr.rs b/vw-htcl/src/repr.rs new file mode 100644 index 0000000..cf1dbb6 --- /dev/null +++ b/vw-htcl/src/repr.rs @@ -0,0 +1,987 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Compiler-emitted `repr` procs for typed runtime values. +//! +//! The REPL (and other consumers) ask "give me Tcl source that, when +//! applied to a value of type T, produces a display string." That's +//! `dispatch_name(T)`, which always resolves to `::repr` in +//! the running Tcl interpreter: +//! +//! - For **primitives** (`string`, `int`, `bool`, `unit`), `::repr` +//! is shipped once at session start via [`emit_primitive_prelude`]. +//! - For **user-declared newtypes** (`bd_cell`, `widget`, …), `::repr` +//! is the user's own proc — the validator enforces it exists (see +//! [`crate::validate::build_type_decl_table`]). +//! - For **generics** (`list`, `dict`, nested combinations), +//! [`emit_repr`] monomorphizes a per-instantiation `::repr` +//! that delegates to its element / key / value reprs. Each unique +//! nested instantiation gets its own proc. +//! +//! All emission goes through [`vw_quote::quote_tcl!`] so word +//! quoting is handled automatically rather than via `format!` string +//! concatenation. +//! +//! Mangling: dot-free, separator `_`. `dict` → +//! `dict_string_int`; `list>` → +//! `list_dict_string_bd_cell`. The mangled string is used as the +//! namespace of the emitted proc — `dict_string_int::repr`. This +//! corner-collides only when a user declares `type X` whose name +//! happens to equal a mangled compiler-generated namespace (e.g. +//! `type dict_string_int`); pathological in practice. + +use std::collections::{HashMap, HashSet}; + +use vw_quote::quote_tcl; + +use crate::ast::{EnumDecl, EnumVariant, TypeDecl, TypeExpr}; + +/// Output of [`emit_repr`]: the per-type Tcl procs to ship (in +/// dependency order) and the dispatch name to invoke after they're +/// in scope. +#[derive(Clone, Debug)] +pub struct ReprEmission { + /// Tcl proc declarations to ship to the worker before any + /// expression that needs them. Each entry is a complete + /// `proc ::repr { v } { … }` source. + pub procs: Vec, + /// Fully-qualified Tcl proc to invoke: `::repr`. + pub dispatch: String, +} + +/// Mangled namespace name for `ty`. The compiler-emitted repr proc +/// for this type lives at `::repr`. +pub fn mangle(ty: &TypeExpr) -> String { + match ty { + TypeExpr::Named { name, .. } => name.clone(), + TypeExpr::Generic { name, args, .. } => { + let mut out = String::with_capacity(name.len() + args.len() * 8); + out.push_str(name); + for arg in args { + out.push('_'); + out.push_str(&mangle(arg)); + } + out + } + TypeExpr::Qualified { + namespace, variant, .. + } => { + // Qualified names that reached codegen at a value + // position resolve to a namespaced newtype — the + // validator has already verified the name refers to a + // declared `type ns::T = …`. Enum-variant Qualifieds + // never flow here (they're only legal as the dispatch + // first-arg annotation, which mangling doesn't touch). + // Mangle by joining with `::` so the resulting Tcl + // proc name matches the newtype's declared namespace. + format!("{namespace}::{variant}") + } + } +} + +/// Fully-qualified Tcl name of `ty`'s repr proc — what a caller +/// invokes on a value to format it. +pub fn dispatch_name(ty: &TypeExpr) -> String { + format!("{}::repr", mangle(ty)) +} + +/// Fully-qualified Tcl name of `ty`'s `to_raw` proc — the +/// boundary-lowering helper that flattens a typed htcl value +/// down to the bare-Tcl form Vivado consumes through `extern::`. +/// Used by [`emit_to_raw_arm`] and by wrappers that explicitly +/// invoke a type's lowering on a typed arg before forwarding to +/// `extern::`. +pub fn to_raw_dispatch_name(ty: &TypeExpr) -> String { + format!("{}::to_raw", mangle(ty)) +} + +/// Fully-qualified Tcl name of `ty`'s `from_raw` proc — the +/// boundary-lifting helper that wraps a raw extern-returned +/// value into the typed form htcl downstream consumes. +pub fn from_raw_dispatch_name(ty: &TypeExpr) -> String { + format!("{}::from_raw", mangle(ty)) +} + +/// Whether `name` is a primitive type the compiler ships repr for. +/// Anything else is either a user-declared newtype (whose triplet is +/// validated separately) or a generic instantiation (whose repr is +/// emitted by [`emit_repr`]). +pub fn is_primitive(name: &str) -> bool { + matches!(name, "string" | "int" | "bool" | "unit") +} + +/// Emit the primitive prelude — Tcl source for the +/// `string` / `int` / `bool` / `unit` triplets (`repr` + `from` + +/// `to`). Shipped once at session start so every typed expression +/// downstream can rely on the primitives being defined. +/// +/// Each type's procs are wrapped in an explicit `namespace eval` +/// block. `string` is a Tcl built-in command, so the otherwise- +/// implicit `proc string::repr` namespace-creation hits a +/// "unknown namespace" error from the interpreter; wrapping in +/// `namespace eval string {...}` sidesteps that (we're operating +/// on the namespace as a Tcl namespace, not as a command class). +/// The same wrapping is applied uniformly to `int` / `bool` / +/// `unit` for consistency and so a future Tcl that promotes +/// `bool` or `int` to a built-in doesn't silently break us. +/// +/// `from` / `to` for primitives are identity (or coerce to the +/// canonical representation, e.g. `expr {int(...)}` for `int`). +pub fn emit_primitive_prelude() -> Vec { + // Compiler-emitted reprs share the same kwargs envelope as + // user-written newtype reprs (`proc ::repr {v: T} string + // { … }` lowers to `proc repr {args} { ::vw::kwargs $args + // {v ""}; … }`). The dispatch site (see + // `vw-repl::lower::wrap_with_repr`) always calls them with + // `-v ` so the kwargs envelope binds `$v` uniformly. + // Without this uniformity, user-written reprs (which can't + // avoid the kwargs wrap) would error on positional calls. + vec![ + // string: identity at every slot — including to_raw / from_raw, + // since the Tcl runtime representation of a string IS the raw + // value the extern boundary expects. + quote_tcl!( + "namespace eval string {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n\ + }\n" + ), + // int: format / coerce. to_raw / from_raw mirror to / from + // since Vivado consumes integer-shaped strings. + quote_tcl!( + "namespace eval int {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return [format %d $v] }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return [expr {int($v)}] }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return [expr {int($v)}] }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return [expr {int($v)}] }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return [expr {int($v)}] }\n\ + }\n" + ), + // bool: textual form; 0/1 round-trip for from/to. to_raw / + // from_raw use the same 0/1 form Vivado expects. + quote_tcl!( + "namespace eval bool {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? \"true\" : \"false\"}] }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? 1 : 0}] }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? 1 : 0}] }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? 1 : 0}] }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? 1 : 0}] }\n\ + }\n" + ), + // unit: empty value. The App suppresses on the *type*, not + // on the value — these procs exist so generics over `unit` + // still type-check, even though they're unusual. + quote_tcl!( + "namespace eval unit {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n\ + }\n" + ), + // list: newline-joined for repr. Tcl's default string form + // is space-separated with brace-escaping — technically the + // canonical rendering, but unreadable at scale (lists of + // paths are the common case at the REPL). Rendering one + // entry per line is the pragmatic choice. + // + // `from` / `to` stay identity — the boundary with Vivado + // still expects the Tcl-list byte form, only the human + // rendering changes. `to_raw` / `from_raw` also identity + // for the same reason. + quote_tcl!( + "namespace eval list {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return [join $v \"\\n\"] }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n\ + }\n" + ), + // dict: readable, structure-aware rendering. + // + // - Single-value entries (`llength $v == 1`) render inline + // as `key = value`. + // - Multi-value entries render as `key:` followed by one + // indented item per line. + // + // The `llength > 1` heuristic isn't perfect — a bare + // multi-word string is ambiguous with a Tcl list at this + // level — but the concrete case that motivated this + // (JSON-derived dicts of lists via the RPC) is exactly + // the shape where `llength` accurately distinguishes + // structure from prose. `key = 1 2 3` (all-on-one-line) is + // unreadable for paths; `key:\n a\n b\n c` is exactly + // what a human wants to skim. + // + // `from` / `to` / `to_raw` / `from_raw` are identity so + // the extern boundary still round-trips the Tcl byte + // form. Only human display changes. + quote_tcl!( + "namespace eval dict {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; set _out {}; dict for {_k _dv} $v { if {[llength $_dv] > 1} { lappend _out \"$_k:\"; foreach _item $_dv { lappend _out \" $_item\" } } else { lappend _out \"$_k = $_dv\" } }; return [join $_out \"\\n\"] }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n\ + }\n" + ), + // Runtime `::putr` — fallback for the compile-time putr + // rewrite. `vw-htcl/src/putr.rs` catches every TOP-LEVEL + // `putr ` at parse time and dispatches it to the right + // `T::repr` when the type is statically known. It does + // NOT descend into braced-word bodies of control-flow + // builtins (`foreach { … }`, `dict for { … } { … } { … }`, + // `for { … } { … } { … }`, etc.), so a `putr $x` inside + // one of those lands at Tcl as a literal command call. + // Without this runtime proc, Tcl would report `invalid + // command name \"putr\"`. + // + // Best-effort heuristic dispatch at runtime: multi-element + // values render through `list::repr` (one-per-line); + // scalars land through plain `puts`. Callers who want + // dict rendering call `::dict::repr -v $x` directly. + quote_tcl!( + "proc ::putr {v} { if {[llength $v] > 1} { puts [::list::repr -v $v] } else { puts $v } }\n" + ), + ] +} + +/// Walk `ty` depth-first, emitting one `::repr` proc per +/// unique generic instantiation along the way. Plain [`Named`] types +/// (primitives or user newtypes) don't get codegen here — their +/// reprs come from [`emit_primitive_prelude`] or from the user's +/// own `::repr` proc (validator-enforced). +/// +/// The returned dispatch name is `::repr` — the caller +/// invokes it on a value of type `ty` to get the display string. +pub fn emit_repr(ty: &TypeExpr) -> ReprEmission { + emit_repr_with_types(ty, &HashMap::new()) +} + +/// Same as [`emit_repr`] but also walks user-declared newtypes +/// (`type T = U`) — when the dispatch type is a newtype whose +/// underlying is a generic, the generic's repr needs to be in +/// scope so the user's `proc T::repr` body can call it. +/// +/// Without this recursion, `Properties::repr` (which delegates to +/// `dict_string_Property::repr`) errors at runtime with +/// `invalid command name "dict_string_Property::repr"` because +/// the monomorphized generic was never emitted. +pub fn emit_repr_with_types( + ty: &TypeExpr, + types: &HashMap, +) -> ReprEmission { + let mut procs = Vec::new(); + let mut seen = HashSet::new(); + emit_recursive(ty, &mut procs, &mut seen, types); + ReprEmission { + procs, + dispatch: dispatch_name(ty), + } +} + +/// Emit the auto-generated `namespace eval { … }` prelude +/// for an enum declaration. Contains: +/// +/// - **Constructors** — one per variant. Payload variants take a +/// `v` arg and return `[list $v]`; empty-payload +/// variants take no args and return `[list ]`. +/// - **`tag` / `payload`** — explicit unwrap accessors wrappers +/// use to bridge enum values into bare-Tcl `extern::` calls. +/// - **`repr`** — switches on `[lindex $v 0]`, calls each variant +/// payload type's `repr` and wraps as `()` for +/// payload variants, bare `` for empty ones. +/// - **`from` / `to`** — identity (enum values are already in their +/// canonical tagged-tuple form; the triplet exists so generics +/// over enums type-check uniformly with newtypes). +/// +/// The block is wrapped in `namespace eval` — Tcl auto-creates +/// the namespace on `proc ::` ONLY when nothing else +/// claims the name. For defensiveness (and so users can pick +/// enum names that happen to match a Tcl built-in's namespace +/// later without a confusing failure mode), we use the explicit +/// form, mirroring the primitive prelude. +pub fn emit_enum_prelude(enum_decl: &EnumDecl) -> String { + let Some(name) = enum_decl.name.as_deref() else { + // Anonymous enum — shouldn't happen post-parser, but + // bail rather than emit junk. + return String::new(); + }; + let mut body = String::new(); + body.push_str(&format!("namespace eval {name} {{\n")); + // Constructors — plain positional Tcl, called by user code as + // `Property::Scalar foo` (positional). NOT through the kwargs + // envelope. + for v in &enum_decl.variants { + emit_constructor(&mut body, &v.name, v.payload.is_some()); + } + // tag / payload — also positional; called by wrappers in + // `extern::` bridging code as `Property::payload $v`. + body.push_str(" proc tag {v} { return [lindex $v 0] }\n"); + body.push_str(" proc payload {v} { return [lindex $v 1] }\n"); + // repr / from / to — kwargs envelope so they're callable + // uniformly with all other reprs (the dispatch site emits + // `-v ` form universally; see + // `vw-repl::lower::wrap_with_repr`). + body.push_str(" proc repr {args} {\n"); + body.push_str(" ::vw::kwargs $args {v \"\"}\n"); + body.push_str(" switch -- [lindex $v 0] {\n"); + for v in &enum_decl.variants { + emit_repr_arm(&mut body, v); + } + body.push_str(" default { return \"\" }\n"); + body.push_str(" }\n"); + body.push_str(" }\n"); + // from / to are identity for enums (the constructors are the + // user-facing lift). + body.push_str( + " proc from {args} { ::vw::kwargs $args {v \"\"}; return $v }\n", + ); + body.push_str( + " proc to {args} { ::vw::kwargs $args {v \"\"}; return $v }\n", + ); + // to_raw: lower the tagged enum value to its raw extern-side + // representation. Switch on variant tag; for payload variants + // recurse via the payload type's to_raw; for empty variants + // emit the variant name (the convention extern Vivado calls + // recognize for tag-style values). See [docs/htcl-extern-boundary.md] + // for the rationale on this being mechanical / compiler-emitted. + body.push_str(" proc to_raw {args} {\n"); + body.push_str(" ::vw::kwargs $args {v \"\"}\n"); + body.push_str(" switch -- [lindex $v 0] {\n"); + for v in &enum_decl.variants { + emit_to_raw_arm(&mut body, v); + } + body.push_str( + " default { error \"unknown variant: [lindex $v 0]\" }\n", + ); + body.push_str(" }\n"); + body.push_str(" }\n"); + // from_raw: default lift wraps the raw value as the FIRST + // variant. For sum types where the right variant depends on + // the value's shape (e.g. Property — Scalar vs Nested + // chosen by structural inference), users override via + // `proc ::from_raw` AFTER the compiler-emitted prelude; + // Tcl's last-`proc`-wins lets the user override take + // precedence. + if let Some(first) = enum_decl.variants.first() { + emit_from_raw_default(&mut body, first); + } else { + body.push_str( + " proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n", + ); + } + body.push_str("}\n"); + body +} + +/// Emit one arm of `::to_raw`'s `switch -- [lindex $v 0]` +/// body: for payload variants, recurse via the payload type's +/// `to_raw`; for empty variants, emit the variant name as the +/// raw value (matches how extern Vivado callers receive bare +/// enum-style tags). +fn emit_to_raw_arm(out: &mut String, v: &EnumVariant) { + let variant = &v.name; + match &v.payload { + None => { + out.push_str(&format!( + " {variant} {{ return \"{variant}\" }}\n" + )); + } + Some(payload_ty) => { + let dispatch = to_raw_dispatch_name(payload_ty); + out.push_str(&format!( + " {variant} {{ return [{dispatch} -v [lindex $v 1]] }}\n" + )); + } + } +} + +/// Default `::from_raw` body — wrap input as the first +/// variant. For payload variants, the input flows through the +/// payload type's `from_raw` first. For empty variants, the +/// input is ignored and we return the bare-variant constructor. +fn emit_from_raw_default(out: &mut String, first: &EnumVariant) { + let variant = &first.name; + match &first.payload { + None => { + out.push_str(&format!( + " proc from_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + return [list {variant}]\n \ + }}\n", + )); + } + Some(payload_ty) => { + let dispatch = from_raw_dispatch_name(payload_ty); + out.push_str(&format!( + " proc from_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + return [list {variant} [{dispatch} -v $v]]\n \ + }}\n", + )); + } + } +} + +fn emit_constructor(out: &mut String, variant: &str, has_payload: bool) { + if has_payload { + out.push_str(&format!( + " proc {variant} {{v}} {{ return [list {variant} $v] }}\n" + )); + } else { + out.push_str(&format!( + " proc {variant} {{}} {{ return [list {variant}] }}\n" + )); + } +} + +fn emit_repr_arm(out: &mut String, v: &EnumVariant) { + let variant = &v.name; + match &v.payload { + None => { + // Empty-payload: just the bare variant name. + out.push_str(&format!( + " {variant} {{ return \"{variant}\" }}\n" + )); + } + Some(payload_ty) => { + // Payload variant: `()`. Formatting + // depends on whether the inner repr fits on one line: + // + // single-line: `Variant(inner)` + // multi-line: `Variant(\n line1\n line2\n)` + // + // The multi-line shape (opening paren followed by + // newline + 2-space indent for the first child, + // closing paren on its own line, every inner line + // indented one extra level) keeps deeply-nested + // values readable instead of arrowing off the right + // margin. + // + // 2-space indent applies to ALL inner lines + // (including their pre-existing continuation indents), + // so each nesting level adds exactly 2 spaces of + // indent uniformly. + let dispatch = dispatch_name(payload_ty); + out.push_str(&format!( + " {variant} {{\n \ + set __vw_inner [{dispatch} -v [lindex $v 1]]\n \ + if {{[string first \"\\n\" $__vw_inner] >= 0}} {{\n \ + set __vw_indented [string map [list \\n \"\\n \"] $__vw_inner]\n \ + return \"{variant}(\\n $__vw_indented\\n)\"\n \ + }} else {{\n \ + return \"{variant}($__vw_inner)\"\n \ + }}\n \ + }}\n" + )); + } + } +} + +fn emit_recursive( + ty: &TypeExpr, + out: &mut Vec, + seen: &mut HashSet, + types: &HashMap, +) { + match ty { + TypeExpr::Named { name, .. } => { + // No codegen for plain names directly — `::repr` + // is either a primitive (shipped via + // `emit_primitive_prelude`) or a user newtype + // (validator-enforced to exist). BUT if `name` + // resolves to a user newtype whose underlying is a + // generic, we have to recurse so the underlying's + // monomorphized repr is shipped — the user's + // `proc ::repr` body typically delegates to it. + if let Some(decl) = types.get(name.as_str()) { + if let Some(underlying) = decl.underlying.as_ref() { + emit_recursive(underlying, out, seen, types); + } + } + } + TypeExpr::Generic { name, args, .. } => { + // Depth-first: emit each arg's repr first so this + // proc's body can call them. + for a in args { + emit_recursive(a, out, seen, types); + } + let m = mangle(ty); + if !seen.insert(m.clone()) { + return; // Already emitted this instantiation. + } + let body = match name.as_str() { + "dict" if args.len() == 2 => { + emit_dict_repr(&m, &args[0], &args[1]) + } + "list" if args.len() == 1 => emit_list_repr(&m, &args[0]), + _ => emit_unknown_generic_repr(&m), + }; + out.push(body); + } + TypeExpr::Qualified { + namespace, variant, .. + } => { + // Namespaced newtype reference (`dcmac::GtChProps` and + // friends). Look up by the joined qualified name — the + // types table is keyed exactly that way (see + // `validate::build_type_decl_table`). If found and its + // underlying is a generic, recurse so the generic's + // monomorphized repr ships alongside. + let qualified = format!("{namespace}::{variant}"); + if let Some(decl) = types.get(qualified.as_str()) { + if let Some(underlying) = decl.underlying.as_ref() { + emit_recursive(underlying, out, seen, types); + } + } + } + } +} + +/// `dict::repr` — iterate pairs, format each as +/// ` ` joined with newlines. +/// +/// The body uses braced `expr {…}` and avoids interpolating the +/// dispatch names raw via `quote_tcl!` because Tcl's word quoting +/// would brace the `::` separators (those are bare-safe but the +/// macro's `Word::lit` doesn't know that). The proc names go in via +/// raw substitution at template time instead — they're already +/// valid Tcl, and the macro template's literal regions pass through +/// untouched. +fn emit_dict_repr(mangled: &str, k: &TypeExpr, v: &TypeExpr) -> String { + let key_repr = dispatch_name(k); + let val_repr = dispatch_name(v); + let key_to_raw = to_raw_dispatch_name(k); + let val_to_raw = to_raw_dispatch_name(v); + let key_from_raw = from_raw_dispatch_name(k); + let val_from_raw = from_raw_dispatch_name(v); + // Uses the same kwargs envelope as `emit_primitive_prelude` + // so the dispatch site can uniformly call all reprs with + // `-v `. Sub-element reprs are invoked through the + // same `-v` convention. + // + // to_raw / from_raw are emitted in the SAME namespace so + // callers can dispatch via `::{repr,to_raw,from_raw}` + // uniformly. to_raw walks the dict, applying K::to_raw and + // V::to_raw element-wise and rebuilding as a flat paired + // list (the shape Vivado consumes). from_raw is the inverse + // — walks a paired list, applies K::from_raw / V::from_raw + // element-wise, builds a typed dict. + format!( + "namespace eval {ns} {{\n \ + proc repr {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out \"\"\n \ + set first 1\n \ + foreach {{k val}} $v {{\n \ + if {{!$first}} {{ append out \"\\n\" }}\n \ + set first 0\n \ + append out [{kr} -v $k] \" \" [{vr} -v $val]\n \ + }}\n \ + return $out\n \ + }}\n \ + proc to_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out [list]\n \ + foreach {{k val}} $v {{\n \ + # Recurse into the element to_raw calls with a\n \ + # catch that prepends this level's key on error,\n \ + # so a failure deep inside a nested Properties\n \ + # tree bubbles up as a dotted path (e.g.\n \ + # `LR0_SETTINGS.RX_REFCLK_FREQUENCY.`).\n \ + if {{[catch {{\n \ + set __vw_kraw [{ktr} -v $k]\n \ + set __vw_vraw [{vtr} -v $val]\n \ + }} __vw_msg]}} {{\n \ + # Lowercase the key in the error so\n \ + # nested-property failures surface with\n \ + # the HTCL-surface arg name\n \ + # (`tx_refclk_frequency`) rather than\n \ + # the Vivado SCREAM_CASE dict key\n \ + # (`TX_REFCLK_FREQUENCY`). Innocuous for\n \ + # non-Vivado dicts — lowercasing an\n \ + # already-lowercase key is a no-op.\n \ + error \"[string tolower $k].$__vw_msg\"\n \ + }}\n \ + lappend out $__vw_kraw $__vw_vraw\n \ + }}\n \ + return $out\n \ + }}\n \ + proc from_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out [dict create]\n \ + foreach {{k val}} $v {{\n \ + dict set out [{kfr} -v $k] [{vfr} -v $val]\n \ + }}\n \ + return $out\n \ + }}\n\ + }}\n", + ns = mangled, + kr = key_repr, + vr = val_repr, + ktr = key_to_raw, + vtr = val_to_raw, + kfr = key_from_raw, + vfr = val_from_raw, + ) +} + +/// `list::repr` — iterate elements, format each via `T::repr`, +/// join with newlines. Also emits `to_raw` / `from_raw` element- +/// wise dispatching through `T::to_raw` / `T::from_raw`. +fn emit_list_repr(mangled: &str, elem: &TypeExpr) -> String { + let elem_repr = dispatch_name(elem); + let elem_to_raw = to_raw_dispatch_name(elem); + let elem_from_raw = from_raw_dispatch_name(elem); + format!( + "namespace eval {ns} {{\n \ + proc repr {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out \"\"\n \ + set first 1\n \ + foreach item $v {{\n \ + if {{!$first}} {{ append out \"\\n\" }}\n \ + set first 0\n \ + append out [{er} -v $item]\n \ + }}\n \ + return $out\n \ + }}\n \ + proc to_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out [list]\n \ + foreach item $v {{\n \ + lappend out [{etr} -v $item]\n \ + }}\n \ + return $out\n \ + }}\n \ + proc from_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out [list]\n \ + foreach item $v {{\n \ + lappend out [{efr} -v $item]\n \ + }}\n \ + return $out\n \ + }}\n\ + }}\n", + ns = mangled, + er = elem_repr, + etr = elem_to_raw, + efr = elem_from_raw, + ) +} + +/// Fallback for generic shapes we don't have a specialized shell +/// for (e.g. a hypothetical `tuple<…>` we haven't designed yet). +/// Renders the raw Tcl value — at least the user sees *something* +/// instead of an "unknown generic" error. +fn emit_unknown_generic_repr(mangled: &str) -> String { + format!( + "namespace eval {mangled} {{ \ + proc repr {{args}} {{ ::vw::kwargs $args {{v \"\"}}; return $v }} \ + proc to_raw {{args}} {{ ::vw::kwargs $args {{v \"\"}}; return $v }} \ + proc from_raw {{args}} {{ ::vw::kwargs $args {{v \"\"}}; return $v }} \ + }}\n" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::span::Span; + + fn named(name: &str) -> TypeExpr { + TypeExpr::Named { + name: name.into(), + span: Span::new(0, 0), + } + } + + fn generic(name: &str, args: Vec) -> TypeExpr { + TypeExpr::Generic { + name: name.into(), + name_span: Span::new(0, 0), + args, + span: Span::new(0, 0), + } + } + + #[test] + fn mangle_primitives() { + assert_eq!(mangle(&named("string")), "string"); + assert_eq!(mangle(&named("bd_cell")), "bd_cell"); + assert_eq!(mangle(&named("unit")), "unit"); + } + + #[test] + fn mangle_dict_two_args() { + let ty = generic("dict", vec![named("string"), named("int")]); + assert_eq!(mangle(&ty), "dict_string_int"); + } + + #[test] + fn mangle_list_one_arg() { + let ty = generic("list", vec![named("bd_cell")]); + assert_eq!(mangle(&ty), "list_bd_cell"); + } + + #[test] + fn mangle_nested() { + let inner = generic("dict", vec![named("string"), named("bd_cell")]); + let outer = generic("list", vec![inner]); + assert_eq!(mangle(&outer), "list_dict_string_bd_cell"); + } + + #[test] + fn dispatch_for_primitive_uses_name() { + assert_eq!(dispatch_name(&named("string")), "string::repr"); + assert_eq!(dispatch_name(&named("bd_cell")), "bd_cell::repr"); + } + + #[test] + fn dispatch_for_generic_uses_mangled() { + let ty = generic("dict", vec![named("string"), named("string")]); + assert_eq!(dispatch_name(&ty), "dict_string_string::repr"); + } + + #[test] + fn primitive_prelude_emits_one_namespace_block_per_type() { + let procs = emit_primitive_prelude(); + // 6 types (namespace-eval blocks) plus the standalone + // runtime `::putr` fallback. + assert_eq!(procs.len(), 7); + assert!(procs.iter().any(|p| p.contains("namespace eval string"))); + assert!(procs.iter().any(|p| p.contains("namespace eval int"))); + assert!(procs.iter().any(|p| p.contains("namespace eval bool"))); + assert!(procs.iter().any(|p| p.contains("namespace eval unit"))); + assert!(procs.iter().any(|p| p.contains("namespace eval list"))); + assert!(procs.iter().any(|p| p.contains("namespace eval dict"))); + // Standalone runtime `::putr` fallback for braced-body + // scripts where the compile-time rewrite can't fire. + assert!(procs.iter().any(|p| p.contains("proc ::putr"))); + // Every NAMESPACE-BLOCK entry contains the full + // repr/from/to triplet — the standalone `::putr` entry + // (which is a bare proc, not a namespace block) is + // exempted. + for p in &procs { + if !p.contains("namespace eval") { + continue; + } + assert!(p.contains("proc repr"), "missing repr in: {p}"); + assert!(p.contains("proc from"), "missing from in: {p}"); + assert!(p.contains("proc to"), "missing to in: {p}"); + } + } + + #[test] + fn emit_repr_named_emits_no_procs() { + // Primitives and user newtypes don't need codegen — repr + // lives in the primitive prelude or the user's own proc. + let e = emit_repr(&named("string")); + assert!(e.procs.is_empty()); + assert_eq!(e.dispatch, "string::repr"); + + let e = emit_repr(&named("bd_cell")); + assert!(e.procs.is_empty()); + assert_eq!(e.dispatch, "bd_cell::repr"); + } + + #[test] + fn emit_repr_dict_string_string() { + let ty = generic("dict", vec![named("string"), named("string")]); + let e = emit_repr(&ty); + assert_eq!(e.dispatch, "dict_string_string::repr"); + assert_eq!(e.procs.len(), 1); + let body = &e.procs[0]; + // The proc is defined inside its `namespace eval`, so the + // textual proc name is just `repr` — the namespace is in + // the surrounding `namespace eval dict_string_string`. + assert!(body.contains("namespace eval dict_string_string")); + assert!(body.contains("proc repr {args}")); + assert!(body.contains("::vw::kwargs $args")); + assert!(body.contains("foreach {k val} $v")); + // Element reprs called by their fully-qualified name via + // the universal `-v ` kwargs form. + assert!(body.contains("[string::repr -v $k]")); + assert!(body.contains("[string::repr -v $val]")); + } + + #[test] + fn emit_repr_list_bd_cell() { + let ty = generic("list", vec![named("bd_cell")]); + let e = emit_repr(&ty); + assert_eq!(e.dispatch, "list_bd_cell::repr"); + assert_eq!(e.procs.len(), 1); + let body = &e.procs[0]; + assert!(body.contains("namespace eval list_bd_cell")); + assert!(body.contains("proc repr {args}")); + assert!(body.contains("[bd_cell::repr -v $item]")); + } + + #[test] + fn emit_repr_nested_topologically_orders_sub_procs() { + // dict>: emits list::repr first, + // then dict_string_list_int::repr. + let inner = generic("list", vec![named("int")]); + let outer = generic("dict", vec![named("string"), inner]); + let e = emit_repr(&outer); + assert_eq!(e.dispatch, "dict_string_list_int::repr"); + assert_eq!(e.procs.len(), 2); + // First proc emitted is the inner list, second is the + // outer dict. + assert!(e.procs[0].contains("namespace eval list_int")); + assert!(e.procs[1].contains("namespace eval dict_string_list_int")); + // Outer body calls the inner by its fully-qualified name. + assert!(e.procs[1].contains("[list_int::repr")); + } + + #[test] + fn emit_repr_dedups_repeated_subtypes() { + // dict — bd_cell is a leaf (Named), so no + // codegen for it, but if we had dict, list> + // we'd want list_int::repr emitted only ONCE. + let inner = generic("list", vec![named("int")]); + let outer = generic("dict", vec![inner.clone(), inner]); + let e = emit_repr(&outer); + // list_int's namespace block appears once even though it's + // referenced twice in the outer dict. + let list_int_count = e + .procs + .iter() + .filter(|p| p.contains("namespace eval list_int ")) + .count(); + assert_eq!(list_int_count, 1); + } + + #[test] + fn emit_repr_unknown_generic_falls_back_to_identity() { + let ty = generic("tuple", vec![named("string"), named("int")]); + let e = emit_repr(&ty); + assert_eq!(e.procs.len(), 1); + assert!( + e.procs[0].contains("return $v"), + "expected identity body, got {:?}", + e.procs[0] + ); + } + + #[test] + fn is_primitive_table() { + assert!(is_primitive("string")); + assert!(is_primitive("int")); + assert!(is_primitive("bool")); + assert!(is_primitive("unit")); + assert!(!is_primitive("bd_cell")); + assert!(!is_primitive("widget")); + assert!(!is_primitive("dict")); + } + + // --- enum prelude emission -------------------------------------- + + fn ed_with_variants( + name: &str, + vs: Vec<(&str, Option)>, + ) -> EnumDecl { + EnumDecl { + name: Some(name.into()), + name_span: Span::new(0, 0), + variants: vs + .into_iter() + .map(|(n, p)| EnumVariant { + name: n.into(), + name_span: Span::new(0, 0), + payload: p, + payload_span: Span::new(0, 0), + span: Span::new(0, 0), + }) + .collect(), + body_span: Span::new(0, 0), + } + } + + #[test] + fn enum_prelude_with_payload_variants() { + let ed = ed_with_variants( + "Property", + vec![ + ("Scalar", Some(named("string"))), + ( + "Nested", + Some(generic( + "dict", + vec![named("string"), named("string")], + )), + ), + ], + ); + let p = emit_enum_prelude(&ed); + // Wrapped in namespace eval. + assert!(p.contains("namespace eval Property")); + // Constructors with payload. + assert!(p.contains("proc Scalar {v} { return [list Scalar $v] }")); + assert!(p.contains("proc Nested {v} { return [list Nested $v] }")); + // Accessors. + assert!(p.contains("proc tag {v}")); + assert!(p.contains("proc payload {v}")); + // Repr switch — kwargs envelope around the body. + assert!(p.contains("proc repr {args}")); + assert!(p.contains("::vw::kwargs $args")); + assert!(p.contains("switch -- [lindex $v 0]")); + // Each variant's body now uses an intermediate + // `__vw_inner` after applying the continuation-indent + // `string map` transform. + assert!(p.contains("Scalar($__vw_inner)")); + assert!(p.contains("Nested($__vw_inner)")); + // Payload reprs dispatched via mangled names with `-v`. + assert!(p.contains("string::repr -v")); + assert!(p.contains("dict_string_string::repr -v")); + // Identity from/to — also kwargs envelope. + assert!(p.contains("proc from {args}")); + assert!(p.contains("proc to {args}")); + } + + #[test] + fn enum_prelude_with_empty_payload_variants() { + let ed = ed_with_variants( + "Direction", + vec![ + ("North", None), + ("South", None), + ("East", None), + ("West", None), + ], + ); + let p = emit_enum_prelude(&ed); + // Empty-payload constructors take no args. + assert!(p.contains("proc North {} { return [list North] }")); + assert!(p.contains("proc West {} { return [list West] }")); + // Repr arms render bare variant name (no parens). + assert!(p.contains("North { return \"North\" }")); + assert!(p.contains("West { return \"West\" }")); + // No `(` after variant names in the repr arms. + let arm = "North { return \"North("; + assert!(!p.contains(arm), "shouldn't have parens for empty variants"); + } + + #[test] + fn enum_prelude_mixed_payload_and_empty() { + let ed = ed_with_variants( + "Maybe", + vec![("Some", Some(named("int"))), ("None", None)], + ); + let p = emit_enum_prelude(&ed); + assert!(p.contains("proc Some {v} { return [list Some $v] }")); + assert!(p.contains("proc None {} { return [list None] }")); + // Payload arm uses `__vw_inner` after the + // continuation-indent `string map` transform. + assert!(p.contains("int::repr -v")); + assert!(p.contains("Some($__vw_inner)")); + assert!(p.contains("None { return \"None\" }")); + } +} diff --git a/vw-htcl/src/scope.rs b/vw-htcl/src/scope.rs new file mode 100644 index 0000000..80d3343 --- /dev/null +++ b/vw-htcl/src/scope.rs @@ -0,0 +1,390 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Variable scope resolution shared by goto and hover. +//! +//! Tcl variables are local to a proc (its parameters plus whatever it +//! `set`s / `variable`s); top-level code shares the global scope. That +//! lexical model is enough to point a `$name` reference at its +//! definition. +//! +//! Two entry styles: +//! +//! - [`resolve_var_def`] resolves a name given a known scope — used by +//! the structured path when the reference is a real [`WordPart::VarRef`]. +//! - [`scan_var_ref`] + [`innermost_scope`] recover a reference the +//! structured parser left buried in opaque text (a command +//! substitution body, or an `if`/`while` condition), by reading the +//! raw source at the cursor and locating the enclosing proc by span. + +use crate::ast::{ + Command, CommandKind, Document, Proc, ProcArg, Stmt, TypeDecl, TypeExpr, +}; +use crate::span::Span; + +/// What a `$name` reference resolves to. +#[derive(Clone, Copy, Debug)] +pub enum VarDef<'a> { + /// A parameter of the enclosing proc. + Param(&'a ProcArg), + /// A local established by `set name ...` or `variable name ...`. + /// Carries the span of the defined name. + Local(Span), +} + +impl VarDef<'_> { + /// The span to navigate to / anchor hover on. + pub fn def_span(&self) -> Span { + match self { + VarDef::Param(arg) => arg.name_span, + VarDef::Local(span) => *span, + } + } +} + +/// Resolve `name` within `scope_stmts` (the statements of the current +/// scope), falling back to a parameter of `enclosing`. `offset` biases +/// local resolution toward the last definition at or before the +/// reference (the value in effect there). +pub fn resolve_var_def<'a>( + name: &str, + scope_stmts: &'a [Stmt], + enclosing: Option<&'a Proc>, + offset: u32, +) -> Option> { + let mut best: Option = None; + let mut first: Option = None; + for stmt in scope_stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let Some(def) = local_def_target(cmd, name) else { + continue; + }; + first.get_or_insert(def); + if def.start <= offset { + best = Some(def); + } + } + if let Some(span) = best.or(first) { + return Some(VarDef::Local(span)); + } + + let sig = enclosing?.signature.as_ref()?; + sig.args.iter().find(|a| a.name == name).map(VarDef::Param) +} + +/// If `cmd` defines variable `name` — via `set NAME …` / `variable +/// NAME …` / `foreach NAME …` / `foreach {A B …} …` / `dict for {K +/// V} …` / `catch BODY NAME` — return the span to anchor +/// hover/goto on. Braced varname-list positions (`foreach {a b}`, +/// `dict for {k v}`) return the span of the whole braced word; +/// sub-token spans would need extra parser wiring. +fn local_def_target(cmd: &Command, name: &str) -> Option { + match &cmd.kind { + CommandKind::Set => { + let target = cmd.words.get(1)?; + (target.as_text()? == name).then_some(target.span) + } + CommandKind::Generic => { + let head = cmd.words.first()?.as_text()?; + match head { + "variable" => { + let target = cmd.words.get(1)?; + (target.as_text()? == name).then_some(target.span) + } + "foreach" => { + // Iterator target(s) at word 1 (and every + // second word after, up to but not including + // the body). Matches `unused::collect_foreach_decls`. + let body_idx = cmd.words.len().saturating_sub(1); + let mut i = 1; + while i < body_idx { + if word_declares_name(&cmd.words[i], name) { + return Some(cmd.words[i].span); + } + i += 2; + } + None + } + "dict" => { + // `dict for {K V} DICT BODY` — the kv list is at + // word 2, only for the `for` sub-command. + if cmd.words.get(1)?.as_text()? != "for" { + return None; + } + let target = cmd.words.get(2)?; + if word_declares_name(target, name) { + Some(target.span) + } else { + None + } + } + "catch" => { + // `catch BODY [RESVAR [OPTVAR]]` — words 2/3. + for w in cmd.words.iter().skip(2).take(2) { + if word_declares_name(w, name) { + return Some(w.span); + } + } + None + } + _ => None, + } + } + CommandKind::Proc(_) + | CommandKind::Src(_) + | CommandKind::NamespaceEval(_) + | CommandKind::TypeDecl(_) + | CommandKind::EnumDecl(_) => None, + } +} + +/// True when `word` names `target` — either as a bare identifier +/// (`foreach x …`) or as a whitespace-separated token inside a +/// braced list (`foreach {a b} …`, `dict for {k v} …`). +fn word_declares_name(word: &crate::ast::Word, target: &str) -> bool { + use crate::ast::{WordForm, WordPart}; + // Bare word: exact match. + if word.form == WordForm::Bare { + return word.as_text() == Some(target); + } + // Braced list: whitespace-split the interior Text. + if word.form != WordForm::Braced { + return false; + } + let Some(WordPart::Text { value, .. }) = word.parts.first() else { + return false; + }; + value.split_whitespace().any(|tok| tok == target) +} + +/// The innermost proc whose body contains `offset`, together with that +/// body's statements. `(document.stmts, None)` when `offset` is at the +/// top level. +pub fn innermost_scope( + document: &Document, + offset: u32, +) -> (&[Stmt], Option<&Proc>) { + fn helper(stmts: &[Stmt], offset: u32) -> Option<(&[Stmt], &Proc)> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Proc(proc) = &cmd.kind else { + continue; + }; + if proc.body_span.contains(offset) { + return Some( + helper(&proc.body, offset).unwrap_or((&proc.body, proc)), + ); + } + } + None + } + match helper(&document.stmts, offset) { + Some((stmts, proc)) => (stmts, Some(proc)), + None => (&document.stmts, None), + } +} + +/// If the cursor at `offset` sits on a `$name` (or `${name}`) +/// reference — even one the structured parser left inside opaque text +/// (a command substitution, or an expr condition) — return its name +/// and the span of the whole reference. +pub fn scan_var_ref(source: &str, offset: u32) -> Option<(String, Span)> { + let bytes = source.as_bytes(); + let len = bytes.len(); + let off = (offset as usize).min(len); + let is_ident = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + + // If the cursor sits on the `$` itself, step into the name. + let probe = if off < len && bytes[off] == b'$' { + off + 1 + } else { + off + }; + let probe = probe.min(len); + + let mut start = probe; + while start > 0 && is_ident(bytes[start - 1]) { + start -= 1; + } + let mut end = probe; + while end < len && is_ident(bytes[end]) { + end += 1; + } + if end <= start { + return None; + } + + // `$name` + if start > 0 && bytes[start - 1] == b'$' { + let name = source.get(start..end)?.to_string(); + return Some((name, Span::new((start - 1) as u32, end as u32))); + } + // `${name}` + if start >= 2 + && bytes[start - 1] == b'{' + && bytes[start - 2] == b'$' + && end < len + && bytes[end] == b'}' + { + let name = source.get(start..end)?.to_string(); + return Some((name, Span::new((start - 2) as u32, (end + 1) as u32))); + } + None +} + +/// Walk `document` and return the innermost [`TypeExpr`] whose +/// span contains `offset`. Considers proc-signature arg +/// annotations, proc return-type annotations, `type … = TYPE` +/// underlying, and generic type arguments (recursively). Returns +/// `None` when the cursor isn't on a type-expression position. +pub fn type_expr_at(document: &Document, offset: u32) -> Option<&TypeExpr> { + fn inner(ty: &TypeExpr, offset: u32) -> Option<&TypeExpr> { + if !ty.span().contains(offset) { + return None; + } + // Recurse into generic args first so the innermost match wins. + if let TypeExpr::Generic { args, .. } = ty { + for a in args { + if let Some(hit) = inner(a, offset) { + return Some(hit); + } + } + } + Some(ty) + } + fn walk(stmts: &[Stmt], offset: u32) -> Option<&TypeExpr> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = arg.type_annotation.as_ref() { + if let Some(hit) = inner(ty, offset) { + return Some(hit); + } + } + } + if let Some(ret) = sig.return_type.as_ref() { + if let Some(hit) = inner(ret, offset) { + return Some(hit); + } + } + } + if let Some(hit) = walk(&proc.body, offset) { + return Some(hit); + } + } + CommandKind::TypeDecl(td) => { + if let Some(ty) = td.underlying.as_ref() { + if let Some(hit) = inner(ty, offset) { + return Some(hit); + } + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(hit) = walk(&ns.body, offset) { + return Some(hit); + } + } + _ => {} + } + } + None + } + walk(&document.stmts, offset) +} + +/// Find a top-level `type NAME = …` declaration whose name matches +/// `name`. Handles bare and qualified forms — a caller looking up +/// `dcmac::MacPortProps` and one looking up `Properties` both hit +/// the right decl since parser stores the raw textual name. +pub fn find_type_decl<'a>( + document: &'a Document, + name: &str, +) -> Option<&'a TypeDecl> { + fn walk<'a>(stmts: &'a [Stmt], name: &str) -> Option<&'a TypeDecl> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::TypeDecl(td) + if td.name.as_deref() == Some(name) => + { + return Some(td); + } + CommandKind::NamespaceEval(ns) => { + if let Some(hit) = walk(&ns.body, name) { + return Some(hit); + } + } + CommandKind::Proc(proc) => { + // Types can also be declared inside a proc body + // in principle (though rare). Search anyway. + if let Some(hit) = walk(&proc.body, name) { + return Some(hit); + } + } + _ => {} + } + } + None + } + walk(&document.stmts, name) +} + +/// Extract the qualified/bare identifier a [`TypeExpr`] references, +/// suitable for [`find_type_decl`] lookup. Returns the joined +/// `"namespace::variant"` for [`TypeExpr::Qualified`], and the raw +/// `name` field for the other two shapes. +pub fn type_expr_lookup_name(ty: &TypeExpr) -> String { + match ty { + TypeExpr::Named { name, .. } | TypeExpr::Generic { name, .. } => { + name.clone() + } + TypeExpr::Qualified { + namespace, variant, .. + } => format!("{namespace}::{variant}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scan_finds_bare_var_from_within() { + let src = "puts $kind here"; + // cursor on the `i` of `$kind` + let pos = (src.find("kind").unwrap() + 1) as u32; + let (name, span) = scan_var_ref(src, pos).unwrap(); + assert_eq!(name, "kind"); + assert_eq!(span.slice(src), "$kind"); + } + + #[test] + fn scan_finds_var_on_dollar() { + let src = "x $y"; + let pos = src.find('$').unwrap() as u32; + let (name, _) = scan_var_ref(src, pos).unwrap(); + assert_eq!(name, "y"); + } + + #[test] + fn scan_finds_braced_var() { + let src = "a ${foo} b"; + let pos = (src.find("foo").unwrap() + 1) as u32; + let (name, span) = scan_var_ref(src, pos).unwrap(); + assert_eq!(name, "foo"); + assert_eq!(span.slice(src), "${foo}"); + } + + #[test] + fn scan_returns_none_off_a_var() { + let src = "plain text"; + assert!(scan_var_ref(src, 2).is_none()); + } +} diff --git a/vw-htcl/src/signature_help.rs b/vw-htcl/src/signature_help.rs new file mode 100644 index 0000000..1929262 --- /dev/null +++ b/vw-htcl/src/signature_help.rs @@ -0,0 +1,159 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Signature help for htcl proc calls. +//! +//! When the cursor is inside a call to a known `proc`, report that +//! proc's signature and which parameter is "active" so the editor can +//! highlight it. The active parameter is the one named by the most +//! recent `-flag` typed on the line; before any flag is typed there is +//! no active parameter (the whole signature is shown). +//! +//! Pure analysis, like [`crate::complete`]: the LSP backend turns the +//! returned [`SignatureHelp`] into `lsp_types::SignatureHelp`. + +use crate::ast::{CommandKind, Document, ProcSignature, Stmt}; +use crate::cmdline::{self, CmdLine}; + +#[derive(Clone, Debug)] +pub struct SignatureHelp<'a> { + pub proc_name: String, + pub signature: &'a ProcSignature, + /// Proc-level doc comments (`##` above the declaration). + pub doc_comments: &'a [String], + /// Index into `signature.args` of the parameter under the cursor, + /// if one is determinable. + pub active_parameter: Option, +} + +/// Signature help for the call the cursor at `offset` is inside, or +/// `None` if the cursor isn't in a known proc call. +pub fn signature_help_at<'a>( + document: &'a Document, + source: &str, + offset: u32, +) -> Option> { + let line = cmdline::analyze(source, offset); + // `command_name` is `None` while the cursor is still on the first + // word, which is exactly when there's no call to describe yet. + let name = line.command_name()?; + let (signature, doc_comments) = find_proc(document, name)?; + Some(SignatureHelp { + proc_name: name.to_string(), + signature, + doc_comments, + active_parameter: active_parameter(signature, &line), + }) +} + +fn find_proc<'a>( + document: &'a Document, + name: &str, +) -> Option<(&'a ProcSignature, &'a [String])> { + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Proc(proc) = &cmd.kind else { + continue; + }; + if proc.name.as_deref() == Some(name) { + return Some((proc.signature.as_ref()?, &cmd.doc_comments)); + } + } + None +} + +/// The active parameter is the arg named by the most recent `-flag` +/// token. Complete flags must name an arg exactly; a flag still being +/// typed (the partial word) matches by prefix so the highlight tracks +/// as the user types. +fn active_parameter(sig: &ProcSignature, line: &CmdLine<'_>) -> Option { + let mut active = None; + for word in line.words.iter().skip(1) { + if let Some(flag) = word.strip_prefix('-') { + if let Some(i) = sig.args.iter().position(|a| a.name == flag) { + active = Some(i as u32); + } + } + } + if let Some(flag) = line.partial.strip_prefix('-') { + if !flag.is_empty() { + if let Some(i) = + sig.args.iter().position(|a| a.name.starts_with(flag)) + { + return Some(i as u32); + } + } + } + active +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn cursor(src_with_marker: &str) -> (String, u32) { + let offset = src_with_marker.find('|').expect("no cursor marker"); + (src_with_marker.replacen('|', "", 1), offset as u32) + } + + fn help(src_with_marker: &str) -> Option<(String, Option)> { + let (src, off) = cursor(src_with_marker); + let parsed = parse(&src); + signature_help_at(&parsed.document, &src, off) + .map(|h| (h.proc_name, h.active_parameter)) + } + + #[test] + fn shows_signature_after_name() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg |\n"; + let (name, active) = help(src).unwrap(); + assert_eq!(name, "cfg"); + assert_eq!(active, None); + } + + #[test] + fn active_parameter_follows_last_flag() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -depth |\n"; + let (_, active) = help(src).unwrap(); + assert_eq!(active, Some(1)); + } + + #[test] + fn active_parameter_tracks_partial_flag() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -wid|\n"; + let (_, active) = help(src).unwrap(); + assert_eq!(active, Some(0)); + } + + #[test] + fn none_while_typing_proc_name() { + let src = "\ +proc cfg {\n width\n} { }\n\ +cf|\n"; + assert!(help(src).is_none()); + } + + #[test] + fn none_for_unknown_command() { + let src = "puts |\n"; + assert!(help(src).is_none()); + } + + #[test] + fn works_inside_proc_body() { + let src = "\ +proc helper {\n size\n} { }\n\ +proc outer {} {\n helper -size |\n}\n"; + let (name, active) = help(src).unwrap(); + assert_eq!(name, "helper"); + assert_eq!(active, Some(0)); + } +} diff --git a/vw-htcl/src/span.rs b/vw-htcl/src/span.rs new file mode 100644 index 0000000..5dfb736 --- /dev/null +++ b/vw-htcl/src/span.rs @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Byte-offset spans over source text. + +use std::ops::Range; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Span { + pub start: u32, + pub end: u32, +} + +impl Span { + pub const fn new(start: u32, end: u32) -> Self { + Self { start, end } + } + + pub fn range(self) -> Range { + self.start as usize..self.end as usize + } + + pub fn slice(self, source: &str) -> &str { + &source[self.range()] + } + + pub fn merge(self, other: Span) -> Span { + Span::new(self.start.min(other.start), self.end.max(other.end)) + } + + /// Translate this span by `delta` bytes. Used to lift spans from a + /// sub-parse (e.g. a proc body parsed as its own fragment) back + /// into whole-source coordinates. + pub const fn shifted(self, delta: u32) -> Span { + Span::new(self.start + delta, self.end + delta) + } + + /// True if `offset` lies within this span (start-inclusive, + /// end-inclusive). End-inclusive is the right call for hover and + /// "what's at the cursor" queries: a cursor visually positioned + /// right after a token is still on it. + pub fn contains(self, offset: u32) -> bool { + offset >= self.start && offset <= self.end + } +} + +impl From> for Span { + fn from(range: Range) -> Self { + Self { + start: range.start as u32, + end: range.end as u32, + } + } +} diff --git a/vw-htcl/src/src_path.rs b/vw-htcl/src/src_path.rs new file mode 100644 index 0000000..7b527b9 --- /dev/null +++ b/vw-htcl/src/src_path.rs @@ -0,0 +1,402 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Classification and resolution of `src` import paths. +//! +//! The plan defines three path shapes: +//! +//! - `relative/path` — relative to the importing file's directory. +//! - `/absolute/path` — filesystem-absolute (allowed but discouraged). +//! - `@name/path` — resolved via `vw.toml`'s `[dependencies.]` +//! entry; the cached repo root comes from `vw-lib`'s dependency +//! resolver and `` plus the rest of the path identify a file +//! in that repo. +//! +//! Resolution is split into two stages so the parser/AST side has no +//! filesystem dependency: [`classify`] decides which shape a path is, +//! [`Resolver`] turns a classified path into an actual on-disk file. + +use std::path::{Path, PathBuf}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PathKind { + /// Relative to the importing file's directory. + Relative, + /// Filesystem-absolute (starts with `/`). + Absolute, + /// Resolved via a workspace dependency named `name`. `subpath` is + /// the rest of the path after `@name/` (may be empty). + Named { name: String, subpath: String }, +} + +#[derive(Clone, Debug)] +pub struct ClassifiedPath<'a> { + pub kind: PathKind, + /// The original path text, retained for diagnostics. + pub raw: &'a str, +} + +/// Classify an import path. Doesn't touch the filesystem. +pub fn classify(path: &str) -> ClassifiedPath<'_> { + let kind = if let Some(rest) = path.strip_prefix('@') { + let (name, subpath) = match rest.split_once('/') { + Some((n, s)) => (n.to_string(), s.to_string()), + None => (rest.to_string(), String::new()), + }; + PathKind::Named { name, subpath } + } else if path.starts_with('/') { + PathKind::Absolute + } else { + PathKind::Relative + }; + ClassifiedPath { kind, raw: path } +} + +#[derive(Debug, thiserror::Error)] +pub enum ResolveError { + #[error( + "unknown dependency `{name}` in `src @{name}{}`; \ + add a `[dependencies.{name}]` entry to your workspace's \ + vw.toml or run `vw add` to fetch it", + if .subpath.is_empty() { String::new() } else { format!("/{}", .subpath) } + )] + UnknownDependency { name: String, subpath: String }, + + #[error("imported file does not exist: {path}")] + NotFound { path: PathBuf }, + + #[error( + "import path `{raw}` reduces to an empty file path; \ + a `src` must name a real file" + )] + EmptyPath { raw: String }, +} + +/// Bare `src @` resolves to `/{DEFAULT_MODULE}.htcl`. +/// The convention is intentionally fixed (no `vw.toml` knob) so every +/// htcl module is laid out the same way — a reader can open +/// `module.htcl` and know they're at the entry point. +/// +/// The same convention applies to any directory that appears in a +/// `src` path — `src ip` where `ip/` is a directory resolves to +/// `ip/{DEFAULT_MODULE}.htcl` (analogous to Rust's `mod foo;` +/// picking `foo/mod.rs` when `foo.rs` is absent). +pub const DEFAULT_MODULE: &str = "module"; + +/// Resolver that turns import paths into on-disk file paths. Construct +/// one per workspace and reuse it across imports. +/// +/// Named deps are looked up in `cached_deps`, a `name → cache root` +/// map normally built from `vw.lock` via `vw-lib`. The caller is +/// responsible for filling this in — the htcl crate stays free of +/// `vw-lib` and filesystem-cache concerns. +#[derive(Clone, Debug, Default)] +pub struct Resolver { + cached_deps: std::collections::HashMap, +} + +impl Resolver { + pub fn new() -> Self { + Self::default() + } + + /// Register a dependency's cached root path (typically + /// `~/.vw/deps/-`). + pub fn with_dep(mut self, name: impl Into, root: PathBuf) -> Self { + self.cached_deps.insert(name.into(), root); + self + } + + /// Same as [`with_dep`], but only registers `name` when no + /// entry already exists. Cargo-parity semantic for + /// self-injecting the enclosing workspace as `@` + /// — a user-declared dep with the same name (rare but + /// possible) still wins. + pub fn with_dep_if_absent( + mut self, + name: impl Into, + root: PathBuf, + ) -> Self { + let name = name.into(); + self.cached_deps.entry(name).or_insert(root); + self + } + + /// Iterate the registered dependencies as `(name, root)` pairs. + /// Order is unspecified — callers that care should sort. + pub fn deps(&self) -> impl Iterator { + self.cached_deps + .iter() + .map(|(k, v)| (k.as_str(), v.as_path())) + } + + /// Look up a dependency's cached root by name. + pub fn dep_root(&self, name: &str) -> Option<&Path> { + self.cached_deps.get(name).map(PathBuf::as_path) + } + + /// Resolve `path` (as written in a `src` statement) against the + /// directory containing the importing file. Returns the canonical + /// path to the imported file, with `.htcl` appended if absent. + pub fn resolve( + &self, + importing_file_dir: &Path, + path: &str, + ) -> Result { + let classified = classify(path); + let candidate = match &classified.kind { + PathKind::Relative => importing_file_dir.join(path), + PathKind::Absolute => PathBuf::from(path), + PathKind::Named { name, subpath } => { + let Some(root) = self.cached_deps.get(name) else { + return Err(ResolveError::UnknownDependency { + name: name.clone(), + subpath: subpath.clone(), + }); + }; + // Bare `@` resolves to the dep's default entry + // point — `module.htcl` at the dep root, analogous to + // Rust's `src/lib.rs`. `@/` still picks a + // specific module under the dep. + if subpath.is_empty() { + root.join(DEFAULT_MODULE) + } else { + root.join(subpath) + } + } + }; + + // Directory-as-module: `src ip` where `ip/` is a real + // directory containing `module.htcl` resolves to + // `ip/module.htcl`. Mirrors the bare-`@dep` behavior — a + // dep root and an in-tree subdirectory both use + // `module.htcl` as the entry point — and gives users the + // Rust-style choice between `foo.htcl` and + // `foo/module.htcl` for a growing module. Checked BEFORE + // the `.htcl` append so a directory with a sibling + // `.htcl` file favors the file (predictable when + // both happen to exist during a rename). + if candidate.extension().is_none() { + let with_ext = candidate.with_extension("htcl"); + if with_ext.exists() { + return Ok(with_ext.canonicalize().unwrap_or(with_ext)); + } + if candidate.is_dir() { + let module = + candidate.join(DEFAULT_MODULE).with_extension("htcl"); + if module.exists() { + return Ok(module.canonicalize().unwrap_or(module)); + } + // Directory exists but has no module.htcl — surface + // the module path in the error so the fix is obvious + // ("create ip/module.htcl") rather than pointing at + // the sibling `.htcl` we tried first. + return Err(ResolveError::NotFound { path: module }); + } + return Err(ResolveError::NotFound { path: with_ext }); + } + + // Path already carries an extension — take it verbatim. + if !candidate.exists() { + return Err(ResolveError::NotFound { path: candidate }); + } + Ok(candidate.canonicalize().unwrap_or(candidate)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[test] + fn classify_relative() { + assert_eq!(classify("foo/bar").kind, PathKind::Relative); + assert_eq!(classify("bar").kind, PathKind::Relative); + } + + #[test] + fn classify_absolute() { + assert_eq!(classify("/opt/x/y").kind, PathKind::Absolute); + } + + #[test] + fn classify_named() { + assert_eq!( + classify("@quartz/ip/bacd").kind, + PathKind::Named { + name: "quartz".into(), + subpath: "ip/bacd".into() + } + ); + assert_eq!( + classify("@bare").kind, + PathKind::Named { + name: "bare".into(), + subpath: String::new() + } + ); + } + + fn fixture() -> (tempfile::TempDir, Resolver) { + let dir = tempfile::tempdir().unwrap(); + let dep_root = dir.path().join("dep"); + fs::create_dir_all(dep_root.join("ip")).unwrap(); + fs::write(dep_root.join("ip").join("bacd.htcl"), "## stub\n").unwrap(); + fs::write(dir.path().join("local.htcl"), "## local\n").unwrap(); + let resolver = Resolver::new().with_dep("quartz", dep_root); + (dir, resolver) + } + + #[test] + fn resolve_relative_appends_htcl() { + let (dir, resolver) = fixture(); + let resolved = resolver.resolve(dir.path(), "local").unwrap(); + assert_eq!( + resolved.file_name().and_then(|s| s.to_str()), + Some("local.htcl") + ); + } + + #[test] + fn resolve_named_dependency() { + let (dir, resolver) = fixture(); + let resolved = resolver.resolve(dir.path(), "@quartz/ip/bacd").unwrap(); + assert!(resolved.ends_with("dep/ip/bacd.htcl"), "{resolved:?}"); + } + + #[test] + fn bare_named_dep_resolves_to_module_htcl() { + // `src @quartz` → `/module.htcl` (analogous to + // Rust's `use crate` resolving to `src/lib.rs`). + let dir = tempfile::tempdir().unwrap(); + let dep_root = dir.path().join("dep"); + fs::create_dir_all(&dep_root).unwrap(); + fs::write(dep_root.join("module.htcl"), "# entry\n").unwrap(); + let resolver = Resolver::new().with_dep("quartz", dep_root.clone()); + let resolved = resolver.resolve(dir.path(), "@quartz").unwrap(); + assert!(resolved.ends_with("dep/module.htcl"), "{resolved:?}"); + } + + #[test] + fn unknown_dep_errors_cleanly() { + let (dir, resolver) = fixture(); + let err = resolver.resolve(dir.path(), "@nope/foo").unwrap_err(); + assert!( + matches!(err, ResolveError::UnknownDependency { .. }), + "{err:?}" + ); + } + + #[test] + fn missing_file_errors() { + let (dir, resolver) = fixture(); + let err = resolver.resolve(dir.path(), "does/not/exist").unwrap_err(); + assert!(matches!(err, ResolveError::NotFound { .. }), "{err:?}"); + } + + #[test] + fn directory_with_module_htcl_resolves() { + // `src ip` where `ip/` is a directory with `module.htcl` + // inside → `ip/module.htcl`. The layout the metroid + // project switched to after reorganizing per-IP files + // into per-IP directories. + let dir = tempfile::tempdir().unwrap(); + let ip = dir.path().join("ip"); + fs::create_dir_all(&ip).unwrap(); + fs::write(ip.join("module.htcl"), "# entry\n").unwrap(); + let resolver = Resolver::new(); + let resolved = resolver.resolve(dir.path(), "ip").unwrap(); + assert!(resolved.ends_with("ip/module.htcl"), "{resolved:?}"); + } + + #[test] + fn sibling_htcl_wins_over_directory_module() { + // When both `foo.htcl` and `foo/module.htcl` exist, prefer + // the file. Predictable during a rename: users incrementally + // migrating a single-file module to a directory don't get + // a surprise resolution swap the moment the directory + // sprouts a `module.htcl`. + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("foo")).unwrap(); + fs::write(dir.path().join("foo.htcl"), "# file\n").unwrap(); + fs::write(dir.path().join("foo").join("module.htcl"), "# dir\n") + .unwrap(); + let resolver = Resolver::new(); + let resolved = resolver.resolve(dir.path(), "foo").unwrap(); + assert!(resolved.ends_with("foo.htcl"), "{resolved:?}"); + assert!(!resolved.ends_with("module.htcl"), "{resolved:?}"); + } + + #[test] + fn directory_without_module_htcl_errors_pointing_at_module() { + // Directory exists but lacks `module.htcl` — the error + // path should name the missing `module.htcl`, not the + // sibling `.htcl` the resolver also considered. That's + // what tells the user "add module.htcl here" instead of + // "create a sibling file." + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("ip")).unwrap(); + let resolver = Resolver::new(); + let err = resolver.resolve(dir.path(), "ip").unwrap_err(); + match err { + ResolveError::NotFound { path } => { + assert!(path.ends_with("ip/module.htcl"), "{path:?}"); + } + other => panic!("expected NotFound, got {other:?}"), + } + } + + #[test] + fn named_dep_subpath_can_be_directory_module() { + // `src @quartz/ip` where the dep has `ip/module.htcl` — + // same directory-as-module rule applies to subpaths of a + // named dep, not just to workspace-local paths. + let dir = tempfile::tempdir().unwrap(); + let dep_root = dir.path().join("dep"); + fs::create_dir_all(dep_root.join("ip")).unwrap(); + fs::write(dep_root.join("ip").join("module.htcl"), "# entry\n") + .unwrap(); + let resolver = Resolver::new().with_dep("quartz", dep_root); + let resolved = resolver.resolve(dir.path(), "@quartz/ip").unwrap(); + assert!(resolved.ends_with("dep/ip/module.htcl"), "{resolved:?}"); + } + + #[test] + fn with_dep_if_absent_leaves_existing_alone() { + // A user-declared dep of the same name must shadow the + // self-injection — same policy Cargo uses for the crate- + // self reference. Without this, a library that legitimately + // depends on an external `foo` couldn't also self-reference. + let existing = PathBuf::from("/tmp/existing"); + let new_path = PathBuf::from("/tmp/new"); + let resolver = Resolver::new() + .with_dep("foo", existing.clone()) + .with_dep_if_absent("foo", new_path); + assert_eq!(resolver.dep_root("foo"), Some(existing.as_path())); + } + + #[test] + fn with_dep_if_absent_registers_when_missing() { + let path = PathBuf::from("/tmp/self"); + let resolver = Resolver::new().with_dep_if_absent("self", path.clone()); + assert_eq!(resolver.dep_root("self"), Some(path.as_path())); + } + + #[test] + fn self_referential_workspace_resolves() { + // Simulate a library named `foo` that sources one of its + // own sibling modules via `src @foo/bar`. The resolver has + // `foo` self-injected to the workspace root, so `@foo/bar` + // resolves to `/bar.htcl`. + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("bar.htcl"), "## sib\n").unwrap(); + fs::write(dir.path().join("vw.toml"), "[workspace]\nname = \"foo\"\n") + .unwrap(); + let resolver = + Resolver::new().with_dep_if_absent("foo", dir.path().to_path_buf()); + let resolved = resolver.resolve(dir.path(), "@foo/bar").unwrap(); + assert!(resolved.ends_with("bar.htcl"), "{resolved:?}"); + } +} diff --git a/vw-htcl/src/type_parse.rs b/vw-htcl/src/type_parse.rs new file mode 100644 index 0000000..90e051d --- /dev/null +++ b/vw-htcl/src/type_parse.rs @@ -0,0 +1,457 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Mini-parser for htcl type expressions. +//! +//! Grammar: +//! +//! ```text +//! Type ::= Ident ('::' Ident | '<' Type (',' Type)* '>')? +//! Ident ::= [A-Za-z_] [A-Za-z0-9_]* +//! ``` +//! +//! The `Ident '::' Ident` form yields a [`TypeExpr::Qualified`], +//! used for the `Enum::Variant` annotations on overloaded handler +//! procs. The two forms (qualified vs generic) are mutually +//! exclusive — `Enum::Variant<…>` is a parse error. +//! +//! Whitespace is permitted between tokens but not within identifiers. +//! That's why type expressions with whitespace (`dict`) +//! must be brace-wrapped when used as a single htcl word — `dict` parses as four htcl words at the parent level, but +//! `{dict}` parses as one. The caller of [`parse`] is +//! responsible for that unwrap before handing us the type text. +//! +//! Spans returned are absolute source spans: the caller passes a +//! `base_offset` corresponding to the byte position of the first +//! character of `text` in the original source. + +use crate::ast::TypeExpr; +use crate::span::Span; + +/// One parse-error from the type parser. The caller renders these as +/// regular htcl parse-error diagnostics. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TypeParseError { + pub message: String, + pub span: Span, +} + +/// Parse `text` as a type expression, with absolute source positions +/// rooted at `base_offset`. Returns the parsed expression on success, +/// or the first error encountered. +pub fn parse(text: &str, base_offset: u32) -> Result { + let mut p = Parser::new(text, base_offset); + let ty = p.parse_type()?; + p.skip_ws(); + if !p.eof() { + return Err(TypeParseError { + message: format!( + "unexpected `{}` after type expression", + p.rest().chars().next().unwrap_or('\0') + ), + span: p.here_span(), + }); + } + Ok(ty) +} + +struct Parser<'a> { + text: &'a str, + bytes: &'a [u8], + pos: usize, + base: u32, +} + +impl<'a> Parser<'a> { + fn new(text: &'a str, base: u32) -> Self { + Self { + text, + bytes: text.as_bytes(), + pos: 0, + base, + } + } + + fn eof(&self) -> bool { + self.pos >= self.bytes.len() + } + + fn rest(&self) -> &str { + &self.text[self.pos..] + } + + fn skip_ws(&mut self) { + while self.pos < self.bytes.len() + && self.bytes[self.pos].is_ascii_whitespace() + { + self.pos += 1; + } + } + + fn here(&self) -> u32 { + self.base + self.pos as u32 + } + + /// Zero-width span at the current position — used for "unexpected + /// token" diagnostics where there's no real token to underline. + fn here_span(&self) -> Span { + let h = self.here(); + Span::new(h, h) + } + + fn span_from(&self, start: usize) -> Span { + Span::new(self.base + start as u32, self.base + self.pos as u32) + } + + /// Consume one bare identifier, returning its text and span. + /// Identifiers start with `[A-Za-z_]` and contain `[A-Za-z0-9_]`. + fn parse_ident(&mut self) -> Result<(String, Span), TypeParseError> { + self.skip_ws(); + let start = self.pos; + if self.eof() { + return Err(TypeParseError { + message: "expected type name, found end of input".into(), + span: self.here_span(), + }); + } + let first = self.bytes[self.pos]; + if !(first.is_ascii_alphabetic() || first == b'_') { + return Err(TypeParseError { + message: format!( + "expected type name, found `{}`", + first as char + ), + span: self.here_span(), + }); + } + self.pos += 1; + while self.pos < self.bytes.len() { + let c = self.bytes[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' { + self.pos += 1; + } else { + break; + } + } + let name = self.text[start..self.pos].to_string(); + Ok((name, self.span_from(start))) + } + + fn parse_type(&mut self) -> Result { + let start = self.pos; + self.skip_ws(); + let ident_start = self.pos; + let (name, name_span) = self.parse_ident()?; + self.skip_ws(); + // Optional `::Variant` qualified-path suffix. Mutually + // exclusive with the `<…>` generic form — `E::V` is + // rejected below. + // + // Deeper chains (`A::B::C::D`) collapse into a `Named` type + // whose name is the whole colon-joined string. That's how + // generated wrappers can reference nested-namespace + // newtypes like `gtwiz_versal::intf0::gt_settings::Lr0Settings` + // without teaching the whole validator about + // multi-segment qualified paths (they never carry variant + // semantics — they're just deep newtype references). + if self.pos + 1 < self.bytes.len() + && self.bytes[self.pos] == b':' + && self.bytes[self.pos + 1] == b':' + { + self.pos += 2; // :: + let (variant, variant_span) = self.parse_ident()?; + self.skip_ws(); + // Third `::segment`? Keep going and produce a flat + // Named type with the whole joined path as its name. + if self.pos + 1 < self.bytes.len() + && self.bytes[self.pos] == b':' + && self.bytes[self.pos + 1] == b':' + { + let mut joined = format!("{name}::{variant}"); + while self.pos + 1 < self.bytes.len() + && self.bytes[self.pos] == b':' + && self.bytes[self.pos + 1] == b':' + { + self.pos += 2; + let (seg, _) = self.parse_ident()?; + joined.push_str("::"); + joined.push_str(&seg); + self.skip_ws(); + } + if !self.eof() && self.bytes[self.pos] == b'<' { + return Err(TypeParseError { + message: format!( + "nested-namespace type `{joined}` cannot take \ + generic arguments" + ), + span: self.here_span(), + }); + } + let span = self.span_from(start); + return Ok(TypeExpr::Named { name: joined, span }); + } + // Exactly two segments — the classic `Enum::Variant` + // shape used for overload dispatch. Preserve the + // Qualified form so the validator's variant-reference + // rules kick in. + // + // Reject `E::V<…>` — qualified names don't take generic + // args (their purpose is to name one variant of a + // declared enum, which has no type parameters in v1). + if !self.eof() && self.bytes[self.pos] == b'<' { + return Err(TypeParseError { + message: format!( + "qualified type `{name}::{variant}` cannot take \ + generic arguments" + ), + span: self.here_span(), + }); + } + return Ok(TypeExpr::Qualified { + namespace: name, + variant, + namespace_span: name_span, + variant_span, + span: self.span_from(start), + }); + } + // Optional `<...>` generic argument list. + if !self.eof() && self.bytes[self.pos] == b'<' { + self.pos += 1; // < + let mut args = Vec::new(); + // Allow empty? No — `list<>` is meaningless. Require + // at least one arg. + args.push(self.parse_type()?); + self.skip_ws(); + while !self.eof() && self.bytes[self.pos] == b',' { + self.pos += 1; // , + args.push(self.parse_type()?); + self.skip_ws(); + } + if self.eof() { + return Err(TypeParseError { + message: format!( + "unterminated generic type `{name}<…>`: \ + expected `>` or `,`", + ), + span: self.span_from(ident_start), + }); + } + if self.bytes[self.pos] != b'>' { + return Err(TypeParseError { + message: format!( + "expected `>` or `,` in generic type `{name}<…>`, \ + found `{}`", + self.bytes[self.pos] as char + ), + span: self.here_span(), + }); + } + self.pos += 1; // > + return Ok(TypeExpr::Generic { + name, + name_span, + args, + span: self.span_from(start), + }); + } + Ok(TypeExpr::Named { + name, + span: name_span, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(s: &str) -> TypeExpr { + parse(s, 0).unwrap_or_else(|e| panic!("parse failed: {e:?}")) + } + + #[test] + fn named_simple() { + let ty = p("string"); + match ty { + TypeExpr::Named { name, span } => { + assert_eq!(name, "string"); + assert_eq!(span, Span::new(0, 6)); + } + _ => panic!("expected Named"), + } + } + + #[test] + fn generic_single_arg() { + let ty = p("list"); + let TypeExpr::Generic { + name, args, span, .. + } = ty + else { + panic!("expected Generic"); + }; + assert_eq!(name, "list"); + assert_eq!(span, Span::new(0, 13)); + assert_eq!(args.len(), 1); + assert_eq!(args[0].name(), "bd_cell"); + } + + #[test] + fn generic_two_args() { + let ty = p("dict"); + let TypeExpr::Generic { name, args, .. } = ty else { + panic!(); + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + assert_eq!(args[1].name(), "int"); + } + + #[test] + fn nested_generic() { + let ty = p("dict>"); + let TypeExpr::Generic { args, .. } = ty else { + panic!() + }; + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + let TypeExpr::Generic { + name, args: inner, .. + } = &args[1] + else { + panic!("expected inner generic"); + }; + assert_eq!(name, "list"); + assert_eq!(inner[0].name(), "int"); + } + + #[test] + fn deeply_nested() { + let ty = p("list>"); + let TypeExpr::Generic { name, args, .. } = ty else { + panic!() + }; + assert_eq!(name, "list"); + let TypeExpr::Generic { + name: inner_name, + args: inner_args, + .. + } = &args[0] + else { + panic!(); + }; + assert_eq!(inner_name, "dict"); + assert_eq!(inner_args.len(), 2); + assert_eq!(inner_args[0].name(), "string"); + assert_eq!(inner_args[1].name(), "bd_cell"); + } + + #[test] + fn whitespace_between_tokens_is_fine() { + let ty = p(" dict < string , int > "); + let TypeExpr::Generic { name, args, .. } = ty else { + panic!() + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + } + + #[test] + fn span_uses_base_offset() { + let ty = parse("bd_cell", 100).unwrap(); + let TypeExpr::Named { span, .. } = ty else { + panic!() + }; + assert_eq!(span, Span::new(100, 107)); + } + + #[test] + fn err_empty_input() { + let e = parse("", 0).unwrap_err(); + assert!(e.message.contains("expected type name")); + } + + #[test] + fn err_invalid_ident_start() { + let e = parse("", 0).unwrap_err(); + assert!( + e.message.contains("cannot take generic arguments"), + "{}", + e.message + ); + } + + #[test] + fn err_qualified_missing_variant() { + let e = parse("Property::", 0).unwrap_err(); + assert!(e.message.contains("expected type name"), "{}", e.message); + } + + #[test] + fn err_single_colon_not_qualified() { + // `Property:Scalar` (one colon) — not a qualified form. The + // first ident parses, then the trailing `:Scalar` is junk. + let e = parse("Property:Scalar", 0).unwrap_err(); + assert!(e.message.contains("unexpected")); + } +} diff --git a/vw-htcl/src/undefined.rs b/vw-htcl/src/undefined.rs new file mode 100644 index 0000000..acec5dc --- /dev/null +++ b/vw-htcl/src/undefined.rs @@ -0,0 +1,780 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Undefined-variable error pass. +//! +//! Emits a [`Diagnostic`] with severity [`Severity::Error`] for every +//! `$name` reference whose name isn't in the enclosing scope's decl +//! set. Same scope model as [`crate::unused`] (proc args + `set` + +//! `foreach` + `upvar`, with body-host bodies contributing decls to +//! the enclosing scope), inverted: uses that lack a matching decl. +//! +//! **What we catch.** The motivating case is the typo +//! `set _dcmac [ … ]; return $dcmac` from dcmac.htcl — obvious +//! misspelling, previously only caught at runtime by Vivado. +//! +//! **What we don't catch.** Full Tcl-style scoping (deep `upvar N` +//! traversal, `interp` / child interpreters, `global`/`variable` +//! cross-frame refs) is out of scope. Cross-file globals aren't +//! tracked either — the pass looks at one document at a time. +//! +//! **Escape hatches.** Scope-leak suppression (same policy as +//! `unused.rs`): a scope containing `eval $x` / `uplevel N $x` / +//! `apply $x` / `info exists $var` is opaque, so we emit no undef +//! errors for it. A small implicit-name whitelist (`env`, +//! `errorInfo`, `errorCode`, `tcl_platform`, `tcl_version`, `argv`, +//! `argv0`, `_`) covers Tcl's environment-provided names. + +use std::collections::{HashMap, HashSet}; + +use crate::ast::{ + Command, CommandKind, Document, Stmt, Word, WordForm, WordPart, +}; +use crate::hover::is_body_host; +use crate::span::Span; +use crate::unused::{ + collect_decls, reparse_braced_body, scope_is_leaked, DeclSite, +}; +use crate::validate::{Diagnostic, Severity}; + +/// Names pre-defined in every Tcl scope. Referencing any of these +/// never produces an undefined-variable error. +const IMPLICITS: &[&str] = &[ + "env", + "errorInfo", + "errorCode", + "tcl_platform", + "tcl_version", + "tcl_pkgPath", + "tcl_library", + "tcl_patchLevel", + "argv", + "argv0", + "argc", + "_", +]; + +/// Collect the names of every variable defined at the document's +/// top level. Same collection rules as the undef pass — `set` +/// LHS, `foreach` iterator vars, `upvar` locals, `catch` result +/// vars, `regexp`/`regsub` capture vars, and any of the above +/// inside body-host `if`/`while`/… bodies that run in the top- +/// level frame. Used by the REPL to accumulate a name set across +/// batches so `set p …` in batch N-1 doesn't cause a false- +/// positive `undefined variable $p` in batch N. +pub fn top_level_var_names( + document: &Document, + source: &str, +) -> HashSet { + let mut decls: HashMap = HashMap::new(); + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + crate::unused::collect_decls(cmd, source, &mut decls); + } + decls.into_keys().collect() +} + +/// Companion to [`top_level_var_names`] that also returns the +/// inferred type of each top-level `set VAR ` binding when +/// the RHS's type is statically knowable. Used by the REPL to +/// carry variable-type context across batches so `putr $foo` in +/// batch N sees the type that batch N-1's `set foo […]` produced. +/// +/// Only the whole-word `[proc-call]`, `$var-copy`, and bare +/// `true`/`false` shapes are typed — everything else stays out +/// (matches [`crate::validate::value_type`]'s coverage). Missing +/// entries are fine: the caller merges these into an initial +/// `VarTypeTable` and falls back to plain `puts` when a name +/// isn't present. +pub fn top_level_var_types( + document: &Document, + sig_table: &HashMap, +) -> HashMap { + use crate::ast::CommandKind; + // Threaded var_table so a later `set y $x` picks up the type + // an earlier `set x [typed_proc]` recorded — matches the + // rewrite walker's own scope discipline for consistency. + let mut var_table = crate::validate::VarTypeTable::new(); + // Proc table for return-type INFERENCE on unannotated procs. + // A user proc like `proc configure_gtm {} { set cfg [typed]; …; return $cfg }` + // has no annotated return type — `value_type` alone would + // report None for `[configure_gtm]`. The proc-table lookup + // lets `value_type_with_procs` walk the body's last `return` + // to figure out the type flows out. Without this, `putr + // $_gtm` after `set _gtm [configure_gtm]` falls to plain + // puts and dumps the raw tagged tree. + let proc_table = crate::validate::build_proc_table(document); + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !matches!(cmd.kind, CommandKind::Set) { + continue; + } + let Some(name_word) = cmd.words.get(1) else { + continue; + }; + let Some(value_word) = cmd.words.get(2) else { + continue; + }; + let Some(name) = name_word.as_text() else { + continue; + }; + if let Some(ty) = crate::validate::value_type_with_procs( + value_word, + sig_table, + &var_table, + Some(&proc_table), + ) { + var_table.insert(name.to_string(), ty); + } + } + var_table +} + +/// Top-level entry. Walks the document as one scope (for top-level +/// `set`/`$var` references), then recurses into every proc body / +/// namespace-eval body as an independent scope. +pub fn validate_undefined_vars( + document: &Document, + source: &str, + diags: &mut Vec, +) { + validate_undefined_vars_with_extras( + document, + source, + &HashSet::new(), + diags, + ); +} + +/// Same as [`validate_undefined_vars`], with an extra pool of +/// top-level variable names the caller injects as "already +/// defined" — used by the REPL so a `set p …` in batch N-1 makes +/// `$p` in batch N legal. Only applies at the DOCUMENT top level; +/// proc bodies start with just their own args + `set`s (Tcl +/// semantics — top-level vars aren't visible inside a proc +/// without `global`/`upvar`, so leaking session state in would +/// mask real bugs). +pub fn validate_undefined_vars_with_extras( + document: &Document, + source: &str, + extra_top_level: &HashSet, + diags: &mut Vec, +) { + walk_scope(&document.stmts, source, extra_top_level, diags); +} + +/// One scope pass — collect decls, walk uses (with spans), emit +/// errors for each use that has no matching decl and isn't implicit. +/// The `extra_decls` param is only non-empty for the document top +/// level (see [`validate_undefined_vars_with_extras`]); recursive +/// scope walks pass an empty set. +fn walk_scope( + stmts: &[Stmt], + source: &str, + extra_decls: &HashSet, + diags: &mut Vec, +) { + let mut decls: HashMap = HashMap::new(); + for name in extra_decls { + decls.insert( + name.clone(), + DeclSite { + span: Span::new(0, 0), + kind: crate::unused::DeclKind::Set, + }, + ); + } + let mut use_sites: Vec<(String, Span)> = Vec::new(); + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + collect_decls(cmd, source, &mut decls); + collect_use_sites_in_command(cmd, source, &mut use_sites); + } + if !scope_is_leaked(stmts, source) { + emit_undefined(&decls, &use_sites, diags); + } + // Descend into fresh-frame children (proc bodies, namespace eval + // bodies) regardless — the outer scope's leak doesn't taint them. + // Extras only apply to the document top level; recursive walks + // pass an empty set (see [`validate_undefined_vars_with_extras`] + // docstring for the rationale). + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + descend_scopes(cmd, source, diags); + } +} + +/// Recurse into scope-establishing children of `cmd`. Nested procs +/// and `namespace eval` bodies each get their own `walk_scope`. +/// Mirrors `unused::descend_scopes` but with the undef pass's decl +/// seeding + use collection. +fn descend_scopes(cmd: &Command, source: &str, diags: &mut Vec) { + match &cmd.kind { + CommandKind::Proc(proc) => { + let mut decls: HashMap = HashMap::new(); + let mut use_sites: Vec<(String, Span)> = Vec::new(); + if let Some(sig) = proc.signature.as_ref() { + for arg in &sig.args { + decls.insert( + arg.name.clone(), + DeclSite { + span: arg.name_span, + // The DeclKind values are private to + // unused.rs's warning-message dispatch; + // we never read them here, but the field + // must be set to something. Use ProcArg + // as the closest match — this DeclSite + // exists purely as a "name is defined" + // marker for our lookup. + kind: crate::unused::DeclKind::ProcArg, + }, + ); + } + } + for stmt in &proc.body { + let Stmt::Command(inner) = stmt else { continue }; + collect_decls(inner, source, &mut decls); + collect_use_sites_in_command(inner, source, &mut use_sites); + } + if !scope_is_leaked(&proc.body, source) { + emit_undefined(&decls, &use_sites, diags); + } + for stmt in &proc.body { + let Stmt::Command(inner) = stmt else { continue }; + descend_scopes(inner, source, diags); + } + } + CommandKind::NamespaceEval(ns) => { + walk_scope(&ns.body, source, &HashSet::new(), diags); + } + _ => {} + } +} + +/// Walk `cmd.words` (and any body-host braced-body interiors) and +/// record every `WordPart::VarRef` as a (name, span) pair. +/// +/// Body-host bodies run in the enclosing frame, so their `$foo` +/// references count as uses of the current scope. Skip `proc` and +/// `namespace eval` — those open fresh scopes and `descend_scopes` +/// handles them. +fn collect_use_sites_in_command( + cmd: &Command, + source: &str, + use_sites: &mut Vec<(String, Span)>, +) { + // `set foo` (exactly 2 words) is a read of $foo — mirror the + // unused pass's convention. + if matches!(cmd.kind, CommandKind::Set) && cmd.words.len() == 2 { + if let Some(name) = cmd.words[1].as_text() { + use_sites.push((name.to_string(), cmd.words[1].span)); + } + } + for word in &cmd.words { + collect_use_sites_in_word(word, source, use_sites); + } + // Body-host recursion for uses. Same shape as `collect_decls`'s + // recursion — proc / namespace eval are excluded. + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + if is_body_host(head) + && !matches!( + &cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) + { + for word in cmd.words.iter().skip(1) { + if let Some(stmts) = reparse_braced_body(word, source) { + for stmt in &stmts { + let Stmt::Command(inner) = stmt else { continue }; + collect_use_sites_in_command(inner, source, use_sites); + } + } + } + } + } +} + +fn collect_use_sites_in_word( + word: &Word, + source: &str, + use_sites: &mut Vec<(String, Span)>, +) { + // Braced literals are opaque in Tcl — `puts {$foo}` prints the + // literal string `$foo`, not the value of `$foo`. The parser + // encodes this by making `Braced` words carry a single Text + // part with no VarRef sub-parts, so the loop below naturally + // skips them. + for part in &word.parts { + match part { + WordPart::VarRef { name, span, .. } => { + use_sites.push((name.clone(), *span)); + } + WordPart::CmdSubst { body, .. } => { + for stmt in body { + let Stmt::Command(inner) = stmt else { continue }; + collect_use_sites_in_command(inner, source, use_sites); + } + } + WordPart::Text { .. } | WordPart::Escape { .. } => {} + } + } + // Defensive: on the off-chance a Braced word carries VarRef + // sub-parts (parser change / future extension), skip them — + // Tcl braced-word semantics prevail. If future parsers surface + // sub-refs from a `Quoted` word, they'll flow through above. + if word.form == WordForm::Braced { + // No-op: the loop above already handled the (only) Text + // part. This block exists as documentation and a place to + // add a diagnostic if the invariant ever breaks. + } +} + +/// For each use-site whose name isn't defined and isn't implicit, +/// emit an `undefined variable` error. The message includes a +/// "did you mean" hint when a decl within edit-distance 2 exists. +fn emit_undefined( + decls: &HashMap, + use_sites: &[(String, Span)], + diags: &mut Vec, +) { + // Dedupe by span so a name referenced twice at the same span + // (shouldn't happen, but cheap insurance) only emits once. + let mut seen: HashSet<(String, u32, u32)> = HashSet::new(); + // Emit in source order for stable output. + let mut items: Vec<&(String, Span)> = use_sites.iter().collect(); + items.sort_by_key(|(_, sp)| (sp.start, sp.end)); + for (name, span) in items { + if !seen.insert((name.clone(), span.start, span.end)) { + continue; + } + // The base name for `arr(key)` subscripts — if `arr` is + // defined, the subscript form is a legal reference to it. + let base = match name.find('(') { + Some(idx) => &name[..idx], + None => name.as_str(), + }; + if decls.contains_key(base) || decls.contains_key(name) { + continue; + } + if is_implicit(base) || is_implicit(name) { + continue; + } + // Numeric names ($1, $2) — regex submatch refs and the + // like. Skip. + if name.parse::().is_ok() { + continue; + } + let hint = suggest(name, decls); + let message = match hint { + Some(sug) => { + format!("undefined variable `${name}`; did you mean `${sug}`?") + } + None => format!("undefined variable `${name}`"), + }; + diags.push(Diagnostic { + severity: Severity::Error, + message, + span: *span, + }); + } +} + +fn is_implicit(name: &str) -> bool { + if IMPLICITS.contains(&name) { + return true; + } + // Compiler-provided kwargs presence flags. The vw-htcl lowering + // injects a `::vw::kwargs` shim call at proc entry that sets a + // `__vw_kw__set` boolean for every optional kwarg, so + // proc bodies (both hand-written and generator-emitted) can + // check `${__vw_kw_foo_set}` to see whether the user passed a + // value. These never appear as source-level decls; treat as + // pre-defined in every scope. See vw-ip/src/generate.rs's + // `emit_dict_proc` for the emission side. + if let Some(rest) = name.strip_prefix("__vw_kw_") { + if rest.ends_with("_set") { + return true; + } + } + false +} + +/// Levenshtein-distance suggestion — returns the closest decl name +/// within distance 2, only for names of length ≥ 3 (below that, +/// every 2-char name is within distance 2 of every other, which +/// produces noisy hints). +fn suggest(name: &str, decls: &HashMap) -> Option { + if name.len() < 3 { + return None; + } + let mut best: Option<(usize, String)> = None; + for candidate in decls.keys() { + if candidate.len() < 3 { + continue; + } + let d = edit_distance(name, candidate); + if d > 2 { + continue; + } + if best.as_ref().is_none_or(|(bd, _)| d < *bd) { + best = Some((d, candidate.clone())); + } + } + best.map(|(_, s)| s) +} + +/// Iterative Levenshtein distance. Small strings only — no need +/// to optimize further for our decl-set sizes. +fn edit_distance(a: &str, b: &str) -> usize { + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + let n = a.len(); + let m = b.len(); + if n == 0 { + return m; + } + if m == 0 { + return n; + } + let mut prev: Vec = (0..=m).collect(); + let mut curr: Vec = vec![0; m + 1]; + for i in 1..=n { + curr[0] = i; + for j in 1..=m { + let cost = if a[i - 1] == b[j - 1] { 0 } else { 1 }; + curr[j] = + (prev[j] + 1).min(curr[j - 1] + 1).min(prev[j - 1] + cost); + } + std::mem::swap(&mut prev, &mut curr); + } + prev[m] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn errors(src: &str) -> Vec { + let parsed = parse(src); + assert!( + parsed.errors.is_empty(), + "unexpected parse errors: {:?}", + parsed.errors + ); + let mut out = Vec::new(); + validate_undefined_vars(&parsed.document, src, &mut out); + out.into_iter() + .filter(|d| d.severity == Severity::Error) + .collect() + } + + #[test] + fn dcmac_typo_flagged() { + // The motivating case: `set _dcmac …` then `return $dcmac`. + let src = "proc f {} { set _dcmac 1; return $dcmac }\n"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("undefined variable `$dcmac`"), + "{}", + e[0].message + ); + assert!( + e[0].message.contains("did you mean `$_dcmac`"), + "{}", + e[0].message + ); + } + + #[test] + fn defined_local_clean() { + let src = "proc f {} { set x 1; puts $x }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn proc_arg_clean() { + let src = "proc f {x} { puts $x }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn foreach_var_clean() { + let src = "proc f {xs} { foreach i $xs { puts $i } }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn foreach_iterator_available_after_loop() { + // Tcl semantics: foreach iterator persists after the loop. + let src = "proc f {xs} { foreach i $xs { }; puts $i }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn braced_ref_not_flagged() { + // `puts {$foo}` — braced content is literal in Tcl, so + // `$foo` here is just the string `$foo`, not a var ref. + let src = "proc f {} { puts {$foo} }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn if_else_union_clean() { + // Tcl runs `if`/`else` bodies in the enclosing frame, so a + // `set x` inside either branch defines `x` in the outer + // scope. Reading `$x` after the `if` is legal. + let src = "\ +proc f {c} { + if { $c } { set x 1 } else { set x 2 } + puts $x +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn underscore_prefix_no_alias() { + // `_foo` and `foo` are distinct names. The unused-var + // pass's `_`-prefix escape hatch does NOT create a $foo + // alias for a set _foo decl. + let src = "proc f {} { set _foo 1; puts $foo }\n"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("undefined variable `$foo`"), + "{}", + e[0].message + ); + } + + #[test] + fn implicit_env_clean() { + // `$env(HOME)` is a subscript on the `env` array, which is + // pre-defined by Tcl. + let src = "proc f {} { puts $env(HOME) }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn implicit_errorinfo_clean() { + let src = "proc f {} { puts $errorInfo }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn dynamic_eval_suppresses() { + // Scope leak — we can't tell what `eval $script` might + // reference, so suppress the whole scope's undef check. + let src = "proc f {} { eval $script; puts $mystery }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn top_level_flagged() { + let src = "set x 1\nputs $y\n"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("undefined variable `$y`"), + "{}", + e[0].message + ); + } + + #[test] + fn top_level_defined_clean() { + // Matches the ~/sketch/metroid/project.htcl pattern: + // top-level `set proj [...]` then `... -proj $proj`. + let src = "set proj 1\nputs $proj\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn upvar_local_defined() { + let src = "proc f {} { upvar 1 remote local; puts $local }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn subscript_base_defined() { + // `$arr(key)` — `arr` is defined, so the array subscript + // reference is fine. + let src = "proc f {} { set arr 1; puts $arr(key) }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn cmd_subst_in_body() { + let src = "proc f {x} { set y [list $x]; puts $y }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn nested_scope_isolated() { + // Inner proc's `x` arg is distinct from outer scope. + // Outer `$outer_var` is undefined; inner uses `$x` cleanly. + let src = "\ +proc outer {} { + proc inner {x} { puts $x } + puts $outer_var +} +"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("undefined variable `$outer_var`"), + "{}", + e[0].message + ); + } + + #[test] + fn numeric_names_skipped() { + // `$1`, `$2` are typically regex submatch refs. Not flagged. + let src = "proc f {} { puts $1 }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn typo_with_multiple_candidates() { + // Closest match by edit distance wins. + let src = "\ +proc f {} { + set apple 1 + set apricot 2 + puts $applet +} +"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("did you mean `$apple`"), + "{}", + e[0].message + ); + } + + #[test] + fn port_htcl_repro_minimal() { + // Simplest form of the port.htcl false-positive: an `if + // {...} { continue }` in the foreach body between the + // `foreach` line and the `set val`. + let src = "\ +proc f {obj} { + foreach prop [list $obj] { + if {$prop eq \"\"} { continue } + set val 1 + puts $val + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn port_htcl_repro() { + // The exact shape from ~/src/htcl/amd/vivado-cmd/port.htcl:116-124. + let src = "\ +proc f {obj} { + foreach prop [list $obj] { + if {![string match \"CONFIG.*\" $prop]} { continue } + if {[regexp {^CONFIG\\.foo$} $prop]} { continue } + set val [list $prop $obj] + if {$val eq \"\"} { continue } + puts $val + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn dict_for_binds_key_and_value_vars() { + // `dict for {lib srcs} $deps { … }` — the two brace-list + // names bind for the body's lifetime, same as a `foreach + // {k v} $pairs { … }`. Before the fix, `$lib` / `$srcs` + // inside the body were flagged as undefined. + let src = "\ +proc f {deps} { + dict for {lib srcs} $deps { + puts $lib + puts $srcs + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn set_inside_foreach_body_defines_in_outer_scope() { + // Repro for the port.htcl false-positive: `foreach x [...] { + // set val [...]; puts $val }`. The `set val` is inside the + // foreach body, which runs in the enclosing frame per Tcl + // semantics, so `$val` on the next line is a legal ref. + let src = "\ +proc f {xs} { + foreach x $xs { + set val 1 + puts $val + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn catch_result_var_is_a_decl() { + // `catch { … } n` — Tcl's catch binds the caught body's + // result into `n` (the enclosing scope's frame). Repro for + // lift.htcl:29 false-positive. + let src = "\ +proc f {raw} { + if {[catch {llength $raw} n]} { return 0 } + return $n +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn regexp_var_captures_are_decls() { + // `regexp {pattern} $s var1 var2 …` — every trailing var + // arg is a decl. Repro for props.htcl:121. + let src = "\ +proc f {s} { + if {[regexp {(.*)=(.*)} $s _all key val]} { + puts $key + puts $val + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn vw_kwargs_flags_are_implicit() { + // The vw-htcl lowering injects `__vw_kw__set` boolean + // sentinels via a `::vw::kwargs` shim at proc entry — every + // optional kwarg gets one. These never appear as source- + // level decls, and generator output relies on them + // heavily. Treat as pre-defined in every scope. + let src = "\ +proc f {} { + if {${__vw_kw_config_c0_set}} { puts hi } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn short_names_no_suggestion_noise() { + // Names < 3 chars don't participate in the suggestion. + let src = "proc f {} { set ab 1; puts $xy }\n"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!(!e[0].message.contains("did you mean"), "{}", e[0].message); + } +} diff --git a/vw-htcl/src/unused.rs b/vw-htcl/src/unused.rs new file mode 100644 index 0000000..e716fca --- /dev/null +++ b/vw-htcl/src/unused.rs @@ -0,0 +1,1086 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Unused-variable warning pass. +//! +//! Emits a [`Diagnostic`] with severity [`Severity::Warning`] for every +//! local binding whose name never surfaces in the same scope's uses. +//! Slice 1 (current): proc args + `set` decls at both the top level +//! and inside proc bodies. No brace-body reparse yet (bodies of +//! `if`/`while`/`foreach`/etc. are opaque and can hide uses), so the +//! pass tolerates false-negatives but never emits false-positives. +//! Later slices add brace-body reparse and per-construct escape +//! hatches for `upvar`/`uplevel`/`eval`/`apply`/`info`. +//! +//! **Escape hatch.** A leading `_` on a name suppresses the warning +//! for that decl — `_ignored`, `_unused`, `_` alone all count. + +use std::collections::{HashMap, HashSet}; + +use crate::ast::{ + Command, CommandKind, Document, Stmt, Word, WordForm, WordPart, +}; +use crate::hover::is_body_host; +use crate::span::Span; +use crate::validate::{Diagnostic, Severity}; + +/// Kind of local binding — drives the diagnostic message. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum DeclKind { + ProcArg, + Set, + ForeachVar, + Upvar, +} + +/// Where a name was declared and by what construct. +#[derive(Clone, Copy, Debug)] +pub(crate) struct DeclSite { + pub(crate) span: Span, + pub(crate) kind: DeclKind, +} + +/// Top-level entry. Walks the document as one scope (for top-level +/// `set` decls), then recurses into every proc body / namespace-eval +/// body as its own independent scope. +pub fn validate_unused_vars( + document: &Document, + source: &str, + diags: &mut Vec, +) { + walk_scope(&document.stmts, source, diags); +} + +/// Collect decls + uses over a flat list of statements as one scope, +/// then emit warnings for decls whose names never appear as uses. +/// Recurses into `NamespaceEval.body` and each `Proc.body` as fresh +/// scopes. +/// +/// If the scope contains a dynamic-script construct we can't see +/// through (`eval $x`, `uplevel N $x`, `apply $x`) we suppress the +/// warnings for this scope but still descend into nested scopes — +/// those are unaffected. +fn walk_scope(stmts: &[Stmt], source: &str, diags: &mut Vec) { + let mut decls: HashMap = HashMap::new(); + let mut uses: HashSet = HashSet::new(); + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + collect_decls(cmd, source, &mut decls); + collect_uses_in_command(cmd, source, &mut uses); + } + if !scope_is_leaked(stmts, source) { + emit_unused(&decls, &uses, diags); + } + // Descend into nested scopes regardless — a leak in the outer + // scope doesn't taint an inner proc's locals. + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + descend_scopes(cmd, source, diags); + } +} + +/// True when this scope contains a construct that could reference +/// locals by dynamically-computed names. Presence of any of the +/// following counts as a leak: +/// +/// - `eval` with a non-literal script arg (`$x`, `"…$x…"`). +/// - `uplevel LEVEL` with a non-literal script arg (any LEVEL — +/// even LEVEL=0 with a dynamic body is unpeekable). +/// - `apply` with a non-literal envelope word (`apply $x …`). +/// - `info level`, `info vars`, `info exists` with a dynamic arg. +/// +/// Scans this scope's statements plus any brace-body interiors +/// that Slice 2's reparse would walk — same-frame constructs +/// (`if`/`while`/etc.) can leak from inside their bodies too. +pub(crate) fn scope_is_leaked(stmts: &[Stmt], source: &str) -> bool { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if command_leaks(cmd) { + return true; + } + // Recurse into any brace-body interior — same frame, so + // a leak there leaks the outer scope. Only body-hosts + // (`if`/`while`/`foreach`/…) have such bodies. + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if let Some(inner) = reparse_braced_body(word, source) { + if scope_is_leaked(&inner, source) { + return true; + } + } + } + } + } + } + false +} + +/// True when a single command is a scope-leak site. +fn command_leaks(cmd: &Command) -> bool { + let Some(head) = cmd.words.first().and_then(Word::as_text) else { + return false; + }; + match head { + "eval" | "uplevel" | "apply" => { + // Any non-literal arg → leak. Walk args, skipping the + // command name; if any word contains a VarRef or a + // CmdSubst it's dynamic. + cmd.words.iter().skip(1).any(word_is_dynamic) + } + "info" => { + // `info level`, `info vars`, `info exists` — all three + // are introspection over the current frame. When their + // arg is dynamic we can't tell which locals get named, + // so bail. Static forms (`info exists foo`) don't leak + // (they're a use of `foo`; `collect_uses_in_command` + // could later grow to record them, but for now the + // conservative treatment is to treat as leak only when + // dynamic). + let sub = cmd.words.get(1).and_then(Word::as_text); + match sub { + Some("level") | Some("vars") | Some("exists") => { + cmd.words.iter().skip(2).any(word_is_dynamic) + } + _ => false, + } + } + _ => false, + } +} + +/// True when `word` isn't a pure literal (contains a `$var` or +/// `[…]` substitution). +pub(crate) fn word_is_dynamic(word: &Word) -> bool { + word.parts.iter().any(|p| { + matches!(p, WordPart::VarRef { .. } | WordPart::CmdSubst { .. }) + }) +} + +/// Recurse into scope-establishing children of `cmd`. Nested procs +/// and `namespace eval` bodies each get their own `walk_scope` call. +fn descend_scopes(cmd: &Command, source: &str, diags: &mut Vec) { + match &cmd.kind { + CommandKind::Proc(proc) => { + // Fresh scope. Seed it with the proc's args before + // walking the body's statements. + let mut decls: HashMap = HashMap::new(); + let mut uses: HashSet = HashSet::new(); + if let Some(sig) = proc.signature.as_ref() { + for arg in &sig.args { + decls.insert( + arg.name.clone(), + DeclSite { + span: arg.name_span, + kind: DeclKind::ProcArg, + }, + ); + } + } + for stmt in &proc.body { + let Stmt::Command(inner) = stmt else { continue }; + collect_decls(inner, source, &mut decls); + collect_uses_in_command(inner, source, &mut uses); + } + if !scope_is_leaked(&proc.body, source) { + emit_unused(&decls, &uses, diags); + } + // And recurse into nested scopes inside the body. + for stmt in &proc.body { + let Stmt::Command(inner) = stmt else { continue }; + descend_scopes(inner, source, diags); + } + } + CommandKind::NamespaceEval(ns) => { + walk_scope(&ns.body, source, diags); + } + _ => {} + } +} + +/// If `cmd` binds a local, add it to `decls`. Recognizes: +/// - `set X value` (`CommandKind::Set` with `words.len() >= 3`) +/// - `foreach X list body` (Generic command with head `"foreach"`). +/// Both single-var (`words[1]` bare) and multi-var brace-list +/// (`words[1]` Braced containing whitespace-separated names) +/// forms are handled. The iterator var is declared in the +/// *enclosing* scope's frame per Tcl semantics — a `foreach x $list +/// {}` binding is visible after the loop returns. That means adding +/// the iterator to the same scope's decl map is correct. +/// +/// **Recursion into body-hosts.** Tcl's `if`/`while`/`for`/`foreach`/ +/// `catch`/`try` bodies run in the enclosing frame — a `set foo …` +/// inside `if { … } { … }` binds `foo` in the outer scope. So we +/// reparse each braced-body argument of a body-host and recurse. +/// Only `proc` and `namespace eval` bodies open fresh frames; those +/// are handled by [`descend_scopes`], not here. +pub(crate) fn collect_decls( + cmd: &Command, + source: &str, + decls: &mut HashMap, +) { + match &cmd.kind { + CommandKind::Set => { + // 2-word `set` is a read (`set foo` returns $foo). Only + // 3+-word forms are decls. + if cmd.words.len() < 3 { + return; + } + let target = &cmd.words[1]; + let Some(name) = target.as_text() else { + return; + }; + // First decl in the scope wins — Tcl reassignment + // doesn't create a new binding, and pointing at the + // original decl is what the user recognizes when + // hunting an unused local. + decls.entry(name.to_string()).or_insert(DeclSite { + span: target.span, + kind: DeclKind::Set, + }); + } + CommandKind::Generic => { + // Head-based recognition. A command whose first word + // isn't a plain identifier (e.g. `[cmd-subst]`) has + // no head we can dispatch on — skip this stage, but + // fall through to the body-host and cmd-subst recursion + // below (which walk the WORDS regardless). + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + match head { + "foreach" => collect_foreach_decls(cmd, decls), + "upvar" => collect_upvar_decls(cmd, decls), + "catch" => collect_catch_decls(cmd, decls), + "regexp" | "regsub" => collect_regexp_decls(cmd, decls), + // `dict for {kv} DICT BODY` — the varname list + // at words[2] binds one or two locals in the + // enclosing frame, same as `foreach`. + "dict" => collect_dict_for_decls(cmd, decls), + _ => {} + } + } + } + _ => {} + } + // Body-host recursion: `if`/`while`/`foreach`/`for`/`catch`/`try`/… + // bodies run in the enclosing frame. Reparse each braced-body arg + // and recurse — a `set foo …` inside binds `foo` in this scope. + // Skip `proc` and `namespace eval` (they open fresh scopes). + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + if is_body_host(head) + && !matches!( + &cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) + { + for word in cmd.words.iter().skip(1) { + if let Some(stmts) = reparse_braced_body(word, source) { + for stmt in &stmts { + let Stmt::Command(inner) = stmt else { continue }; + collect_decls(inner, source, decls); + } + } + } + } + } + // Command-substitution recursion: `[set x 1]`, `if {[catch {…} n]} …`, + // and any other `[…]` embedded in a word runs in the enclosing + // frame. Its `set`/`catch`/`regexp`/… count as decls here. + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + for stmt in body { + let Stmt::Command(inner) = stmt else { continue }; + collect_decls(inner, source, decls); + } + } + } + } +} + +/// Extract the *local* names from an `upvar` command. Syntax: +/// `upvar [LEVEL] remote local ?remote local ...?` +/// LEVEL is optional; when present it's a bare numeric or `#N` +/// prefix on the first arg. Rather than parse it precisely, we +/// probe: if the first arg after `upvar` looks like a level +/// (leading digit or `#`), skip it; then take pairs (remote, local). +/// +/// Every `local` becomes a decl. The `remote` names are opaque — +/// they refer to an outer frame we can't see. Dynamic-remote form +/// (`upvar $var local`) is fine: we can still see the *local* half +/// literally as a decl. +pub(crate) fn collect_upvar_decls( + cmd: &Command, + decls: &mut HashMap, +) { + let mut idx = 1; + // Skip the optional LEVEL: bare numeric or `#`-prefixed. + if let Some(w) = cmd.words.get(idx) { + if let Some(t) = w.as_text() { + if t.starts_with('#') + || t.chars().next().is_some_and(|c| c.is_ascii_digit()) + { + idx += 1; + } + } + } + // Now consume (remote, local) pairs. + while idx + 1 < cmd.words.len() { + let local_word = &cmd.words[idx + 1]; + if let Some(name) = local_word.as_text() { + decls.entry(name.to_string()).or_insert(DeclSite { + span: local_word.span, + kind: DeclKind::Upvar, + }); + } + idx += 2; + } +} + +/// Extract the result-var and options-var names from a `catch`. +/// +/// Syntax: `catch script ?resultVarName? ?optionsVarName?`. Both +/// trailing args (when literal identifiers) are decls in the +/// enclosing scope — catch runs the script in the current frame +/// and stores the return value into `resultVarName`. Dynamic +/// forms (`catch script $var`) are opaque; we skip them. +pub(crate) fn collect_catch_decls( + cmd: &Command, + decls: &mut HashMap, +) { + // `catch script` (2 words) — no result var. + // `catch script name` (3 words) — name is result decl. + // `catch script name opts` (4 words) — both are decls. + for i in [2, 3] { + let Some(w) = cmd.words.get(i) else { break }; + let Some(name) = w.as_text() else { continue }; + decls.entry(name.to_string()).or_insert(DeclSite { + span: w.span, + kind: DeclKind::Set, + }); + } +} + +/// Extract capture-var names from a `regexp`/`regsub` command. +/// +/// `regexp ?switches? pattern string ?matchVar? ?subVar ...?` +/// Everything after the pattern + string that's a bare identifier +/// becomes a capture-var decl in the enclosing scope. Switches +/// (leading `-`) are skipped up to `--` or the first non-switch +/// word; the switch/pattern boundary heuristic here is: the first +/// non-`-` word is the pattern, the next is the string, and +/// everything else is a capture-var. That's the standard Tcl +/// shape; we skip precise switch-list parsing. +/// +/// `regsub` shares the same trailing-vars shape (the last arg is +/// the result-var). +pub(crate) fn collect_regexp_decls( + cmd: &Command, + decls: &mut HashMap, +) { + // Skip the command word (index 0). Skip leading switches (any + // word beginning with `-` that isn't `--`). After the pattern + // + string, the rest are capture vars. + let mut i = 1; + while let Some(w) = cmd.words.get(i) { + let Some(t) = w.as_text() else { break }; + if t == "--" { + i += 1; + break; + } + if !t.starts_with('-') { + break; + } + i += 1; + } + // i is now the pattern arg. Skip it + the string arg. + i += 2; + // Remaining words are capture-var names (bare identifiers). + while let Some(w) = cmd.words.get(i) { + if let Some(name) = w.as_text() { + decls.entry(name.to_string()).or_insert(DeclSite { + span: w.span, + kind: DeclKind::Set, + }); + } + i += 1; + } +} + +/// Extract the iterator variable(s) from a `foreach` command. +/// `foreach var $list {…}` — single var at `words[1]`. +/// `foreach {a b c} $list {…}` — multi-var brace list at `words[1]` +/// containing whitespace-separated names. +/// `foreach a $la b $lb {…}` — pairs form. We take every 2nd word +/// starting at 1 as an iterator target: words[1], words[3], … up to +/// `words.len() - 2` (last two words are the final list-value and +/// the body). +pub(crate) fn collect_foreach_decls( + cmd: &Command, + decls: &mut HashMap, +) { + if cmd.words.len() < 4 { + // `foreach var list body` minimum. Malformed: give up + // gracefully rather than emit a spurious decl. + return; + } + // The last word is the body; strip it, then every even-indexed + // remaining word (skipping the leading `foreach`) is an iter + // target. Odd-indexed remainders are the list values. + let body_idx = cmd.words.len() - 1; + let mut i = 1; + while i < body_idx { + let target = &cmd.words[i]; + add_foreach_target(target, decls); + i += 2; + } +} + +/// Extract the key/value binding names from `dict for {kv} DICT +/// BODY`. Only recognizes the `for` sub-command; other `dict` +/// forms (`dict get`, `dict set`, …) don't introduce bindings. +/// +/// The varname list at `words[2]` follows the same shape as +/// `foreach`'s first-arg target — either a bare word (rare; Tcl +/// requires exactly two names in the braced form for `dict for` +/// but the parser accepts anything) or a braced whitespace- +/// separated list. Both shapes flow through `add_foreach_target`. +pub(crate) fn collect_dict_for_decls( + cmd: &Command, + decls: &mut HashMap, +) { + // `dict for {kv} DICT BODY` needs at least 5 words. If the + // second word isn't `for`, this isn't the binding form — skip. + if cmd.words.len() < 5 { + return; + } + if cmd.words.get(1).and_then(Word::as_text) != Some("for") { + return; + } + add_foreach_target(&cmd.words[2], decls); +} + +fn add_foreach_target(target: &Word, decls: &mut HashMap) { + if target.form == WordForm::Braced { + // Multi-var brace list. The interior is a single Text part + // (the parser doesn't sub-split braced words). Whitespace- + // split and treat each token as a decl. + let Some(WordPart::Text { value, span }) = target.parts.first() else { + return; + }; + // Each token gets a fresh DeclSite whose span points at + // the containing braced word — good enough for a "the + // culprit is here" underline; sub-token spans would need + // extra parser wiring. + for name in value.split_whitespace() { + decls.entry(name.to_string()).or_insert(DeclSite { + span: *span, + kind: DeclKind::ForeachVar, + }); + } + return; + } + // Bare form: whole word is the iterator name. + let Some(name) = target.as_text() else { + return; + }; + decls.entry(name.to_string()).or_insert(DeclSite { + span: target.span, + kind: DeclKind::ForeachVar, + }); +} + +/// Walk `cmd.words` and every command substitution nested inside, +/// adding every `WordPart::VarRef` name to `uses`. If `cmd` is a +/// body-host construct (`if`/`while`/`foreach`/…), each `Braced` +/// argument is reparsed as a script fragment and its interior is +/// walked recursively — that reparse is what recovers false- +/// negatives from Slice 1 (variables used inside `if { $x > 0 } +/// { … }` etc.). +pub(crate) fn collect_uses_in_command( + cmd: &Command, + source: &str, + uses: &mut HashSet, +) { + // Scope-opening commands (`proc`, `namespace eval`) live in + // their own frame — a `$y` in a nested proc's body doesn't + // reference the outer scope's `y`. `descend_scopes` walks + // those bodies with a fresh decls/uses table; skip them here + // so the outer scope doesn't count their body-word contents + // as its own uses. + if matches!( + cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) { + return; + } + // Special case: `set foo` (exactly 2 words) is a *read* of `foo`, + // not a decl. `collect_decls` correctly ignores this shape, but + // we also need to count it here as a use so a `set foo 1; set foo` + // doesn't warn `foo` as unused. + if matches!(cmd.kind, CommandKind::Set) && cmd.words.len() == 2 { + if let Some(name) = cmd.words[1].as_text() { + uses.insert(name.to_string()); + } + } + for word in &cmd.words { + collect_uses_in_word(word, source, uses); + } + // Body-host commands hide scripts inside braced words. Reparse + // each such word and walk its statements as if they were part + // of the current scope — Tcl runs them in the current frame + // (for `if`/`while`/`foreach`/`for`/`catch`/`try` bodies at + // least), so their VarRefs count as uses here. + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if let Some(stmts) = reparse_braced_body(word, source) { + for stmt in &stmts { + let Stmt::Command(inner) = stmt else { + continue; + }; + collect_uses_in_command(inner, source, uses); + } + } + } + } + } +} + +/// If `word` is a braced word, reparse its interior as a script +/// fragment (same recipe as `hover_in_braced_bodies`). Returns +/// `None` for non-braced words (`Bare`, `Quoted`) or when the +/// interior isn't a single text part. +pub(crate) fn reparse_braced_body( + word: &Word, + source: &str, +) -> Option> { + if word.form != WordForm::Braced { + return None; + } + let WordPart::Text { value, span } = word.parts.first()? else { + return None; + }; + // Body-host bodies (`if`/`while`/`foreach`/`for`/`catch`/`try` + // {…}) are Tcl SCRIPTS — newline is a statement separator, not + // whitespace. Reparse in `Mode::Toplevel` so each `set foo` / + // `puts $bar` / etc. lands as its own Command. Using BracketBody + // here would merge every statement into a single mega-command + // whose head is the first statement's head, causing both decl + // collection AND use collection to miss everything past the + // first statement (repros against port.htcl's `foreach` body). + let (mut stmts, mut errs) = crate::parser::parse_fragment( + value.as_str(), + crate::parser::Mode::Toplevel, + ); + let delta = span.start; + for s in &mut stmts { + crate::parser::shift_stmt(s, delta); + } + // Errors from reparse would surface as duplicate parser + // diagnostics; we discard them here since the top-level + // parser has already flagged real issues. The unused-var + // pass is best-effort. + crate::parser::populate_procs(&mut stmts, source, &mut errs); + Some(stmts) +} + +fn collect_uses_in_word(word: &Word, source: &str, uses: &mut HashSet) { + for part in &word.parts { + match part { + WordPart::VarRef { name, .. } => { + // `${foo(bar)}` lands here as a single name + // `"foo(bar)"`. We record the whole string. A + // decl `set foo(bar) …` would match; a decl + // `set foo …` won't. Rare enough to defer. + uses.insert(name.clone()); + // Also record the base name (before the `(`) so + // that `${arr(key)}` counts as a use of a decl + // `set arr …` — the array-vs-scalar distinction + // is Tcl-internal, not a decl-scope question. + if let Some(paren) = name.find('(') { + uses.insert(name[..paren].to_string()); + } + } + WordPart::CmdSubst { body, .. } => { + // Nested command substitution: its interior stmts + // run in the *current* frame (Tcl semantics), so + // their VarRefs count as uses of the outer scope. + for stmt in body { + let Stmt::Command(inner) = stmt else { continue }; + collect_uses_in_command(inner, source, uses); + } + } + WordPart::Text { value, .. } => { + // Braced words (`expr { $kind eq "..." }`, + // `if { $x == 1 } {...}`, condition bodies of + // `while`/`for`, etc.) are stored as opaque Text + // — the parser doesn't split their `$var` + // substrings into VarRefs because inside `{...}` + // Tcl doesn't perform variable substitution at + // parse time. But at runtime, `expr` (and the + // implicit-`expr` bodies of `if`/`while`/`for`) + // DO interpolate variables. Scan the text for + // `$IDENT` patterns and record each as a use so + // a `set kind ""; catch { set kind [...] }; + // expr { $kind == "x" }` shape doesn't warn + // `kind` as unused. + // + // Conservative false-positive rate is fine here: + // the worst case is a literal `$foo` in a text + // that ALSO happens to match a same-named local, + // suppressing a legitimate warning. Missing real + // uses is worse (that's a spurious warning users + // learn to ignore). + if word.form == WordForm::Braced { + scan_brace_var_refs(value, uses); + } + } + WordPart::Escape { .. } => {} + } + } +} + +/// Scan `text` for `$IDENT` and `${IDENT}` occurrences and record +/// each identifier as a use. Handles arrays (`$arr(key)` → both +/// `arr(key)` and `arr`), namespace qualifiers (`$ns::x`), and +/// escaped dollars (`\$foo` → skipped). Doesn't try to be Tcl- +/// precise — the goal is to catch the common variable-reference +/// shapes in expr and control-flow bodies. +fn scan_brace_var_refs(text: &str, uses: &mut HashSet) { + let bytes = text.as_bytes(); + let mut i = 0; + let is_ident = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + while i < bytes.len() { + if bytes[i] != b'$' { + i += 1; + continue; + } + // Skip escaped `\$`. + if i > 0 && bytes[i - 1] == b'\\' { + i += 1; + continue; + } + let name_start; + let name_end; + if i + 1 < bytes.len() && bytes[i + 1] == b'{' { + // `${...}` — read until matching `}`. + name_start = i + 2; + let mut j = name_start; + while j < bytes.len() && bytes[j] != b'}' { + j += 1; + } + name_end = j; + i = j + 1; + } else { + name_start = i + 1; + let mut j = name_start; + while j < bytes.len() && is_ident(bytes[j]) { + j += 1; + } + // Include an `(...)` array-index suffix if present. + if j < bytes.len() && bytes[j] == b'(' { + while j < bytes.len() && bytes[j] != b')' { + j += 1; + } + if j < bytes.len() { + j += 1; + } + } + name_end = j; + i = j; + } + if name_end > name_start { + let name = &text[name_start..name_end]; + uses.insert(name.to_string()); + if let Some(paren) = name.find('(') { + uses.insert(name[..paren].to_string()); + } + } + } +} + +/// Emit one warning per decl whose name isn't in `uses` and isn't +/// underscore-prefixed. +fn emit_unused( + decls: &HashMap, + uses: &HashSet, + diags: &mut Vec, +) { + // Sort by span so diagnostic order is stable across runs — + // HashMap iteration order isn't. + let mut items: Vec<(&String, &DeclSite)> = decls.iter().collect(); + items.sort_by_key(|(_, d)| d.span.start); + for (name, decl) in items { + if name.starts_with('_') { + continue; + } + if uses.contains(name) { + continue; + } + let message = match decl.kind { + DeclKind::ProcArg => format!("unused proc arg '{name}'"), + DeclKind::Set => format!("unused local '{name}'"), + DeclKind::ForeachVar => { + format!("unused foreach var '{name}'") + } + DeclKind::Upvar => { + format!("unused upvar binding '{name}'") + } + }; + diags.push(Diagnostic { + severity: Severity::Warning, + message, + span: decl.span, + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn diags(src: &str) -> Vec { + let parsed = parse(src); + assert!( + parsed.errors.is_empty(), + "unexpected parse errors: {:?}", + parsed.errors + ); + let mut out = Vec::new(); + validate_unused_vars(&parsed.document, src, &mut out); + out + } + + fn warning_messages(d: &[Diagnostic]) -> Vec { + d.iter() + .filter(|dd| dd.severity == Severity::Warning) + .map(|dd| dd.message.clone()) + .collect() + } + + #[test] + fn unused_proc_arg_is_flagged() { + let src = "proc f {x} { return 1 }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused proc arg 'x'"]); + } + + #[test] + fn used_proc_arg_is_not_flagged() { + let src = "proc f {x} { return $x }\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn underscore_prefix_suppresses_arg_warning() { + let src = "proc f {_ignored} { return 1 }\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn unused_local_set_is_flagged() { + let src = "proc f {} { set y 1; return 2 }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused local 'y'"]); + } + + #[test] + fn used_local_set_is_not_flagged() { + let src = "proc f {} { set y 1; return $y }\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn underscore_prefix_suppresses_local_warning() { + let src = "proc f {} { set _tmp 1; return 2 }\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn use_inside_command_substitution_counts() { + // `$x` at the top level of `[…]` runs in the same frame — + // counts as a use. The braced `{$x + 1}` argument of `expr` + // is opaque in Slice 1; use a form where `$x` sits as a + // direct word so Slice 1's walker sees it. + let src = "proc f {x} { return [list $x] }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn nested_proc_scope_does_not_leak() { + // Outer `y` is unused. Inner proc uses its own `y` — that + // shouldn't count as a use of the outer. + let src = "\ +proc outer {} { + set y 1 + proc inner {y} { return $y } + return 2 +} +"; + let msgs = warning_messages(&diags(src)); + // Both `y`s should be OK now — outer 'y' unused (warned), + // inner 'y' used (not warned). + assert_eq!(msgs, vec!["unused local 'y'"], "{:?}", diags(src)); + } + + #[test] + fn top_level_unused_set_is_flagged() { + let src = "set foo 1\nputs hello\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused local 'foo'"]); + } + + #[test] + fn top_level_used_set_is_not_flagged() { + let src = "set foo 1\nputs $foo\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn top_level_underscore_prefix_suppresses() { + let src = "set _bar 1\nputs hi\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn two_word_set_is_read_not_decl() { + // `set foo` (2 words) *reads* $foo. It's a use, not a decl. + // So the declared `foo` from earlier IS used by the bare + // `set foo` — no warning. + let src = "set foo 1\nset foo\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn subscript_reference_counts_as_use_of_base() { + // `$arr(key)` should count as a use of `arr`. + let src = "proc f {} { set arr 1; return $arr(key) }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn arg_used_in_namespace_eval_body_counts() { + let src = "\ +namespace eval ns { + proc f {x} { return $x } +} +"; + assert!(diags(src).is_empty()); + } + + // ---------- Slice 2 tests ---------- + + #[test] + fn use_inside_if_body_is_reached() { + let src = "proc f {x} { if {1} { return $x } }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_inside_while_body_is_reached() { + let src = "proc f {x} { while {0} { puts $x } }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_inside_nested_if_else_is_reached() { + let src = "\ +proc f {x y} { + if {1} { + return $x + } else { + return $y + } +} +"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn foreach_iterator_used_in_body_is_not_flagged() { + let src = "proc f {} { foreach z {1 2 3} { puts $z } }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn foreach_iterator_unused_is_flagged() { + let src = "proc f {} { foreach z {1 2 3} { puts hi } }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused foreach var 'z'"]); + } + + #[test] + fn foreach_multi_var_all_used() { + let src = "\ +proc f {} { + foreach {a b} {1 2 3 4} { + puts $a + puts $b + } +} +"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn foreach_multi_var_partial_unused() { + let src = "\ +proc f {} { + foreach {a b} {1 2 3 4} { + puts $a + } +} +"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused foreach var 'b'"]); + } + + #[test] + fn foreach_pairs_form_all_used() { + let src = "\ +proc f {} { + foreach a {1 2} b {3 4} { + puts $a + puts $b + } +} +"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_via_expr_braced_arg_is_reached() { + // The braced `{$x > 0}` is a body-host (`if`) arg — reparse + // catches the `$x`. + let src = "proc f {x} { if {$x > 0} { puts hi } }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + // ---------- Slice 3 tests ---------- + + #[test] + fn upvar_local_used_is_not_flagged() { + let src = "proc f {} { upvar 1 remote local; return $local }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn upvar_local_unused_is_flagged() { + let src = "proc f {} { upvar 1 remote local; return 1 }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused upvar binding 'local'"]); + } + + #[test] + fn upvar_multi_pair_partial_unused() { + let src = "\ +proc f {} { + upvar 1 remoteA localA remoteB localB + return $localA +} +"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused upvar binding 'localB'"]); + } + + #[test] + fn dynamic_eval_leaks_scope() { + // Proc contains `eval $script` — we can't see the script, + // so the unused local `q` may or may not actually be + // referenced. Conservatively: no warning. + let src = "proc f {} { set q 1; eval $script }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn literal_eval_does_not_leak() { + // `eval { puts hi }` — literal body, no dynamic ref. Unused + // `q` should still warn. + let src = "proc f {} { set q 1; eval { puts hi } }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused local 'q'"], "{:?}", diags(src)); + } + + #[test] + fn dynamic_uplevel_leaks_scope() { + let src = "proc f {} { set q 1; uplevel 1 $script }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn dynamic_apply_leaks_scope() { + let src = "proc f {} { set q 1; apply $lambda 42 }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn dynamic_info_exists_leaks_scope() { + let src = "proc f {} { set q 1; info exists $name }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn nested_proc_scope_not_tainted_by_outer_leak() { + // Outer scope has `eval $script` (leaked). Inner proc + // has an unused arg `x` — that should still warn since the + // inner scope is a fresh frame and doesn't inherit the + // leak. + let src = "\ +proc outer {} { + eval $script + proc inner {x} { return 1 } +} +"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused proc arg 'x'"]); + } + + #[test] + fn use_via_expr_arg_of_expr_command() { + // Bare-form `expr $x + 1` — the `$x` word is a proper + // VarRef part, no brace-scanning needed. + let src = "proc f {x} { return [expr $x + 1] }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_via_expr_braced_arg() { + // Braced form `expr { $x + 1 }` — the `$x` inside `{...}` + // is stored as opaque text by the parser (Tcl doesn't + // substitute inside braces at parse time), but at runtime + // `expr` interpolates it. Regression against the + // `lift::vivado_property` shape where a `set kind ""` + // + `catch { set kind [...] }` + `expr { $kind eq ... }` + // was falsely flagged as unused. Scan the braced text + // for `$IDENT` patterns and count them as uses. + let src = "proc f {} {\n\ + set x 1\n\ + return [expr {$x + 1}]\n\ + }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_via_expr_braced_arg_with_string_ops() { + // Real-world shape: guard-init + reassign-in-catch + read + // in expr braces. Matches the lift.htcl `vivado_property` + // proc that was warning `unused local 'kind'`. + let src = "proc f {} {\n\ + set kind \"\"\n\ + catch { set kind hello }\n\ + return [expr {$kind eq \"string\" || $kind eq \"bool\"}]\n\ + }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_via_braced_var_with_namespace_qualifier() { + let src = "proc f {} {\n\ + set ns::x 1\n\ + return [expr {$ns::x + 1}]\n\ + }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } +} diff --git a/vw-htcl/src/validate.rs b/vw-htcl/src/validate.rs new file mode 100644 index 0000000..408b968 --- /dev/null +++ b/vw-htcl/src/validate.rs @@ -0,0 +1,5026 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Signature-aware call-site validation. +//! +//! Builds a {proc_name → ProcSignature} table from the top-level +//! procs in a document, then walks every call site in the same +//! document and checks the keyword arguments against the declared +//! signature. Diagnostics are language-neutral; downstream (the LSP, +//! `vw check`) maps them to the appropriate display form. + +use std::collections::HashMap; + +use crate::ast::{ + Attribute, AttributeValue, Command, CommandKind, Document, EnumDecl, + OverloadInfo, OverloadVariant, Proc, ProcArg, ProcSignature, Stmt, + TypeDecl, TypeExpr, Word, WordPart, +}; +use crate::span::Span; + +/// Side-table produced alongside the signature table by +/// [`build_signature_table_with_overloads`]. Maps each public proc +/// name that resolves to an enum-overload set to its [`OverloadInfo`]. +/// Names not in this map are regular (non-overloaded) procs. +pub type OverloadTable = HashMap; + +/// Mangle a specialization's internal name. The `__` prefix is +/// reserved (the validator rejects user procs whose names start +/// with `__`) so mangled names don't collide with anything +/// user-written. +/// +/// For namespaced public names (`Property::as_nested`), the +/// prefix goes on the LEAF, not the whole name — otherwise the +/// mangled form (`__Property::as_nested__Nested`) puts the proc +/// in a fictional `__Property` namespace Tcl hasn't created, and +/// `proc` errors with "unknown namespace." Keeping the leaf-only +/// prefix (`Property::__as_nested__Nested`) places the +/// specialization inside the SAME namespace as its public +/// dispatcher, which the enum prelude or user `namespace eval` +/// already declared. +pub fn mangle_specialization(public_name: &str, variant: &str) -> String { + match public_name.rsplit_once("::") { + Some((ns, leaf)) => format!("{ns}::__{leaf}__{variant}"), + None => format!("__{public_name}__{variant}"), + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Severity { + Error, + Warning, +} + +#[derive(Clone, Debug)] +pub struct Diagnostic { + pub severity: Severity, + pub message: String, + pub span: Span, +} + +pub fn validate(document: &Document, source: &str) -> Vec { + validate_with_signatures(document, source, &HashMap::new()) +} + +/// Same as [`validate`], but resolves unknown calls against an +/// additional pool of signatures supplied by the caller — used by +/// the REPL to make procs declared in earlier session batches +/// visible to a new batch without re-parsing the whole prelude. +/// +/// Merge rules: +/// +/// - The document's own signatures shadow `extra`. Redefining a +/// proc in `document` overrides the prior version (Tcl +/// semantics — a second `proc` redefines). +/// - Duplicate-definition diagnostics only fire for collisions +/// **within** `document`. A new batch that re-`src`s a wrapper +/// already loaded earlier shouldn't warn on every input. +pub fn validate_with_signatures<'doc>( + document: &'doc Document, + source: &str, + extra: &HashMap, +) -> Vec { + validate_with_extras(document, source, extra, &HashMap::new()) +} + +/// Full validation entry point: same as [`validate_with_signatures`] +/// but also takes a pool of newtype declarations from prior session +/// batches. Lets the REPL drop a `proc bd_cell::repr` in batch N +/// without re-tripping "type bd_cell missing repr" diagnostics for +/// the `type bd_cell = string` declaration in batch N-1. +pub fn validate_with_extras<'doc>( + document: &'doc Document, + source: &str, + extra_sigs: &HashMap, + extra_types: &HashMap, +) -> Vec { + validate_with_all_extras( + document, + source, + extra_sigs, + extra_types, + &HashMap::new(), + ) +} + +/// Full validation entry point. Accepts a prior-batch pool of +/// signatures, type declarations, AND enum declarations, so the +/// REPL can split an `enum E = …` decl across batches from the +/// procs that dispatch on it. +pub fn validate_with_all_extras<'doc>( + document: &'doc Document, + source: &str, + extra_sigs: &HashMap, + extra_types: &HashMap, + extra_enums: &HashMap, +) -> Vec { + validate_with_all_extras_and_vars( + document, + source, + extra_sigs, + extra_types, + extra_enums, + &std::collections::HashSet::new(), + &std::collections::HashSet::new(), + ) +} + +/// Same as [`validate_with_all_extras`], plus: +/// +/// - `extra_top_level_vars` — top-level variable names known to +/// be defined in prior batches. The undef-variable pass merges +/// these into its top-level decl set, so a `set p …` in REPL +/// batch N-1 makes `$p` in batch N legal. Proc-body scopes +/// ignore the pool (Tcl locals don't inherit top-level scope), +/// so this only affects the document's own top-level statements. +/// - `extra_dep_names` — workspace-dependency names the caller +/// registered with its `vw_htcl::Resolver`. Each `src @` +/// statement in the document is checked against this pool; a +/// name that's not in the set fires an `Error` diagnostic +/// spanned to the `@` text. Empty set → the check +/// no-ops (unit tests and non-workspace-aware callers). +pub fn validate_with_all_extras_and_vars<'doc>( + document: &'doc Document, + source: &str, + extra_sigs: &HashMap, + extra_types: &HashMap, + extra_enums: &HashMap, + extra_top_level_vars: &std::collections::HashSet, + extra_dep_names: &std::collections::HashSet, +) -> Vec { + let mut diags = Vec::new(); + // Type-table FIRST — its keys feed the overloaded-proc-arm + // detection in `build_signature_table_with_overloads` so a proc + // whose first arg is a newtype-Qualified name (e.g. + // `-config: versal_cips::PsPmcConfig` inside `namespace eval + // versal_cips`) isn't misclassified as an enum-overload arm. + // Duplicate-decl diagnostics from this pass are held aside and + // re-emitted after signature collection so ordering matches the + // pre-refactor rendering. + let mut type_prescan_diags = Vec::new(); + let mut type_table = + build_type_decl_table(document, &mut type_prescan_diags); + for (name, td) in extra_types { + type_table.entry(name.clone()).or_insert(*td); + } + let newtype_qualified_names: std::collections::HashSet = + type_table.keys().cloned().collect(); + let (mut table, _overloads) = build_signature_table_with_overloads( + document, + &newtype_qualified_names, + &mut diags, + ); + // Prior-batch signatures fill in the gaps. The doc's own entries + // win because `entry().or_insert(...)` is a no-op on present keys. + for (name, sig) in extra_sigs { + table.entry(name.clone()).or_insert(*sig); + } + diags.extend(type_prescan_diags); + let mut enum_table = build_enum_decl_table(document, &mut diags); + for (name, ed) in extra_enums { + enum_table.entry(name.clone()).or_insert(*ed); + } + validate_type_decl_triplets(&type_table, &table, &mut diags); + validate_enum_decls(&enum_table, &type_table, &mut diags); + // Set of every declared newtype's qualified name — passed to + // the qualified-position validator so `Qualified` references + // that resolve to a real newtype pass through instead of + // hitting the enum-variant-focused reject. + let newtype_names: std::collections::HashSet = + type_table.keys().cloned().collect(); + validate_qualified_positions(document, &newtype_names, &mut diags); + let mut var_table = VarTypeTable::new(); + // Build a proc table once so the return-type check can walk + // any proc's body without re-scanning the document per call. + let proc_table = build_proc_table(document); + validate_stmts( + &document.stmts, + source, + &table, + &proc_table, + &newtype_qualified_names, + &mut var_table, + &mut diags, + ); + // Undefined-variable check. Errors (fail `vw check` / red LSP + // squiggle), mirror shape to unused-var pass but with the + // set-operation flipped. `extra_top_level_vars` seeds the + // top-level decl set so REPL batches see prior-batch vars. + crate::undefined::validate_undefined_vars_with_extras( + document, + source, + extra_top_level_vars, + &mut diags, + ); + // Warning-level pass: unused-variable check. Runs last so the + // hard-error diagnostics keep priority visually and any short- + // circuit in earlier passes is unaffected by the walk here. + crate::unused::validate_unused_vars(document, source, &mut diags); + // Undefined-src-module check. `src @` where `` + // isn't a registered dep name in the caller's `Resolver` fires + // a spanned Error diagnostic; without this the LSP silently + // drops the import (workspace.rs::collect_imports) and `vw + // check` only surfaces the failure via the loader's hard-abort + // path (no span). The check no-ops when `extra_dep_names` is + // empty — unit tests and non-workspace callers skip it. + validate_src_imports(document, extra_dep_names, &mut diags); + validate_test_attributes(document, &mut diags); + diags +} + +/// `@test` semantic checks. Fires warnings (not errors) so +/// misused tags surface in the LSP + `vw check` without blocking +/// execution. Rules: +/// +/// - `@test(X)` where X isn't the literal ident `dedicated-eda` +/// (the only recognized value today). +/// - `@test` on a proc with a non-empty parameter list — tests +/// are zero-arg for the MVP runner. +/// - `@test` on a nested proc (declared inside another proc's +/// body) — only top-level `@test` procs are discoverable by +/// `vw test`. +/// +/// Doesn't check for `@test` on non-proc statements — that's +/// caught at parse time with a spanned error. +fn validate_test_attributes(document: &Document, diags: &mut Vec) { + walk_procs_for_test_check( + &document.stmts, + /*inside_proc=*/ false, + diags, + ); +} + +fn walk_procs_for_test_check( + stmts: &[Stmt], + inside_proc: bool, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(attr) = proc.attribute("test") { + if inside_proc { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test` on a nested proc — only \ + top-level `@test`-annotated procs \ + are discoverable by `vw test`" + .into(), + span: attr.span, + }); + } + let mut has_dedicated = false; + let mut target_key_present = false; + let mut variant_key_present = false; + for value in &attr.values { + match value { + crate::ast::AttributeValue::Ident { + value: v, + .. + } if v == "dedicated-eda" => { + has_dedicated = true; + } + crate::ast::AttributeValue::Keyed { + key, .. + } if key == "target" => { + target_key_present = true; + } + crate::ast::AttributeValue::Keyed { + key, .. + } if key == "variant" => { + variant_key_present = true; + } + crate::ast::AttributeValue::Keyed { + key, .. + } => { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "`@test(...)` — unrecognized key \ + `{key}` (recognized: `target=`, \ + `variant=`)" + ), + span: attr.span, + }); + } + _ => { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "`@test(…)` value must be the \ + `dedicated-eda` marker, \ + `target=`, or `variant=`; \ + got `{}`", + render_attribute_value(value), + ), + span: attr.span, + }); + } + } + } + if target_key_present && !has_dedicated { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test(target=…)` requires the \ + `dedicated-eda` marker — shared-bucket \ + tests cannot override the \ + auto-project's `-part`" + .into(), + span: attr.span, + }); + } + if variant_key_present && !has_dedicated { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test(variant=…)` requires the \ + `dedicated-eda` marker — shared-bucket \ + tests cannot switch design surfaces" + .into(), + span: attr.span, + }); + } + if target_key_present && variant_key_present { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test(...)` — pick one of \ + `target=` or `variant=`, \ + not both (variants own their parts)" + .into(), + span: attr.span, + }); + } + if let Some(sig) = &proc.signature { + if !sig.args.is_empty() { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test` procs must take zero \ + arguments — parameterized tests \ + aren't supported yet" + .into(), + span: attr.span, + }); + } + } + } + walk_procs_for_test_check(&proc.body, true, diags); + } + CommandKind::NamespaceEval(ns) => { + walk_procs_for_test_check(&ns.body, inside_proc, diags); + } + _ => {} + } + } +} + +fn render_attribute_value(v: &crate::ast::AttributeValue) -> String { + v.to_tcl_literal() +} + +/// Walk every top-level `src @` statement in `document` and +/// emit an Error diagnostic for any `` not present in +/// `known_deps`. Relative and absolute path imports (non-`@` +/// forms) are skipped — those get their existence checked +/// downstream by the loader's filesystem probe. +fn validate_src_imports( + document: &Document, + known_deps: &std::collections::HashSet, + diags: &mut Vec, +) { + // The empty-set case covers unit tests (no workspace) and + // downstream callers that don't hook up a Resolver. Short- + // circuit rather than walking every statement for nothing. + if known_deps.is_empty() { + return; + } + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(src) = &cmd.kind else { + continue; + }; + // Missing path (contains `$var` / `[cmd]` substitution) — + // handled by other passes; don't double-flag here. + let Some(path) = src.path.as_deref() else { + continue; + }; + let classified = crate::src_path::classify(path); + let crate::src_path::PathKind::Named { name, subpath } = + classified.kind + else { + continue; + }; + if known_deps.contains(&name) { + continue; + } + // Message mirrors `ResolveError::UnknownDependency`'s text + // in `src_path.rs` — same hint keeps the CLI hard-abort + // path (which still fires) and the analyzer diagnostic + // pointing at the same fix. + let subpath_hint = if subpath.is_empty() { + String::new() + } else { + format!("/{subpath}") + }; + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "unknown src module `{name}` in `src @{name}{subpath_hint}`; \ + add a `[dependencies.{name}]` entry to your workspace's \ + vw.toml or run `vw add` to fetch it" + ), + span: src.path_span, + }); + } +} + +/// Validate every command in `stmts`, descending into proc bodies so +/// that calls nested inside a proc are checked just like top-level +/// ones. The signature table is document-wide, so a call resolves to +/// its (top-level) proc at any depth. +/// Variable-type table keyed by name. Populated as `validate_stmts` +/// walks `set VAR ` and proc-parameter bindings; consulted by +/// `value_type` when it hits a `$var` reference at a call site. +/// +/// Nominal / strict: entries hold the DECLARED `TypeExpr` (from a +/// proc's `return_type` or a `ProcArg.type_annotation`) without any +/// alias-walking. Comparing two entries via [`types_match`] gives +/// newtype identity — `Quad0Ch1Props` and `Quad1Ch0Props` are +/// distinct even though both alias to `Properties`. +/// +/// Scope discipline: each proc body owns its own table (created in +/// the `CommandKind::Proc` arm of `validate_stmts`). Nested +/// `namespace eval` blocks share the enclosing table (matches Tcl +/// semantics — `namespace eval` creates a namespace but doesn't +/// open a new local-variable scope). Bodies of `[ … ]` command +/// substitutions also share the enclosing table so `set` inside +/// brackets is visible outside. +pub(crate) type VarTypeTable = HashMap; + +/// Infer the type of a value word — the argument on the right of a +/// `-flag` at a call site, or the RHS of a `set VAR`. +/// +/// Covers the two forms the call-site type-check actually needs: +/// +/// - A whole-word command substitution `[proc-call …]` returns the +/// called proc's `return_type`. Multi-command bodies (`[a; b]`) +/// take the LAST command's return type — matches Tcl's "value of +/// the last command wins" for `[…]` substitution. +/// - A whole-word variable reference `$foo` / `${foo}` returns the +/// type recorded in `var_table` (from a prior `set` or a proc +/// parameter with a `type_annotation`). +/// +/// Anything else — literals, quoted strings, mixed compounds like +/// `prefix-$var` — returns `None`. Callers treat `None` as "unknown +/// type, skip the check" (gradual typing). We don't error on what +/// we can't infer. +pub(crate) fn value_type( + word: &crate::ast::Word, + sig_table: &HashMap, + var_table: &VarTypeTable, +) -> Option { + value_type_with_procs(word, sig_table, var_table, None) +} + +/// Companion to [`value_type`] that also has access to a proc +/// table for return-type inference on unannotated procs. When a +/// `[proc-call]` word hits a signature whose `return_type` is +/// `None`, the caller can supply the corresponding [`Proc`] node +/// via `proc_table` and this function walks the body's last +/// `return` statement to infer the type — handles the common +/// pattern where a user proc doesn't declare a return type but +/// its body ends with `return $x` or `return [typed_proc]`. +pub(crate) fn value_type_with_procs( + word: &crate::ast::Word, + sig_table: &HashMap, + var_table: &VarTypeTable, + proc_table: Option<&HashMap>, +) -> Option { + use crate::ast::{Stmt, TypeExpr, WordPart}; + match word.parts.as_slice() { + [WordPart::VarRef { name, .. }] => var_table.get(name).cloned(), + [WordPart::CmdSubst { body, .. }] => { + let last_cmd = body.iter().rev().find_map(|s| match s { + Stmt::Command(c) => Some(c), + _ => None, + })?; + let call_name = last_cmd.words.first()?.as_text()?; + let sig = sig_table.get(call_name)?; + // Annotated return type wins. + if let Some(ty) = &sig.return_type { + return Some(ty.clone()); + } + // Fallback: walk the proc body to infer. Only fires + // when a proc_table is supplied — top-level callers + // (per-batch var-type builders, putr rewrite) pass one + // through; internal callers that just want fast + // annotation-based lookup pass `None`. + let procs = proc_table?; + let proc = procs.get(call_name)?; + infer_return_type_from_body(proc, sig_table, procs) + } + // Bare `true` / `false` literals — the ONLY textual values + // whose type we infer, and only because they're the + // canonical HTCL bool literals. Everything else (bare + // words, quoted strings, mixed compounds) stays untyped + // (gradual typing). Position matters: this makes the + // check symmetric so `set flag true` binds `flag: bool` + // and a subsequent `-slot $flag` at a `bool` arg matches. + [WordPart::Text { value, .. }] + if value == "true" || value == "false" => + { + Some(TypeExpr::Named { + name: "bool".into(), + span: word.span, + }) + } + _ => None, + } +} + +/// Validate that every `return X` statement in `proc`'s body +/// produces a value whose type matches the proc's declared +/// return type. Only fires when the proc has a `return_type` +/// annotation (untyped procs skip the check entirely). +/// +/// Descends into the braced bodies of `if`/`elseif`/`else`/ +/// `while`/`for`/`foreach`/`catch` — a `return` buried inside +/// an early-exit branch still gets checked. Nested control +/// blocks recurse through the same walker so an arbitrary +/// depth of `if { if { return X } }` still catches wrong-typed +/// returns. +/// +/// Bare `return` (no argument) in an annotated proc is a hard +/// error — the annotation is a promise to produce a value. +pub(crate) fn validate_proc_returns( + proc: &crate::ast::Proc, + source: &str, + sig_table: &HashMap, + proc_table: &HashMap, + newtype_names: &std::collections::HashSet, + diags: &mut Vec, +) { + // Newtype-triplet exemption: `T::from`, `T::to`, `T::repr`, + // `T::empty` are compiler-emitted (or generator-emitted) + // identity conversions between a newtype and its underlying. + // Under strict nominal identity, `return $v` where `$v: string` + // in a proc returning `T` would flag as a mismatch — but + // that's the WHOLE POINT of the from/to/repr layer: cross + // the newtype boundary via `return $v` (Tcl-level identity). + // Skip the check when the proc name is `T::` for a + // declared newtype `T` and a triplet suffix. Applies to both + // annotated and unannotated forms — hand-written triplets + // often omit the annotation and rely on the identity-shape. + if let Some(name) = proc.name.as_deref() { + if is_newtype_triplet_name(name, newtype_names) { + return; + } + } + let Some(declared) = &proc.return_type else { + // No return-type annotation → the proc is implicitly a + // side-effect-only op. `return X` in a side-effect proc is + // a structural mismatch: the caller has nothing to receive + // it, and the missing annotation tells readers the same. Flag + // every `return X` with a value. Bare `return` is fine. + walk_returns_without_annotation(&proc.body, source, diags); + return; + }; + // Enum-overload-arm exemption: `proc f {v: E::A} string { … }` + // is the specialization shape for the overload dispatcher — + // `v` is the enum variant's payload, which at Tcl runtime is + // just the underlying type's raw value (a `string` for + // `E::A: string`). Returning it as its underlying is + // structurally identity, same rationale as the newtype + // triplet. Detect by first-arg type being `Qualified` — + // that's the only shape overload arms use. + if let Some(sig) = &proc.signature { + if let Some(first) = sig.args.first() { + if matches!( + first.type_annotation, + Some(crate::ast::TypeExpr::Qualified { .. }) + ) { + return; + } + } + } + // Seed a var table with typed parameters. Walker updates it + // as it visits `set` bindings so downstream `return $VAR` + // resolves via the same scope-aware inference the outer + // arg-type check uses. + let mut local_vars = VarTypeTable::new(); + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + local_vars.insert(arg.name.clone(), ty.clone()); + } + } + } + walk_returns( + &proc.body, + source, + sig_table, + proc_table, + &mut local_vars, + declared, + diags, + ); + + // Must-return: an annotated proc that isn't `unit` must + // reach a `return` on every path (or end with a + // last-expression whose type matches the annotation, per + // Tcl's implicit-return rule). Runs AFTER walk_returns so + // the per-return type errors surface first if both fire. + let is_unit = matches!( + declared, + crate::ast::TypeExpr::Named { name, .. } if name == "unit" + ); + if !is_unit { + // walk_returns mutated local_vars — snapshot a fresh + // seed for the must-return pass so we start from the + // proc's typed parameters, matching the walker's own + // starting state. + let mut fresh_vars = VarTypeTable::new(); + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + fresh_vars.insert(arg.name.clone(), ty.clone()); + } + } + } + if !paths_always_return( + &proc.body, + source, + declared, + sig_table, + proc_table, + &mut fresh_vars, + ) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "proc annotated `{}` may fall through without \ + returning a value of the right type; every code \ + path must end with `return $X` (or a final \ + expression whose type matches)", + render_type_inline(declared), + ), + span: proc.name_span, + }); + } + } +} + +/// True when `proc_name` matches the newtype-triplet pattern +/// `::` where `T` is a declared newtype and `suffix` +/// is one of `from`, `to`, `repr`, `empty`. These procs are +/// identity conversions across the newtype boundary, so their +/// `return $v` bodies would trip the strict-nominal check by +/// design; the check exempts them. +fn is_newtype_triplet_name( + proc_name: &str, + newtype_names: &std::collections::HashSet, +) -> bool { + for suffix in ["from", "to", "repr", "empty"] { + let marker = format!("::{suffix}"); + if let Some(prefix) = proc_name.strip_suffix(&marker) { + if newtype_names.contains(prefix) { + return true; + } + } + } + false +} + +/// Recursive walker used by [`validate_proc_returns`]. Tracks +/// `set` bindings, records `return X` statements it finds, and +/// descends into parsed control-flow bodies. +fn walk_returns( + stmts: &[crate::ast::Stmt], + source: &str, + sig_table: &HashMap, + proc_table: &HashMap, + var_table: &mut VarTypeTable, + declared: &crate::ast::TypeExpr, + diags: &mut Vec, +) { + use crate::ast::{Stmt, WordForm}; + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // Track `set VAR ` bindings the same way the + // rewrite walker does — later `return $VAR` needs to see + // the type. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type_with_procs( + value_word, + sig_table, + var_table, + Some(proc_table), + ) { + var_table.insert(name.to_string(), ty); + } + } + } + } + // `return X` — the actual check. + let head_text = cmd.words.first().and_then(|w| w.as_text()); + if head_text == Some("return") { + check_return( + cmd, sig_table, proc_table, var_table, declared, diags, + ); + } + // Descend into control-flow braced bodies. Heuristic: + // for known control-flow heads, parse every WordForm:: + // Braced word as a candidate body. Condition-shaped + // braces (like `if {$x == 1}`) parse without errors but + // don't contain `return` calls, so they contribute + // nothing — a benign no-op. `for INIT COND NEXT BODY`'s + // INIT and NEXT can hold `set` calls that would affect + // var_table if we tracked them; we don't, since the + // walker isn't a live-execution simulator and Tcl's + // control-flow scope semantics are already muddy. + if matches!( + head_text, + Some( + "if" | "elseif" + | "else" + | "while" + | "for" + | "foreach" + | "catch" + ) + ) { + for word in cmd.words.iter().skip(1) { + if word.form != WordForm::Braced { + continue; + } + // The word's span covers `{...}` including the + // outer braces. Strip 1 byte from each end for + // the interior text; parse as a fragment; shift + // spans by the interior start. + let word_start = word.span.start as usize; + let word_end = word.span.end as usize; + if word_end <= word_start + 2 { + // `{}` — empty body, nothing to check. + continue; + } + let interior_start = word_start + 1; + let interior_end = word_end - 1; + let body_text = &source[interior_start..interior_end]; + let (mut body_stmts, _errs) = crate::parser::parse_fragment( + body_text, + crate::parser::Mode::Toplevel, + ); + for s in &mut body_stmts { + crate::parser::shift_stmt(s, interior_start as u32); + } + // Populate procs INSIDE the parsed body so nested + // structures behave — mostly irrelevant for + // returns but keeps recursion consistent. + crate::parser::populate_procs( + &mut body_stmts, + source, + &mut Vec::new(), + ); + walk_returns( + &body_stmts, + source, + sig_table, + proc_table, + var_table, + declared, + diags, + ); + } + } + } +} + +/// Walk `stmts` looking for `return X` statements with a value in a +/// proc that has NO declared return type. Every such `return X` is +/// an error — the proc's shape declares "side effects only," and a +/// value-carrying return contradicts that. +/// +/// Descends into control-flow braced bodies the same way +/// [`walk_returns`] does — a value-return buried in an `if`/`else` +/// branch is caught. Bare `return` is fine (side-effect procs +/// naturally use it for early exits). +fn walk_returns_without_annotation( + stmts: &[crate::ast::Stmt], + source: &str, + diags: &mut Vec, +) { + use crate::ast::{Stmt, WordForm}; + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let head_text = cmd.words.first().and_then(|w| w.as_text()); + if head_text == Some("return") && cmd.words.len() >= 2 { + diags.push(Diagnostic { + severity: Severity::Error, + message: "`return` with a value in a proc that has no \ + declared return type — add a return-type \ + annotation (`proc NAME { args } TYPE { ... }`) \ + or drop the returned value" + .to_string(), + span: cmd.span, + }); + } + if matches!( + head_text, + Some( + "if" | "elseif" + | "else" + | "while" + | "for" + | "foreach" + | "catch" + ) + ) { + for word in cmd.words.iter().skip(1) { + if word.form != WordForm::Braced { + continue; + } + let word_start = word.span.start as usize; + let word_end = word.span.end as usize; + if word_end <= word_start + 2 { + continue; + } + let interior_start = word_start + 1; + let interior_end = word_end - 1; + let body_text = &source[interior_start..interior_end]; + let (mut body_stmts, _errs) = crate::parser::parse_fragment( + body_text, + crate::parser::Mode::Toplevel, + ); + for s in &mut body_stmts { + crate::parser::shift_stmt(s, interior_start as u32); + } + crate::parser::populate_procs( + &mut body_stmts, + source, + &mut Vec::new(), + ); + walk_returns_without_annotation(&body_stmts, source, diags); + } + } + } +} + +/// Check a single `return X` (or bare `return`) against the +/// declared return type. Emits diagnostics directly. +fn check_return( + cmd: &crate::ast::Command, + sig_table: &HashMap, + proc_table: &HashMap, + var_table: &VarTypeTable, + declared: &crate::ast::TypeExpr, + diags: &mut Vec, +) { + // `return` alone — bare — violates the annotation's promise + // to produce a value UNLESS the declared type is `unit`, + // which means "no meaningful value" and matches bare-return + // semantics. Common in side-effecting procs that early-out. + let Some(arg) = cmd.words.get(1) else { + let is_unit = matches!( + declared, + crate::ast::TypeExpr::Named { name, .. } if name == "unit" + ); + if !is_unit { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "bare `return` in proc annotated `{}` — the return \ + type requires a value; use `return $X`", + render_type_inline(declared), + ), + span: cmd.span, + }); + } + return; + }; + // Try to infer the returned expression's type. If we can't, + // skip — gradual typing (same policy as the arg-type check). + let Some(actual) = + value_type_with_procs(arg, sig_table, var_table, Some(proc_table)) + else { + return; + }; + if !types_match(declared, &actual) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "return type mismatch: proc declared `{}`, but this \ + `return` produces `{}`", + render_type_inline(declared), + render_type_inline(&actual), + ), + span: cmd.span, + }); + } +} + +/// Whether every code path through `stmts` reaches an explicit +/// `return`, OR ends with a final statement whose result type +/// matches `declared` (Tcl's implicit-last-expression return). +/// +/// Empty `stmts` → false. This is the primary failure case for +/// the must-return check: a proc annotated with a non-`unit` +/// return type whose body is completely empty (or ends with a +/// side-effecting `puts`) has no path that produces a value. +/// +/// Descends into control-flow braced bodies via +/// `parser::parse_fragment` + `shift_stmt` — the same reparse +/// dance `walk_returns` already uses (see the identical pattern +/// at line ~570). +/// +/// Coverage of control commands: +/// +/// - `return` (any form) → this path terminates. +/// - `set VAR X` → records VAR's inferred type in `local_vars` +/// so a downstream implicit-last-expression `$VAR` gets typed. +/// - `if COND BODY [elseif COND BODY]* [else BODY]` → terminates +/// iff there IS an `else` AND every branch body terminates. +/// - `while` / `for` / `foreach` → conservative false (body may +/// not execute at runtime). +/// - `switch X { pat body pat body … [default body] }` → +/// terminates iff a `default` arm exists AND every arm's body +/// terminates. Fallthrough arms (`pat -`) inherit the next +/// arm's body. +/// - `catch` → conservative false (body may error out). +/// - Anything else → non-terminating individually; keep scanning. +fn paths_always_return( + stmts: &[Stmt], + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &mut VarTypeTable, +) -> bool { + if stmts.is_empty() { + return false; + } + let mut last_cmd_index: Option = None; + for (i, stmt) in stmts.iter().enumerate() { + let Stmt::Command(cmd) = stmt else { continue }; + let head_text = cmd.words.first().and_then(|w| w.as_text()); + + // Track `set VAR ` bindings so a trailing `$VAR` + // (implicit last-expression) sees the right type. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type_with_procs( + value_word, + sig_table, + local_vars, + Some(proc_table), + ) { + local_vars.insert(name.to_string(), ty); + } + } + } + } + last_cmd_index = Some(i); + + // Explicit `return` — the path terminates here. + if head_text == Some("return") { + return true; + } + // `error` — unwinds the stack, so control never falls + // through. Counts as terminating for the must-return + // analysis (the proc can't reach its end after this). + if head_text == Some("error") { + return true; + } + // `if` — check if all branches terminate AND an `else` exists. + if head_text == Some("if") + && if_command_terminates( + cmd, source, declared, sig_table, proc_table, local_vars, + ) + { + return true; + } + // `switch` — check for `default` arm and all-arm termination. + if head_text == Some("switch") + && switch_command_terminates( + cmd, source, declared, sig_table, proc_table, local_vars, + ) + { + return true; + } + // `try { body } [on ... handler]* [finally script]` — + // terminates iff the body AND every handler + // path-terminate. This is what the generator's + // wrap-body pattern emits (`try { return X } on error + // { error "prefix.$msg" }`). + if head_text == Some("try") + && try_command_terminates( + cmd, source, declared, sig_table, proc_table, local_vars, + ) + { + return true; + } + // `while` / `for` / `foreach` / `catch` — never + // guaranteed to run their body, so they can't be sole + // terminators. Keep scanning. + } + + // Implicit-last-expression rule: if we've made it here + // without hitting a `return`, look at the last command's + // last word. If that word's type matches `declared`, this + // path counts as terminating. + let Some(last_idx) = last_cmd_index else { + return false; + }; + let Stmt::Command(last_cmd) = &stmts[last_idx] else { + return false; + }; + let expr_word = if last_cmd.words.len() == 1 { + last_cmd.words.first() + } else { + // A multi-word command (e.g. `set _ [...]; $_`) parses + // as a single `$_` command with one word — but a + // multi-word command like `puts $x` has 2 words. Use + // the FIRST word only when there's a single word (a + // pure `$var` or `[proc-call]` expression); otherwise + // treat the last statement as an expression via its + // *head* word only when that head IS a proc-call — + // handled below. + last_cmd.words.first() + }; + let Some(expr_word) = expr_word else { + return false; + }; + // For a bare command like `some_typed_proc arg1 arg2`, we + // want to check the CALL's return type. `value_type_with_procs` + // won't help directly on the head word (it's a bare-text + // word, not a CmdSubst). But we CAN look up the head in the + // sig_table. If the head is a known proc AND its return + // type matches `declared`, treat as implicit return. + if let Some(head) = last_cmd.words.first().and_then(|w| w.as_text()) { + // `extern::name` is the caller's opt-out: "this is a raw + // Tcl proc; I'm not declaring its type, trust me." Same + // policy as `validate_command`'s check at line 1946. + // Trailing extern call = trust for the must-return check. + if crate::lower::is_extern_call(head) { + return true; + } + if let Some(sig) = sig_table.get(head) { + if let Some(ret_ty) = &sig.return_type { + if types_match(declared, ret_ty) { + return true; + } + } + } + } + // Otherwise: try value_type_with_procs on the expression + // word itself (handles `$var` and `[proc-call]` shapes). + if let Some(ty) = value_type_with_procs( + expr_word, + sig_table, + local_vars, + Some(proc_table), + ) { + if types_match(declared, &ty) { + return true; + } + } + false +} + +/// Does a single `if COND BODY [elseif COND BODY]* [else BODY]` +/// command terminate? Yes iff every branch body terminates AND +/// there IS an `else` (a chain with no `else` may fall through). +/// +/// Word layout, positions counted from 0: +/// - [0] = "if" +/// - [1] = condition +/// - [2] = body +/// - [3] = "elseif" | "else" (or end) +/// - [4] = condition (after elseif) | body (after else) +/// - … +fn if_command_terminates( + cmd: &crate::ast::Command, + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &VarTypeTable, +) -> bool { + let mut i = 1; + let mut has_else = false; + let mut branch_bodies: Vec<&crate::ast::Word> = Vec::new(); + while i < cmd.words.len() { + // Skip the condition word. + if i + 1 >= cmd.words.len() { + return false; + } + branch_bodies.push(&cmd.words[i + 1]); + i += 2; + if i >= cmd.words.len() { + break; + } + match cmd.words[i].as_text() { + Some("elseif") => { + i += 1; + // Loop continues with i at condition position. + } + Some("else") => { + has_else = true; + i += 1; + if i >= cmd.words.len() { + return false; + } + branch_bodies.push(&cmd.words[i]); + break; + } + _ => { + // Something unexpected — conservative: don't + // treat as terminating. + return false; + } + } + } + if !has_else { + return false; + } + for body_word in branch_bodies { + if !branch_body_terminates( + body_word, source, declared, sig_table, proc_table, local_vars, + ) { + return false; + } + } + true +} + +/// Does a `switch X { pat body … [default body] }` terminate? +/// Yes iff there's a `default` arm AND every arm's body +/// terminates. Fallthrough arms (`pat -` where the body word is +/// literally `-`) inherit the next arm's body. +fn switch_command_terminates( + cmd: &crate::ast::Command, + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &VarTypeTable, +) -> bool { + // Find the switch body — the LAST braced word in the command + // (the argument list before it is `[options] value`, which + // we don't parse). + let body_word = cmd + .words + .iter() + .rev() + .find(|w| w.form == crate::ast::WordForm::Braced); + let Some(body_word) = body_word else { + return false; + }; + let word_start = body_word.span.start as usize; + let word_end = body_word.span.end as usize; + if word_end <= word_start + 2 { + return false; + } + let interior_start = word_start + 1; + let interior_end = word_end - 1; + let body_text = &source[interior_start..interior_end]; + let (mut arm_stmts, _errs) = + crate::parser::parse_fragment(body_text, crate::parser::Mode::Toplevel); + for s in &mut arm_stmts { + crate::parser::shift_stmt(s, interior_start as u32); + } + // Each arm parses as one command with words = [pat, body]. + // Collect them as pairs, resolving fallthrough (`pat -`) + // arms to the next arm's body. + let mut pairs: Vec<(String, &crate::ast::Word)> = Vec::new(); + let mut pending_pats: Vec = Vec::new(); + for stmt in &arm_stmts { + let Stmt::Command(arm_cmd) = stmt else { + continue; + }; + if arm_cmd.words.len() < 2 { + return false; + } + let pat = match arm_cmd.words[0].as_text() { + Some(p) => p.to_string(), + None => return false, + }; + let body_arg = &arm_cmd.words[1]; + if body_arg.as_text() == Some("-") { + // Fallthrough: this pattern inherits the next + // resolved arm's body. + pending_pats.push(pat); + continue; + } + // Resolve pending fallthroughs to this arm's body too. + for pending in pending_pats.drain(..) { + pairs.push((pending, body_arg)); + } + pairs.push((pat, body_arg)); + } + if !pending_pats.is_empty() { + // Trailing `pat -` without a resolving arm — malformed. + return false; + } + // Must have a `default` arm. + let has_default = pairs.iter().any(|(p, _)| p == "default"); + if !has_default { + return false; + } + for (_pat, body_word) in &pairs { + if !branch_body_terminates( + body_word, source, declared, sig_table, proc_table, local_vars, + ) { + return false; + } + } + true +} + +/// Does a `try BODY [on CODE VAR HANDLER]* [trap PATS VAR HANDLER]* +/// [finally SCRIPT]` terminate? Yes iff BODY terminates AND every +/// handler body terminates. `finally` doesn't participate in +/// termination (it runs REGARDLESS of what the body/handlers did, +/// so it can't turn a non-terminating structure into a terminating +/// one — but it also doesn't invalidate one). +fn try_command_terminates( + cmd: &crate::ast::Command, + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &VarTypeTable, +) -> bool { + // Body is the first argument (word[1]). + let Some(body_word) = cmd.words.get(1) else { + return false; + }; + if !branch_body_terminates( + body_word, source, declared, sig_table, proc_table, local_vars, + ) { + return false; + } + // Walk the remaining words looking for handler bodies. Each + // `on CODE VAR HANDLER` or `trap PATS VAR HANDLER` clause + // occupies 4 words; each `finally SCRIPT` occupies 2. + let mut i = 2; + while i < cmd.words.len() { + let head = cmd.words[i].as_text(); + match head { + Some("on") | Some("trap") => { + // handler body is at position i+3. + let Some(handler_body) = cmd.words.get(i + 3) else { + return false; + }; + if !branch_body_terminates( + handler_body, + source, + declared, + sig_table, + proc_table, + local_vars, + ) { + return false; + } + i += 4; + } + Some("finally") => { + // Finally script doesn't affect the terminator + // analysis — skip past its word. + i += 2; + } + _ => { + // Malformed / something we don't recognize; + // conservative false. + return false; + } + } + } + true +} + +/// Does the branch body word (a `WordForm::Braced` script) +/// terminate? Reparses the interior as a fragment and delegates +/// to `paths_always_return`. Non-braced body words (unusual — +/// only shows up when the parser hits malformed input) → false. +fn branch_body_terminates( + body_word: &crate::ast::Word, + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &VarTypeTable, +) -> bool { + if body_word.form != crate::ast::WordForm::Braced { + return false; + } + let word_start = body_word.span.start as usize; + let word_end = body_word.span.end as usize; + if word_end <= word_start + 2 { + return false; + } + let interior_start = word_start + 1; + let interior_end = word_end - 1; + let body_text = &source[interior_start..interior_end]; + let (mut body_stmts, _errs) = + crate::parser::parse_fragment(body_text, crate::parser::Mode::Toplevel); + for s in &mut body_stmts { + crate::parser::shift_stmt(s, interior_start as u32); + } + crate::parser::populate_procs(&mut body_stmts, source, &mut Vec::new()); + // Branch bodies share the outer scope's var table (Tcl + // control-flow doesn't create a new frame). + let mut branch_vars = local_vars.clone(); + paths_always_return( + &body_stmts, + source, + declared, + sig_table, + proc_table, + &mut branch_vars, + ) +} + +/// Walk `proc`'s body left-to-right, tracking `set VAR ` +/// bindings via [`value_type_with_procs`], then find the last +/// `return X` statement and resolve `X`'s type. `None` when the +/// body doesn't end with an inferrable return. +/// +/// Recursion depth is bounded implicitly by the finite proc-set +/// in a document: we don't cache visited procs here since v1 +/// documents don't hit deep recursion in practice. If a real +/// program starts driving this in circles, add a `HashSet<&str>` +/// guard on the proc name. +fn infer_return_type_from_body( + proc: &crate::ast::Proc, + sig_table: &HashMap, + proc_table: &HashMap, +) -> Option { + use crate::ast::{CommandKind, Stmt}; + let mut local_vars = VarTypeTable::new(); + // Seed the local var table with the proc's typed parameters — + // so a body like `proc pass_through {x: MyType} { return $x }` + // resolves through `$x` to `MyType`. + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + local_vars.insert(arg.name.clone(), ty.clone()); + } + } + } + // Walk `set` bindings in body order. + let mut return_word: Option<&crate::ast::Word> = None; + for stmt in &proc.body { + let Stmt::Command(cmd) = stmt else { continue }; + // Track `set VAR `. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type_with_procs( + value_word, + sig_table, + &local_vars, + Some(proc_table), + ) { + local_vars.insert(name.to_string(), ty); + } + } + } + } + // Track the last `return ` we see. Body execution + // ordinarily halts at `return`, but syntactically it's + // valid to have more code after (dead code); take the + // LAST occurrence since that's what the user's intent + // most likely reflects when reading the body. + if cmd.words.first().and_then(|w| w.as_text()) == Some("return") { + return_word = cmd.words.get(1); + } + } + let word = return_word?; + value_type_with_procs(word, sig_table, &local_vars, Some(proc_table)) +} + +/// Build a name → `Proc` lookup for return-type inference. Walks +/// top-level statements plus namespace-eval bodies (using the +/// namespace as a `::` prefix, matching how +/// [`build_signature_table`] qualifies proc names). +pub(crate) fn build_proc_table( + document: &crate::ast::Document, +) -> HashMap { + let mut out = HashMap::new(); + collect_procs(&document.stmts, "", &mut out); + out +} + +fn collect_procs<'doc>( + stmts: &'doc [Stmt], + prefix: &str, + out: &mut HashMap, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(name) = proc.name.as_deref() { + let qualified = qualify(prefix, name); + // Later `proc` shadows earlier — same as sig_table. + out.insert(qualified, proc); + } + } + CommandKind::NamespaceEval(ns) => { + let ns_name = ns.name.as_deref().unwrap_or(""); + let new_prefix = qualify(prefix, ns_name); + collect_procs(&ns.body, &new_prefix, out); + } + _ => {} + } + } +} + +/// True when a `TypeExpr` names the HTCL `bool` primitive. Kept +/// as a helper (rather than inlined) so the check has a single +/// point of change if we ever alias `bool` under a namespace +/// (e.g. `htcl::bool`). +fn is_bool_type(ty: &crate::ast::TypeExpr) -> bool { + matches!( + ty, + crate::ast::TypeExpr::Named { name, .. } if name == "bool" + ) +} + +fn validate_stmts( + stmts: &[Stmt], + source: &str, + table: &HashMap, + proc_table: &HashMap, + newtype_names: &std::collections::HashSet, + var_table: &mut VarTypeTable, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // Bind `set VAR ` into the var table BEFORE + // recursing — so downstream `$VAR` references in the same + // scope see the type. `validate_command` bails on + // `CommandKind::Set` (not a "call"), so this is the only + // place set-binding is observed. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type(value_word, table, var_table) { + var_table.insert(name.to_string(), ty); + } + } + } + } + validate_command(cmd, source, table, var_table, diags); + match &cmd.kind { + CommandKind::Proc(proc) => { + // Return-type check: fires only when the proc has + // an annotated return type. Every `return X` in + // the body (including inside control-flow braced + // bodies) must produce a value whose inferred + // type matches the annotation. + validate_proc_returns( + proc, + source, + table, + proc_table, + newtype_names, + diags, + ); + // Fresh scope per proc body. Seed with typed + // parameters so `-slot $arg` inside the body knows + // `arg`'s declared type without the caller having + // to `set` it locally. + let mut proc_scope = VarTypeTable::new(); + if let Some(sig) = &proc.signature { + for a in &sig.args { + if let Some(ty) = &a.type_annotation { + proc_scope.insert(a.name.clone(), ty.clone()); + } + } + } + validate_stmts( + &proc.body, + source, + table, + proc_table, + newtype_names, + &mut proc_scope, + diags, + ); + } + CommandKind::NamespaceEval(ns) => { + // Calls inside the namespace body are validated the + // same way; the signature-table is document-wide so + // a call to `project::set_target_language` from + // anywhere resolves to the same entry. (Bare, + // sibling-relative calls inside a namespace body + // aren't auto-qualified yet — write the qualified + // name explicitly.) Var scope is shared with the + // enclosing frame, matching Tcl's rule that + // `namespace eval` creates a namespace but not a + // fresh local-variable scope. + validate_stmts( + &ns.body, + source, + table, + proc_table, + newtype_names, + var_table, + diags, + ); + } + _ => {} + } + // Also descend into any `[ … ]` command substitutions on this + // command's words so calls written inline get validated the + // same as top-level ones. Var-table shared with the + // enclosing scope — a `set X …` inside `[…]` is visible + // outside, per Tcl. + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + validate_stmts( + body, + source, + table, + proc_table, + newtype_names, + var_table, + diags, + ); + } + } + } + } +} + +/// Build a name → signature map from every proc declaration in +/// the document, including those nested inside `namespace eval` +/// blocks (which register under `::`, matching Tcl's +/// namespace semantics). Duplicate names raise a diagnostic and the +/// later declaration wins, again matching Tcl (a second `proc` +/// redefines). +pub fn build_signature_table<'doc>( + document: &'doc Document, + diags: &mut Vec, +) -> HashMap { + let (table, _overloads) = build_signature_table_with_overloads( + document, + &std::collections::HashSet::new(), + diags, + ); + table +} + +/// Same as [`build_signature_table`] but also returns the +/// [`OverloadTable`] side-map. Callers that need to know whether a +/// given proc name resolves through enum-overload dispatch (codegen, +/// hover, signature help) consult this. +pub fn build_signature_table_with_overloads<'doc>( + document: &'doc Document, + newtype_qualified_names: &std::collections::HashSet, + diags: &mut Vec, +) -> (HashMap, OverloadTable) { + // First pass: collect every proc decl per qualified name, + // preserving order so a "first wins" / "last wins" choice is + // unambiguous when we have to make one. Multi-decl entries are + // candidate overload sets; single-decl entries are normal + // procs. + let mut multi: HashMap> = + HashMap::new(); + collect_signatures_multi( + &document.stmts, + "", + newtype_qualified_names, + &mut multi, + diags, + ); + + let mut table: HashMap = HashMap::new(); + let mut overloads: OverloadTable = HashMap::new(); + + for (qualified, decls) in multi { + match decls.len() { + 0 => { /* impossible */ } + 1 => { + let (proc, sig) = decls[0]; + check_reserved_proc_name(&qualified, proc.name_span, diags); + table.insert(qualified, sig); + } + _ => { + // Multi-decl: classify as enum-overload OR emit + // hard error for ad-hoc overloading. + match classify_overload_set(&qualified, &decls, diags) { + Some(info) => { + // Each specialization registers under its + // mangled name so analyzer drill-down + the + // dispatcher's runtime switch can find it. + for v in &info.variants { + // Find the decl whose first arg is this + // variant. We computed the mangled name + // from it during classify, so the order + // matches by construction. + for (_proc, sig) in &decls { + let Some(first) = sig.args.first() else { + continue; + }; + if matches!( + &first.type_annotation, + Some(TypeExpr::Qualified { variant, .. }) + if variant == &v.variant_name + ) { + // Mangled names are compiler- + // generated — they're allowed to + // start with `__` (that's the + // whole point). Skip the + // reserved-name check here. + table.insert( + v.mangled_proc_name.clone(), + sig, + ); + } + } + } + // Public name resolves to the first overload's + // sig as a representative. Analyzer / callers + // that want the "true" public interface + // consult `overloads`. + let (proc, sig) = decls[0]; + check_reserved_proc_name( + &qualified, + proc.name_span, + diags, + ); + table.insert(qualified.clone(), sig); + overloads.insert(qualified, info); + } + None => { + // classify_overload_set already emitted the + // diagnostic; for table consistency, fall + // back to "last wins" so downstream + // validation keeps working. Check the + // reserved prefix on each. + let (proc, sig) = *decls.last().unwrap(); + check_reserved_proc_name( + &qualified, + proc.name_span, + diags, + ); + table.insert(qualified, sig); + } + } + } + } + } + + (table, overloads) +} + +/// User procs whose qualified name starts with `__` would collide +/// with the compiler's overload-specialization mangling +/// (`____`). Reject them up front. +fn check_reserved_proc_name( + qualified: &str, + name_span: Span, + diags: &mut Vec, +) { + // Look at the last segment after the final `::`. Tcl's + // namespace separator is part of the qualified name, so e.g. + // `vivado_cmd::__foo` has its "leaf" name as `__foo` — the + // collision risk is on the leaf, not the prefix. + let leaf = qualified.rsplit("::").next().unwrap_or(qualified); + if leaf.starts_with("__") { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "proc name `{qualified}` is reserved: names starting with \ + `__` are used by the compiler for overload-specialization \ + mangling (e.g. `__handle_prop__Scalar`). Rename to avoid \ + collisions." + ), + span: name_span, + }); + } +} + +fn collect_signatures_multi<'doc>( + stmts: &'doc [Stmt], + prefix: &str, + newtype_qualified_names: &std::collections::HashSet, + multi: &mut HashMap>, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let Some(sig) = proc.signature.as_ref() else { + continue; + }; + // v1 restriction: enum-overloaded procs must be + // declared at the top level. Inside a `namespace + // eval` block, the REPL's batch-prepare layer + // doesn't re-route to the mangled-name + dispatcher + // pipeline, so an overload arm inside a namespace + // would silently lose its dispatch semantics. + // + // Only ENUM-Qualified first args (`Foo::Variant` where + // `Foo` is a declared enum) count as overload-arm + // shape. A `Qualified` type that resolves to a + // declared NEWTYPE (e.g. `-config: versal_cips::Config` + // in a `namespace eval versal_cips { proc create … }` + // block) is just a typed newtype ref — legal + // everywhere. The newtype-set peeked in by the caller + // disambiguates. + let overload_shape_first = sig + .args + .first() + .and_then(|a| a.type_annotation.as_ref()) + .and_then(|t| match t { + TypeExpr::Qualified { + namespace, variant, .. + } => Some(format!("{namespace}::{variant}")), + _ => None, + }) + .filter(|qname| !newtype_qualified_names.contains(qname)) + .is_some(); + if overload_shape_first && !prefix.is_empty() { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overloaded proc `{name}` is declared inside \ + `namespace eval {prefix}` — v1 enum-overloads \ + must be declared at the top level. Move the \ + overload arms out of the namespace block." + ), + span: proc.name_span, + }); + continue; + } + let qualified = qualify(prefix, name); + multi.entry(qualified).or_default().push((proc, sig)); + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + // `extern` is reserved by htcl's lowering as the + // prefix for runtime-Tcl-proc disambiguation + // (`extern::foo` → `__vw_extern_foo`). A user- + // defined namespace named `extern` would silently + // collide with that rewrite at call sites; reject + // it up front. + if name == "extern" { + diags.push(Diagnostic { + severity: Severity::Error, + message: "`extern` is a reserved namespace name in \ + htcl (used for runtime-Tcl-proc \ + disambiguation); pick a different name" + .into(), + span: ns.name_span, + }); + continue; + } + let nested = qualify(prefix, name); + collect_signatures_multi( + &ns.body, + &nested, + newtype_qualified_names, + multi, + diags, + ); + } + _ => {} + } + } +} + +/// Classify a multi-decl proc-name set. Returns `Some(OverloadInfo)` +/// if every member's first arg is a distinct variant of the same +/// enum AND the tail args / return type agree; returns `None` and +/// emits a diagnostic if it's not a valid overload (ad-hoc +/// overloading, missing variant, tail mismatch, etc.). +fn classify_overload_set<'doc>( + public_name: &str, + decls: &[(&'doc Proc, &'doc ProcSignature)], + diags: &mut Vec, +) -> Option { + // Each decl's first arg must be `Qualified { namespace: E, variant: V }`. + // Collect (enum_name, variant_name, dispatch_arg_span) per decl. + let mut dispatch_infos: Vec<(String, String, Span, &Proc, &ProcSignature)> = + Vec::with_capacity(decls.len()); + for (proc, sig) in decls { + let Some(first) = sig.args.first() else { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "proc `{public_name}` is declared multiple times; for \ + this to be a valid enum-overload set, every \ + declaration's first argument must be annotated with \ + a qualified variant type like `E::V`. This one has \ + no arguments." + ), + span: proc.name_span, + }); + return None; + }; + match &first.type_annotation { + Some(TypeExpr::Qualified { + namespace, variant, .. + }) => { + dispatch_infos.push(( + namespace.clone(), + variant.clone(), + first.name_span, + proc, + sig, + )); + } + _ => { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "proc `{public_name}` is declared multiple times \ + with first-arg types that aren't all variants \ + of a common enum; ad-hoc overloading on arbitrary \ + types is not supported. Use an enum or rename \ + one of the procs." + ), + span: first.name_span, + }); + return None; + } + } + } + // All overloads must dispatch on the same enum. + let enum_name = dispatch_infos[0].0.clone(); + for (ns, _, sp, _, _) in &dispatch_infos[1..] { + if ns != &enum_name { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload set for proc `{public_name}` mixes enums: \ + `{enum_name}` and `{ns}`. All overloads in a set \ + must dispatch on the same enum." + ), + span: *sp, + }); + return None; + } + } + // Variants must be distinct. + { + let mut seen: std::collections::HashSet<&str> = + std::collections::HashSet::new(); + for (_, v, sp, _, _) in &dispatch_infos { + if !seen.insert(v.as_str()) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload set for proc `{public_name}` has two \ + arms dispatching on the same variant \ + `{enum_name}::{v}`. Each variant must have at \ + most one arm." + ), + span: *sp, + }); + return None; + } + } + } + // Tail-arg agreement: v1 restricts every arm to exactly one + // arg (the dispatched variant). Multi-arg overloads are + // future work — kwargs / specialization-binding interactions + // get hairy and the property-display motivating case doesn't + // need them. + let (_, _, _, first_proc, first_sig) = &dispatch_infos[0]; + if first_sig.args.len() != 1 { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload arm `{public_name}` declares {} args; v1 \ + enum-overloads support exactly ONE arg (the dispatched \ + variant). Additional tail args are future work — model \ + the tail as a payload field on the enum variant for now.", + first_sig.args.len() + ), + span: first_sig.span, + }); + return None; + } + for (ns, v, _, _, sig) in &dispatch_infos[1..] { + if sig.args.len() != 1 { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload arm `{public_name}` for `{ns}::{v}` declares \ + {} args; v1 enum-overloads support exactly ONE arg \ + (the dispatched variant).", + sig.args.len() + ), + span: sig.span, + }); + return None; + } + } + // Return-type agreement: every annotated return type must match. + // Mixed annotated/unannotated → error. + let first_ret = first_sig.return_type.as_ref(); + for (ns, v, _, _, sig) in &dispatch_infos[1..] { + match (first_ret, sig.return_type.as_ref()) { + (None, None) => {} + (Some(a), Some(b)) if types_match(a, b) => {} + _ => { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload arm `{public_name}` for `{ns}::{v}` \ + declares a different return type than the other \ + arms. All arms must agree on the return type \ + (annotate every arm with the same type, or none)." + ), + span: sig.span, + }); + return None; + } + } + } + // Arg-name agreement: every arm must use the same first-arg + // name so the dispatcher can pass the payload via kwargs as + // `- `. Cheaper than per-arm dispatch + // tracking and matches user convention (everyone writes `v`). + let dispatch_arg_name = first_sig.args[0].name.clone(); + for (ns, v, _, _, sig) in &dispatch_infos[1..] { + if sig.args[0].name != dispatch_arg_name { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload arm `{public_name}` for `{ns}::{v}` names its \ + dispatch arg `{}`; other arms name it `{dispatch_arg_name}`. \ + All arms must use the same arg name (convention: `v`).", + sig.args[0].name + ), + span: sig.args[0].name_span, + }); + return None; + } + } + // Build the OverloadInfo. Variant order matches source order + // of the overloads. + let variants = dispatch_infos + .iter() + .map(|(_, v, sp, _, _)| OverloadVariant { + variant_name: v.clone(), + mangled_proc_name: mangle_specialization(public_name, v), + dispatch_arg_span: *sp, + }) + .collect(); + Some(OverloadInfo { + public_name: public_name.to_string(), + enum_name, + dispatch_arg_name, + variants, + anchor_span: first_proc.name_span, + }) +} + +// `tails_match` / `attr_values_equal` lived here for the multi-arg +// overload tail-agreement check. v1 restricts overloads to a single +// arg (see `classify_overload_set`), so we don't compare tails. The +// helpers are kept as a record in git history; restore when adding +// multi-arg overloads. + +/// Collect every `type NAME = UNDERLYING` declaration in `document`, +/// qualified by enclosing `namespace eval` prefix (so a `type widget` +/// declared inside `namespace eval foo {}` registers as `foo::widget`, +/// matching how procs already qualify). Duplicate declarations emit +/// a warning and the later one wins — same shape as duplicate-proc +/// handling above. +pub fn build_type_decl_table<'doc>( + document: &'doc Document, + diags: &mut Vec, +) -> HashMap { + let mut table = HashMap::new(); + collect_type_decls(&document.stmts, "", &mut table, diags); + table +} + +fn collect_type_decls<'doc>( + stmts: &'doc [Stmt], + prefix: &str, + table: &mut HashMap, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::TypeDecl(td) => { + let Some(name) = td.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + if table.insert(qualified.clone(), td).is_some() { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "duplicate definition of type {qualified}; \ + later definition wins" + ), + span: td.name_span, + }); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + if name == "extern" { + continue; + } + let nested = qualify(prefix, name); + collect_type_decls(&ns.body, &nested, table, diags); + } + CommandKind::Proc(proc) => { + // Nested type decls inside proc bodies are unusual + // but not illegal — walk them so they register. + collect_type_decls(&proc.body, prefix, table, diags); + } + _ => {} + } + } +} + +/// Mirror of [`build_type_decl_table`] for `enum NAME = { ... }` +/// declarations. Duplicate enums warn and the later one wins — +/// same shape as type-decl handling. +pub fn build_enum_decl_table<'doc>( + document: &'doc Document, + diags: &mut Vec, +) -> HashMap { + let mut table = HashMap::new(); + collect_enum_decls(&document.stmts, "", &mut table, diags); + table +} + +fn collect_enum_decls<'doc>( + stmts: &'doc [Stmt], + prefix: &str, + table: &mut HashMap, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::EnumDecl(ed) => { + let Some(name) = ed.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + if table.insert(qualified.clone(), ed).is_some() { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "duplicate definition of enum {qualified}; \ + later definition wins" + ), + span: ed.name_span, + }); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + if name == "extern" { + continue; + } + let nested = qualify(prefix, name); + collect_enum_decls(&ns.body, &nested, table, diags); + } + CommandKind::Proc(proc) => { + collect_enum_decls(&proc.body, prefix, table, diags); + } + _ => {} + } + } +} + +/// Per-enum sanity checks. v1: variants must have distinct names; +/// payload types are syntactically valid (already enforced by +/// `enum_parse`); a payload that references an unknown user type +/// is a soft warning for now (could be defined cross-batch). +fn validate_enum_decls( + enum_table: &HashMap, + _type_table: &HashMap, + diags: &mut Vec, +) { + for (qualified, ed) in enum_table { + let mut seen: std::collections::HashSet<&str> = + std::collections::HashSet::new(); + for v in &ed.variants { + if !seen.insert(v.name.as_str()) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "enum `{qualified}` declares variant `{}` more than \ + once. Each variant name must be unique within an \ + enum.", + v.name + ), + span: v.name_span, + }); + } + } + } +} + +/// Walk the document and reject [`TypeExpr::Qualified`] anywhere +/// other than as a proc's first-arg type annotation. Qualified +/// types (`E::V`) are only meaningful as overload-dispatch +/// indicators unless they resolve to a declared newtype — those +/// pass through as regular namespaced type references. +/// +/// `newtype_names` carries the qualified names of every declared +/// newtype in the document (built via +/// [`build_type_decl_table`]); it's the disambiguator between +/// enum-variant refs and namespaced newtype refs. +fn validate_qualified_positions( + document: &Document, + newtype_names: &std::collections::HashSet, + diags: &mut Vec, +) { + fn walk_stmts( + stmts: &[Stmt], + newtype_names: &std::collections::HashSet, + diags: &mut Vec, + ) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = proc.signature.as_ref() { + for (i, arg) in sig.args.iter().enumerate() { + if let Some(ty) = arg.type_annotation.as_ref() { + // The first arg may be Qualified; + // tail args may NOT (unless a + // known-newtype ref, handled by the + // reject fn itself). + let allow_qualified = i == 0; + reject_nested_qualified( + ty, + allow_qualified, + newtype_names, + diags, + ); + } + } + if let Some(ret) = sig.return_type.as_ref() { + reject_nested_qualified( + ret, + false, + newtype_names, + diags, + ); + } + } + walk_stmts(&proc.body, newtype_names, diags); + } + CommandKind::NamespaceEval(ns) => { + walk_stmts(&ns.body, newtype_names, diags); + } + CommandKind::TypeDecl(td) => { + if let Some(ty) = td.underlying.as_ref() { + reject_nested_qualified( + ty, + false, + newtype_names, + diags, + ); + } + } + CommandKind::EnumDecl(ed) => { + for v in &ed.variants { + if let Some(ty) = v.payload.as_ref() { + reject_nested_qualified( + ty, + false, + newtype_names, + diags, + ); + } + } + } + _ => {} + } + } + } + walk_stmts(&document.stmts, newtype_names, diags); +} + +fn reject_nested_qualified( + ty: &TypeExpr, + allow_top_qualified: bool, + newtype_names: &std::collections::HashSet, + diags: &mut Vec, +) { + match ty { + TypeExpr::Named { .. } => {} + TypeExpr::Generic { args, .. } => { + // Inside a generic, nested Qualified is never allowed — + // except for known-newtype references, which are just + // namespaced type names. + for a in args { + reject_nested_qualified(a, false, newtype_names, diags); + } + } + TypeExpr::Qualified { + namespace, + variant, + span, + .. + } => { + // A qualified name that resolves to a declared newtype + // is a regular namespaced type reference — legal + // wherever a Named type is legal, including return + // types and generic args. + let qualified = format!("{namespace}::{variant}"); + if newtype_names.contains(&qualified) { + return; + } + if !allow_top_qualified { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "qualified type `{namespace}::{variant}` is only \ + legal as the first-argument type annotation on an \ + overloaded handler proc. It can't appear as a \ + return type, generic argument, type-decl \ + underlying, or enum-variant payload." + ), + span: *span, + }); + } + } + } +} + +/// For each newtype declaration `T`, verify the user provided the +/// required `T::repr`, `T::from`, `T::to` procs with the correct +/// shapes: +/// +/// - `T::repr` takes one arg named `v` of type `T` (or untyped), +/// returns `string` (or untyped). +/// - `T::from` takes one arg named `v` of type `` (or +/// untyped), returns `T` (or untyped). +/// - `T::to` takes one arg named `v` of type `T` (or untyped), +/// returns `` (or untyped). +/// +/// Type annotations are *optional* on these procs — an untyped +/// arg or return slot is accepted as a "trust the user" form +/// (some procs ship pre-arg-types and were authored before the +/// shape check existed). The arg COUNT and NAME (`v`) are +/// always enforced; the type slots get a stricter check only +/// when the user opted in by annotating them. +fn validate_type_decl_triplets( + type_table: &HashMap, + sig_table: &HashMap, + diags: &mut Vec, +) { + use crate::ast::TypeExpr; + for (qualified_name, td) in type_table { + let underlying = td.underlying.as_ref(); + let slots: &[(&str, Option<&TypeExpr>, Option<&str>)] = &[ + // (slot, arg type expected, return type expected as + // type-name). We pass the type-name via Option<&str> + // and compare with TypeExpr::Named's name; that's + // sufficient for the v1 set (all involved types are + // either named primitives or named newtypes; no + // generics in repr/from/to signatures). + ( + "repr", + // arg should be T + Some(&named_lit(qualified_name)), + Some("string"), + ), + ( + "from", + // arg should be + underlying, + // return should be T + Some(qualified_name.as_str()), + ), + ( + "to", + // arg should be T + Some(&named_lit(qualified_name)), + // return should be + underlying.and_then(|u| match u { + TypeExpr::Named { name, .. } => Some(name.as_str()), + _ => None, + }), + ), + ]; + for (slot, expected_arg, expected_ret) in slots { + let want = format!("{qualified_name}::{slot}"); + let Some(sig) = sig_table.get(&want) else { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "newtype `{qualified_name}` is missing required \ + proc `{qualified_name}::{slot}` (see \ + docs/htcl-return-types.md)." + ), + span: td.name_span, + }); + continue; + }; + // Arg count + name. + if sig.args.len() != 1 || sig.args[0].name != "v" { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "newtype proc `{qualified_name}::{slot}` must \ + take exactly one argument named `v`" + ), + span: sig.span, + }); + continue; + } + // Arg type — only checked when the user annotated it. + if let (Some(actual), Some(expected)) = + (sig.args[0].type_annotation.as_ref(), expected_arg) + { + if !types_match(actual, expected) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "newtype proc `{qualified_name}::{slot}`: \ + arg `v` is declared `{}` but should be \ + `{}`", + render_type_inline(actual), + render_type_inline(expected) + ), + span: sig.args[0].name_span, + }); + } + } + // Return type — only checked when the user annotated it + // and we know what to compare against. + if let (Some(actual), Some(want_name)) = + (sig.return_type.as_ref(), expected_ret) + { + // Compare on the identifier's name. For Qualified + // (namespaced newtype refs like `dcmac::GtChProps`) + // we join the parts so the compare matches the + // qualified-name key `want_name` carries. + let actual_name_owned: String; + let actual_name = match actual { + TypeExpr::Named { name, .. } => name.as_str(), + TypeExpr::Generic { name, .. } => name.as_str(), + TypeExpr::Qualified { + namespace, variant, .. + } => { + actual_name_owned = format!("{namespace}::{variant}"); + actual_name_owned.as_str() + } + }; + if actual_name != *want_name { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "newtype proc `{qualified_name}::{slot}` \ + returns `{}` but should return `{want_name}`", + render_type_inline(actual) + ), + span: sig.span, + }); + } + } + } + } +} + +/// Build a one-shot `TypeExpr::Named` literal for comparison +/// purposes. The span is meaningless here — we only ever +/// inspect the name. +fn named_lit(name: &str) -> crate::ast::TypeExpr { + crate::ast::TypeExpr::Named { + name: name.to_string(), + span: Span::new(0, 0), + } +} + +/// Structural equality on type expressions, ignoring spans. +/// +/// `Qualified { ns, var }` is treated as equivalent to +/// `Named { name: "ns::var" }` — the two forms describe the same +/// identifier and callers that need to compare an ast-parsed +/// annotation against a synthetic Named expected type (e.g. +/// `named_lit(qualified_name)` inside newtype-triplet validation) +/// shouldn't see a spurious mismatch. +fn types_match(a: &crate::ast::TypeExpr, b: &crate::ast::TypeExpr) -> bool { + use crate::ast::TypeExpr; + match (a, b) { + ( + TypeExpr::Named { name: an, .. }, + TypeExpr::Named { name: bn, .. }, + ) => an == bn, + ( + TypeExpr::Generic { + name: an, args: aa, .. + }, + TypeExpr::Generic { + name: bn, args: ba, .. + }, + ) => { + an == bn + && aa.len() == ba.len() + && aa.iter().zip(ba.iter()).all(|(x, y)| types_match(x, y)) + } + // Cross-form equivalence for qualified newtype references + // (`dcmac::GtChProps` on one side, `Named("dcmac::GtChProps")` + // on the other). Commutative. + ( + TypeExpr::Qualified { + namespace, variant, .. + }, + TypeExpr::Named { name, .. }, + ) + | ( + TypeExpr::Named { name, .. }, + TypeExpr::Qualified { + namespace, variant, .. + }, + ) => *name == format!("{namespace}::{variant}"), + ( + TypeExpr::Qualified { + namespace: ans, + variant: av, + .. + }, + TypeExpr::Qualified { + namespace: bns, + variant: bv, + .. + }, + ) => ans == bns && av == bv, + _ => false, + } +} + +/// Render a type expression for inclusion in a diagnostic message. +/// Mirrors `vw-analyzer/src/htcl_backend.rs::render_type` — kept +/// in sync by convention since the analyzer can't depend on +/// validate.rs. +fn render_type_inline(ty: &crate::ast::TypeExpr) -> String { + use crate::ast::TypeExpr; + match ty { + TypeExpr::Named { name, .. } => name.clone(), + TypeExpr::Generic { name, args, .. } => { + let inner: Vec = + args.iter().map(render_type_inline).collect(); + format!("{name}<{}>", inner.join(",")) + } + TypeExpr::Qualified { + namespace, variant, .. + } => { + format!("{namespace}::{variant}") + } + } +} + +/// Tcl core builtins that legitimately take either `-flag` +/// arguments natively (`string match -nocase`, `regexp -line`, +/// `lsort -unique`) or take positional list arguments that +/// commonly start with `-` (e.g. `lappend cmd -ruledeck $x` where +/// `-ruledeck` is being appended as a literal token, not parsed +/// by `lappend`). Calls to anything in this list pass the +/// unknown-call check unconditionally. +/// +/// Keep this small but pragmatic: a missed builtin produces a +/// pestering error on calls that work fine; an over-included name +/// hides a real "you forgot to src @x" mistake. The set below is +/// the standard Tcl core surface most htcl bodies actually use. +fn is_known_tcl_builtin(name: &str) -> bool { + matches!( + name, + // Container ops whose positional args often look like flags. + "lappend" + | "lset" + | "linsert" + | "lreplace" + | "lrange" + | "lindex" + | "list" + | "llength" + | "dict" + | "array" + | "set" + | "unset" + | "incr" + | "append" + | "concat" + // String / regex / sort builtins that accept `-flag`s natively. + | "string" + | "regexp" + | "regsub" + | "lsort" + | "lsearch" + | "switch" + | "format" + | "scan" + | "binary" + // Flow / introspection / interp. + | "after" + | "eval" + | "uplevel" + | "upvar" + | "apply" + | "info" + | "package" + | "catch" + | "try" + | "throw" + | "error" + | "return" + | "expr" + // I/O & filesystem. + | "puts" + // `putr` is our compile-time repr-dispatching shim: + // `putr $x` gets rewritten in `crate::putr::rewrite` + // to `puts [T::repr -v $x]` when the argument's type + // is statically known, else to plain `puts $x`. The + // rewrite fires before any code reaches Tcl, so at + // eval time `putr` isn't a real command — the + // analyzer needs to recognize it as a builtin so + // undefined-proc checks don't flag the call sites. + | "putr" + | "gets" + | "read" + | "close" + | "open" + | "file" + | "exec" + | "fconfigure" + | "fileevent" + | "flush" + // Channels / Tk-style. + | "namespace" + | "variable" + | "global" + | "rename" + | "interp" + ) +} + +/// Join a namespace prefix with a member name using Tcl's `::` +/// separator. The empty prefix yields the bare name (used at the +/// document root where there's no enclosing namespace). +fn qualify(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + } +} + +fn validate_command( + cmd: &Command, + source: &str, + table: &HashMap, + var_table: &VarTypeTable, + diags: &mut Vec, +) { + let call_name = match &cmd.kind { + CommandKind::Generic => match cmd.words.first() { + Some(w) => match w.as_text() { + Some(t) => t, + None => return, + }, + None => return, + }, + // Don't validate inside declarations themselves — those + // aren't calls. (NamespaceEval is a declaration; its body's + // statements are validated by the recursion in + // `validate_stmts`.) + CommandKind::Proc(_) + | CommandKind::Set + | CommandKind::Src(_) + | CommandKind::NamespaceEval(_) + | CommandKind::TypeDecl(_) + | CommandKind::EnumDecl(_) => { + return; + } + }; + // `extern::name` is the user's opt-out: "this call resolves + // to a runtime Tcl proc, don't analyze its signature." Lowering + // strips the prefix and aliases the underlying proc into place. + if crate::lower::is_extern_call(call_name) { + return; + } + let Some(sig) = table.get(call_name) else { + // Unknown call. Two paths fire an error: + // + // 1. The call uses `-flag` keyword arguments. Almost + // always the user meant an htcl wrapper that isn't + // loaded — shipping it raw to the EDA backend either + // errors cryptically or misinterprets the args. + // + // 2. The unqualified name has a matching namespaced + // proc in scope (e.g., `get_bd_addr_spaces` when + // `vivado_cmd::get_bd_addr_spaces` exists). That's a + // missed namespace prefix on the same wrapper the + // user is calling elsewhere with the qualified name. + // Catching this even for positional-only calls is + // what makes the analyzer's behavior consistent — + // otherwise `assign_bd_address` errors (has `-flag` + // args) but `[get_bd_addr_spaces X]` inside its arg + // silently passes, which reads as an analyzer gap. + // + // A positional-only call to a bare Tcl builtin (`llength`, + // `dict`, etc.) still passes cleanly: `is_known_tcl_builtin` + // filters those, and there's no namespaced homonym. + let uses_keyword = cmd.words.iter().skip(1).any(|w| { + w.as_text() + .is_some_and(|t| t.starts_with('-') && t.len() > 1) + }); + let namespaced_match = if !call_name.contains("::") { + let suffix = format!("::{call_name}"); + table.keys().find(|k| k.ends_with(&suffix)).cloned() + } else { + None + }; + let should_flag = !is_known_tcl_builtin(call_name) + && (uses_keyword || namespaced_match.is_some()); + if should_flag { + // Prefer the exact namespaced match as the "did you + // mean" — it's a stronger signal than the fuzzy + // Levenshtein suggestion, which for the bare-name + // case would surface the same or a nearby name + // anyway. + let hint = if let Some(qualified) = &namespaced_match { + format!(" — did you mean `{qualified}`?") + } else { + match suggest_name(call_name, table.keys()) { + Some(s) => format!(" — did you mean `{s}`?"), + None => String::new(), + } + }; + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "undefined proc `{call_name}`{hint}; either \ + `src` a module that defines it or use \ + `extern::{call_name}` to call the underlying \ + Tcl proc directly" + ), + span: cmd.words[0].span, + }); + } + return; + }; + + // Parse keyword args from the command's words. The first word is + // the call name; the remaining words alternate -flag/value. + let mut idx = 1usize; + let mut seen: HashMap = HashMap::new(); + while idx < cmd.words.len() { + let word = &cmd.words[idx]; + let flag_text = match word.as_text() { + Some(t) if t.starts_with('-') => &t[1..], + Some(t) => { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!("expected keyword argument, found {t}"), + span: word.span, + }); + idx += 1; + continue; + } + None => { + diags.push(Diagnostic { + severity: Severity::Error, + message: "expected keyword argument".into(), + span: word.span, + }); + idx += 1; + continue; + } + }; + let flag_name = flag_text.to_string(); + let value_word = cmd.words.get(idx + 1); + + match sig.find(&flag_name) { + None => { + let known: Vec<&str> = + sig.args.iter().map(|a| a.name.as_str()).collect(); + let hint = if known.is_empty() { + String::new() + } else { + format!(". Possible values are {}", known.join(", ")) + }; + diags.push(Diagnostic { + severity: Severity::Error, + message: format!("undefined argument -{flag_name}{hint}"), + span: word.span, + }); + } + Some(arg) => { + if let Some(prev) = seen.insert(flag_name.clone(), word.span) { + let _ = prev; + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!("duplicate argument -{flag_name}"), + span: word.span, + }); + } + if let Some(value) = value_word { + validate_value(call_name, arg, value, source, diags); + // Nominal type check for the value expression. + // Only fires when BOTH sides have known types — + // the caller wrote a `: TYPE` annotation on the + // arg (populated by proc_args parsing into + // `ProcArg.type_annotation`) AND the value word + // is one whose type `value_type` can infer + // (`[proc-call]` return type or `$var` binding). + // Literals and mixed compounds silently skip + // (gradual typing). + // + // Identity via `types_match` — no alias walking + // (see `VarTypeTable` docs). `Quad0Ch1Props ≠ + // Quad1Ch0Props ≠ Properties` even when both + // alias the same underlying; that's what + // catches the copy-paste-wrong-constructor bug + // this check exists for. + if let Some(declared) = &arg.type_annotation { + if let Some(actual) = + value_type(value, table, var_table) + { + if !types_match(declared, &actual) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "type mismatch for -{}: expected \ + `{}`, found `{}`", + flag_name, + render_type_inline(declared), + render_type_inline(&actual), + ), + span: value.span, + }); + } + } else if is_bool_type(declared) { + // `bool`-typed slot with a value the + // `value_type` pass can't infer. The + // ONLY textual values that produce a + // known `bool` are the bare `true` / + // `false` literals (see `value_type`); + // any other whole-word text literal at + // this slot is a mistyped bool + // (`-flag 1`, `-flag yes`, `-flag + // potato`). Reject with a message + // naming the offending literal so the + // caller can rewrite it. + // + // Skips vars/cmdsubst and compound + // words: those return None from + // value_type because we couldn't + // deduce the type, not because they're + // definitively wrong. + if let Some(lit) = value.as_text() { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "type mismatch for -{}: expected \ + `bool`, found literal `{}` (use \ + `true` or `false`)", + flag_name, lit, + ), + span: value.span, + }); + } + } + } + } else { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "argument -{flag_name} is missing a value" + ), + span: word.span, + }); + } + } + } + // Step past the flag and its value. + idx += if value_word.is_some() { 2 } else { 1 }; + } + + // Build canonical `@one_of` groups. Each arg's `@one_of(...)` + // declares an alternatives set: the arg itself plus the named + // siblings. We collapse declarations from each direction (sib A + // says `@one_of(B)` and sib B says `@one_of(A)`) into one + // canonical group, then check that **exactly one** arg from each + // group is supplied at the call site. + // + // Args participating in a group are treated as optional for the + // missing-required check below — the group rule is the source of + // truth for "must supply something." + let one_of_groups = collect_one_of_groups(sig); + let in_one_of: std::collections::HashSet<&str> = one_of_groups + .iter() + .flat_map(|g| g.iter().map(String::as_str)) + .collect(); + + // Required-args check. An arg is required when it has no + // `@default` to fall back to — the user must supply a value. + // Args in an `@one_of` group are governed by the group rule + // instead, so skip them here. + for arg in &sig.args { + if seen.contains_key(&arg.name) { + continue; + } + if in_one_of.contains(arg.name.as_str()) { + continue; + } + let is_required = arg.attribute("default").is_none(); + if is_required { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "missing required argument -{name}", + name = arg.name, + ), + span: cmd.span, + }); + } + } + + // `@one_of` groups: exactly one alternative must be present. + for group in &one_of_groups { + let present: Vec<&str> = group + .iter() + .filter(|n| seen.contains_key(n.as_str())) + .map(String::as_str) + .collect(); + if present.len() == 1 { + continue; + } + let opts: Vec = group.iter().map(|n| format!("-{n}")).collect(); + let message = if present.is_empty() { + format!( + "missing required argument — exactly one of {} must be \ + supplied", + opts.join(", ") + ) + } else { + let got: Vec = + present.iter().map(|n| format!("-{n}")).collect(); + format!( + "exactly one of {} may be supplied, got {}", + opts.join(", "), + got.join(", ") + ) + }; + diags.push(Diagnostic { + severity: Severity::Error, + message, + span: cmd.span, + }); + } + + // Inter-arg deps for present args. + for (flag_name, flag_span) in &seen { + let Some(arg) = sig.find(flag_name) else { + continue; + }; + if let Some(req) = arg.attribute("requires") { + for value in &req.values { + let referenced = match value { + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value.as_str(), + AttributeValue::Integer { .. } + | AttributeValue::Keyed { .. } => continue, + }; + if !seen.contains_key(referenced) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "argument -{flag_name} requires -{referenced} \ + to also be set" + ), + span: *flag_span, + }); + } + } + } + if let Some(conflicts) = arg.attribute("conflicts") { + for value in &conflicts.values { + let referenced = match value { + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value.as_str(), + AttributeValue::Integer { .. } + | AttributeValue::Keyed { .. } => continue, + }; + if seen.contains_key(referenced) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "argument -{flag_name} conflicts with \ + -{referenced}" + ), + span: *flag_span, + }); + } + } + } + if arg.attribute("deprecated").is_some() { + let msg = arg + .attribute("deprecated") + .and_then(|a| a.values.first()) + .map(|v| v.as_str().to_string()) + .unwrap_or_default(); + let m = if msg.is_empty() { + format!("argument -{flag_name} is deprecated") + } else { + format!("argument -{flag_name} is deprecated: {msg}") + }; + diags.push(Diagnostic { + severity: Severity::Warning, + message: m, + span: *flag_span, + }); + } + } +} + +/// Collect canonical `@one_of` alternatives groups for a signature. +/// +/// Each arg's `@one_of(sib1, sib2, ...)` declares that exactly one +/// of `{arg, sib1, sib2, ...}` must be supplied at the call site. +/// We treat the declaration as symmetric (both `dict @one_of(name)` +/// and `name @one_of(dict)` describe the same group), so a `BTreeSet` +/// of the participating names canonicalizes each group regardless of +/// which direction (or which redundant copies) the author wrote. +fn collect_one_of_groups( + sig: &ProcSignature, +) -> Vec> { + use std::collections::BTreeSet; + let mut seen: std::collections::HashSet> = + std::collections::HashSet::new(); + let mut out: Vec> = Vec::new(); + for arg in &sig.args { + let Some(attr) = arg.attribute("one_of") else { + continue; + }; + let mut group: BTreeSet = BTreeSet::new(); + group.insert(arg.name.clone()); + for value in &attr.values { + match value { + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => { + group.insert(value.clone()); + } + AttributeValue::Integer { .. } + | AttributeValue::Keyed { .. } => continue, + } + } + if group.len() >= 2 && seen.insert(group.clone()) { + out.push(group); + } + } + out +} + +fn validate_value( + call_name: &str, + arg: &ProcArg, + value_word: &Word, + _source: &str, + diags: &mut Vec, +) { + // For Phase 2 we only validate literal-text values. Word forms + // that include `$var` or `[cmd]` are runtime-dynamic; we let them + // through silently. Future work can teach the validator about + // values produced by known builtins. + let Some(literal) = literal_value(value_word) else { + return; + }; + + if let Some(enum_attr) = arg.attribute("enum") { + check_enum( + call_name, &arg.name, enum_attr, &literal, value_word, diags, + ); + } + if let Some(range_attr) = arg.attribute("range") { + check_range( + call_name, &arg.name, range_attr, &literal, value_word, diags, + ); + } +} + +fn literal_value(word: &Word) -> Option { + let mut out = String::new(); + for part in &word.parts { + match part { + WordPart::Text { value, .. } => out.push_str(value), + WordPart::Escape { value, .. } => out.push(*value), + WordPart::VarRef { .. } | WordPart::CmdSubst { .. } => { + // Dynamic content — not a literal. + return None; + } + } + } + Some(out) +} + +fn check_enum( + _call_name: &str, + arg_name: &str, + enum_attr: &Attribute, + literal: &str, + value_word: &Word, + diags: &mut Vec, +) { + let allowed: Vec = enum_attr + .values + .iter() + .map(|v| match v { + AttributeValue::Integer { value, .. } => value.to_string(), + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value.clone(), + // `key=value` in @enum(…) doesn't make semantic sense + // (enum values are positional). Include the literal + // `key=value` string so a runtime match against the + // raw arg still works if someone actually did that. + AttributeValue::Keyed { .. } => v.to_tcl_literal(), + }) + .collect(); + if !allowed.iter().any(|a| a == literal) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "value {literal} for -{arg_name} is not in @enum. Possible \ + values are {}", + allowed.join(", ") + ), + span: value_word.span, + }); + } +} + +fn check_range( + _call_name: &str, + arg_name: &str, + range_attr: &Attribute, + literal: &str, + value_word: &Word, + diags: &mut Vec, +) { + let (Some(min), Some(max)) = + (range_attr.values.first(), range_attr.values.get(1)) + else { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "@range on -{arg_name} should have two numeric bounds" + ), + span: range_attr.span, + }); + return; + }; + let ( + AttributeValue::Integer { value: min, .. }, + AttributeValue::Integer { value: max, .. }, + ) = (min, max) + else { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!("@range on -{arg_name} has non-integer bounds"), + span: range_attr.span, + }); + return; + }; + let Ok(n) = literal.parse::() else { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "argument -{arg_name} expects an integer, found {literal}" + ), + span: value_word.span, + }); + return; + }; + if n < *min || n > *max { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "value {n} for -{arg_name} is out of @range({min}, {max})" + ), + span: value_word.span, + }); + } +} + +/// Standard compiler-style "did you mean X?" suggestion: pick the +/// in-scope name with the smallest edit distance from `target`, +/// within a length-scaled threshold. Returns `None` when no +/// candidate is close enough (so unknown calls that aren't +/// near-misses don't get nonsense suggestions tacked on). +fn suggest_name<'a, I>(target: &str, candidates: I) -> Option +where + I: IntoIterator, +{ + // rustc-style threshold: scales with name length so single-char + // typos count for short names, but a 12-char identifier + // tolerates a few keystroke errors. Floor at 1, ceiling at 3 — + // anything past 3 starts producing surprising suggestions. + let threshold = (target.chars().count() / 3).clamp(1, 3); + let target_len = target.chars().count(); + let mut best: Option<(usize, &str)> = None; + for cand in candidates { + // Length-difference lower-bound: levenshtein(a, b) ≥ ||a| - |b||. + // Skip candidates whose length already exceeds the threshold — + // no amount of substitution/insertion can bring the distance + // in range. This alone cuts the ~5000-candidate scan on + // gtwiz-versal (proc names 20-80 chars, target ~15 chars) + // down to a small handful of viable comparisons. + let cand_len = cand.chars().count(); + let len_diff = target_len.abs_diff(cand_len); + if len_diff > threshold { + continue; + } + let d = levenshtein_capped(target, cand, threshold); + if d == 0 || d > threshold { + continue; + } + if best.map(|(b, _)| d < b).unwrap_or(true) { + best = Some((d, cand.as_str())); + } + } + best.map(|(_, s)| s.to_string()) +} + +/// Length-capped Levenshtein — returns `max+1` (or larger) as soon +/// as the running row minimum exceeds `max`, avoiding the full +/// O(m*n) sweep when the caller only cares whether the distance +/// is ≤ `max`. Every call from `suggest_name` cares about a +/// threshold of at most 3, so this ends most comparisons after a +/// handful of cells — critical when the candidate table has +/// thousands of entries. +fn levenshtein_capped(a: &str, b: &str, max: usize) -> usize { + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + let m = a.len(); + let n = b.len(); + if m == 0 { + return n; + } + if n == 0 { + return m; + } + let mut prev: Vec = (0..=n).collect(); + let mut cur = vec![0usize; n + 1]; + let sentinel = max + 1; + for i in 1..=m { + cur[0] = i; + let mut row_min = cur[0]; + for j in 1..=n { + let sub = if a[i - 1] == b[j - 1] { 0 } else { 1 }; + cur[j] = (prev[j] + 1).min(cur[j - 1] + 1).min(prev[j - 1] + sub); + if cur[j] < row_min { + row_min = cur[j]; + } + } + if row_min > max { + return sentinel; + } + std::mem::swap(&mut prev, &mut cur); + } + prev[n] +} + +/// Standard Levenshtein edit distance — number of single-character +/// insertions, deletions, or substitutions to turn `a` into `b`. +/// Two-row rolling table; O(n*m) time, O(n) space. +#[allow(dead_code)] +fn levenshtein(a: &str, b: &str) -> usize { + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + let m = a.len(); + let n = b.len(); + if m == 0 { + return n; + } + if n == 0 { + return m; + } + let mut prev: Vec = (0..=n).collect(); + let mut cur = vec![0usize; n + 1]; + for i in 1..=m { + cur[0] = i; + for j in 1..=n { + let sub = if a[i - 1] == b[j - 1] { 0 } else { 1 }; + cur[j] = (prev[j] + 1).min(cur[j - 1] + 1).min(prev[j - 1] + sub); + } + std::mem::swap(&mut prev, &mut cur); + } + prev[n] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn diags(src: &str) -> Vec { + let parsed = parse(src); + // Parse errors shouldn't be present in these tests; assert + // so that test failures point at the right layer. + assert!( + parsed.errors.is_empty(), + "unexpected parse errors: {:?}", + parsed.errors + ); + // Filter out unused-variable warnings AND undefined-variable + // errors. This module tests the arg / type / enum / + // qualified-position validators; fixtures typically declare + // test-only procs with unused args and reference free vars + // to keep the snippets short. The unused / undefined passes + // have their own tests in `unused::tests` / `undefined::tests`. + validate(&parsed.document, src) + .into_iter() + .filter(|d| { + let unused = d.severity == Severity::Warning + && (d.message.starts_with("unused proc arg ") + || d.message.starts_with("unused local ")); + let undefined = d.severity == Severity::Error + && d.message.starts_with("undefined variable "); + !(unused || undefined) + }) + .collect() + } + + fn proc_decl(body: &str, call: &str) -> String { + format!("proc axis_interface {{\n{body}\n}} {{ # body\n}}\n{call}\n") + } + + #[test] + fn happy_path_no_diagnostics() { + let src = proc_decl( + " @default(0) has_tkeep\n @default(8) tdata_num_bytes", + "axis_interface -has_tkeep 1 -tdata_num_bytes 16", + ); + assert!(diags(&src).is_empty()); + } + + #[test] + fn unknown_arg() { + let src = + proc_decl(" @default(0) has_tkeep", "axis_interface -has_typo 1"); + let d = diags(&src); + assert_eq!(d.len(), 1); + assert!( + d[0].message.contains("undefined argument -has_typo"), + "{:?}", + d + ); + assert!(d[0].message.contains("Possible values are has_tkeep")); + } + + #[test] + fn missing_required() { + let src = proc_decl(" @required width", "axis_interface"); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("missing required"))); + } + + #[test] + fn enum_rejects_unlisted_value() { + let src = proc_decl( + " @enum(1, 2, 4, 8) tdata_num_bytes", + "axis_interface -tdata_num_bytes 3", + ); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("@enum"))); + } + + #[test] + fn enum_accepts_listed_value() { + let src = proc_decl( + " @enum(1, 2, 4, 8) tdata_num_bytes", + "axis_interface -tdata_num_bytes 4", + ); + assert!(diags(&src).is_empty()); + } + + #[test] + fn range_check() { + let src = + proc_decl(" @range(1, 16) width", "axis_interface -width 32"); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("out of @range"))); + } + + #[test] + fn requires_dependency() { + let src = proc_decl( + " @default(0) has_tuser\n @requires(has_tuser) tuser_width", + "axis_interface -tuser_width 8", + ); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("requires")), "{:?}", d); + } + + #[test] + fn conflicts_dependency() { + let src = proc_decl( + " has_a\n @conflicts(has_a) has_b", + "axis_interface -has_a 1 -has_b 1", + ); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("conflicts"))); + } + + #[test] + fn one_of_requires_exactly_one_alternative() { + // Two args in an @one_of group — neither supplied → error. + let src = proc_decl( + " @default(\"\") @one_of(b) a\n @default(\"\") @one_of(a) b", + "axis_interface", + ); + let d = diags(&src); + assert!( + d.iter().any(|m| m.message.contains("exactly one of -a, -b") + && m.message.contains("must be supplied")), + "{:?}", + d + ); + } + + #[test] + fn one_of_satisfied_by_either_alternative() { + let src = proc_decl( + " @default(\"\") @one_of(b) a\n @default(\"\") @one_of(a) b", + "axis_interface -a 1", + ); + assert!(diags(&src).is_empty()); + } + + #[test] + fn one_of_rejects_both_alternatives() { + // Both supplied — should be reported once (group rule). + let src = proc_decl( + " @default(\"\") @one_of(b) a\n @default(\"\") @one_of(a) b", + "axis_interface -a 1 -b 2", + ); + let d = diags(&src); + assert!( + d.iter().any(|m| m.message.contains("got -a, -b")), + "{:?}", + d + ); + } + + #[test] + fn one_of_arg_is_not_treated_as_required() { + // An @one_of arg without @default should NOT trigger the + // separate "missing required" error — the group rule + // supersedes individual required-ness. + let src = + proc_decl(" @one_of(b) a\n @one_of(a) b", "axis_interface -a 1"); + let d = diags(&src); + assert!( + d.iter() + .all(|m| !m.message.contains("missing required argument -a") + && !m.message.contains("missing required argument -b")), + "{:?}", + d + ); + } + + #[test] + fn one_of_declarations_are_symmetric() { + // Declaring `@one_of(b)` on `a` alone is enough; we don't need + // the reverse on `b`. + let src = proc_decl( + " @default(\"\") @one_of(b) a\n @default(\"\") b", + "axis_interface", + ); + let d = diags(&src); + let group_errors: Vec<_> = d + .iter() + .filter(|m| { + m.message.contains("exactly one of") + && m.message.contains("must be supplied") + }) + .collect(); + assert_eq!(group_errors.len(), 1, "{:?}", d); + } + + #[test] + fn namespace_eval_proc_validates_at_qualified_name() { + // A proc declared inside `namespace eval project { ... }` + // should be reachable from the validator at its qualified + // name (`project::set_target_language`), so `@enum` + // constraints on its args still catch bad values at call + // sites — exactly like a top-level proc declaration would. + let src = "\ +namespace eval project { + proc set_target_language { + proj + @enum(VHDL, Verilog) language + } { } +} +project::set_target_language -proj p -language Klingon +"; + let d = diags(src); + assert!( + d.iter().any(|m| m.message.contains("Klingon") + && m.message.contains("@enum")), + "{:?}", + d + ); + } + + #[test] + fn namespaced_proc_satisfied_by_valid_args() { + let src = "\ +namespace eval project { + proc set_target_language { + proj + @enum(VHDL, Verilog) language + } { } +} +project::set_target_language -proj p -language VHDL +"; + assert!(diags(src).is_empty()); + } + + #[test] + fn nested_namespace_eval_qualifies_recursively() { + let src = "\ +namespace eval outer { + namespace eval inner { + proc foo { @enum(a, b) x } { } + } +} +outer::inner::foo -x bogus +"; + let d = diags(src); + assert!(d.iter().any(|m| m.message.contains("bogus")), "{:?}", d); + } + + #[test] + fn unknown_call_gets_did_you_mean_suggestion() { + // The exact shape that caught the user's typo in metroid: + // a single-char edit-distance miss against a known proc + // should produce a `did you mean ...` suggestion. + let src = "\ +namespace eval port { + proc plumb_if_pin { + name + pin + } { } +} +port::plum_if_pin -name p -pin q +"; + let d = diags(src); + let err = d.iter().find(|m| m.severity == Severity::Error).unwrap(); + assert!( + err.message.contains("did you mean `port::plumb_if_pin`"), + "{}", + err.message + ); + } + + #[test] + fn unrelated_unknown_call_has_no_suggestion() { + // A name with no near-miss should NOT get a fake suggestion + // tacked on — that's just misleading noise. + let src = "totally_made_up_thing -arg 1\n"; + let d = diags(src); + let err = d.iter().find(|m| m.severity == Severity::Error).unwrap(); + assert!(!err.message.contains("did you mean"), "{}", err.message); + } + + #[test] + fn unknown_keyword_call_is_an_error() { + // No proc declaration in scope, no `extern::` prefix — the + // call uses `-flag` shape so the validator demands the user + // be explicit about the dependency. + let src = "create_project -in_memory 1 -name foo\n"; + let d = diags(src); + assert!( + d.iter().any(|m| m.severity == Severity::Error + && m.message.contains("create_project") + && m.message.contains("extern::")), + "{:?}", + d + ); + } + + #[test] + fn extern_prefixed_call_skips_unknown_check() { + // `extern::` is the user's opt-out: they're calling a raw + // Tcl proc deliberately. No diagnostic even though the + // name isn't in the signature table. + let src = "extern::create_project -name foo\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn positional_unknown_call_is_allowed() { + // No `-flag` args → looks like a positional Tcl builtin + // call (puts, set, etc.). Pass through silently. + let src = "puts hello\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn positional_call_with_namespaced_match_is_flagged() { + // `get_thing X` positional-only would normally pass (raw + // Tcl assumption), but here a `foo::get_thing` is in + // scope — the unqualified form is almost certainly a + // missed namespace prefix, worth flagging with a + // "did you mean" that names the exact match. + let src = "\ +namespace eval foo { + proc get_thing {name} { return $name } +} +puts [get_thing X] +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|e| e.message.contains("undefined proc `get_thing`")) + .collect(); + assert_eq!( + errs.len(), + 1, + "expected one undefined-proc diag, got {d:?}" + ); + assert!( + errs[0].message.contains("foo::get_thing"), + "expected `foo::get_thing` suggestion, got: {}", + errs[0].message, + ); + } + + #[test] + fn positional_call_with_no_namespaced_match_still_passes() { + // No matching namespaced proc → keep the "raw Tcl + // builtin assumption" semantics for positional calls. + // A bare `some_native X` with nothing named `*::some_native` + // in scope stays silent. + let src = "puts [some_native X]\n"; + let d = diags(src); + assert!( + d.iter().all(|e| !e.message.contains("undefined proc")), + "unexpected diag: {d:?}", + ); + } + + #[test] + fn known_tcl_builtin_with_keyword_args_is_allowed() { + // `string match -nocase ...` is a legitimate Tcl-core + // pattern; the allowlist keeps it from triggering the + // unknown-call error. + let src = "string match -nocase pat str\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn namespace_eval_extern_is_rejected() { + let src = "namespace eval extern { proc foo {} { } }\n"; + let d = diags(src); + assert!( + d.iter() + .any(|m| m.message.contains("reserved namespace name")), + "{:?}", + d + ); + } + + #[test] + fn duplicate_arg_warns() { + let src = proc_decl(" has_a", "axis_interface -has_a 1 -has_a 2"); + let d = diags(&src); + assert!( + d.iter().any(|d| d.message.contains("duplicate argument")), + "{:?}", + d + ); + } + + #[test] + fn dynamic_value_skips_enum_check() { + let src = + proc_decl(" @enum(1, 2, 4) width", "axis_interface -width $x"); + // $x is runtime; we don't statically know it's outside the + // enum, so no enum diagnostic. + let d = diags(&src); + assert!(d.iter().all(|d| !d.message.contains("@enum"))); + } + + #[test] + fn validates_call_inside_proc_body() { + // A bad flag on a call nested in another proc's body should be + // diagnosed, same as at the top level. + let src = "\ +proc if_tport {\n type\n name\n} { }\n\ +proc axis_if {\n kind\n} {\n if_tport -type t -namze m\n}\n"; + let d = diags(src); + assert!( + d.iter() + .any(|d| d.message.contains("undefined argument -namze")), + "{:?}", + d + ); + } + + #[test] + fn validates_call_inside_command_substitution() { + // The user case: `set cell [create_cpm5 -foo bar]`. The + // validator must descend into `[…]` so the bad flag is caught + // the same way it is at the top level. + let src = "\ +proc create_cpm5 {\n @default(0) name\n} { }\n\ +set cell [create_cpm5 -foo bar]\n"; + let d = diags(src); + assert!( + d.iter() + .any(|d| d.message.contains("undefined argument -foo")), + "{:?}", + d + ); + } + + #[test] + fn arg_with_no_default_is_implicitly_required() { + // `name` has neither `@default` nor `@required` — calling the + // proc without a value for it should still error. + let src = "\ +proc create_cpm5 {\n name\n} { }\n\ +create_cpm5\n"; + let d = diags(src); + assert!( + d.iter() + .any(|d| d.message.contains("missing required argument -name")), + "{:?}", + d + ); + } + + #[test] + fn implicit_required_satisfied_when_supplied() { + let src = "\ +proc create_cpm5 {\n name\n} { }\n\ +create_cpm5 -name x\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn extra_signatures_resolve_unknown_calls_from_prior_batches() { + // The REPL session case: a wrapper declared in a prior + // batch is in `extra`; the new batch's bare call to it must + // resolve (no `extern::` error) and its keyword args must + // validate against the prior signature. + let prior_src = "\ +namespace eval vivado { + proc create_project { + @default(\"\") name + @enum(0, 1) @default(0) in_memory + } { } +} +"; + let prior_parsed = parse(prior_src); + assert!(prior_parsed.errors.is_empty()); + let mut sink = Vec::new(); + let prior_table = + build_signature_table(&prior_parsed.document, &mut sink); + + // New batch: bare `vivado::create_project -name foo`. No + // declaration in scope here — only the prior batch's table + // saves it from the unknown-keyword-call error. + let new_src = "vivado::create_project -name foo\n"; + let new_parsed = parse(new_src); + let diags = validate_with_signatures( + &new_parsed.document, + new_src, + &prior_table, + ); + assert!( + diags.iter().all(|d| d.severity != Severity::Error), + "{:?}", + diags + ); + + // And the keyword-args still get validated — a bad enum + // value should still error even though the sig came in + // through `extra`. + let bad_src = "vivado::create_project -in_memory bogus\n"; + let bad_parsed = parse(bad_src); + let bad_diags = validate_with_signatures( + &bad_parsed.document, + bad_src, + &prior_table, + ); + assert!( + bad_diags + .iter() + .any(|d| d.message.contains("bogus") + && d.message.contains("@enum")), + "{:?}", + bad_diags + ); + } + + #[test] + fn doc_signatures_shadow_extra_without_warning() { + // Re-declaring a proc in the new batch should NOT raise the + // "duplicate definition" warning against the prior-batch + // signature — that's a normal `src @lib` reload case in the + // REPL and would be noisy. The new declaration takes + // precedence. + let prior_src = "proc foo { @default(0) x } { }\n"; + let prior_parsed = parse(prior_src); + let mut sink = Vec::new(); + let prior_table = + build_signature_table(&prior_parsed.document, &mut sink); + + let new_src = "proc foo { @default(1) y } { }\nfoo -y 2\n"; + let new_parsed = parse(new_src); + let diags = validate_with_signatures( + &new_parsed.document, + new_src, + &prior_table, + ); + assert!( + diags.iter().all(|d| !d.message.contains("duplicate")), + "{:?}", + diags + ); + // And the new sig is the one that resolved: `-y` is + // accepted, `-x` would have been the prior sig's arg. + assert!( + diags.iter().all(|d| d.severity != Severity::Error), + "{:?}", + diags + ); + } + + #[test] + fn unknown_positional_call_is_not_validated() { + // Bare positional call to an unknown name (could be a Tcl + // builtin) is silently accepted. Unknown calls with + // `-flag` args are the *only* unknown-call case that + // errors — see `unknown_keyword_call_is_an_error`. + let src = "axis_interface tkeep_yes 1\n"; + assert!(diags(src).is_empty()); + } + + // --- type-decl triplet enforcement (step 1b) ---------------- + + /// Build a valid type+triplet block — bd_cell with all three + /// procs present so the validator should accept it. + fn full_triplet_src() -> &'static str { + "type bd_cell = string\n\ + proc bd_cell::repr {v} { return $v }\n\ + proc bd_cell::from {v} { return $v }\n\ + proc bd_cell::to {v} { return $v }\n" + } + + #[test] + fn type_decl_with_full_triplet_passes() { + let src = full_triplet_src(); + let d = diags(src); + assert!( + d.iter().all(|d| !d.message.contains("missing required")), + "unexpected diagnostics: {:?}", + d + ); + } + + #[test] + fn type_decl_missing_repr_emits_diagnostic() { + let src = "type bd_cell = string\n\ + proc bd_cell::from {v} { return $v }\n\ + proc bd_cell::to {v} { return $v }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| d.message.contains("missing required")) + .expect("expected diagnostic"); + assert!(hit.message.contains("bd_cell::repr"), "{:?}", hit); + assert_eq!(hit.severity, Severity::Error); + } + + #[test] + fn type_decl_missing_all_three_lists_each() { + let src = "type widget = string\n"; + let d = diags(src); + // Now each missing slot emits its own diagnostic, so we + // assert each one shows up separately. + let missing: Vec<&str> = d + .iter() + .filter(|d| d.message.contains("missing required proc")) + .map(|d| d.message.as_str()) + .collect(); + assert!(missing.iter().any(|m| m.contains("widget::repr"))); + assert!(missing.iter().any(|m| m.contains("widget::from"))); + assert!(missing.iter().any(|m| m.contains("widget::to"))); + } + + #[test] + fn type_decl_wrong_arg_type_emits_diagnostic() { + // Annotate the v arg with the wrong type and expect a + // shape-mismatch diagnostic. + let src = "type widget = string\n\ + proc widget::repr {v: int} string { return $v }\n\ + proc widget::from {v: string} widget { return $v }\n\ + proc widget::to {v: widget} string { return $v }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| d.message.contains("widget::repr")) + .expect("expected mismatch diagnostic"); + assert!( + hit.message.contains("`int`") || hit.message.contains("int"), + "{:?}", + hit + ); + assert!(hit.message.contains("widget"), "{:?}", hit); + } + + #[test] + fn type_decl_wrong_return_type_emits_diagnostic() { + let src = "type widget = string\n\ + proc widget::repr {v: widget} int { return 0 }\n\ + proc widget::from {v: string} widget { return $v }\n\ + proc widget::to {v: widget} string { return $v }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| d.message.contains("returns")) + .expect("expected return-type mismatch diagnostic"); + assert!(hit.message.contains("widget::repr"), "{:?}", hit); + assert!(hit.message.contains("string"), "{:?}", hit); + } + + #[test] + fn type_decl_unannotated_triplet_still_passes() { + // Existence-only check stays a fallback when the user + // hasn't annotated the procs yet. + let src = "type widget = string\n\ + proc widget::repr {v} { return $v }\n\ + proc widget::from {v} { return $v }\n\ + proc widget::to {v} { return $v }\n"; + let d = diags(src); + assert!( + d.iter().all(|d| d.severity != Severity::Error), + "got: {:?}", + d + ); + } + + #[test] + fn type_decl_in_namespace_qualifies() { + let src = "namespace eval x {\n\ + type widget = string\n\ + }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| d.message.contains("missing required")) + .expect("expected diagnostic"); + // The namespace-qualified name should appear in the message. + assert!(hit.message.contains("x::widget"), "{:?}", hit); + assert!(hit.message.contains("x::widget::repr")); + } + + #[test] + fn prior_batch_procs_satisfy_current_batch_type_decl() { + // Batch 1: just declares the procs. + let prior_src = "proc bd_cell::repr {v} { return $v }\n\ + proc bd_cell::from {v} { return $v }\n\ + proc bd_cell::to {v} { return $v }\n"; + let prior = parse(prior_src); + let mut prior_diags = Vec::new(); + let prior_sigs = + build_signature_table(&prior.document, &mut prior_diags); + // Batch 2: declares the type — should NOT complain because + // the procs live in the prior batch's signature table. + let new_src = "type bd_cell = string\n"; + let new_parsed = parse(new_src); + let diags = validate_with_signatures( + &new_parsed.document, + new_src, + &prior_sigs, + ); + assert!( + diags + .iter() + .all(|d| !d.message.contains("missing required")), + "got: {:?}", + diags + ); + } + + #[test] + fn prior_batch_type_decl_does_not_re_trigger_in_current_batch() { + // Batch 1: declares the type, no procs yet (would error + // in isolation). + let prior_src = "type bd_cell = string\n"; + let prior = parse(prior_src); + let mut prior_diags = Vec::new(); + let prior_types = + build_type_decl_table(&prior.document, &mut prior_diags); + // Batch 2: adds the procs. The type is in `extra_types`, + // and the procs are in batch 2's signature table. Putting + // them together via validate_with_extras should pass. + let new_src = "proc bd_cell::repr {v} { return $v }\n\ + proc bd_cell::from {v} { return $v }\n\ + proc bd_cell::to {v} { return $v }\n"; + let new_parsed = parse(new_src); + let empty_sigs: HashMap = HashMap::new(); + let d = validate_with_extras( + &new_parsed.document, + new_src, + &empty_sigs, + &prior_types, + ); + assert!( + d.iter().all(|d| !d.message.contains("missing required")), + "got: {:?}", + d + ); + } + + // --- enum + overload classifier (step 3) ---------------------- + + #[test] + fn enum_decl_with_unique_variants_passes() { + let src = "enum Direction = {\n North\n South\n East\n West\n}\n"; + let d = diags(src); + assert!(d.is_empty(), "got: {:?}", d); + } + + #[test] + fn enum_decl_with_duplicate_variants_errors() { + let src = "enum Bad = {\n A: int\n B: string\n A: bool\n}\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| { + d.severity == Severity::Error + && d.message.contains("variant `A`") + }) + .expect("expected duplicate-variant diagnostic"); + assert!(hit.message.contains("more than once"), "{:?}", hit); + } + + #[test] + fn overload_set_with_exhaustive_arms_classifies() { + let src = "\ +enum Property = {\n Scalar: string\n Nested: int\n}\n\ +proc handle {v: Property::Scalar} { return $v }\n\ +proc handle {v: Property::Nested} { return $v }\n"; + let parsed = parse(src); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + let mut diags = Vec::new(); + let (sig_table, overloads) = build_signature_table_with_overloads( + &parsed.document, + &std::collections::HashSet::new(), + &mut diags, + ); + assert!( + diags.iter().all(|d| d.severity != Severity::Error), + "got: {:?}", + diags + ); + let info = overloads.get("handle").expect("overload info for `handle`"); + assert_eq!(info.enum_name, "Property"); + assert_eq!(info.variants.len(), 2); + let names: Vec<&str> = info + .variants + .iter() + .map(|v| v.variant_name.as_str()) + .collect(); + assert!(names.contains(&"Scalar")); + assert!(names.contains(&"Nested")); + // Public-name entry exists in the sig table. + assert!(sig_table.contains_key("handle")); + // Specializations also register under mangled names so + // analyzer drill-down works. + assert!(sig_table.contains_key("__handle__Scalar")); + assert!(sig_table.contains_key("__handle__Nested")); + } + + #[test] + fn ad_hoc_overload_emits_hard_error() { + let src = "\ +proc foo {v: int} { return $v }\n\ +proc foo {v: string} { return $v }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| { + d.severity == Severity::Error + && d.message.contains("ad-hoc overloading") + }) + .expect("expected ad-hoc-overloading diagnostic"); + assert!(hit.message.contains("foo"), "{:?}", hit); + } + + #[test] + fn overload_with_mismatched_enums_errors() { + let src = "\ +enum A = {\n X\n Y\n}\n\ +enum B = {\n P\n Q\n}\n\ +proc foo {v: A::X} { }\n\ +proc foo {v: B::P} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("mixes enums")), + "got: {:?}", + d + ); + } + + #[test] + fn overload_with_duplicate_variant_errors() { + let src = "\ +enum E = {\n A: int\n B: int\n}\n\ +proc foo {v: E::A} { }\n\ +proc foo {v: E::A} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("two") + && d.message.contains("E::A")), + "got: {:?}", + d + ); + } + + #[test] + fn overload_with_extra_args_errors() { + // v1 restricts overloaded procs to exactly one arg (the + // dispatched variant). Extra tail args trip the arity + // check. + let src = "\ +enum E = {\n A: int\n B: int\n}\n\ +proc foo {\n v: E::A\n x\n} { }\n\ +proc foo {\n v: E::B\n y\n} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("exactly ONE arg")), + "got: {:?}", + d + ); + } + + #[test] + fn overload_with_mismatched_return_type_errors() { + let src = "\ +enum E = {\n A: int\n B: int\n}\n\ +proc foo {v: E::A} int { return 0 }\n\ +proc foo {v: E::B} string { return \"\" }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("return type")), + "got: {:?}", + d + ); + } + + #[test] + fn reserved_prefix_user_proc_errors() { + let src = "proc __foo {v} { return $v }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("reserved") + && d.message.contains("__")), + "got: {:?}", + d + ); + } + + #[test] + fn qualified_type_in_return_position_errors() { + let src = "\ +enum E = {\n A\n}\n\ +proc bad {} E::A { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("qualified") + && d.message.contains("only legal")), + "got: {:?}", + d + ); + } + + #[test] + fn qualified_type_in_tail_arg_errors() { + let src = "\ +enum E = {\n A\n B\n}\n\ +proc bad {\n v: E::A\n x: E::B\n} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("qualified")), + "got: {:?}", + d + ); + } + + #[test] + fn qualified_type_inside_generic_errors() { + let src = "\ +enum E = {\n A\n}\n\ +proc bad {x: list} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("qualified")), + "got: {:?}", + d + ); + } + + /// A qualified name that resolves to a declared newtype + /// (`dcmac::GtChProps`) is legal wherever a Named type is + /// legal — including return-type slots on the newtype's own + /// `from`/`empty` helpers. + #[test] + fn namespaced_newtype_return_type_allowed() { + let src = "\ +namespace eval dcmac {}\n\ +namespace eval dcmac::T {}\n\ +type dcmac::T = string\n\ +proc dcmac::T::repr {v: dcmac::T} string { return $v }\n\ +proc dcmac::T::from {v: string} dcmac::T { return $v }\n\ +proc dcmac::T::to {v: dcmac::T} string { return $v }\n"; + let d = diags(src); + let errs: Vec<_> = + d.iter().filter(|d| d.severity == Severity::Error).collect(); + assert!(errs.is_empty(), "unexpected errors: {errs:?}"); + } + + /// A namespaced newtype used as a tail-arg type annotation + /// (non-first-arg position, previously ruled out for + /// Qualified) passes when the name resolves to a real newtype. + #[test] + fn namespaced_newtype_tail_arg_allowed() { + // `use` returns via `dcmac::T::to` so the return-type + // check sees a `string`-typed value matching the + // declared `string` return. A raw `return $slot` (leaking + // the newtype out as its underlying) is now correctly + // flagged as a type mismatch — the analyzer wants + // callers to cross the newtype boundary through the + // explicit `T::to` conversion. + let src = "\ +namespace eval dcmac {}\n\ +namespace eval dcmac::T {}\n\ +type dcmac::T = string\n\ +proc dcmac::T::repr {v: dcmac::T} string { return $v }\n\ +proc dcmac::T::from {v: string} dcmac::T { return $v }\n\ +proc dcmac::T::to {v: dcmac::T} string { return $v }\n\ +proc use {name slot: dcmac::T} string { return [dcmac::T::to -v $slot] }\n"; + let d = diags(src); + let errs: Vec<_> = + d.iter().filter(|d| d.severity == Severity::Error).collect(); + assert!(errs.is_empty(), "unexpected errors: {errs:?}"); + } + + /// Unknown qualified names (no matching `type` decl) still + /// get rejected — the disambiguator only clears names that + /// resolve to a real newtype. + #[test] + fn unknown_qualified_still_rejected() { + let src = "proc bad {name} Unknown::Thing { return $name }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("qualified") + && d.message.contains("only legal")), + "got: {:?}", + d + ); + } + + #[test] + fn recursive_enum_passes() { + // Inner generic with whitespace needs brace-wrapping at + // the word level — that's the existing type-decl rule. + let src = "\ +enum Property = {\n Scalar: string\n Nested: Properties\n}\n\ +type Properties = {dict}\n\ +proc Properties::repr {v} { return $v }\n\ +proc Properties::from {v} { return $v }\n\ +proc Properties::to {v} { return $v }\n"; + let d = diags(src); + // No errors — Property/Properties cycle is fine (Tcl + // resolves at call time) and the triplet exists for the + // type-decl side. + assert!( + d.iter().all(|d| d.severity != Severity::Error), + "got: {:?}", + d + ); + } + + // ------------------------------------------------------------------ + // Undefined `src @` module check. + // ------------------------------------------------------------------ + + fn src_diags(src: &str, known_deps: &[&str]) -> Vec { + let parsed = crate::parser::parse(src); + assert!( + parsed.errors.is_empty(), + "unexpected parse errors: {:?}", + parsed.errors + ); + let deps: std::collections::HashSet = + known_deps.iter().map(|s| s.to_string()).collect(); + validate_with_all_extras_and_vars( + &parsed.document, + src, + &HashMap::new(), + &HashMap::new(), + &HashMap::new(), + &std::collections::HashSet::new(), + &deps, + ) + .into_iter() + .filter(|d| d.message.starts_with("unknown src module")) + .collect() + } + + #[test] + fn src_at_named_unresolved_flagged() { + let src = "src @gtwiz-versal\n"; + let d = src_diags(src, &["vivado-cmd", "cpm5"]); + assert_eq!(d.len(), 1); + assert_eq!(d[0].severity, Severity::Error); + assert!(d[0].message.contains("gtwiz-versal"), "{}", d[0].message); + assert!( + d[0].message.contains("[dependencies.gtwiz-versal]"), + "{}", + d[0].message + ); + // Span should cover the `@gtwiz-versal` text. + let bytes = + &src.as_bytes()[d[0].span.start as usize..d[0].span.end as usize]; + assert_eq!(std::str::from_utf8(bytes).unwrap(), "@gtwiz-versal"); + } + + #[test] + fn src_at_named_resolved_clean() { + let src = "src @vivado-cmd\n"; + let d = src_diags(src, &["vivado-cmd"]); + assert!(d.is_empty(), "unexpected diags: {d:?}"); + } + + #[test] + fn src_at_named_multi_flagged() { + let src = "src @foo\nsrc @bar\nsrc @cpm5\n"; + let d = src_diags(src, &["cpm5"]); + assert_eq!(d.len(), 2); + // Distinct spans. + assert_ne!(d[0].span.start, d[1].span.start); + } + + #[test] + fn src_bare_path_not_flagged() { + // Relative path form — never triggers the `@` check + // even when known_deps is empty. Filesystem existence gets + // validated downstream by the loader; the analyzer's + // job here is only the dep-name lookup. + let src = "src ./ports.htcl\n"; + let d = src_diags(src, &[]); + assert!(d.is_empty(), "unexpected diags: {d:?}"); + } + + #[test] + fn src_subpath_reported_with_hint() { + // `@foo/sub` — the subpath appears in the diagnostic + // message so the user sees the exact directive that + // failed. Also verifies subpath doesn't confuse the + // classifier. + let src = "src @gtwiz-versal/module\n"; + let d = src_diags(src, &["cpm5"]); + assert_eq!(d.len(), 1); + assert!( + d[0].message.contains("@gtwiz-versal/module"), + "{}", + d[0].message + ); + } + + #[test] + fn empty_deps_no_check() { + // The check must no-op when the caller doesn't know about + // any deps — matches the behavior for unit tests / non- + // workspace-aware callers who invoke `validate` directly. + let src = "src @gtwiz-versal\nsrc @other\n"; + let d = src_diags(src, &[]); + assert!(d.is_empty(), "unexpected diags: {d:?}"); + } + + // ------ call-site type check --------------------------------- + // + // Three fixtures exercise the arg-type check: + // + // 1. Matching newtypes → no diagnostic (happy path). + // 2. Mismatched newtypes (the copy-paste-wrong-constructor + // bug in ip/gtm.htcl) → one diagnostic naming both types. + // 3. Unknown-type value (literal string) → silent skip + // (gradual typing). Sanity-checks that the check doesn't + // over-fire on values we can't infer. + + /// Set up a minimal document with two newtypes and one taker + /// proc that accepts a `-slot` of each. Callers append their + /// own top-level call. + fn typed_slots_fixture(tail: &str) -> String { + format!( + "type ns::TypeA = Properties\n\ + type ns::TypeB = Properties\n\ + proc make_a {{}} ns::TypeA {{ return {{}} }}\n\ + proc make_b {{}} ns::TypeB {{ return {{}} }}\n\ + proc take {{ @default(\"\") slot: ns::TypeA }} {{ }}\n\ + {tail}", + ) + } + + #[test] + fn type_check_matching_types_no_diagnostic() { + // `-slot [make_a]` — expected ns::TypeA, actual ns::TypeA. + let src = typed_slots_fixture("take -slot [make_a]\n"); + let d = diags(&src); + assert!( + d.iter().all(|x| !x.message.contains("type mismatch")), + "unexpected type diags: {d:?}", + ); + } + + #[test] + fn type_check_mismatched_types_errors() { + // `-slot [make_b]` — expected ns::TypeA, actual ns::TypeB. + // This is the class of bug the check exists to catch. + let src = typed_slots_fixture("take -slot [make_b]\n"); + let d = diags(&src); + let type_errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("type mismatch")) + .collect(); + assert_eq!(type_errs.len(), 1, "expected 1 type diag, got {d:?}"); + let msg = &type_errs[0].message; + assert!( + msg.contains("ns::TypeA") && msg.contains("ns::TypeB"), + "message should name both types: {msg}", + ); + assert!(msg.contains("-slot"), "message should name the arg: {msg}",); + } + + #[test] + fn type_check_unknown_value_is_silent() { + // `-slot hello` — value is a plain literal, type unknown. + // The check must NOT fire (gradual typing). + let src = typed_slots_fixture("take -slot hello\n"); + let d = diags(&src); + assert!( + d.iter().all(|x| !x.message.contains("type mismatch")), + "unexpected type diag on untyped literal: {d:?}", + ); + } + + #[test] + fn type_check_var_binding_flows_through_set() { + // `set x [make_b]; take -slot $x` — actual type flows + // through the `set` binding into the `$x` reference. + let src = typed_slots_fixture("set x [make_b]\ntake -slot $x\n"); + let d = diags(&src); + let type_errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("type mismatch")) + .collect(); + assert_eq!( + type_errs.len(), + 1, + "expected 1 type diag through `set`, got {d:?}", + ); + } + + // ------ bool literal check ---------------------------------- + // + // Sanity check the four legs of the bool-typed slot machinery: + // `true` and `false` land clean, `1` and arbitrary garbage + // both error with a message that names the offending literal + // and the arg. + + fn bool_slot_fixture(value: &str) -> String { + format!( + "proc take {{ @default(false) flag: bool }} {{ }}\n\ + take -flag {value}\n", + ) + } + + #[test] + fn bool_literal_true_no_diagnostic() { + let src = bool_slot_fixture("true"); + let d = diags(&src); + assert!( + d.iter().all(|x| !x.message.contains("type mismatch")), + "unexpected type diag on `true`: {d:?}", + ); + } + + #[test] + fn bool_literal_false_no_diagnostic() { + let src = bool_slot_fixture("false"); + let d = diags(&src); + assert!( + d.iter().all(|x| !x.message.contains("type mismatch")), + "unexpected type diag on `false`: {d:?}", + ); + } + + #[test] + fn bool_literal_integer_1_errors() { + // The specific class of bug this pass exists to catch — + // `-enable_reg_interface 1` accepted silently today. + let src = bool_slot_fixture("1"); + let d = diags(&src); + let type_errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("type mismatch")) + .collect(); + assert_eq!(type_errs.len(), 1, "expected 1 diag, got {d:?}"); + let msg = &type_errs[0].message; + assert!(msg.contains("bool"), "message names type: {msg}"); + assert!(msg.contains("1"), "message names literal: {msg}"); + assert!(msg.contains("-flag"), "message names arg: {msg}"); + } + + #[test] + fn bool_literal_arbitrary_string_errors() { + // Guards against `potato` / `yes` / `on` sliding through + // as "Tcl also accepts this as truthy" — HTCL's bool + // surface is exactly `true` / `false`. + let src = bool_slot_fixture("potato"); + let d = diags(&src); + let type_errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("type mismatch")) + .collect(); + assert_eq!(type_errs.len(), 1, "expected 1 diag, got {d:?}"); + assert!( + type_errs[0].message.contains("potato"), + "message names offending literal: {}", + type_errs[0].message, + ); + } + + // ------ return-type check ------------------------------------ + // + // Annotated procs must produce a value whose type matches the + // declared return type on every `return X` in the body — + // including returns buried inside `if`/`while`/etc bodies. + + #[test] + fn return_type_matching_annotation_no_diag() { + let src = "\ +type ns::TypeA = Properties +proc make_a {} ns::TypeA { return {} } +proc use_a {} ns::TypeA { + set x [make_a] + return $x +} +"; + let d = diags(src); + assert!( + d.iter() + .all(|x| !x.message.contains("return type mismatch")), + "unexpected diags: {d:?}", + ); + } + + #[test] + fn return_type_mismatch_errors() { + // Body returns `ns::TypeA` but annotation says `ns::TypeB`. + // This is the shape the user hit with `configure_gtm` + // wrongly annotated `cpm5::Config`. + let src = "\ +type ns::TypeA = Properties +type ns::TypeB = Properties +proc make_a {} ns::TypeA { return {} } +proc mismatched {} ns::TypeB { + set x [make_a] + return $x +} +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("return type mismatch")) + .collect(); + assert_eq!(errs.len(), 1, "expected 1 diag, got {d:?}"); + let msg = &errs[0].message; + assert!( + msg.contains("ns::TypeA") && msg.contains("ns::TypeB"), + "message names both types: {msg}", + ); + } + + #[test] + fn return_type_check_descends_into_if_body() { + // Wrong-typed return buried inside an `if` body — the + // walker parses the braced body and finds the return. + let src = "\ +type ns::TypeA = Properties +type ns::TypeB = Properties +proc make_a {} ns::TypeA { return {} } +proc branchy {} ns::TypeB { + if 1 { + set x [make_a] + return $x + } + return {} +} +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("return type mismatch")) + .collect(); + assert!(!errs.is_empty(), "expected mismatch inside if, got {d:?}"); + } + + #[test] + fn bare_return_in_annotated_proc_errors() { + let src = "\ +type ns::TypeA = Properties +proc bad {} ns::TypeA { + return +} +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("bare `return`")) + .collect(); + assert_eq!(errs.len(), 1, "expected bare-return diag, got {d:?}"); + } + + #[test] + fn bare_return_in_unit_proc_is_ok() { + // `unit` return type = "no meaningful value"; bare + // `return` is idiomatic for side-effecting procs that + // early-out on a condition. + let src = "\ +proc side_effect {x} unit { + if $x { return } + return +} +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("bare `return`")), + "unexpected bare-return diag: {d:?}", + ); + } + + #[test] + fn unannotated_proc_return_with_value_errors() { + // No return annotation + `return X` → an error. The proc's + // shape declares "side effects only," and a value-carrying + // return contradicts that. + let src = "\ +proc anything {} { return 42 } +"; + let d = diags(src); + assert!( + d.iter() + .any(|x| x.message.contains("no declared return type")), + "expected diag, got: {d:?}", + ); + } + + #[test] + fn unannotated_proc_bare_return_ok() { + // Bare `return` is fine in a side-effect proc — common + // early-exit pattern. + let src = "\ +proc anything {} { puts hi; return } +"; + let d = diags(src); + assert!( + d.iter() + .all(|x| !x.message.contains("no declared return type")), + "unexpected diag, got: {d:?}", + ); + } + + #[test] + fn unannotated_proc_return_value_inside_if_errors() { + // A value-return buried in a control-flow branch still fires. + let src = "\ +proc anything { x: int } { if {$x > 0} { return 42 } } +"; + let d = diags(src); + assert!( + d.iter() + .any(|x| x.message.contains("no declared return type")), + "expected diag, got: {d:?}", + ); + } + + // ─── must-return / fallthrough analysis ───────────────────── + + fn has_fallthrough_diag(d: &[Diagnostic]) -> bool { + d.iter().any(|x| x.message.contains("may fall through")) + } + + #[test] + fn empty_body_annotated_proc_errors() { + // The user's `configure_txr1` case: annotated with a + // real type but the body is empty. Must-return should + // flag it. + let src = "\ +type MyType = string +proc configure_txr1 {} MyType { } +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn single_puts_body_annotated_proc_errors() { + // Body has a side-effecting `puts` and no return; the + // last statement's result isn't a MyType. + let src = "\ +type MyType = string +proc foo {} MyType { + puts hello +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn if_no_else_annotated_proc_errors() { + let src = "\ +type MyType = string +proc bad {} MyType { + if 1 { + return {} + } +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn while_body_return_annotated_proc_errors() { + // Even with a `return` inside the loop body, the loop + // may not execute — must-return still fires. + let src = "\ +type MyType = string +proc bad {} MyType { + while 1 { + return {} + } +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn if_else_both_return_no_error() { + let src = "\ +type MyType = string +proc ok {} MyType { + if 1 { + return {} + } else { + return {} + } +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn if_elseif_else_all_return_no_error() { + let src = "\ +type MyType = string +proc ok {} MyType { + if 1 { + return {} + } elseif 2 { + return {} + } else { + return {} + } +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn implicit_last_expression_typed_proc_call_no_error() { + // Trailing typed proc-call is an implicit return in Tcl. + let src = "\ +type MyType = string +proc make_it {} MyType { return {} } +proc user {} MyType { + make_it +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn implicit_last_expression_extern_call_no_error() { + // Trailing extern call is the user's opt-out for raw + // Tcl. Trust it as a valid implicit return. + let src = "\ +type MyType = string +proc user {} MyType { + extern::some_raw_tcl_proc -foo bar +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn switch_with_default_all_return_no_error() { + let src = "\ +type MyType = string +proc ok {} MyType { + switch $x { + a { return {} } + default { return {} } + } +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn switch_no_default_errors() { + let src = "\ +type MyType = string +proc bad {} MyType { + switch $x { + a { return {} } + b { return {} } + } +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn switch_default_falls_through_errors() { + let src = "\ +type MyType = string +proc bad {} MyType { + switch $x { + a { return {} } + default { puts hi } + } +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn try_body_and_handler_terminate_no_error() { + // Matches the vw-ip generator's wrap pattern: + // `try { return X } on error { error "…" }`. + let src = "\ +type MyType = string +proc gen {} MyType { + try { + return {} + } on error {msg} { + error \"foo.$msg\" + } +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn unit_annotated_empty_body_no_error() { + // `unit` return type doesn't require a value. + let src = "\ +proc side_effect {} unit { } +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn newtype_triplet_empty_body_no_error() { + // T::from and friends are exempted before the must-return + // check fires. + let src = "\ +type ns::T = string +proc ns::T::from {v: string} ns::T { return $v } +proc ns::T::empty {} ns::T { } +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn enum_overload_arm_empty_body_no_error() { + // First-arg-Qualified shape (overload arm) is exempted + // before the must-return check fires. + let src = "\ +enum E = { + A: string + B: string +} +proc handle {v: E::A} string { } +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + // ─── @test attribute semantic checks ─────────────────────── + + #[test] + fn test_attribute_dedicated_eda_ok() { + let src = "@test(dedicated-eda)\nproc t {} { }\n"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("dedicated-eda")), + "unexpected diag: {d:?}", + ); + } + + #[test] + fn test_attribute_wrong_positional_value_warns() { + let src = "@test(bogus)\nproc t {} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("dedicated-eda")), + "expected `dedicated-eda`-related warning: {d:?}", + ); + } + + #[test] + fn test_attribute_on_proc_with_args_warns() { + // Zero-arg only for MVP. + let src = "@test\nproc t {x: int} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("zero arguments")), + "expected zero-arg warning: {d:?}", + ); + } + + #[test] + fn test_attribute_on_nested_proc_warns() { + // Only top-level @test procs are runnable. + let src = "\ +proc outer {} { + @test + proc inner {} { puts hi } +} +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("nested proc")), + "expected nested-proc warning: {d:?}", + ); + } + + // ─── @test(target=…) keyed-item checks ───────────────────── + + #[test] + fn test_attribute_with_dedicated_eda_and_target_ok() { + let src = "\ +@test(dedicated-eda target=\"xcvm3358-vsvh1747-2M-e-S\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("`@test") + && !x.message.contains("target=")), + "unexpected @test diag: {d:?}", + ); + } + + #[test] + fn test_target_without_dedicated_eda_warns() { + // Shared bucket can't honor per-test parts. + let src = "\ +@test(target=\"xcvm3358-vsvh1747-2M-e-S\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("dedicated-eda")), + "expected dedicated-eda requirement warning: {d:?}", + ); + } + + #[test] + fn test_attribute_unknown_key_warns() { + let src = "\ +@test(dedicated-eda family=versal) +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("unrecognized key")), + "expected unrecognized-key warning: {d:?}", + ); + } + + #[test] + fn test_attribute_variant_with_dedicated_eda_ok() { + let src = "\ +@test(dedicated-eda variant=\"vpk120\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("`@test")), + "unexpected @test diag: {d:?}", + ); + } + + #[test] + fn test_variant_without_dedicated_eda_warns() { + let src = "\ +@test(variant=\"vpk120\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("dedicated-eda")), + "expected dedicated-eda requirement warning: {d:?}", + ); + } + + #[test] + fn test_target_and_variant_together_warns() { + // Mutually exclusive within `@test` — variants own their + // parts, so specifying both is a config bug. + let src = "\ +@test(dedicated-eda target=\"xcv...\" variant=\"vpk120\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("pick one of")), + "expected pick-one-of warning: {d:?}", + ); + } +} diff --git a/vw-ip/Cargo.toml b/vw-ip/Cargo.toml new file mode 100644 index 0000000..9846a73 --- /dev/null +++ b/vw-ip/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "vw-ip" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +ipxact.workspace = true +vw-htcl = { path = "../vw-htcl" } +vw-quote = { path = "../vw-quote" } +thiserror.workspace = true +serde.workspace = true +quick-xml = { version = "0.37", features = ["serialize"] } +regex.workspace = true +toml.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-ip/src/cips_dict.rs b/vw-ip/src/cips_dict.rs new file mode 100644 index 0000000..cf4bdee --- /dev/null +++ b/vw-ip/src/cips_dict.rs @@ -0,0 +1,886 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Schema loader for Xilinx's `structured_tcldict` IP-XACT parameters. +//! +//! Some IP-XACT parameters in CIPS-family components are declared with +//! `structured_tcldict`: +//! the IP-XACT value is just an opaque space-separated `KEY VAL …` +//! dict string. The real schema for those inner fields lives in +//! out-of-band data files Vivado ships: +//! +//! - `versal/flows/automation/cipsToPsWiz_Porting/csv_files/` +//! - `param_mapping_direct.csv` — `(KEY, {DEFAULT}, …)` per row. +//! - `param_mapping_presets.csv` — preset-bundle layout for +//! `mode`-style selector fields like `CLOCK_MODE`, `BOOT_MODE`. +//! - `versal/cips_hip//guidata/ParamInfo.xml` — per-field +//! `` text, used as a doc comment. +//! - `versal/cips_hip//global/global_preset*.xml` and +//! `versal/cips_hip//presets/**/*.xml` — `` entries used to widen `@enum(…)` lists, same format +//! already parsed by [`crate::presets`]. +//! +//! We deliberately ignore the deprecated +//! `flows/automation/deprecated/cips_pswiz_key_and_value.csv` — its +//! content is a subset of the two supported CSVs above. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::overrides::OverridesFile; +use crate::paired_list::{parse_paired_list, PairedValue}; + +#[derive(Debug, Clone, Default)] +pub struct DictSchema { + /// Scalar fields at this level — each becomes a typed + /// `@default(…) name` proc arg in the emitted constructor. + pub fields: Vec, + /// Sub-schemas for nested paired-list slots at this level. Keyed + /// by the same upper-snake XML name the field would carry; the + /// emitter picks a nested-namespace proc name from that key. + /// `LR0_SETTINGS` → sub-schema whose `fields` are `RX_HD_EN`, + /// `TX_HD_EN`, etc., emitted as + /// `gtwiz_versal::intf::gt_settings::lr0_settings`. + pub sub_schemas: BTreeMap, +} + +#[derive(Debug, Clone)] +pub struct DictField { + /// IP-XACT-style upper-snake name, e.g. `PCIE_APERTURES_DUAL_ENABLE`. + pub name: String, + /// Default value as recorded in the supporting data files. May be + /// empty when no default is known (rare); the generator treats it + /// the same as any other defaultless arg. + pub default: String, + /// Display-name / one-line description from `ParamInfo.xml`, when + /// present. + pub description: Option, + /// `@enum(…)` choices we were able to recover from preset files + /// or from `param_mapping_presets.csv`. Empty when no enum data + /// was found. + pub enum_values: BTreeSet, +} + +impl DictSchema { + /// Build a nested `DictSchema` from a paired-list default value. + /// + /// This is the entry point for typed-constructor emission on + /// Properties-shaped params that AREN'T sourced from Xilinx's + /// `structured_tcldict` CSVs — gtwiz-versal's + /// `INTF*_TXRX_OPTIONAL_PORTS`, `INTF*_CHANNEL_MAP`, etc. The + /// paired-list parser turns the XML default into a tree of + /// `PairedValue::Scalar` (leaves) and `PairedValue::Nested` + /// (sub-slots); this walks that tree, applying `overrides` at + /// each level to attach `@enum(...)` restrictions and default + /// refinements where the XML is silent. + /// + /// `shape_path` is the `::`-separated ident chain the caller + /// will use to look up overrides — e.g. `"intf::gt_settings"` + /// for the top-level `gt_settings` slot on the `intf` family. + /// Sub-schemas recurse with the current field name appended + /// (`"intf::gt_settings::lr0_settings"` for the LR0 slot). + pub fn from_paired_default( + default: &str, + shape_path: &str, + overrides: &OverridesFile, + ) -> Self { + let pairs = parse_paired_list(default); + Self::from_pairs(&pairs, shape_path, overrides) + } + + /// Walk `self`'s fields and re-apply `overrides` at the given + /// `shape_path`. Used when a schema is COPIED from an anchor + /// param to a different slot — the anchor's fields were built + /// with the anchor's shape path, but at the copy site the same + /// fields are surfaced under a different shape path that may + /// have its own overrides. Idempotent: a field with no matching + /// override in either place stays unchanged. + pub fn reapply_overrides( + &mut self, + shape_path: &str, + overrides: &OverridesFile, + ) { + for f in &mut self.fields { + let field_lookup = lower(&f.name); + let Some(fo) = overrides.field(shape_path, &field_lookup) else { + continue; + }; + if let Some(default) = &fo.default { + f.default = default.clone(); + } + if let Some(enum_values) = &fo.enum_values { + f.enum_values = enum_values.iter().cloned().collect(); + } + } + for (sub_name, sub) in &mut self.sub_schemas { + let sub_path = format!("{shape_path}::{}", lower(sub_name)); + sub.reapply_overrides(&sub_path, overrides); + } + } + + fn from_pairs( + pairs: &[(String, PairedValue)], + shape_path: &str, + overrides: &OverridesFile, + ) -> Self { + let mut fields = Vec::new(); + let mut sub_schemas = BTreeMap::new(); + for (raw_name, value) in pairs { + match value { + PairedValue::Nested(inner) => { + // Nested slot — recurse. Sub-shape path extends + // the current path with the lowercase form of + // this key, matching the convention the emitter + // uses when writing the sub-proc's namespace + // segment. + let sub_path = if shape_path.is_empty() { + lower(raw_name) + } else { + format!("{shape_path}::{}", lower(raw_name)) + }; + let sub = Self::from_pairs(inner, &sub_path, overrides); + sub_schemas.insert(raw_name.clone(), sub); + } + PairedValue::Scalar(default) => { + // Merge overrides on top of the XML default. + // Field key inside the override file is the + // lowercase form (matching the emitted arg name); + // the underlying DictField preserves the raw + // upper-snake name for downstream `set_property` + // key composition. + let field_lookup = lower(raw_name); + let field_override = + overrides.field(shape_path, &field_lookup); + let (final_default, enum_values) = match field_override { + Some(fo) => { + let d = fo + .default + .clone() + .unwrap_or_else(|| default.clone()); + let e = fo + .enum_values + .clone() + .map(|v| v.into_iter().collect::>()) + .unwrap_or_default(); + (d, e) + } + None => (default.clone(), BTreeSet::new()), + }; + fields.push(DictField { + name: raw_name.clone(), + default: final_default, + description: None, + enum_values, + }); + } + } + } + Self { + fields, + sub_schemas, + } + } +} + +/// Lowercase form of a field/slot name used for override lookup and +/// (downstream) as the emitted arg / namespace segment. Uppercase +/// letters map to lowercase; underscores and digits pass through +/// verbatim. Matches `generate::lowercase_ident`'s behavior on +/// upper-snake input (no digit-suffix handling needed here — Xilinx +/// keys don't start with digits). +fn lower(s: &str) -> String { + s.to_ascii_lowercase() +} + +/// Returns the schema for each `structured_tcldict` parameter we can +/// find data for. Keys are the IP-XACT parameter names +/// (`PS_PMC_CONFIG`, …); the matching `_INTERNAL` variants point at +/// the same schema. +/// +/// Empty when the Xilinx `data/` ancestor can't be located. +pub fn load_schemas(component_path: &Path) -> HashMap { + let mut out = HashMap::new(); + let Some(data_root) = find_data_root(component_path) else { + return out; + }; + if let Some(schema) = load_ps_pmc_schema(&data_root) { + out.insert("PS_PMC_CONFIG".to_string(), schema.clone()); + out.insert("PS_PMC_CONFIG_INTERNAL".to_string(), schema); + } + out +} + +/// Inputs scoped to a CIPS `PS_PMC_CONFIG`. +fn load_ps_pmc_schema(data_root: &Path) -> Option { + let pspmc = data_root.join("versal/cips_hip/pspmc"); + let csv_dir = + data_root.join("versal/flows/automation/cipsToPsWiz_Porting/csv_files"); + if !pspmc.is_dir() || !csv_dir.is_dir() { + return None; + } + + let mut fields: HashMap = HashMap::new(); + parse_direct_csv(&csv_dir.join("param_mapping_direct.csv"), &mut fields); + parse_presets_csv(&csv_dir.join("param_mapping_presets.csv"), &mut fields); + + // Drop keys that belong to a different `structured_tcldict`. + fields.retain(|name, _| { + !name.starts_with("CPM_") + && !name.starts_with("XRAM_") + && !is_cips_toplevel(name) + }); + if fields.is_empty() { + return None; + } + + layer_param_info(&pspmc.join("guidata/ParamInfo.xml"), &mut fields); + layer_presets(&pspmc, &mut fields); + + let mut sorted: Vec = fields.into_values().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + Some(DictSchema { + fields: sorted, + // PS_PMC_CONFIG is a flat CSV-driven schema — no nested + // sub-slots. The from_paired_default path is what populates + // sub_schemas for XML-derived schemas. + sub_schemas: BTreeMap::new(), + }) +} + +/// Names of `` entries that live at the top level of +/// the CIPS IP-XACT — we don't want to re-emit them as inner dict +/// fields. (Recovered by `vw ip generate` separately, but easier to +/// hard-code the small list than to thread the IP-XACT through here.) +fn is_cips_toplevel(name: &str) -> bool { + matches!( + name, + "AURORA_LINE_RATE_GPBS" + | "BOOT_SECONDARY_PCIE_ENABLE" + | "Component_Name" + | "GT_REFCLK_MHZ" + | "PMC_REF_CLK_FREQMHZ" + | "PS_PMC_CONFIG" + | "PS_PMC_CONFIG_INTERNAL" + | "PS_PMC_CONFIG_APPLIED" + | "CPM_CONFIG" + | "CPM_CONFIG_INTERNAL" + | "XRAM_CONFIG" + | "XRAM_CONFIG_INTERNAL" + ) +} + +/// `param_mapping_direct.csv` layout: `,{CIPS_DEFAULT},,{PSWIZ_DEFAULT}`. +/// The `{…}` value cells routinely contain commas (Tcl list syntax), +/// so we tokenize comma-separated columns at brace depth 0 rather than +/// splitting on every comma. Rows whose value has unbalanced braces +/// (the Xilinx CSV does ship a handful of those — line-wrapped or +/// truncated by the vendor) keep the field name but no default. +fn parse_direct_csv(path: &Path, fields: &mut HashMap) { + let Ok(text) = fs::read_to_string(path) else { + return; + }; + let text = text.strip_prefix('\u{feff}').unwrap_or(&text); + for line in text.lines() { + let cols = split_brace_aware(line); + let (Some(key), Some(raw_default)) = ( + cols.first().map(|s| s.trim()), + cols.get(1).map(|s| s.trim()), + ) else { + continue; + }; + if key.is_empty() || !is_safe_key(key) { + continue; + } + let stripped = unwrap_one_brace(raw_default); + let default = if braces_balanced(stripped) { + stripped.to_string() + } else { + String::new() + }; + fields.entry(key.to_string()).or_insert_with(|| DictField { + name: key.to_string(), + default, + description: None, + enum_values: BTreeSet::new(), + }); + } +} + +/// Split a CSV row on commas at brace depth 0. Treats `{` and `}` as +/// Tcl-style grouping characters so that `KEY,{a,b,c},…` splits into +/// three columns rather than five. +fn split_brace_aware(line: &str) -> Vec<&str> { + let mut out = Vec::new(); + let bytes = line.as_bytes(); + let mut start = 0usize; + let mut depth: i32 = 0; + for (i, b) in bytes.iter().enumerate() { + match b { + b'{' => depth += 1, + b'}' => depth -= 1, + b',' if depth == 0 => { + out.push(&line[start..i]); + start = i + 1; + } + _ => {} + } + } + out.push(&line[start..]); + out +} + +fn braces_balanced(s: &str) -> bool { + let mut depth: i32 = 0; + for b in s.bytes() { + match b { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth < 0 { + return false; + } + } + _ => {} + } + } + depth == 0 +} + +/// `param_mapping_presets.csv` layout: a single header row lists +/// preset-selector field names (e.g. `BOOT_MODE,CLOCK_MODE,…`); each +/// data row has the selector name in column 0 and one valid value in +/// column 1. The CSV repeats the header row between sections — we +/// detect those repeats and skip them so the header names don't get +/// mistakenly recorded as values of each other. +fn parse_presets_csv(path: &Path, fields: &mut HashMap) { + let Ok(text) = fs::read_to_string(path) else { + return; + }; + let text = text.strip_prefix('\u{feff}').unwrap_or(&text); + let mut lines = text.lines().filter(|l| !l.trim().is_empty()); + let Some(header_line) = lines.next() else { + return; + }; + let headers: BTreeSet = header_line + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty() && is_safe_key(s)) + .map(str::to_string) + .collect(); + + let mut by_key: HashMap> = HashMap::new(); + for line in lines { + let cols: Vec<&str> = line.split(',').map(str::trim).collect(); + let Some(first) = cols.first().filter(|c| !c.is_empty()) else { + continue; + }; + // Skip repeated header rows: every non-empty cell is itself a + // declared header name. + let is_header_row = + cols.iter().all(|c| c.is_empty() || headers.contains(*c)); + if is_header_row { + continue; + } + if !headers.contains(*first) { + continue; + } + let Some(val) = cols.get(1).filter(|v| !v.is_empty()) else { + continue; + }; + by_key + .entry((*first).to_string()) + .or_default() + .insert(unwrap_one_brace(val).to_string()); + } + + for name in headers { + let mut enums = by_key.remove(&name).unwrap_or_default(); + // Vivado UI convention: preset-selector fields always offer + // `Custom` as the "configure each inner field manually" choice + // even when the CSV doesn't enumerate it. It's also the most + // useful default — picking a preset bundle locks the inner + // fields, picking `Custom` lets the user override them. + enums.insert("Custom".to_string()); + let default = "Custom".to_string(); + let f = fields.entry(name.clone()).or_insert_with(|| DictField { + name: name.clone(), + default: default.clone(), + description: None, + enum_values: BTreeSet::new(), + }); + for v in enums { + f.enum_values.insert(v); + } + } +} + +/// Strip one layer of Tcl-style braces from a value if present: +/// `{0}` → `0`, `{{ENABLE 0}}` → `{ENABLE 0}`. Leaves unbalanced or +/// unbraced inputs alone. +fn unwrap_one_brace(s: &str) -> &str { + let s = s.trim(); + if s.len() >= 2 && s.starts_with('{') && s.ends_with('}') { + // Verify the outer braces actually pair with each other (i.e. + // depth reaches 0 only at the final `}`). + let mut depth: i32 = 0; + for (i, b) in s.bytes().enumerate() { + match b { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 && i != s.len() - 1 { + return s; // not a single outer pair + } + } + _ => {} + } + } + return &s[1..s.len() - 1]; + } + s +} + +/// Conservative IP-XACT-style identifier check: starts with a letter +/// or `_`, then alphanumerics / `_`. Anything else is data we don't +/// understand and should ignore (rather than mistake for a field). +fn is_safe_key(s: &str) -> bool { + let mut chars = s.chars(); + let Some(c0) = chars.next() else { + return false; + }; + if !(c0.is_ascii_alphabetic() || c0 == '_') { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// Layer `X` text from a `ParamInfo.xml` +/// onto matching field descriptions. Uses a tiny line-oriented scan +/// — the schema is too irregular to demand a full XML parser. +fn layer_param_info(path: &Path, fields: &mut HashMap) { + let Ok(text) = fs::read_to_string(path) else { + return; + }; + let mut current: Option = None; + for line in text.lines() { + if let Some(start) = line.find("") { + let after = &line[start + "".len()..]; + if let Some(end) = after.find("") { + let text = after[..end].trim(); + if !text.is_empty() { + if let Some(f) = fields.get_mut(name) { + f.description = Some(text.to_string()); + } + } + } + } + if line.contains("") { + current = None; + } + } + } +} + +/// Layer enum values from preset XML files onto matching fields. +/// We only consume `` entries — those are +/// genuine selector-style enumerations (BOOT_MODE, SMON_ALARMS, …) +/// where Vivado's UI offers a fixed dropdown. The XMLs also contain +/// `` entries, but those are concrete +/// values *applied* by a parent preset; they're not an exhaustive +/// list of valid values. For a numeric field like +/// `PMC_CRP_PL0_REF_CTRL_FREQMHZ` the user can supply any frequency +/// the clock generator can synthesize (e.g. `250`, `195`), so +/// treating `` values as an `@enum` would wrongly reject those. +fn layer_presets(pspmc_dir: &Path, fields: &mut HashMap) { + let mut paths = Vec::new(); + paths.push(pspmc_dir.join("global/global_preset.xml")); + paths.push(pspmc_dir.join("global/global_presetForNonPS.xml")); + walk_for_xml(&pspmc_dir.join("presets"), &mut paths); + + for p in paths { + let Ok(text) = fs::read_to_string(&p) else { + continue; + }; + for line in text.lines() { + if let Some((param, val)) = + extract_two_attrs(line, "` works). +fn extract_two_attrs<'a>( + line: &'a str, + tag: &str, + attr_a: &str, + attr_b: &str, +) -> Option<(&'a str, &'a str)> { + let tag_idx = line.find(tag)?; + let after_tag = &line[tag_idx + tag.len()..]; + let (a, rest) = scan_attr(after_tag, attr_a)?; + let (b, _) = scan_attr(rest, attr_b)?; + Some((a, b)) +} + +fn scan_attr<'a>(s: &'a str, name: &str) -> Option<(&'a str, &'a str)> { + let needle_eq = format!("{name}=\""); + let idx = s.find(&needle_eq)?; + let after = &s[idx + needle_eq.len()..]; + let end = after.find('"')?; + Some((&after[..end], &after[end + 1..])) +} + +fn walk_for_xml(dir: &Path, out: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let path = e.path(); + if path.is_dir() { + walk_for_xml(&path, out); + } else if path.extension().and_then(|s| s.to_str()) == Some("xml") { + out.push(path); + } + } +} + +/// Walk up `start` looking for an ancestor literally named `data`. +fn find_data_root(start: &Path) -> Option { + for ancestor in start.ancestors() { + if ancestor.file_name().and_then(|s| s.to_str()) == Some("data") { + return Some(ancestor.to_path_buf()); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_brace_aware_respects_tcl_groups() { + assert_eq!(split_brace_aware("a,b,c"), vec!["a", "b", "c"]); + // Commas inside `{…}` stay attached to that cell. + assert_eq!( + split_brace_aware("KEY,{a,b,c},NEXT"), + vec!["KEY", "{a,b,c}", "NEXT"] + ); + // Nested braces. + assert_eq!( + split_brace_aware("K,{{x,y} {z,w}},end"), + vec!["K", "{{x,y} {z,w}}", "end"] + ); + } + + #[test] + fn parse_direct_csv_drops_unbalanced_brace_defaults() { + use std::io::Write; + let f = tempfile::NamedTempFile::new().unwrap(); + // First row is well-formed; second has an unterminated brace + // group (matches a real bug in Xilinx's vendor CSV). + writeln!(&f, "GOOD_KEY,{{0}},GOOD_KEY,{{0}}").unwrap(); + writeln!(&f, "BAD_KEY,{{a,b,c").unwrap(); + let mut m: HashMap = HashMap::new(); + parse_direct_csv(f.path(), &mut m); + assert_eq!(m["GOOD_KEY"].default, "0"); + // BAD_KEY is recorded as a field but with no default. + assert!(m.contains_key("BAD_KEY")); + assert_eq!(m["BAD_KEY"].default, ""); + } + + #[test] + fn unwrap_one_brace_strips_outer_pair_only() { + assert_eq!(unwrap_one_brace("{0}"), "0"); + assert_eq!(unwrap_one_brace("{{ENABLE 0}}"), "{ENABLE 0}"); + assert_eq!(unwrap_one_brace("Custom"), "Custom"); + // Unbalanced — leave alone. + assert_eq!(unwrap_one_brace("{a"), "{a"); + // Two adjacent groups — not a single outer pair. + assert_eq!(unwrap_one_brace("{a}{b}"), "{a}{b}"); + } + + #[test] + fn safe_key_rejects_anything_with_braces_or_special_chars() { + assert!(is_safe_key("PS_USE_PMCPL_CLK0")); + assert!(is_safe_key("_misc")); + assert!(!is_safe_key("{0}")); + assert!(!is_safe_key("0_LEADS_WITH_DIGIT")); + assert!(!is_safe_key("")); + assert!(!is_safe_key("has space")); + } + + #[test] + fn discovery_returns_empty_outside_xilinx_layout() { + let dir = tempfile::tempdir().unwrap(); + let loose = dir.path().join("component.xml"); + std::fs::write(&loose, "").unwrap(); + assert!(load_schemas(&loose).is_empty()); + } + + /// Build a tempdir mimicking the Xilinx layout closely enough to + /// exercise the loader end-to-end without a Vivado install. + #[test] + fn loads_minimum_viable_schema_from_synthetic_layout() { + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("data"); + + let pspmc = data.join("versal/cips_hip/pspmc"); + let csvs = + data.join("versal/flows/automation/cipsToPsWiz_Porting/csv_files"); + let global = pspmc.join("global"); + let guidata = pspmc.join("guidata"); + let presets = pspmc.join("presets"); + std::fs::create_dir_all(&csvs).unwrap(); + std::fs::create_dir_all(&global).unwrap(); + std::fs::create_dir_all(&guidata).unwrap(); + std::fs::create_dir_all(&presets).unwrap(); + + std::fs::write( + csvs.join("param_mapping_direct.csv"), + "\ +PCIE_APERTURES_DUAL_ENABLE,{0},PCIE_APERTURES_DUAL_ENABLE,{0} +PS_PCIE_RESET,{{ENABLE 0}},PS_PCIE_RESET,{ENABLE 0 IO PS_MIO_18:19} +SMON_ALARMS,{Set_Alarms_On},SMON_ALARMS,{Set_Alarms_On} +CPM_PCIE0_MODES,{None},CPM_PCIE0_MODES,{None} +", + ) + .unwrap(); + std::fs::write( + csvs.join("param_mapping_presets.csv"), + "\ +BOOT_MODE,CLOCK_MODE +BOOT_MODE,JTAG Boot +BOOT_MODE,Master Mode +CLOCK_MODE,Custom +CLOCK_MODE,REF CLK 33.33 MHz +", + ) + .unwrap(); + std::fs::write( + guidata.join("ParamInfo.xml"), + r#" + + + What do you want to do with Alarms? + + +"#, + ) + .unwrap(); + std::fs::write( + presets.join("sysmon.xml"), + r#" + + + +"#, + ) + .unwrap(); + // Empty global presets so the loader still finds the file. + std::fs::write(global.join("global_preset.xml"), "").unwrap(); + + let component = data.join("ip/xilinx/versal_cips_v3_4/component.xml"); + std::fs::create_dir_all(component.parent().unwrap()).unwrap(); + std::fs::write(&component, "").unwrap(); + + let schemas = load_schemas(&component); + assert!( + schemas.contains_key("PS_PMC_CONFIG"), + "schemas: {schemas:?}" + ); + assert!(schemas.contains_key("PS_PMC_CONFIG_INTERNAL")); + let s = &schemas["PS_PMC_CONFIG"]; + let by_name: HashMap<&str, &DictField> = + s.fields.iter().map(|f| (f.name.as_str(), f)).collect(); + // From direct.csv (with CPM_ filtered out): + assert!(by_name.contains_key("PCIE_APERTURES_DUAL_ENABLE")); + assert_eq!(by_name["PCIE_APERTURES_DUAL_ENABLE"].default, "0"); + assert_eq!( + by_name["PS_PCIE_RESET"].default, "{ENABLE 0}", + "should strip one brace layer" + ); + // CPM_ keys are filtered out. + assert!(!by_name.contains_key("CPM_PCIE0_MODES")); + // From ParamInfo: description present for SMON_ALARMS. + assert_eq!( + by_name["SMON_ALARMS"].description.as_deref(), + Some("What do you want to do with Alarms?") + ); + // From presets: enum widened. + assert!(by_name["SMON_ALARMS"].enum_values.contains("Set_Alarms_On")); + assert!(by_name["SMON_ALARMS"] + .enum_values + .contains("Set_Alarms_Off")); + // From presets.csv: CLOCK_MODE present with "Custom" as default. + assert!(by_name.contains_key("CLOCK_MODE")); + assert_eq!(by_name["CLOCK_MODE"].default, "Custom"); + assert!(by_name["CLOCK_MODE"].enum_values.contains("Custom")); + // BOOT_MODE's CSV row only lists "JTAG Boot" and "Master Mode" — + // we should still inject `Custom` automatically (Vivado convention). + assert_eq!(by_name["BOOT_MODE"].default, "Custom"); + assert!(by_name["BOOT_MODE"].enum_values.contains("Custom")); + assert!(by_name["BOOT_MODE"].enum_values.contains("JTAG Boot")); + } + + // ------------------------------------------------------------------ + // DictSchema::from_paired_default — XML-driven schema extraction. + // ------------------------------------------------------------------ + + #[test] + fn from_paired_default_flat_schema() { + // The `intf::channel_map` shape — flat pairs, no nesting. + let src = "INTF0_RX0 QUAD0_RX0 INTF0_TX0 QUAD0_TX0"; + let s = DictSchema::from_paired_default( + src, + "intf::channel_map", + &OverridesFile::default(), + ); + assert_eq!(s.fields.len(), 2); + assert!(s.sub_schemas.is_empty()); + assert_eq!(s.fields[0].name, "INTF0_RX0"); + assert_eq!(s.fields[0].default, "QUAD0_RX0"); + assert!(s.fields[0].enum_values.is_empty()); + } + + #[test] + fn from_paired_default_nested_lr_schema() { + // `INTF_LR_SETTINGS` shape — the nested-slot case. + let src = "LR0_SETTINGS {RX_HD_EN 0 TX_HD_EN 0} LR1_SETTINGS { }"; + let s = DictSchema::from_paired_default( + src, + "intf::txrx_optional_ports", + &OverridesFile::default(), + ); + // No scalar fields at this level — every entry is a nested + // slot. Both LR0_SETTINGS and LR1_SETTINGS become sub-schemas. + assert!(s.fields.is_empty()); + assert_eq!(s.sub_schemas.len(), 2); + let lr0 = s.sub_schemas.get("LR0_SETTINGS").expect("LR0 present"); + assert_eq!(lr0.fields.len(), 2); + assert_eq!(lr0.fields[0].name, "RX_HD_EN"); + assert_eq!(lr0.fields[0].default, "0"); + let lr1 = s.sub_schemas.get("LR1_SETTINGS").expect("LR1 present"); + assert!(lr1.fields.is_empty()); + assert!(lr1.sub_schemas.is_empty()); + } + + #[test] + fn from_paired_default_two_levels_matches_txrx_shape() { + // `INTF0_TXRX_OPTIONAL_PORTS` shape distilled — flat outer + // pairs terminating in a nested INTF_LR_SETTINGS whose + // values are themselves paired dicts. + let src = "GT_TYPE GTY GT_DIRECTION DUPLEX \ + INTF_LR_SETTINGS {LR0_SETTINGS {RX_HD_EN 0}}"; + let s = DictSchema::from_paired_default( + src, + "intf::txrx_optional_ports", + &OverridesFile::default(), + ); + // Two flat scalar fields, one nested sub-schema. + assert_eq!(s.fields.len(), 2); + assert_eq!(s.fields[0].name, "GT_TYPE"); + assert_eq!(s.fields[1].name, "GT_DIRECTION"); + assert_eq!(s.sub_schemas.len(), 1); + let intf_lr = s + .sub_schemas + .get("INTF_LR_SETTINGS") + .expect("INTF_LR_SETTINGS present"); + assert_eq!(intf_lr.sub_schemas.len(), 1); + let lr0 = intf_lr + .sub_schemas + .get("LR0_SETTINGS") + .expect("LR0 sub-schema"); + assert_eq!(lr0.fields.len(), 1); + assert_eq!(lr0.fields[0].name, "RX_HD_EN"); + } + + #[test] + fn overrides_apply_to_matching_shape_path() { + // Field-level enum refinement on a specific shape path. + // XML default is silent on RX_PAM_SEL's bounds; the + // override attaches `@enum(NRZ, PAM4)`. + use crate::overrides::{FieldOverride, OverridesFile, ShapeOverrides}; + let mut ov = OverridesFile::default(); + let mut fields = std::collections::HashMap::new(); + fields.insert( + "rx_pam_sel".to_string(), + FieldOverride { + enum_values: Some(vec!["NRZ".into(), "PAM4".into()]), + default: None, + }, + ); + ov.shapes.insert( + "intf::gt_settings::lr0_settings".into(), + ShapeOverrides { fields }, + ); + // Note the shape path passed at the root is the outer shape; + // the LR0_SETTINGS sub-schema descends to + // `intf::gt_settings::lr0_settings`. + let src = "LR0_SETTINGS {RX_PAM_SEL NRZ RX_HD_EN 0}"; + let s = DictSchema::from_paired_default(src, "intf::gt_settings", &ov); + let lr0 = s + .sub_schemas + .get("LR0_SETTINGS") + .expect("LR0_SETTINGS sub-schema"); + let rx_pam = lr0 + .fields + .iter() + .find(|f| f.name == "RX_PAM_SEL") + .expect("RX_PAM_SEL field"); + assert_eq!( + rx_pam.enum_values, + ["NRZ".to_string(), "PAM4".to_string()] + .into_iter() + .collect() + ); + // Non-overridden field keeps the XML default and empty enum. + let rx_hd = lr0 + .fields + .iter() + .find(|f| f.name == "RX_HD_EN") + .expect("RX_HD_EN field"); + assert!(rx_hd.enum_values.is_empty()); + assert_eq!(rx_hd.default, "0"); + } + + #[test] + fn override_default_replaces_xml_default() { + use crate::overrides::{FieldOverride, OverridesFile, ShapeOverrides}; + let mut ov = OverridesFile::default(); + let mut fields = std::collections::HashMap::new(); + fields.insert( + "rx_line_rate".to_string(), + FieldOverride { + enum_values: None, + default: Some("25.78125".into()), + }, + ); + ov.shapes + .insert("intf::lr0_settings".into(), ShapeOverrides { fields }); + let src = "RX_LINE_RATE 10.3125 RX_HD_EN 0"; + let s = DictSchema::from_paired_default(src, "intf::lr0_settings", &ov); + let rx_line = s + .fields + .iter() + .find(|f| f.name == "RX_LINE_RATE") + .expect("RX_LINE_RATE field"); + assert_eq!(rx_line.default, "25.78125"); + } +} diff --git a/vw-ip/src/family.rs b/vw-ip/src/family.rs new file mode 100644 index 0000000..846c3ff --- /dev/null +++ b/vw-ip/src/family.rs @@ -0,0 +1,424 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Detect **indexed families** among sibling tree nodes. +//! +//! An indexed family is a set of sibling nodes whose labels differ +//! only by a trailing digit run — e.g. `MAC_PORT0`, `MAC_PORT1`, …, +//! `MAC_PORT5` — AND whose per-node parameter shapes are identical +//! once the digit is stripped. When present, the family's N sibling +//! nodes collapse into one constructor proc plus N kwargs on the +//! parent, and the parent's body emits ONE atomic +//! `set_property -dict` across everything. +//! +//! One guardrail keeps the detection safe: +//! +//! - **Strict direct-param shape match.** After stripping the +//! `_` prefix from each direct param, the resulting +//! name-set must be identical across members, and each name's +//! `default_value` / `choice_ref` / `is_user_configurable` triple +//! must agree. Any divergence falls back to per-N. +//! +//! Detection walks the whole tree — the family need not sit at +//! root level, because intermediate grouping nodes (e.g. DCMAC's +//! `MAC` node aggregating `MAC_PORT0..5`) don't emit their own proc. +//! Detected families ALWAYS land as kwargs on the top-level +//! `::create` proc; the sibling sub-procs that today set the +//! direct params of each family member (`::mac_port0..5`) +//! disappear in favor of the composed constructor. +//! +//! Sub-nodes UNDER family members (e.g. `MAC_PORT0_RX`, +//! `MAC_PORT0_TX` — each their own tree node with their own direct +//! params) continue to emit as per-N sub-procs. They set properties +//! deeper in the CONFIG namespace than the family constructor +//! covers, so leaving them per-N is safe — the atomicity bug only +//! affects the family's direct-param slice. +//! +//! Callers can opt out further via [`DetectOptions::excluded_stems`] +//! when a specific stem needs to stay per-N. + +use std::collections::{BTreeMap, HashMap}; + +use ipxact::Parameter; + +use crate::tree::{strip_prefix, Node}; + +/// Options for [`detect_families`]. Every knob defaults to "detect +/// everything the guardrails allow"; the caller can subtract from +/// there via `excluded_stems` when a specific family causes trouble. +#[derive(Clone, Debug, Default)] +pub struct DetectOptions { + /// Family stems (e.g. `"MAC_PORT"`) to leave per-N even when + /// the guardrails would otherwise collapse them. Populated by + /// the CLI's `--no-collapse=STEM,STEM,…` flag. Empty by default. + pub excluded_stems: Vec, +} + +/// A detected indexed family — enough information for the emitter +/// to produce a constructor and weave the kwargs into the parent +/// proc. +#[derive(Clone, Debug)] +pub struct IndexedFamily<'a> { + /// The common label prefix with trailing digits stripped + /// (`"MAC_PORT"`, `"GT_CH"`, `"FEC_SLICE"`, …). Used to name + /// the emitted `::` constructor and the + /// `::` newtype. + pub stem: String, + /// The concrete digit indices present in the source, in + /// ascending order (e.g. `[0, 1, 2, 3, 4, 5]` for DCMAC's + /// MAC_PORT family). Members can be non-contiguous. + pub indices: Vec, + /// Direct-params from the FIRST member. The emitter treats + /// these as canonical shape; enum values are still unioned + /// across all members (see [`Self::members_direct`]) so per- + /// member enum-choice differences don't leak into the + /// constructor's arg constraints. + pub shape: Vec<&'a Parameter>, + /// All members' direct-param lists, in index order (parallel + /// to [`Self::indices`]). Emission uses this to union enum + /// values across members — some IPs give the same logical + /// field different `choiceRef` sets per index (DCMAC's + /// MAC_PORT0 has 27 CONFIG_C0 choices while MAC_PORT1 has 12), + /// and the collapsed constructor needs to accept the superset + /// so callers can populate any port they want. + pub members_direct: Vec>, + /// Parent node's label (empty for root children). Used to + /// place the family's kwargs on the correct parent proc. + pub parent_label: String, + /// Original label of the shape-carrier member (e.g. + /// `"MAC_PORT0"`). Used to strip param-name prefixes at + /// emission time via [`strip_prefix`]. + pub shape_member_label: String, + /// Parallel to [`Self::indices`] — each member's own label + /// (e.g. `["MAC_PORT0", "MAC_PORT1", …, "MAC_PORT5"]`). + /// Emission uses this to strip the correct prefix from each + /// member's params when unioning enum values. + pub member_labels: Vec, +} + +/// Walk `root`'s subtree, collecting every indexed family whose +/// members' direct params shape-match AND whose enclosing parent +/// chain contains no already-indexed node. Sub-nodes UNDER family +/// members are ignored by this pass — they keep emitting per-N. +/// +/// The "no indexed ancestor" rule is what the user's "follow suit +/// for nested" instruction means in practice: DCMAC's +/// `MAC_PORT0..5` collapse (parent chain is root → `MAC`, +/// neither indexed) but CPM5's `PF0_BAR0..N` don't (parent chain +/// runs through `PCIE0` / `PF0`, both digit-indexed themselves). +pub fn detect_families<'a>( + root: &Node<'a>, + opts: &DetectOptions, +) -> Vec> { + let mut out = Vec::new(); + detect_at(root, opts, /* indexed_ancestor */ false, &mut out); + out +} + +fn detect_at<'a>( + node: &Node<'a>, + opts: &DetectOptions, + indexed_ancestor: bool, + out: &mut Vec>, +) { + // Group this node's children by stem. + let mut by_stem: BTreeMap)>> = BTreeMap::new(); + for child in &node.children { + let Some((stem, idx)) = split_trailing_digits(&child.label) else { + continue; + }; + by_stem.entry(stem).or_default().push((idx, child)); + } + // Try to collapse each stem-group. Members that don't pass the + // guardrails fall through untouched — the emitter keeps + // producing per-N procs for them. + for (stem, mut members) in by_stem { + if members.len() < 2 { + continue; + } + if opts.excluded_stems.iter().any(|s| s == &stem) { + continue; + } + if indexed_ancestor { + // Nested-under-indexed context. Follow-suit rule: + // keep emitting per-N. + continue; + } + // Shape match on direct params: same name-set across + // members + same (default, is_user_configurable) per + // field. `choice_ref` intentionally NOT compared — + // per-member enum-choice differences are semantically + // OK for the family constructor as long as we union + // them at emit time (see IndexedFamily::members_direct). + members.sort_by_key(|(idx, _)| *idx); + if !shapes_match(&members) { + continue; + } + let (_, first) = members[0]; + // Skip empty-shape families — nothing to hoist into the + // constructor. Their direct-params list is empty because + // all their params live in sub-nodes; the per-N sub-procs + // handle those and there's no atomicity benefit to + // emitting a stub constructor. + if first.direct.is_empty() { + continue; + } + out.push(IndexedFamily { + stem: stem.clone(), + indices: members.iter().map(|(idx, _)| *idx).collect(), + shape: first.direct.clone(), + members_direct: members + .iter() + .map(|(_, n)| n.direct.clone()) + .collect(), + parent_label: node.label.clone(), + shape_member_label: first.label.clone(), + member_labels: members + .iter() + .map(|(_, n)| n.label.clone()) + .collect(), + }); + } + // Recurse — a family may live under an intermediate grouping + // node (DCMAC's `MAC` node aggregates `MAC_PORT0..5`). + for child in &node.children { + // Once we cross into an indexed node, everything below + // inherits "nested" status and stays per-N. + let child_indexed = + indexed_ancestor || split_trailing_digits(&child.label).is_some(); + detect_at(child, opts, child_indexed, out); + } +} + +/// If `label` ends in a run of ASCII digits, return `(stem, index)`. +/// Empty stem or non-digit-suffix labels return `None` — those aren't +/// indexed family members. +fn split_trailing_digits(label: &str) -> Option<(String, u32)> { + let bytes = label.as_bytes(); + let mut cut = bytes.len(); + while cut > 0 && bytes[cut - 1].is_ascii_digit() { + cut -= 1; + } + if cut == bytes.len() || cut == 0 { + // No trailing digits OR digits are the whole label + // (e.g. `"0"` alone — not a family we can name after a stem). + return None; + } + let stem = label[..cut].to_string(); + let idx: u32 = label[cut..].parse().ok()?; + Some((stem, idx)) +} + +/// Compare all members' direct-param sets against member[0]'s. +/// True when every member has the same set of index-stripped param +/// names AND each name's triple matches. +fn shapes_match(members: &[(u32, &Node<'_>)]) -> bool { + let mut per_member: Vec>> = + Vec::with_capacity(members.len()); + for (_, n) in members { + let mut map = HashMap::new(); + for p in &n.direct { + let short = strip_prefix(&p.name, &n.label).to_string(); + map.insert( + short, + ShapeSlot { + default: p.value.default_value(), + user_config: p.value.is_user_configurable(), + }, + ); + } + per_member.push(map); + } + let first = &per_member[0]; + per_member.iter().skip(1).all(|m| { + m.len() == first.len() && first.iter().all(|(k, v)| m.get(k) == Some(v)) + }) +} + +/// The subset of a `Parameter`'s state that a family constructor +/// treats as the source of truth. Comparing these tuples across +/// members is our shape-equality check. +/// +/// Deliberately NOT included: +/// - `description` — human-authored prose that may reasonably vary +/// per member ("port 0 config" vs "port 1 config") without +/// changing the semantic shape. +/// - `choice_ref` — some IPs give the same logical field +/// different enum-choice sets per index (DCMAC's MAC_PORT0 vs +/// MAC_PORT1..5 CONFIG_C0). The atomicity-fix goal requires +/// allowing these to collapse; per-member choice_refs are +/// unioned at emit time via [`IndexedFamily::members_direct`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ShapeSlot<'a> { + default: &'a str, + user_config: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tree::{build_tree, TreeOptions}; + + fn p(name: &str, default: &str, choice_ref: Option<&str>) -> Parameter { + Parameter { + name: name.into(), + value: ipxact::ParamValue { + text: default.into(), + choice_ref: choice_ref.map(Into::into), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + } + } + + #[test] + fn strips_trailing_digits() { + assert_eq!( + split_trailing_digits("MAC_PORT0"), + Some(("MAC_PORT".into(), 0)) + ); + assert_eq!( + split_trailing_digits("MAC_PORT12"), + Some(("MAC_PORT".into(), 12)) + ); + assert_eq!(split_trailing_digits("MAC_PORT"), None); + assert_eq!(split_trailing_digits("0"), None); + assert_eq!(split_trailing_digits(""), None); + } + + /// Six sibling nodes with matching shape collapse to one family. + /// Uses 3-segment param names so the recursive tree lands the + /// per-port params directly on `MAC_PORT` (no sub-children), + /// which is the shape DCMAC's real params produce. + #[test] + fn dcmac_like_leaf_family_collapses() { + let mut params = Vec::new(); + for n in 0..6 { + params.push(p(&format!("MAC_PORT{n}_CONFIG"), "200GAUI-4", None)); + params.push(p(&format!("MAC_PORT{n}_ENABLE"), "0", None)); + params.push(p(&format!("MAC_PORT{n}_MODE"), "static", None)); + } + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert_eq!(families.len(), 1, "{families:#?}"); + let fam = &families[0]; + assert_eq!(fam.stem, "MAC_PORT"); + assert_eq!(fam.indices, vec![0, 1, 2, 3, 4, 5]); + assert_eq!(fam.shape.len(), 3); + } + + /// Nested siblings (each member has its own children) do NOT + /// collapse — the leaf-only rule keeps them per-N. + #[test] + fn nested_family_does_not_collapse() { + let mut params = Vec::new(); + // Two BARs, each with enough sub-params under `_BRIDGE` and + // `_QDMA` to trigger sub-nodes. + for n in 0..2 { + for kind in ["BRIDGE", "QDMA"] { + for i in 0..5 { + params.push(p( + &format!("PF0_BAR{n}_{kind}_FIELD{i}"), + "0", + None, + )); + } + } + } + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert!( + families.iter().all(|f| f.stem != "PF0_BAR"), + "{families:#?}" + ); + } + + /// Sibling shapes that disagree (one has an extra field) fall + /// back to per-N. + #[test] + fn shape_mismatch_falls_back_to_per_n() { + // port 0 has 3 fields; port 1 has only 2 (extra `_C` missing). + let params = [ + p("MAC_PORT0_A", "0", None), + p("MAC_PORT0_B", "0", None), + p("MAC_PORT0_C", "0", None), + p("MAC_PORT1_A", "0", None), + p("MAC_PORT1_B", "0", None), + ]; + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert!(families.is_empty(), "{families:#?}"); + } + + /// Sibling shapes with different defaults for the same field + /// also fall back to per-N — the collapsed constructor would + /// mis-report a shared default. + #[test] + fn different_defaults_prevent_collapse() { + let params = [ + p("MAC_PORT0_CONFIG", "200GAUI-4", None), + p("MAC_PORT1_CONFIG", "400GAUI-8", None), + ]; + let opts = TreeOptions { min_split_size: 1 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert!(families.is_empty(), "{families:#?}"); + } + + /// Different `choice_ref` values across members SHOULD still + /// collapse — the constructor unions enum values at emit time. + /// (See ShapeSlot's doc comment for the rationale.) + #[test] + fn different_choice_refs_do_collapse() { + let mut params = Vec::new(); + for n in 0..3 { + params.push(p( + &format!("MAC_PORT{n}_CONFIG"), + "100CAUI-4", + Some(&format!("port{n}_choices")), + )); + params.push(p(&format!("MAC_PORT{n}_ENABLE"), "0", None)); + } + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert_eq!(families.len(), 1, "{families:#?}"); + } + + /// Single-shape IPs (no sibling groups at all) produce zero + /// families. + #[test] + fn single_shape_ip_produces_no_families() { + let params = [ + p("ONE", "0", None), + p("TWO", "0", None), + p("THREE", "0", None), + ]; + let tree = build_tree(params.iter(), &TreeOptions::default()); + let families = detect_families(&tree, &DetectOptions::default()); + assert!(families.is_empty(), "{families:#?}"); + } + + /// `excluded_stems` skips detection for named stems even when + /// the guardrails would collapse them. + #[test] + fn excluded_stem_stays_per_n() { + let mut params = Vec::new(); + for n in 0..3 { + params.push(p(&format!("MAC_PORT{n}_A"), "0", None)); + params.push(p(&format!("MAC_PORT{n}_B"), "0", None)); + } + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let opts = DetectOptions { + excluded_stems: vec!["MAC_PORT".into()], + }; + let families = detect_families(&tree, &opts); + assert!(families.is_empty(), "{families:#?}"); + } +} diff --git a/vw-ip/src/generate.rs b/vw-ip/src/generate.rs new file mode 100644 index 0000000..ca36bbf --- /dev/null +++ b/vw-ip/src/generate.rs @@ -0,0 +1,3648 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Emit an htcl wrapper proc for an IP-XACT component. +//! +//! Two shapes, picked by `split_threshold`. Every emitted proc lives +//! in the IP's own namespace (`::…`) so name collisions across +//! IPs are structural and callers reach helpers by tab-completing +//! `::`. +//! +//! - **Single-proc** (small IPs like CIPS with 19 params): one +//! `::create` proc whose structured args mirror the IP's +//! parameters. Each arg gets `@default()` from the IP-XACT +//! default; `@enum(...)` when the parameter has a `choiceRef`. The +//! body emits `set_property -dict [list ...]` mapping each arg back +//! to its `CONFIG.` key. +//! +//! - **Split** (large IPs like CPM5 with ~8700 params): a top +//! `::create` proc that just creates the bd_cell and returns +//! its handle, plus one `::` sub-proc per parameter +//! prefix group. Each sub-proc takes the cell handle as its first +//! arg, then its own group's parameters. Small groups +//! (< `min_group_size`) collapse into a `_misc` sub-proc so we end +//! up with a manageable handful rather than dozens of singletons. +//! +//! Call-site composition: +//! ```tcl +//! set cps [cpm5::create cps] +//! cpm5::pcie1 $cps -max_link_speed "32.0_GT/s" -modes PCIE +//! ``` + +use std::fmt::Write; + +use ipxact::{Component, Parameter}; +use vw_htcl::emit::{Command, Doc, Item, Word}; + +use crate::family::{detect_families, DetectOptions, IndexedFamily}; +use crate::tree::{build_tree, strip_prefix, Node, TreeOptions}; + +#[derive(Clone, Debug)] +pub struct GenerateOptions { + /// Emit `## ` doc comments for parameters that have a + /// description in IP-XACT. + pub include_descriptions: bool, + /// Skip auto-resolve parameters; emit only user-configurable ones. + pub user_configurable_only: bool, + /// When the parameter count exceeds this, the generator emits a + /// hierarchy of procs instead of one. Tuned so CIPS (19) stays + /// single and CPM5 (8673) splits. + pub split_threshold: usize, + /// Don't split a subgroup into its own child proc unless it has at + /// least this many parameters. Smaller subgroups stay as direct + /// args of the parent so we don't get a long tail of singleton + /// procs. + pub min_split_size: usize, + /// Indexed-family stems to leave per-N (skip the composed- + /// constructor collapse). Passed through to + /// [`crate::family::DetectOptions::excluded_stems`]. Empty by + /// default; populated from the CLI's `--no-collapse=STEM,STEM,…` + /// flag when a specific stem needs to opt out. + pub no_collapse: Vec, + /// Per-IP TOML overrides file. Attaches `@enum(…)` refinements + /// and per-field default overrides to XML-derived DictSchemas + /// (see [`crate::overrides::OverridesFile`]). Empty when no + /// override file is present — the generator falls back to + /// XML-only defaults. + pub overrides: crate::overrides::OverridesFile, +} + +impl Default for GenerateOptions { + fn default() -> Self { + Self { + include_descriptions: true, + user_configurable_only: true, + split_threshold: 100, + min_split_size: 8, + no_collapse: Vec::new(), + overrides: crate::overrides::OverridesFile::default(), + } + } +} + +/// Multi-file generation output. +/// +/// Big IPs (gtwiz-versal at 160k lines, cpm5 at 40k) blow past +/// tree-sitter's default incremental-parse budget when squeezed +/// into one file. Splitting per split-node keeps every file under +/// ~20k lines — still large, but within reach of downstream tools. +/// +/// `main` is the primary `module.htcl` content, which sources every +/// entry in `subfiles` via `src ./` lines. Each subfile +/// contains one split-node's dict-schema newtypes + its constructor +/// proc, emitted with fully-qualified proc names so the file +/// stands alone (no namespace-block wrapping needed). +#[derive(Debug, Clone, Default)] +pub struct MultiFileOutput { + pub main: String, + /// `(basename, content)` pairs — basename is the file name + /// (no leading `./`, no directory prefix); the CLI writes each + /// to `/`. + pub subfiles: Vec<(String, String)>, +} + +impl MultiFileOutput { + /// Merge every subfile into `main` and return the concatenated + /// text. Preserves the pre-split single-file shape so unit tests + /// and callers that don't care about file layout can keep + /// treating the output as one string. + pub fn into_single(mut self) -> String { + for (_, sub) in &self.subfiles { + self.main.push('\n'); + self.main.push_str(sub); + } + self.main + } +} + +/// Generate the htcl wrapper text for `component`. +/// +/// `presets` carries supplementary parameter-value information loaded +/// from out-of-band sources (e.g. Vivado's `cpm_preset*.xml`); pass an +/// empty map when there are none. Values from `presets` are merged +/// with the IP-XACT `` entries when emitting `@enum(...)`. +pub fn generate( + component: &Component, + presets: &crate::presets::PresetMap, + dict_schemas: &std::collections::HashMap, + opts: &GenerateOptions, +) -> MultiFileOutput { + let parameters: Vec<&Parameter> = component + .component_parameters() + .filter(|p| { + !opts.user_configurable_only || p.value.is_user_configurable() + }) + .collect(); + let mut out = if parameters.len() > opts.split_threshold { + generate_split(component, presets, ¶meters, opts, dict_schemas) + } else { + generate_single(component, presets, ¶meters, opts, dict_schemas) + }; + // Only the CSV-driven top-level schemas are surfaced here. + // XML-derived schemas for split-node params get emitted inline + // inside `emit_split_node_constructor`, and top-level XML + // schemas (unclaimed by split nodes) get emitted inside + // `generate_split` / `generate_single` themselves — see the + // dedicated merge steps there. + if !dict_schemas.is_empty() { + append_dict_sub_procs(&mut out, component, dict_schemas, opts); + } + // Peel off large split-node blocks into sibling files so + // tree-sitter (and any other line-oriented consumer) doesn't + // choke on the aggregate. See [`split_into_files`] for the + // peel heuristic and file-shape contract. + split_into_files(out) +} + +/// Emit one compositional-value constructor per IP-XACT +/// `structured_tcldict` parameter. Each schema gets a newtype +/// prelude (`namespace eval`, `type = Properties`, +/// `::repr`/`::from`/`::to`/`::empty`) plus a pure `::` +/// constructor that returns the typed value. The top proc's +/// matching kwarg (e.g. `-ps_pmc_config`) then takes the newtype +/// and unwraps it into the atomic `set_property -dict` — no +/// separate mutator call. +fn append_dict_sub_procs( + out: &mut String, + component: &Component, + dict_schemas: &std::collections::HashMap, + opts: &GenerateOptions, +) { + let ip_name = sanitize_ident(&component.name); + let mut keys: Vec<&String> = dict_schemas.keys().collect(); + keys.sort(); + for param_name in keys { + let schema = &dict_schemas[param_name]; + if schema.fields.is_empty() { + continue; + } + writeln!(out).unwrap(); + // Top-level dict-schema procs (PS_PMC_CONFIG, etc.) live + // directly under `::` — pass an empty namespace prefix. + // Sub-schemas emitted while recursing pick up their own + // prefix chain from `emit_dict_sub_schemas`. + emit_dict_props_prelude(out, &ip_name, &[], param_name); + writeln!(out).unwrap(); + emit_dict_sub_proc(out, &ip_name, &[], param_name, schema, opts); + } +} + +/// Emit the newtype declaration + `::from`/`::to`/`::repr`/`::empty` +/// helper procs for one dict-schema param. Mirror of +/// [`emit_family_prelude`] — same shape, different naming source. +/// +/// `namespace_prefix` composes intermediate namespace segments +/// between the IP name and the param name — for a nested +/// sub-constructor emitted under `::intf::gt_settings::`, pass +/// `["intf", "gt_settings"]`. Empty slice reproduces the original +/// flat `::` shape used by PS_PMC_CONFIG. +fn emit_dict_props_prelude( + out: &mut String, + ip_name: &str, + namespace_prefix: &[&str], + param_name: &str, +) { + let qualified = dict_props_name(ip_name, namespace_prefix, param_name); + let ctor_lower = param_name.to_ascii_lowercase(); + let scope_display = if namespace_prefix.is_empty() { + format!("[{ip_name}::create]") + } else { + format!("[{ip_name}::{}]", namespace_prefix.join("::")) + }; + writeln!( + out, + "## Typed configuration value for {scope_display}'s \ + `-{ctor_lower}` slot. Construct with [{}::{ctor_lower}].", + proc_scope(ip_name, namespace_prefix) + ) + .unwrap(); + // Every ancestor namespace segment needs `namespace eval` so + // downstream `proc ::a::b::c` declarations resolve. Emit the + // whole chain from `` down to the newtype's own namespace. + writeln!(out, "namespace eval {ip_name} {{}}").unwrap(); + for i in 0..namespace_prefix.len() { + let chain = std::iter::once(ip_name) + .chain(namespace_prefix[..=i].iter().copied()) + .collect::>() + .join("::"); + writeln!(out, "namespace eval {chain} {{}}").unwrap(); + } + writeln!(out, "namespace eval {qualified} {{}}").unwrap(); + writeln!(out, "type {qualified} = Properties").unwrap(); + writeln!( + out, + "proc {qualified}::repr {{ v: {qualified} }} string \ + {{ return [Properties::repr -v [{qualified}::to -v $v]] }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::from {{ v: Properties }} {qualified} \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::to {{ v: {qualified} }} Properties \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::empty {{}} {qualified} \ + {{ return [{qualified}::from -v [Properties::empty]] }}" + ) + .unwrap(); +} + +/// Compose the parent-namespace form for a nested sub-proc, without +/// the leaf. `("gtwiz_versal", ["intf", "gt_settings"])` → +/// `"gtwiz_versal::intf::gt_settings"`. Used by doc comments to +/// point at the containing scope. +fn proc_scope(ip_name: &str, namespace_prefix: &[&str]) -> String { + if namespace_prefix.is_empty() { + ip_name.to_string() + } else { + format!("{ip_name}::{}", namespace_prefix.join("::")) + } +} + +/// Emit the value-constructor `::` for one +/// dict-schema. Pure — no cell, no `-bd`, no `set_property`. Builds +/// a `Properties`-shaped dict from the supplied field kwargs and +/// wraps it in the newtype. The atomic materialization happens +/// later in `::create`'s body, where the matching +/// `-` kwarg gets unwrapped through `::to` + +/// `Properties::to_raw` and merged into the single +/// `set_property -dict` call. +fn emit_dict_sub_proc( + out: &mut String, + ip_name: &str, + namespace_prefix: &[&str], + param_name: &str, + schema: &crate::DictSchema, + opts: &GenerateOptions, +) { + // Recurse into sub-schemas FIRST so nested newtypes are declared + // before the outer proc references them. The outer proc's slot + // args carry types like `Intf0GtSettingsLr0Settings` — those + // types must exist in the analyzer's view by the time the outer + // proc's signature is checked, so the emission order (deepest + // first, then parent) matches lexical declaration order. + let sub_ctors = emit_dict_sub_schemas( + out, + ip_name, + namespace_prefix, + param_name, + schema, + opts, + ); + + let ctor_local = param_name.to_ascii_lowercase(); + let ctor_scope = proc_scope(ip_name, namespace_prefix); + let ctor_name = format!("{ctor_scope}::{ctor_local}"); + let ret_ty = dict_props_name(ip_name, namespace_prefix, param_name); + + let mut doc = Doc::new(); + let scope_display = if namespace_prefix.is_empty() { + format!("[{ip_name}::create]") + } else { + format!("[{}]", proc_scope(ip_name, namespace_prefix)) + }; + doc.push(Item::DocComment(format!( + "Configuration value for {scope_display}'s \ + `-{ctor_local}` slot (`CONFIG.{param_name}`). Composes into \ + the top proc so every provided field lands in ONE atomic \ + `set_property -dict` call.", + ))); + if !schema.fields.is_empty() || !sub_ctors.is_empty() { + doc.push(Item::Blank); + } + for f in &schema.fields { + emit_dict_field_arg(&mut doc, f, opts); + } + // Typed slots for nested sub-schemas — the outer proc takes + // one arg per LRn slot (or equivalent), and the runtime merges + // each slot's `::to` unwrap into the top-level Properties + // dict under the slot's XML key. + for (raw_name, sub_ret_ty) in &sub_ctors { + let arg = lowercase_ident(raw_name); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("{arg}: {sub_ret_ty}")), + ], + body: None, + })); + } + + let mut body = String::new(); + writeln!(body, "set _vw_d [dict create]").unwrap(); + for f in &schema.fields { + let arg = lowercase_ident(&f.name); + writeln!( + body, + "if {{${{__vw_kw_{arg}_set}}}} \ + {{ dict set _vw_d {} ${arg} }}", + f.name + ) + .unwrap(); + } + // Sub-slot merges: unwrap the typed newtype to its raw paired + // dict via `::to`, and stash it under the slot's original + // XML key. The sub-dict IS the raw paired-list form Vivado + // accepts for tcldict-typed compound properties — a + // `Properties::to_raw` wrap here would try to flatten as a + // TAGGED tree (`Property::Scalar` / `Property::Nested`), but + // the wrapper's `_vw_d` holds bare-string leaves (see the + // scalar `dict set _vw_d KEY $arg` above), not tagged + // `Property::Scalar(x)`. Match the sibling emission at + // `emit_split_node_constructor` — same shape, same reason. + for (raw_name, sub_ret_ty) in &sub_ctors { + let arg = lowercase_ident(raw_name); + // Wrap the dict-set in a `catch` that prepends the HTCL + // arg name on error. Historically fired on the + // `Properties::to_raw` mistake above; retained + // defensively — if a caller reaches this proc with a + // malformed sub-value some future code path relies on, + // the message still points at the offending slot by its + // HTCL surface name (`lr0_settings.…`). + writeln!( + body, + "if {{${{__vw_kw_{arg}_set}}}} {{\n \ + if {{[catch {{\n \ + dict set _vw_d {raw_name} \ + [{sub_ret_ty}::to -v ${arg}]\n \ + }} __vw_msg]}} {{\n \ + error \"{arg}.$__vw_msg\"\n \ + }}\n \ + }}", + ) + .unwrap(); + } + writeln!( + body, + "return [{ret_ty}::from -v [Properties::from -v $_vw_d]]" + ) + .unwrap(); + emit_proc(out, &ctor_name, &doc, Some(&ret_ty), &body); +} + +/// Recursively emit the sub-constructor procs for `schema`'s nested +/// slots. Each sub-schema gets its own prelude + proc, declared in +/// a namespace one level deeper than the outer schema +/// (`::::::`). Returns +/// `(raw_slot_name, sub_ret_ty)` pairs so the outer proc's emitter +/// can declare typed slot args referencing these newtypes. +fn emit_dict_sub_schemas( + out: &mut String, + ip_name: &str, + namespace_prefix: &[&str], + outer_param: &str, + schema: &crate::DictSchema, + opts: &GenerateOptions, +) -> Vec<(String, String)> { + let mut acc = Vec::new(); + if schema.sub_schemas.is_empty() { + return acc; + } + // Extend the namespace prefix with the outer param's lowercase + // form — every sub-slot lives one level under the outer proc's + // scope: `::::`. + let outer_lower = outer_param.to_ascii_lowercase(); + let mut deeper: Vec<&str> = namespace_prefix.to_vec(); + deeper.push(&outer_lower); + for (raw_name, sub_schema) in &schema.sub_schemas { + // Empty sub-schemas (no fields + no further sub-slots) + // arise when an anchor's inner slot is a placeholder — e.g. + // TXRX_OPTIONAL_PORTS's `INTF_LR_SETTINGS` has LR0_SETTINGS + // populated but LR1_SETTINGS..LR15_SETTINGS empty. Emitting + // a proc for those would produce a body-less arg list + // (`proc … { ## doc only }`) which the htcl parser rejects + // as "doc comment with no following argument". Skip them — + // the outer proc's typed slot becomes bare (no sub-slot + // arg) and the pair simply isn't set. + if sub_schema.fields.is_empty() && sub_schema.sub_schemas.is_empty() { + continue; + } + writeln!(out).unwrap(); + emit_dict_props_prelude(out, ip_name, &deeper, raw_name); + writeln!(out).unwrap(); + emit_dict_sub_proc(out, ip_name, &deeper, raw_name, sub_schema, opts); + let sub_ret_ty = dict_props_name(ip_name, &deeper, raw_name); + acc.push((raw_name.clone(), sub_ret_ty)); + } + acc +} + +fn emit_dict_field_arg( + doc: &mut Doc, + f: &crate::DictField, + opts: &GenerateOptions, +) { + if opts.include_descriptions { + if let Some(desc) = f.description.as_deref().filter(|s| !s.is_empty()) { + for line in desc.lines() { + doc.push(Item::DocComment(line.trim_end().into())); + } + } + } + let mut words = Vec::new(); + if !f.enum_values.is_empty() { + let formatted: Vec = f + .enum_values + .iter() + .map(|v| format_attribute_value(v)) + .collect(); + words.push(Word::Raw(format!("@enum({})", formatted.join(", ")))); + } + // Dict fields are always optional: Vivado treats an unset + // inner key as "use the IP's implicit default", so make every + // arg defaultable. When the Xilinx CSV didn't yield a default — + // either the row was missing one or the value had unbalanced + // braces and was rejected — fall back to an empty string so the + // user can omit the arg and let Vivado decide. + words.push(Word::Raw(format!( + "@default({})", + format_attribute_value(&f.default) + ))); + let lowered = lowercase_ident(&f.name); + words.push(Word::Bare(lowered)); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words, + body: None, + })); +} + +// --------------------------------------------------------------------------- +// Single-proc shape. +// --------------------------------------------------------------------------- + +fn generate_single( + component: &Component, + presets: &crate::presets::PresetMap, + parameters: &[&Parameter], + opts: &GenerateOptions, + dict_schemas: &std::collections::HashMap, +) -> String { + let vlnv = component.vlnv(); + let ip_name = sanitize_ident(&component.name); + + let mut out = String::new(); + emit_file_header(&mut out, component, &vlnv); + writeln!( + out, + "## ({} configurable parameter{})", + parameters.len(), + if parameters.len() == 1 { "" } else { "s" } + ) + .unwrap(); + + // `configure`'s arg-doc — the whole documented kwarg pile. + // Same shape as the old inline-`create` version, minus the + // `-name` / `-bd` slots which are now `create`-only. + let mut configure_doc = Doc::new(); + for p in parameters { + emit_arg_decl( + &mut configure_doc, + component, + presets, + p, + opts, + "", + &ip_name, + dict_schemas, + &[], + ); + } + + // `create`'s arg-doc — the narrow surface. Just the + // instantiation-mode args + a single typed `-config`. + let mut create_doc = Doc::new(); + create_doc.push(Item::DocComment( + "Project-level IP module name, or (when `-bd 1`) the \ + instance name in the block design." + .into(), + )); + create_doc.push(Item::Command(Command::call( + "name", + std::iter::empty::(), + ))); + push_bd_switch_arg(&mut create_doc); + create_doc.push(Item::DocComment(format!( + "Typed configuration value. Construct with [{ip_name}::configure]. \ + Defaults to an empty config (all parameters take their IP defaults)." + ))); + create_doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("config: {ip_name}::Config")), + ], + body: None, + })); + + // Config prelude ships at file TOP LEVEL — namespace-eval + // double-prefix bug means qualified proc names like + // `::Config::from` inside `namespace eval {…}` end + // up as `::::Config::from`. See emit_family_prelude + // for the same trick. + writeln!(out).unwrap(); + emit_config_prelude(&mut out, &ip_name); + writeln!(out).unwrap(); + + let dict_schema_newtypes = + build_dict_schema_newtypes(&ip_name, dict_schemas); + let configure_body = build_single_configure_body( + component, + parameters, + &ip_name, + &dict_schema_newtypes, + ); + let configure_body = + wrap_body_qualified_prefix(&configure_body, &ip_name, "configure"); + let create_body = build_single_create_body(&vlnv, &ip_name); + + // Emit both procs inside `namespace eval { … }` so + // `configure` and `create` register as `::configure` and + // `::create`. Same wrap idiom as vivado-cmd's log.htcl. + let mut procs = String::new(); + emit_proc( + &mut procs, + "configure", + &configure_doc, + Some(&config_name(&ip_name)), + &configure_body, + ); + writeln!(procs).unwrap(); + emit_proc( + &mut procs, + "create", + &create_doc, + Some("string"), + &create_body, + ); + write_namespace_block(&mut out, &ip_name, &procs); + out +} + +/// Wrap `body` in `namespace eval { … }`, indenting each +/// non-empty line by two spaces. The wrapper lets the enclosed +/// procs use bare names (`create`, `mac_port`, …) while remaining +/// externally addressable as `::` — exactly the shape +/// vivado-cmd's hand-written `log.htcl` / `ip.htcl` use. +fn write_namespace_block(out: &mut String, ip_name: &str, body: &str) { + writeln!(out, "namespace eval {ip_name} {{").unwrap(); + for line in body.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + writeln!(out, "}}").unwrap(); +} + +/// Emit the `-bd` switch as a proc-arg declaration. +/// +/// `-bd false` (default) → `create_ip` (project-level IP module); +/// `-bd true` → `create_bd_cell` (block-design cell). Project IP +/// is the default because it's the shape Vivado's own +/// `write_ip_tcl`-generated scripts use, and most external tools +/// (simulators, downstream regeneration flows) expect wrappers +/// that create discoverable IP source objects. Wrappers going +/// into a block design still work — the caller passes +/// `-bd true`. Both paths return a plain `string` handle at +/// runtime (a bd_cell path or a project-IP module name); the +/// wrapper's declared return type is `string` so both flows +/// type-check under strict nominal identity, and callers that +/// need `bd_cell` semantics can promote via `bd_cell::from` at +/// their own use site. +fn push_bd_switch_arg(doc: &mut Doc) { + doc.push(Item::Blank); + doc.push(Item::DocComment( + "Create the IP as a project-level module (`false`, \ + default) via `create_ip`, or as a block-design cell \ + (`true`) via `create_bd_cell`. Returns the underlying \ + handle as a string in either case — a `/…` block-\ + design cell path when `true`, a raw module name when \ + `false`. Downstream `set_property -dict …` works on \ + both shapes." + .into(), + )); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(false)".into()), + Word::Bare("bd: bool".into()), + ], + body: None, + })); +} + +/// Build `::create`'s body (single shape). Just cell-creation, +/// config unwrap, `set_property` finalize, and return. All the +/// dict-assembly happens in `::configure` — see +/// [`build_single_configure_body`]. +/// +/// The `-bd` switch chooses between `create_bd_cell` (default +/// bd=1) for block-design usage and `create_ip` (bd=0) for +/// project-IP usage. Returns `$cell` or `$name` accordingly per +/// the two-mode contract [`emit_config_finalize`] and its callers +/// depend on. +fn build_single_create_body(vlnv: &str, ip_name: &str) -> String { + let mut out = String::new(); + // Guard: `-config` defaults to `""` because the analyzer's + // `@default(...)` grammar rejects bracket-expressions like + // `[::empty]`. Coerce to a real empty Config value at the + // top of the body so downstream `::to` unwrap works. + // Family / split-node kwargs on the old shape used the same + // placeholder pattern (see the old generate_split arg-decl + // block for the family precedent). + writeln!(out, "if {{$config eq \"\"}} {{").unwrap(); + writeln!(out, " set config [{ip_name}::Config::empty]").unwrap(); + writeln!(out, "}}").unwrap(); + // Two-mode create: `-bd true` produces a bd_cell path, `-bd + // false` (default) a project-IP module name. Both wind up in + // `$handle` as a plain string; the wrapper's declared return + // type is `string` so the strict-nominal-identity return-type + // check passes in either branch. Callers that need a + // `bd_cell` newtype at their use site promote with + // `bd_cell::from` (which validates the `/…` shape); callers + // that don't just carry the string through. + writeln!(out, "if {{$bd}} {{").unwrap(); + writeln!( + out, + " set handle [vivado_cmd::create_bd_cell -type ip -vlnv {vlnv} -name $name]" + ) + .unwrap(); + writeln!(out, "}} else {{").unwrap(); + writeln!( + out, + " set handle [vivado_cmd::create_ip -vlnv {vlnv} -module_name $name]" + ) + .unwrap(); + // `create_ip` returns an XCI transcript, not the module + // name — but downstream `set_property` and callers want to + // reference the module by name. Overwrite `handle` with + // `$name` so both branches yield the "thing you address + // set_property against." + writeln!(out, " set handle $name").unwrap(); + writeln!(out, "}}").unwrap(); + emit_config_finalize(&mut out, ip_name); + writeln!(out, "return $handle").unwrap(); + out +} + +/// Wrap a proc's assembled body with a `try/on error` that +/// prepends `..` to any error message escaping +/// from the body — so a nested Properties tree failure at +/// `::configure`'s outermost scope surfaces to the user as +/// +/// ```text +/// gtwiz_versal.configure.intf0.gt_settings.lr0_settings.tx_refclk_frequency.unknown variant: 156.25 +/// ``` +/// +/// The per-arg dict-set catches inside intermediate constructor +/// procs (see `emit_dict_sub_proc` and the sibling emissions in +/// `write_dict_assembly` / `write_dict_assembly_with_splits`) +/// grow the arg-name path; this wrapper adds the top-level +/// `.` prefix so the message reads as a fully +/// qualified dotted path from IP name to leaf field. +/// +/// `try` (Tcl 8.6+) propagates the body's `return` code cleanly +/// through the default `on ok` behavior — the error handler only +/// fires when the body errors, not when it returns normally. +fn wrap_body_qualified_prefix( + body: &str, + ip_name: &str, + proc_local: &str, +) -> String { + format!( + "try {{\n\ + {body}\ + }} on error {{__vw_msg}} {{\n \ + error \"{ip_name}.{proc_local}.$__vw_msg\"\n\ + }}\n" + ) +} + +/// Build `::configure`'s body (single shape). Pure dict +/// assembly + wrap in `::Config`. Zero side effects. +fn build_single_configure_body( + component: &Component, + parameters: &[&Parameter], + ip_name: &str, + dict_schema_newtypes: &std::collections::HashMap, +) -> String { + let mut out = String::new(); + write_dict_assembly( + &mut out, + component, + parameters, + "", + &[], + dict_schema_newtypes, + ); + let config_ty = config_name(ip_name); + // Lift the assembled `_vw_d` (flat `CONFIG. value` pairs + // with bare-string values) into a NESTED, tagged Properties + // tree — CONFIG at the top wraps `Property::Nested` containing + // every `` sub-key as `Property::Scalar`. Matches the + // shape `props::get` returns from a live BD cell, so consumers + // can use `dict get [::Config::to -v $cfg] CONFIG` + + // `Property::as_nested -v ...` to extract sub-trees. + writeln!( + out, + "return [{config_ty}::from -v [Properties::from_dotted_pairs -v $_vw_d]]" + ) + .unwrap(); + out +} + +// --------------------------------------------------------------------------- +// Split shape: top proc + one sub-proc per prefix group. +// --------------------------------------------------------------------------- + +fn generate_split( + component: &Component, + presets: &crate::presets::PresetMap, + parameters: &[&Parameter], + opts: &GenerateOptions, + dict_schemas: &std::collections::HashMap, +) -> String { + let vlnv = component.vlnv(); + let ip_name = sanitize_ident(&component.name); + let top_proc = format!("{ip_name}::create"); + + let tree = build_tree( + parameters.iter().copied(), + &TreeOptions { + min_split_size: opts.min_split_size, + }, + ); + + // Merge XML-derived DictSchemas for the top-level Properties- + // shaped params (`tree.direct` only, so we don't clobber + // schemas the split-node emitter builds for its own params). + // These end up in `dict_schemas` for both the top-proc + // arg-decl (`emit_arg_decl` picks up the typed newtype) and + // the tail `append_dict_sub_procs` call in `generate()` (which + // emits the newtype prelude + constructor proc for each). + let mut dict_schemas: std::collections::HashMap = + dict_schemas.clone(); + for p in &tree.direct { + if dict_schemas.contains_key(&p.name) { + continue; + } + // Anchor lookup: some Vivado top-level params carry a scalar + // sentinel default (`0`) while a sibling `` + // of the same name holds the actual paired-list schema. Use + // the model-param default as the anchor when the top-level + // default doesn't parse as paired. Enables typed-constructor + // emission for INTF_PARENT_PIN_LIST and similar — where the + // slot names live in the internal HDL-generic view rather + // than on the user-facing property. + let anchor_default = model_param_anchor_default(component, &p.name); + let use_anchor = !is_properties_shaped_param(p) + && anchor_default.as_deref().is_some_and(|d| { + !crate::paired_list::parse_paired_list(d).is_empty() + }); + if !is_properties_shaped_param(p) && !use_anchor { + continue; + } + let default = if use_anchor { + anchor_default.as_deref().unwrap() + } else { + p.value.default_value() + }; + let shape_path = lowercase_ident(&p.name); + let mut schema = crate::DictSchema::from_paired_default( + default, + &shape_path, + &opts.overrides, + ); + // Extrapolate `QUAD_` keys across every quad the IP + // ships (5 for gtwiz-versal) so the constructor exposes + // ALL slots, not just quad0's — the model-param default + // only enumerates one quad's worth as a template. + // Also attaches auto-derived pin-path enums when the key + // shape matches `QUAD_`. See + // `extrapolate_quad_schema` for the details. + if use_anchor { + extrapolate_quad_schema(&mut schema, component, &shape_path, opts); + } + if schema.fields.is_empty() && schema.sub_schemas.is_empty() { + continue; + } + dict_schemas.insert(p.name.clone(), schema); + } + // Take a reference to what all downstream code expects. + let dict_schemas = &dict_schemas; + + let families = detect_families( + &tree, + &DetectOptions { + excluded_stems: opts.no_collapse.clone(), + }, + ); + // Set of node labels whose per-N sub-proc we should NOT emit — + // they collapsed into a family constructor. Their sub-nodes + // (`MAC_PORT0_RX`, etc.) still emit; only the direct-param + // per-N node vanishes. + let collapsed_labels: std::collections::HashSet = families + .iter() + .flat_map(|f| { + f.indices.iter().map(move |i| stem_index_label(&f.stem, *i)) + }) + .collect(); + + // Collect every node that will emit a proc — the root for the + // top-level `::create` and every non-root node that has at + // least one direct parameter to configure AND hasn't been + // collapsed into a family. + let all_nodes = tree.collect(); + let emit_nodes: Vec<&Node> = all_nodes + .iter() + .copied() + .filter(|n| n.label.is_empty() || !n.direct.is_empty()) + .filter(|n| !collapsed_labels.contains(&n.label)) + .collect(); + + // Family-side lookups the top-proc emitter uses. + let family_merges: Vec> = families + .iter() + .map(|f| FamilyMerge { + stem: f.stem.clone(), + stem_lower: lowercase_ident(&f.stem), + indices: f.indices.clone(), + newtype_qualified: stem_props_name(&ip_name, &f.stem), + marker: std::marker::PhantomData, + }) + .collect(); + + let dict_schema_newtypes = + build_dict_schema_newtypes(&ip_name, dict_schemas); + + let mut out = String::new(); + emit_file_header(&mut out, component, &vlnv); + // Emit newtype declarations + constructor procs for XML-derived + // top-level dict schemas. These live flat under `::` (like + // PS_PMC_CONFIG) — the CSV-driven ones the caller supplied get + // emitted by `append_dict_sub_procs` at the tail of `generate`. + // Emitting them here keeps the top-proc's typed arg references + // (`hnic_pipe_parameters: ::HnicPipeParameters`) resolvable + // during validation. + for p in &tree.direct { + let Some(schema) = dict_schemas.get(&p.name) else { + continue; + }; + // Skip CSV-driven schemas (`append_dict_sub_procs` emits + // those at the tail). We differentiate two rails: + // * XML-derived schemas — populated above from `tree.direct` + // via `is_properties_shaped_param` OR via the + // model-param anchor path. Both are keyed under + // `p.name`; caller's original `dict_schemas` didn't + // hold them yet. We check by whether the schema has + // content — CSV and XML both do, so that's ambiguous. + // * Anchor-derived: `p` isn't structurally Properties + // but a sibling model param is. Emit iff we DID pick + // up an anchor for it. + // Simplest gate: emit whenever `dict_schemas` has an + // entry that wasn't in the original caller-supplied map. + // We express that indirectly via the two-track OR: + // Properties-shaped (usual path) OR model-param anchor + // present (INTF_PARENT_PIN_LIST-shaped path). + let has_anchor = model_param_anchor_default(component, &p.name) + .as_deref() + .is_some_and(|d| { + !crate::paired_list::parse_paired_list(d).is_empty() + }); + if !is_properties_shaped_param(p) && !has_anchor { + continue; + } + writeln!(&mut out).unwrap(); + emit_dict_props_prelude(&mut out, &ip_name, &[], &p.name); + writeln!(&mut out).unwrap(); + emit_dict_sub_proc(&mut out, &ip_name, &[], &p.name, schema, opts); + } + writeln!( + out, + "## {} configurable parameter{} across {} proc{}.", + parameters.len(), + if parameters.len() == 1 { "" } else { "s" }, + emit_nodes.len(), + if emit_nodes.len() == 1 { "" } else { "s" } + ) + .unwrap(); + writeln!(out, "##").unwrap(); + writeln!(out, "## Usage:").unwrap(); + writeln!(out, "## set cell [{top_proc} ]").unwrap(); + writeln!( + out, + "## $cell - ... ;# tab-complete by prefix" + ) + .unwrap(); + + // `configure`'s arg-doc — the whole documented kwarg pile + // (direct + family + split-node). Zero cell handles, zero + // `-name`/`-bd`. The old inline-`create` doc had all of these + // mixed with `-name`/`-bd`; the split lifts them out into + // `configure` so `::create`'s surface stays tiny. + let mut configure_doc = Doc::new(); + for p in &tree.direct { + emit_arg_decl( + &mut configure_doc, + component, + presets, + p, + opts, + "", + &ip_name, + dict_schemas, + &[], + ); + } + // Family kwargs: one `-` per family member, + // typed as the newtype, with a doc comment referencing the + // constructor via `[…]` for semantic-goto. + // + // `@default("")` is a placeholder — the analyzer's + // `@default(...)` grammar rejects bracket-expressions like + // `[::empty]`, so we can't declare the semantically-correct + // default in the annotation. The body's `__vw_kw__set` + // guard skips the merge loop when the caller didn't pass the + // slot, so `$` is never dereferenced with the placeholder + // value — the empty string never reaches the newtype machinery. + if !families.is_empty() { + if !tree.direct.is_empty() { + configure_doc.push(Item::Blank); + } + for f in &families { + let ctor = format!("{ip_name}::{}", lowercase_ident(&f.stem)); + let ty = stem_props_name(&ip_name, &f.stem); + for i in &f.indices { + let arg = format!("{}{i}", lowercase_ident(&f.stem)); + configure_doc.push(Item::DocComment(format!( + "Configuration for {} slot {i}. Construct with [{ctor}].", + f.stem + ))); + configure_doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("{arg}: {ty}")), + ], + body: None, + })); + } + } + } + // Family newtype preludes ship at the FILE TOP LEVEL — inside + // `namespace eval ` blocks, the analyzer double-prefixes + // qualified proc names (a proc `::T::from` inside `namespace + // eval ` becomes `::::T::from`, so external + // references never find it). Keeping them at top level with + // fully-qualified names avoids that and remains legal now that + // the validator accepts qualified newtype names (see Slice 4's + // changes to `validate::reject_nested_qualified`). An empty + // `namespace eval :: {}` prelude keeps Tcl happy. + if !families.is_empty() { + for f in &families { + writeln!(out).unwrap(); + emit_family_prelude(&mut out, &ip_name, f); + } + writeln!(out).unwrap(); + } + + // One value-constructor per non-root node that has direct + // parameters. Each is a pure function: takes typed field + // kwargs, returns a `::` newtype value that the + // top proc composes into its atomic `set_property -dict` call. + // + // `emit_nodes` already excludes labels collapsed into a + // family — those emit through `emit_family_constructor` above. + let split_nodes: Vec<&Node<'_>> = emit_nodes + .iter() + .filter(|n| !n.label.is_empty()) + .copied() + .collect(); + + // Split-shape newtype preludes are emitted INSIDE each + // split-node's own subfile (see `emit_split_node_constructor`) + // rather than up here at module.htcl top-level. That way + // opening `intf7.htcl` standalone in the LSP still finds the + // `gtwiz_versal::Intf7Props` declaration the file needs. + // module.htcl still sees them because it `src`s each subfile + // via the `## ==== split-file: ... ==== ##` peel-off pass. + + // Split-node kwargs on configure: one per split-shape + // constructor, typed as the node's newtype so configure + // composes ALL configuration into one Config value. + // Semantic-ref doc comment points at the constructor for + // goto/hover. + if !split_nodes.is_empty() { + if !tree.direct.is_empty() || !families.is_empty() { + configure_doc.push(Item::Blank); + } + for n in &split_nodes { + let ctor_suffix = sanitize_ident(&n.label.to_ascii_lowercase()); + let ctor = format!("{ip_name}::{ctor_suffix}"); + let ty = split_props_name(&ip_name, &n.label); + configure_doc.push(Item::DocComment(format!( + "Configuration for the {} sub-tree. Construct with \ + [{ctor}].", + n.label + ))); + configure_doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("{ctor_suffix}: {ty}")), + ], + body: None, + })); + } + } + + // Config prelude at file top level (namespace-eval double- + // prefix workaround). Emit BEFORE the namespace-eval block + // opens so `::Config` is available when the enclosed + // `::configure` returns it and `::create` accepts it. + writeln!(out).unwrap(); + emit_config_prelude(&mut out, &ip_name); + writeln!(out).unwrap(); + + // `configure` body: pure dict assembly + wrap. No cell handle, + // no `set_property`, no `-bd` branch. + let mut configure_body = String::new(); + write_dict_assembly_with_splits( + &mut configure_body, + component, + &tree.direct, + &family_merges, + &dict_schema_newtypes, + &split_nodes, + &ip_name, + ); + let config_ty = config_name(&ip_name); + // Lift flat `CONFIG.` keys into a nested tagged Properties + // tree (see `build_single_configure_body` for the rationale). + writeln!( + configure_body, + "return [{config_ty}::from -v [Properties::from_dotted_pairs -v $_vw_d]]" + ) + .unwrap(); + + // `create`'s arg-doc — narrow surface: -name, -bd, -config. + let mut create_doc = Doc::new(); + create_doc.push(Item::DocComment( + "Project-level IP module name, or (when `-bd 1`) the \ + instance name in the block design." + .into(), + )); + create_doc.push(Item::Command(Command::call( + "name", + std::iter::empty::(), + ))); + push_bd_switch_arg(&mut create_doc); + create_doc.push(Item::DocComment(format!( + "Typed configuration value. Construct with [{ip_name}::configure]. \ + Defaults to an empty config (all parameters take their IP defaults)." + ))); + create_doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("config: {config_ty}")), + ], + body: None, + })); + + let create_body = build_single_create_body(&vlnv, &ip_name); + + // Assemble the `namespace eval { … }` body in + // families → splits → configure → create order. + let mut procs = String::new(); + for (i, f) in families.iter().enumerate() { + if i > 0 { + writeln!(procs).unwrap(); + } + emit_family_constructor( + &mut procs, &ip_name, component, presets, opts, f, + ); + } + if !families.is_empty() { + writeln!(procs).unwrap(); + } + for (i, n) in split_nodes.iter().enumerate() { + if !families.is_empty() || i > 0 { + writeln!(procs).unwrap(); + } + // The `out` buffer collects dict-schema newtypes and + // constructor procs for this node's Properties args. They + // live at fully-qualified names like + // `gtwiz_versal::intf0::ChannelMap::from`, which Tcl only + // resolves correctly when NOT inside a `namespace eval + // { … }` block. So `emit_split_node_constructor` writes them + // to the outer `out` buffer (which is emitted at the top + // level of the file, before `write_namespace_block` wraps + // `procs`). + emit_split_node_constructor( + &mut procs, &mut out, &ip_name, component, presets, opts, n, + ); + } + if !families.is_empty() || !split_nodes.is_empty() { + writeln!(procs).unwrap(); + } + let configure_body = + wrap_body_qualified_prefix(&configure_body, &ip_name, "configure"); + emit_proc( + &mut procs, + "configure", + &configure_doc, + Some(&config_ty), + &configure_body, + ); + writeln!(procs).unwrap(); + emit_proc( + &mut procs, + "create", + &create_doc, + Some("string"), + &create_body, + ); + + write_namespace_block(&mut out, &ip_name, &procs); + out +} + +/// Emit the newtype prelude for a split-shape node. Same shape as +/// [`emit_family_prelude`] / [`emit_dict_props_prelude`] — one +/// `namespace eval {}` + `type = Properties` + the four +/// helper procs. Consumed by [`emit_split_node_constructor`] and +/// the top-proc merge loop in +/// [`write_set_property_dict_with_splits`]. +/// Emit the top-level `::Config = Properties` newtype at file +/// top level (outside `namespace eval {…}` to sidestep the +/// analyzer's double-prefix bug on qualified proc names inside +/// namespace-eval blocks). Structural mirror of +/// [`emit_split_props_prelude`] / [`emit_family_prelude`] / +/// [`emit_dict_props_prelude`] — same four helpers (`empty`, +/// `from`, `to`, `repr`) with identity implementations. The name +/// is always `::Config` — one per generated wrapper — so the +/// callsite pattern `set cfg [::configure -foo x]` returns a +/// value the analyzer knows about and `::create` can accept +/// via its typed `-config ::Config` param. +fn emit_config_prelude(out: &mut String, ip_name: &str) { + let qualified = config_name(ip_name); + writeln!( + out, + "## Typed configuration value for [{ip_name}::create]. \ + Construct with [{ip_name}::configure].", + ) + .unwrap(); + writeln!(out, "namespace eval {ip_name} {{}}").unwrap(); + writeln!(out, "namespace eval {qualified} {{}}").unwrap(); + writeln!(out, "type {qualified} = Properties").unwrap(); + // Values are a properly nested tagged Properties tree by the + // time they reach Config (see `build_single_configure_body` — + // configure's return path lifts through + // `Properties::from_dotted_pairs`). Delegate to + // `Properties::repr` for uniform tagged-Property rendering — + // same `KEY Scalar(VALUE)` / `KEY Nested(…)` shape the REPL's + // syntax highlighter colours specially. + writeln!( + out, + "proc {qualified}::repr {{ v: {qualified} }} string \ + {{ return [Properties::repr -v [{qualified}::to -v $v]] }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::from {{ v: Properties }} {qualified} \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::to {{ v: {qualified} }} Properties \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::empty {{}} {qualified} \ + {{ return [{qualified}::from -v [Properties::empty]] }}" + ) + .unwrap(); +} + +/// Qualified name of the top-level Config newtype for `ip_name`. +fn config_name(ip_name: &str) -> String { + format!("{ip_name}::Config") +} + +fn emit_split_props_prelude(out: &mut String, ip_name: &str, label: &str) { + let qualified = split_props_name(ip_name, label); + let ctor_lower = label.to_ascii_lowercase(); + writeln!( + out, + "## Typed configuration value for [{ip_name}::create]'s \ + `-{ctor_lower}` slot. Construct with [{ip_name}::{ctor_lower}].", + ) + .unwrap(); + writeln!(out, "namespace eval {ip_name} {{}}").unwrap(); + writeln!(out, "namespace eval {qualified} {{}}").unwrap(); + writeln!(out, "type {qualified} = Properties").unwrap(); + writeln!( + out, + "proc {qualified}::repr {{ v: {qualified} }} string \ + {{ return [Properties::repr -v [{qualified}::to -v $v]] }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::from {{ v: Properties }} {qualified} \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::to {{ v: {qualified} }} Properties \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::empty {{}} {qualified} \ + {{ return [{qualified}::from -v [Properties::empty]] }}" + ) + .unwrap(); +} + +/// Emit the value-constructor for a split-shape node. Bare proc +/// name (``) — becomes `::` via the +/// enclosing `namespace eval { … }`. Pure — no cell handle, +/// no `-bd`, no `set_property`. Body builds a `Properties`-shaped +/// dict from the supplied field kwargs (index-stripped keys) and +/// wraps in the newtype. +fn emit_split_node_constructor( + out: &mut String, + // Buffer for output that must live OUTSIDE the ip-scoped + // `namespace eval { … }` block that wraps `out`. Dict-schema + // sub-procs (which use fully-qualified names like + // `gtwiz_versal::intf0::ChannelMap::from`) go here — placing them + // inside the `namespace eval` block causes Tcl to double the ip + // prefix (`gtwiz_versal::gtwiz_versal::intf0::…`) and the + // procs become undiscoverable. + outer_out: &mut String, + ip_name: &str, + component: &Component, + presets: &crate::presets::PresetMap, + opts: &GenerateOptions, + n: &Node<'_>, +) { + let ctor_local = sanitize_ident(&n.label.to_ascii_lowercase()); + let ret_ty = split_props_name(ip_name, &n.label); + + let mut doc = Doc::new(); + doc.push(Item::DocComment(format!( + "Configuration value for [{ip_name}::create]'s \ + `-{ctor_local}` slot ({} sub-tree). Composes into the top \ + proc so every provided field lands in ONE atomic \ + `set_property -dict` call.", + n.label + ))); + if !n.direct.is_empty() { + doc.push(Item::Blank); + } + // Build the XML-driven dict-schema map for this split-node's + // Properties-shaped params (channel_map, gt_settings, etc. on an + // `intf` node). Each schema gets emitted as a nested-namespace + // typed sub-proc under `::::`, and the arg-decl + // references the typed newtype instead of raw `Properties`. + // See [`build_split_dict_schemas`] for the extraction logic. + let node_local = sanitize_ident(&n.label.to_ascii_lowercase()); + let local_dict_schemas = build_split_dict_schemas(n, &node_local, opts); + for p in &n.direct { + emit_arg_decl( + &mut doc, + component, + presets, + p, + opts, + &n.label, + ip_name, + &local_dict_schemas, + &[node_local.as_str()], + ); + } + + // Split-file marker: everything between OPEN and CLOSE ends up + // in a sibling `.htcl` file (see `split_into_files`) with the + // main module.htcl gaining a `src ./` line at this + // spot. Basename derives from the split-node's local ident so + // gtwiz-versal's 8 intfN nodes land in `intf0.htcl`…`intf7.htcl`. + // For small IPs the marker still emits but every subfile stays + // tiny (a handful of lines) which the tree-sitter parser handles + // instantly. + writeln!(outer_out, "## ==== split-file: {node_local}.htcl ==== ##") + .unwrap(); + // Every subfile needs its own `src @vivado-cmd` so it can be + // analyzed (and hovered / go-to-def'd) standalone in the LSP + // without the analyzer opening it via module.htcl. The dict- + // schema prelude references `Properties::repr`, `Properties::empty`, + // and similar procs that live in the vivado-cmd library; without + // the src line the analyzer flags every reference as undefined. + writeln!(outer_out, "src @vivado-cmd").unwrap(); + writeln!(outer_out).unwrap(); + + // Split-shape newtype prelude for THIS node — moved from + // module.htcl into the subfile so opening the subfile + // standalone in the LSP finds the type declaration the file's + // own return-type annotations reference. + emit_split_props_prelude(outer_out, ip_name, &n.label); + + // Emit the per-node typed sub-procs first so their newtypes + // are visible before this proc's signature references them. + // Each sub-proc lives at `::::`; + // its newtype at `::::`. + // Placed on the outer buffer so the fully-qualified proc names + // (`gtwiz_versal::intf0::…`) don't get an accidental extra + // `gtwiz_versal::` prefix from Tcl's namespace resolution. + emit_split_dict_sub_procs( + outer_out, + ip_name, + &node_local, + &n.label, + &n.direct, + &local_dict_schemas, + opts, + ); + + let mut body = String::new(); + writeln!(body, "set _vw_d [dict create]").unwrap(); + for p in &n.direct { + let field_key = strip_prefix(&p.name, &n.label); + let arg = lowercase_ident(field_key); + // Dict-schema-backed slot: unwrap the typed newtype to its + // raw paired dict via `::to`. Other Properties-shaped + // slots (none in gtwiz-versal after this change, but the + // path still handles legacy IPs whose defaults slip past + // the schema extractor) arrive as raw paired dicts; store + // `$arg` verbatim without `Properties::to_raw` (which would + // try to strip Scalar/Nested tags that aren't present). + let value_expr = if local_dict_schemas.contains_key(field_key) { + let ret_ty = + dict_props_name(ip_name, &[node_local.as_str()], field_key); + format!("[{ret_ty}::to -v ${arg}]") + } else if let Some(kind) = bool_kind(component, p) { + // Split-node scalar bool field — pure or pseudo-bool + // (kind picks the wire form). Same `if`-shape as the + // top-proc and composed-family paths so `$arg` stays + // a whole-word VarRef the usage walker sees. + bool_value_expr(&arg, kind) + } else { + format!("${arg}") + }; + // Wrap the dict-set in a catch prepending the HTCL arg + // name — see the sibling emission in + // `emit_dict_sub_proc` for the diagnostic-path + // rationale. + writeln!( + body, + "if {{${{__vw_kw_{arg}_set}}}} {{\n \ + if {{[catch {{\n \ + dict set _vw_d {field_key} {value_expr}\n \ + }} __vw_msg]}} {{\n \ + error \"{arg}.$__vw_msg\"\n \ + }}\n \ + }}" + ) + .unwrap(); + } + writeln!( + body, + "return [{ret_ty}::from -v [Properties::from -v $_vw_d]]" + ) + .unwrap(); + // Emit the split-node's own constructor proc into the outer + // buffer (same file as its dict-schema sub-procs) with a fully + // qualified name — so it lives at `::` without + // needing to sit inside a `namespace eval {…}` block. That + // lets the whole split-node emission end up in ONE sibling file + // wrapped by split markers, instead of being torn between the + // main-file namespace-block and the outer-scope schema block. + let ctor_qualified = format!("{ip_name}::{ctor_local}"); + emit_proc(outer_out, &ctor_qualified, &doc, Some(&ret_ty), &body); + + // Split-file close marker — see the matching OPEN above and + // `split_into_files` for the peel logic. + writeln!( + outer_out, + "## ==== end split-file: {node_local}.htcl ==== ##" + ) + .unwrap(); + + // Consume `out` (the namespace-block buffer) so the reader can + // see we intentionally skipped writing into it — the split-node + // no longer contributes short-name procs there. + let _ = out; +} + +/// Build the XML-driven DictSchema map for a split-node's +/// Properties-shaped direct params. Runs once per split-node +/// emission; the returned map is keyed by raw IP-XACT parameter +/// name (matches the arg-decl / body lookup pattern). +/// +/// The shape-path passed to `DictSchema::from_paired_default` is +/// `::`, where `stem` is the un-indexed stem of +/// the node's label (`INTF0` → `intf`, `QUAD0_CH0` → `quad_ch`), +/// so overrides written against `intf::gt_settings` apply across +/// every `intf0`..`intf7` sibling. Matches the "one override entry +/// per family" convention documented in +/// [`crate::overrides::OverridesFile`]. +fn build_split_dict_schemas( + n: &Node<'_>, + _node_local: &str, + opts: &GenerateOptions, +) -> std::collections::HashMap { + let stem_lower = split_stem_lower(&n.label); + let mut out = std::collections::HashMap::new(); + + // Pass 1 — extract each Properties-shaped param's own schema + // from its default value. Empty results (from `0` / `""` / + // `NA NA` sentinel defaults) still land in the map so pass 2 + // can decide whether to synthesize from an anchor. + for p in &n.direct { + if !is_properties_shaped_param(p) { + continue; + } + let field_key = strip_prefix(&p.name, &n.label).to_string(); + let field_lower = lowercase_ident(&field_key); + let shape_path = if stem_lower.is_empty() { + field_lower + } else { + format!("{stem_lower}::{field_lower}") + }; + let schema = crate::DictSchema::from_paired_default( + p.value.default_value(), + &shape_path, + &opts.overrides, + ); + out.insert(field_key, schema); + } + + // Pass 2 — resolve schemas whose own default was uninformative. + // The gtwiz-versal LR{n}_SETTINGS / GT_SETTINGS / GT_INTERNAL + // shapes all share a field vocabulary that only lives inside + // TXRX_OPTIONAL_PORTS's `INTF_LR_SETTINGS.LR0_SETTINGS` payload + // (see the Explore report). Find that payload once, then use it + // to backfill: + // - each trivial-schema `LR{n}_SETTINGS` slot (16 of them), + // - the tcldict-tagged `GT_SETTINGS` / `GT_INTERNAL` params, + // synthesized as wrappers holding 16 LR sub-slots. + let anchor_lr = out + .get("TXRX_OPTIONAL_PORTS") + .and_then(|s| s.sub_schemas.get("INTF_LR_SETTINGS")) + .and_then(|s| s.sub_schemas.get("LR0_SETTINGS")) + .cloned(); + let params_by_key: std::collections::HashMap<&str, &&Parameter> = n + .direct + .iter() + .map(|p| (strip_prefix(&p.name, &n.label), p)) + .collect(); + if let Some(lr_template) = anchor_lr.as_ref() { + for i in 0..16 { + let key = format!("LR{i}_SETTINGS"); + // Only backfill if the slot exists on the node (LR0..15 + // are all declared params on gtwiz-versal's intf nodes). + if !params_by_key.contains_key(key.as_str()) { + continue; + } + // Always replace with the anchor template — the LR* + // params' own defaults are Xilinx sentinels (`NA NA` + // extracts to a bogus `NA=NA` field, hardly trivial by + // fields.is_empty() but useless as a schema). The + // TXRX_OPTIONAL_PORTS anchor carries the real field + // vocabulary that Vivado actually accepts. + let mut copy = lr_template.clone(); + // The anchor's fields were built under the txrx shape + // path; reapply the destination slot's overrides so + // `intf::lrN_settings` gets its own refinements. + let dst_shape_path = if stem_lower.is_empty() { + lowercase_ident(&key) + } else { + format!("{stem_lower}::{}", lowercase_ident(&key)) + }; + copy.reapply_overrides(&dst_shape_path, &opts.overrides); + out.insert(key, copy); + } + // Synthesize wrappers for tcldict GT_SETTINGS / + // GT_INTERNAL. Both take the same LR0..LR15 sub-slot shape + // (Vivado accepts `CONFIG.INTF*_GT_SETTINGS(LR_SETTINGS) + // {...}` — the parenthesized sub-key IS the wrapper slot). + for wrapper_key in ["GT_SETTINGS", "GT_INTERNAL"] { + let Some(p) = params_by_key.get(wrapper_key) else { + continue; + }; + if !p.has_parameter_type("tcldict") { + continue; + } + let existing = out.get(wrapper_key); + if existing.is_some_and(|s| !schema_is_trivial(s)) { + continue; + } + let mut sub = std::collections::BTreeMap::new(); + for i in 0..16 { + let mut copy = lr_template.clone(); + // Sub-slot's shape path is `::::lrN_settings` + // — refines overrides written for that specific path + // (e.g. `intf::gt_settings::lr0_settings`), separate + // from the top-level `intf::lrN_settings` slot. + let wrapper_lower = wrapper_key.to_ascii_lowercase(); + let lr_lower = format!("lr{i}_settings"); + let sub_shape_path = if stem_lower.is_empty() { + format!("{wrapper_lower}::{lr_lower}") + } else { + format!("{stem_lower}::{wrapper_lower}::{lr_lower}") + }; + copy.reapply_overrides(&sub_shape_path, &opts.overrides); + sub.insert(format!("LR{i}_SETTINGS"), copy); + } + out.insert( + wrapper_key.to_string(), + crate::DictSchema { + fields: Vec::new(), + sub_schemas: sub, + }, + ); + } + } + + // Drop trivial schemas — params whose default was `0` / `""` + // sentinels with no anchor available. Fall back to raw + // Properties for those; a bare typed slot with no fields + // would just clutter the LSP surface. + out.retain(|_, schema| !schema_is_trivial(schema)); + out +} + +/// A schema is "trivial" when it has no fields AND no sub-slots — +/// derived from an empty default (`0`, `""`) or a nonsense `NA NA` +/// where the parser found no meaningful structure. Callers use +/// this to decide whether to backfill from an anchor param. +fn schema_is_trivial(s: &crate::DictSchema) -> bool { + s.fields.is_empty() && s.sub_schemas.is_empty() +} + +/// Emit the dict-schema sub-procs (prelude + constructor + any +/// nested sub-slots) for each entry in `dict_schemas`, all under +/// `::::`. Deep recursion happens inside +/// `emit_dict_sub_proc` via `emit_dict_sub_schemas`, so multi-level +/// XML shapes (`INTF0_TXRX_OPTIONAL_PORTS` → `INTF_LR_SETTINGS` → +/// `LR0_SETTINGS`) unfold naturally. +fn emit_split_dict_sub_procs( + out: &mut String, + ip_name: &str, + node_local: &str, + node_label: &str, + params: &[&Parameter], + dict_schemas: &std::collections::HashMap, + opts: &GenerateOptions, +) { + // Emit in the same order params appear so review diffs are + // deterministic and consumers can visually pair the newtype + // with its Properties arg in the outer proc. + for p in params { + let stripped = strip_prefix(&p.name, node_label); + let Some(schema) = dict_schemas.get(stripped) else { + continue; + }; + writeln!(out).unwrap(); + emit_dict_props_prelude(out, ip_name, &[node_local], stripped); + writeln!(out).unwrap(); + emit_dict_sub_proc(out, ip_name, &[node_local], stripped, schema, opts); + } +} + +/// Strip the trailing digit(s) from a split-node label and +/// lowercase — `INTF0` → `"intf"`, `QUAD0_CH1` → `"quad_ch"`. +/// Used as the shape-path stem when looking up overrides so a +/// single override entry applies across every indexed instance in +/// the family. +fn split_stem_lower(label: &str) -> String { + let mut segs: Vec = Vec::new(); + for seg in label.split('_') { + // Strip a trailing digit run from each segment. Keeps + // multi-word stems (`QUAD_CH`) intact while collapsing + // `INTF0` → `INTF`, `QUAD0_CH1` → `QUAD_CH`. + let trimmed = seg.trim_end_matches(|c: char| c.is_ascii_digit()); + if trimmed.is_empty() { + continue; + } + segs.push(trimmed.to_ascii_lowercase()); + } + segs.join("_") +} + +/// Fully-qualified newtype name for a split-shape node. +/// `("dcmac", "MAC_PORT0_RX")` → `"dcmac::MacPort0RxProps"`. +fn split_props_name(ip_name: &str, label: &str) -> String { + format!("{ip_name}::{}", split_props_local(label)) +} + +fn split_props_local(label: &str) -> String { + let mut out = String::new(); + for seg in label.split('_').filter(|s| !s.is_empty()) { + out.push_str(&pascal_case(seg)); + } + out.push_str("Props"); + out +} + +/// Extended top-proc dict writer that also merges split-shape +/// value-constructor outputs into the atomic dict. Mirrors +/// [`write_set_property_dict`]'s structure but weaves the +/// split-node merges in between the top-level knobs, family +/// merges, and the finalization. +#[allow(clippy::too_many_arguments)] +fn write_dict_assembly_with_splits( + out: &mut String, + component: &Component, + parameters: &[&Parameter], + families: &[FamilyMerge<'_>], + dict_schema_newtypes: &std::collections::HashMap, + split_nodes: &[&Node<'_>], + ip_name: &str, +) { + writeln!(out, "set _vw_d [list]").unwrap(); + for p in parameters { + let arg = lowercase_ident(&p.name); + // Type-driven value unwrap: + // - Dict-schema newtype: the constructor stores bare-string + // values in a paired-list dict (see + // [`emit_dict_sub_proc`]), which is EXACTLY what Vivado + // expects at `CONFIG.`. So just unwrap the newtype + // via `::to` — do NOT pipe through `Properties::to_raw`, + // which would try to dispatch on `Property::Scalar`/ + // `Nested` tags our stored values don't carry. + // - Plain Properties (paired-dict-shaped default without a + // schema): assume the caller passed a properly-tagged + // Properties value; unwrap through `Properties::to_raw`. + // - Scalar: `$arg` as-is. + let value_expr = + if let Some(newtype) = dict_schema_newtypes.get(&p.name) { + format!("[{newtype}::to -v ${arg}]") + } else if is_properties_shaped_param(p) { + // Properties-typed args now arrive as tagged trees + // (from other IPs' `configure` procs, or extracted + // via `dict get $props CONFIG` + `Property::as_nested` + // by the caller). Unwrap tags to a raw paired list + // via `Properties::to_raw` so + // `Properties::from_dotted_pairs` (which lifts the + // whole `_vw_d` at the end of the configure body) + // sees consistent bare-string values across all + // sub-slots. Without this the tagged-tree elements + // would confuse the shim's structural lifter. + format!("[Properties::to_raw -v ${arg}]") + } else if let Some(kind) = bool_kind(component, p) { + // Marshal the HTCL bool literal to whichever + // lexical form the IP-XACT declaration expects: + // * `Pure` (`spirit:format="bool"`) → true/false + // * `Pseudo` (long+choice{0,1}) → 1/0 + // Uses `[if $arg …]` (not `[expr {$arg?…}]`) so + // `$arg` appears as a whole-word VarRef in the + // emitted AST — the brace-wrapped form would put + // the reference inside a braced-text part where + // the analyzer's usage walker can't see it, + // producing spurious "unused proc arg" warnings + // for every bool arg. Defense-in-depth for + // callers reaching here via `extern::` (analyzer + // bypassed): Tcl's `if` still coerces via + // `string is boolean` semantics on non-canonical + // input. + bool_value_expr(&arg, kind) + } else { + format!("${arg}") + }; + // Wrap the merge in a `catch` prepending the HTCL arg + // name on error — mirrors the sub-proc emission in + // `emit_dict_sub_proc`. Only nested-Properties args + // actually risk erroring here (`Properties::to_raw` + // walks a tagged tree that might have a mistyped + // leaf); scalar/bool args store bare strings that + // can't fail at `lappend` time, so the catch is + // no-op-cheap for them. + writeln!( + out, + "if {{${{__vw_kw_{arg}_set}}}} {{\n \ + if {{[catch {{\n \ + lappend _vw_d CONFIG.{name} {value_expr}\n \ + }} __vw_msg]}} {{\n \ + error \"{arg}.$__vw_msg\"\n \ + }}\n\ + }}", + name = p.name, + ) + .unwrap(); + } + // Composed-family merges. + for fam in families { + for idx in &fam.indices { + let arg = format!("{}{}", fam.stem_lower, idx); + let prefix = format!("CONFIG.{}{}_", fam.stem, idx); + writeln!(out, "if {{${{__vw_kw_{arg}_set}}}} {{").unwrap(); + writeln!( + out, + " foreach {{_vw_f _vw_v}} [{}::to -v ${arg}] {{", + fam.newtype_qualified + ) + .unwrap(); + writeln!(out, " lappend _vw_d \"{prefix}$_vw_f\" $_vw_v") + .unwrap(); + writeln!(out, " }}").unwrap(); + writeln!(out, "}}").unwrap(); + } + } + // Split-shape merges — each provided value-constructor result + // gets its dict entries flattened into the atomic dict with + // the `CONFIG._` prefix so property names round- + // trip to Vivado exactly as they were in IP-XACT. + for n in split_nodes { + let arg = sanitize_ident(&n.label.to_ascii_lowercase()); + let prefix = format!("CONFIG.{}_", n.label); + let newtype = split_props_name(ip_name, &n.label); + writeln!(out, "if {{${{__vw_kw_{arg}_set}}}} {{").unwrap(); + writeln!( + out, + " foreach {{_vw_f _vw_v}} [{newtype}::to -v ${arg}] {{" + ) + .unwrap(); + writeln!(out, " lappend _vw_d \"{prefix}$_vw_f\" $_vw_v").unwrap(); + writeln!(out, " }}").unwrap(); + writeln!(out, "}}").unwrap(); + } + // No finalization here — callers wrap the assembled `_vw_d` + // in a typed `::Config` value. The `set_property` finalize + // has moved to `emit_config_finalize` on the `::create` + // side, which unwraps the caller's `-config` param and applies. +} + +// --------------------------------------------------------------------------- +// Shared helpers. +// --------------------------------------------------------------------------- + +/// Break the concatenated generator output into one main file plus +/// zero-or-more sibling files. +/// +/// Peel rule: scan the aggregate for the split marker +/// `## ==== split-file: ==== ##` … (matching close marker). +/// The interior lands in `subfiles[]`; the open/close markers +/// in `main` become a `src ./.htcl` line so the main file's +/// downstream references still resolve at load time. +/// +/// Content without markers stays in `main` untouched. Callers that +/// don't want the split at all (unit tests, single-file consumers) +/// use `MultiFileOutput::into_single()` to re-flatten. +/// +/// Threshold-based auto-splitting layers on top: the emitters wrap +/// each large split-node in markers so this pass does the physical +/// separation. Small nodes don't get markers → they stay inline. +fn split_into_files(aggregate: String) -> MultiFileOutput { + const OPEN: &str = "## ==== split-file: "; + const CLOSE: &str = "## ==== end split-file: "; + let mut main = String::with_capacity(aggregate.len()); + let mut subfiles: Vec<(String, String)> = Vec::new(); + let mut rest = aggregate.as_str(); + while let Some(open_off) = rest.find(OPEN) { + // Everything before the marker stays in main. + main.push_str(&rest[..open_off]); + rest = &rest[open_off + OPEN.len()..]; + let Some(open_end) = rest.find('\n') else { + // Malformed marker — retain the rest in main and bail. + main.push_str(OPEN); + main.push_str(rest); + return MultiFileOutput { main, subfiles }; + }; + let name_line = rest[..open_end].trim(); + // Strip the trailing ` ==== ##` from the marker line. The + // suffix has interleaved whitespace, ``#``, and ``=``; strip + // them all in one pass so partial-suffix boundaries (like + // "==== " with the trailing space blocking the "====" strip) + // don't leak into the filename. + let name = name_line + .trim_end_matches(|c: char| { + c == '#' || c == '=' || c.is_whitespace() + }) + .trim(); + rest = &rest[open_end + 1..]; + // Find the matching close marker. + let close_needle = format!("{CLOSE}{name}"); + let Some(close_off) = rest.find(&close_needle) else { + // Unpaired open — restore in main and stop splitting. + main.push_str(OPEN); + main.push_str(name_line); + main.push('\n'); + main.push_str(rest); + return MultiFileOutput { main, subfiles }; + }; + let body = rest[..close_off].to_string(); + // Advance past the close marker's line (up to and + // including its newline). + let after = &rest[close_off..]; + let close_line_end = + after.find('\n').map(|n| n + 1).unwrap_or(after.len()); + rest = &after[close_line_end..]; + // Emit the src line into main so the loader still walks + // this subfile. Uses a relative `./` path — the CLI + // writes the subfile as a sibling of the main output. + writeln!(main, "src ./{name}").unwrap(); + subfiles.push((name.to_string(), body)); + } + main.push_str(rest); + MultiFileOutput { main, subfiles } +} + +fn emit_file_header(out: &mut String, component: &Component, vlnv: &str) { + // Pull in the whole `vivado-cmd` library — the body uses + // `vivado_cmd::create_bd_cell` and `vivado_cmd::set_property` + // alongside `ip::check`, so `src @vivado-cmd/ip` (just the + // ip sub-module) leaves those references unresolved. The + // analyzer reports the unbound calls; sourcing the full + // package brings everything the emitted body actually uses + // into scope. + writeln!(out, "src @vivado-cmd").unwrap(); + writeln!(out).unwrap(); + writeln!(out, "ip::check -name \"{vlnv}\"").unwrap(); + writeln!(out).unwrap(); + if let Some(desc) = + component.description.as_deref().filter(|s| !s.is_empty()) + { + // Split the IP-XACT description into a one-sentence summary + // plus body so an LSP client can show a short blurb on hover + // / completion without repeating it in the documentation + // popup. Same shape as the cmd-doc generator. + let raw: Vec = + desc.lines().map(|l| l.trim_end().to_string()).collect(); + let summary = vw_htcl::doc::brief(&raw); + let extended = vw_htcl::doc::extended(&raw); + if let Some(s) = summary { + for line in vw_htcl::doc::wrap_paragraph(&s, 78) { + writeln!(out, "## {line}").unwrap(); + } + } + if let Some(body) = extended { + for paragraph in body.split("\n\n") { + writeln!(out, "##").unwrap(); + for line in vw_htcl::doc::wrap_paragraph(paragraph, 78) { + writeln!(out, "## {line}").unwrap(); + } + } + } + writeln!(out, "##").unwrap(); + } + writeln!(out, "## Source IP-XACT: {vlnv}").unwrap(); +} + +/// Emit `proc { } ? { }` with the args +/// and body indented two spaces each. When `return_type` is Some, +/// emits it as the 4th htcl word between args and body. +fn emit_proc( + out: &mut String, + name: &str, + args: &Doc, + return_type: Option<&str>, + body: &str, +) { + let args_text = args.to_string(); + writeln!(out, "proc {name} {{").unwrap(); + for line in args_text.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + match return_type { + Some(ty) => { + let needs_brace = ty.chars().any(char::is_whitespace); + if needs_brace { + writeln!(out, "}} {{{ty}}} {{").unwrap(); + } else { + writeln!(out, "}} {ty} {{").unwrap(); + } + } + None => { + writeln!(out, "}} {{").unwrap(); + } + } + for line in body.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + writeln!(out, "}}").unwrap(); +} + +/// Emit the paired-list assembly loops that build `_vw_d` from +/// direct parameters + composed families + dict-schema newtypes. +/// Arg names are built by stripping `prefix_to_strip` from each +/// parameter's full IP-XACT name; the `CONFIG.` key keeps +/// the full name Vivado expects. +/// +/// Callers: `::configure`'s body (both single and split shapes). +/// The output is a plain `_vw_d` paired list — no finalize, no +/// `set_property` call. `configure` wraps the assembled dict in +/// `[::Config::from -v [Properties::from -v $_vw_d]]` and +/// returns it; `::create` unwraps and applies. See +/// [`emit_config_finalize`] for the corresponding apply side. +fn write_dict_assembly( + out: &mut String, + component: &Component, + parameters: &[&Parameter], + prefix_to_strip: &str, + // Composed families to merge atomically into the same dict. + // Empty for sub-procs; populated on the top proc when the + // generator has collapsed indexed sibling groups into + // `::` constructors + `-` + // kwargs. Each family's per-index kwarg is unwrapped through + // its newtype's `::to` + `Properties::to_raw` and merged into + // `_vw_d` with `CONFIG._` keys. + families: &[FamilyMerge<'_>], + // Dict-schema newtype names, keyed by IP-XACT param name + // (e.g. `PS_PMC_CONFIG` → `versal_cips::PsPmcConfig`). When a + // parameter here matches a top-level top-proc param, its + // value_expr gets the `[::to -v $arg]` unwrap injected + // BEFORE the `Properties::to_raw` step so the type-check + // passes at compile time. Empty for sub-procs / single-shape + // IPs without any schemas. + dict_schema_newtypes: &std::collections::HashMap, +) { + // Build the dict conditionally so only user-supplied args reach + // Vivado. See `emit_dict_proc` for the rationale — unconditionally + // setting all CONFIG.* properties re-validates the whole cell and + // Vivado rejects values whose declared defaults happen to be + // out-of-range for the cell's current state. The + // `__vw_kw__set` flag is set by `::vw::kwargs` (shim helper) + // only when the user passed a value for that arg. + writeln!(out, "set _vw_d [list]").unwrap(); + for p in parameters { + let arg = lowercase_ident(strip_prefix(&p.name, prefix_to_strip)); + // Type-driven value unwrap: + // - Dict-schema newtype (`versal_cips::PsPmcConfig` etc.): + // `[Properties::to_raw -v [::to -v $arg]]`. The extra + // `::to` step satisfies the type-checker at compile + // time; at runtime it's identity on the underlying + // Properties value. + // - Plain Properties (paired-dict-shaped default without + // a registered schema): `[Properties::to_raw -v $arg]`. + // - Scalar: `$arg`. + // Type-driven value unwrap: + // - Dict-schema newtype: the constructor stores bare-string + // values in a paired-list dict (see + // [`emit_dict_sub_proc`]), which is EXACTLY what Vivado + // expects at `CONFIG.`. So just unwrap the newtype + // via `::to` — do NOT pipe through `Properties::to_raw`, + // which would try to dispatch on `Property::Scalar`/ + // `Nested` tags our stored values don't carry. + // - Plain Properties (paired-dict-shaped default without a + // schema): assume the caller passed a properly-tagged + // Properties value; unwrap through `Properties::to_raw`. + // - Scalar: `$arg` as-is. + let value_expr = + if let Some(newtype) = dict_schema_newtypes.get(&p.name) { + format!("[{newtype}::to -v ${arg}]") + } else if is_properties_shaped_param(p) { + // Properties-typed args now arrive as tagged trees + // (from other IPs' `configure` procs, or extracted + // via `dict get $props CONFIG` + `Property::as_nested` + // by the caller). Unwrap tags to a raw paired list + // via `Properties::to_raw` so + // `Properties::from_dotted_pairs` (which lifts the + // whole `_vw_d` at the end of the configure body) + // sees consistent bare-string values across all + // sub-slots. Without this the tagged-tree elements + // would confuse the shim's structural lifter. + format!("[Properties::to_raw -v ${arg}]") + } else if let Some(kind) = bool_kind(component, p) { + // Marshal the HTCL bool literal to whichever + // lexical form the IP-XACT declaration expects: + // * `Pure` (`spirit:format="bool"`) → true/false + // * `Pseudo` (long+choice{0,1}) → 1/0 + // Uses `[if $arg …]` (not `[expr {$arg?…}]`) so + // `$arg` appears as a whole-word VarRef in the + // emitted AST — the brace-wrapped form would put + // the reference inside a braced-text part where + // the analyzer's usage walker can't see it, + // producing spurious "unused proc arg" warnings + // for every bool arg. Defense-in-depth for + // callers reaching here via `extern::` (analyzer + // bypassed): Tcl's `if` still coerces via + // `string is boolean` semantics on non-canonical + // input. + bool_value_expr(&arg, kind) + } else { + format!("${arg}") + }; + // Wrap the merge in a `catch` prepending the HTCL arg + // name on error — mirrors the sub-proc emission in + // `emit_dict_sub_proc`. Only nested-Properties args + // actually risk erroring here (`Properties::to_raw` + // walks a tagged tree that might have a mistyped + // leaf); scalar/bool args store bare strings that + // can't fail at `lappend` time, so the catch is + // no-op-cheap for them. + writeln!( + out, + "if {{${{__vw_kw_{arg}_set}}}} {{\n \ + if {{[catch {{\n \ + lappend _vw_d CONFIG.{name} {value_expr}\n \ + }} __vw_msg]}} {{\n \ + error \"{arg}.$__vw_msg\"\n \ + }}\n\ + }}", + name = p.name, + ) + .unwrap(); + } + // Composed-family merges — each provided `-` + // value gets unwrapped and its `FIELD → value` pairs merged + // into `_vw_d` with the `CONFIG._` prefix. Runs + // BEFORE the `if {llength} { set_property … }` finalization + // so the whole thing lands as ONE atomic call. + for fam in families { + for idx in &fam.indices { + let arg = format!("{}{}", fam.stem_lower, idx); + let prefix = format!("CONFIG.{}{}_", fam.stem, idx); + // Family constructors store bare-string values in + // their dict (see `emit_family_constructor`), so + // iterate the raw dict directly rather than going + // through `Properties::to_raw`. The latter would + // dispatch on `Property::Scalar`/`Nested` tags that + // our stored values don't carry — the constructor + // treats every field as a scalar and this loop has + // to match that convention. + writeln!(out, "if {{${{__vw_kw_{arg}_set}}}} {{").unwrap(); + writeln!( + out, + " foreach {{_vw_f _vw_v}} [{}::to -v ${arg}] {{", + fam.newtype_qualified + ) + .unwrap(); + writeln!(out, " lappend _vw_d \"{prefix}$_vw_f\" $_vw_v") + .unwrap(); + writeln!(out, " }}").unwrap(); + writeln!(out, "}}").unwrap(); + } + } +} + +/// Emit the create-side finalize: unwrap `$config` through +/// `::Config::to`, flatten the nested tagged Properties tree +/// back into the flat `CONFIG. value` paired list +/// `set_property -dict` expects (via [`Properties::to_dotted_flat`]), +/// then apply against the cell handle. Two-mode `-bd` branch: +/// `bd=1` → target `$cell` directly (a bd_cell path); `bd=0` → +/// resolve via `[get_ips $name]` because `create_ip` returns an +/// XCI file path, not an IP handle. +fn emit_config_finalize(out: &mut String, ip_name: &str) { + let config_ty = config_name(ip_name); + writeln!( + out, + "set _dict [Properties::to_dotted_flat -v [{config_ty}::to -v $config]]" + ) + .unwrap(); + // Two-mode set_property target: + // - `-bd true`: `$handle` IS the `/…` bd_cell path, + // `set_property -objects` takes it directly. + // - `-bd false`: `$handle` is a bare module name; the IP + // object is fetched via `[get_ips $handle]`. + // Both branches are semantically identical up to the target + // resolution, matching the pre-collapse behavior. + writeln!(out, "if {{[llength $_dict] > 0}} {{").unwrap(); + writeln!(out, " if {{$bd}} {{").unwrap(); + writeln!( + out, + " vivado_cmd::set_property -dict $_dict -objects $handle" + ) + .unwrap(); + writeln!(out, " }} else {{").unwrap(); + writeln!( + out, + " vivado_cmd::set_property -dict $_dict -objects [get_ips $handle]" + ) + .unwrap(); + writeln!(out, " }}").unwrap(); + writeln!(out, "}}").unwrap(); +} + +// --------------------------------------------------------------------------- +// Indexed-family emission. +// --------------------------------------------------------------------------- + +/// Emit one arg-decl for a family constructor. Mirror of +/// [`emit_arg_decl`] with one addition: the `@enum(...)` list is +/// the **union** of enum choices across every provided member +/// (`per_member`), so the collapsed constructor accepts any value +/// any port takes. Defaults are shape-equal by construction (the +/// shape-match check requires it), so we use `shape_param`'s +/// default verbatim. +fn emit_arg_decl_family( + doc: &mut Doc, + component: &Component, + presets: &crate::presets::PresetMap, + shape_param: &Parameter, + per_member: &[&Parameter], + shape_member_label: &str, + opts: &GenerateOptions, +) { + if opts.include_descriptions { + if let Some(desc) = + shape_param.description.as_deref().filter(|s| !s.is_empty()) + { + for line in desc.lines() { + doc.push(Item::DocComment(line.trim_end().into())); + } + } + } + let mut words = Vec::new(); + // Union enum choices across every per-member Parameter that + // maps to this field. Preserve insertion order (IP-XACT first + // across members, presets after) so hover/completion shows a + // stable list. + let mut seen = std::collections::HashSet::new(); + let mut unioned: Vec = Vec::new(); + for p in per_member { + for v in enum_values_for(component, presets, p) { + if seen.insert(v.clone()) { + unioned.push(v); + } + } + } + if !unioned.is_empty() { + let formatted: Vec = + unioned.iter().map(|v| format_attribute_value(v)).collect(); + words.push(Word::Raw(format!("@enum({})", formatted.join(", ")))); + } + let default = shape_param.value.default_value(); + if !default.is_empty() { + words.push(Word::Raw(format!( + "@default({})", + format_attribute_value(default) + ))); + } + let lowered = + lowercase_ident(strip_prefix(&shape_param.name, shape_member_label)); + let typed_name = if is_properties_shaped_param(shape_param) { + format!("{lowered}: Properties") + } else { + lowered + }; + words.push(Word::Bare(typed_name)); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words, + body: None, + })); +} + +/// Emit the newtype declaration and its `from`/`to`/`repr`/`empty` +/// helper procs for one family. Called from inside a +/// `namespace eval { … }` block, so identifiers use their +/// TOP-LEVEL forms — this fn is called at file top level, NOT +/// inside `namespace eval { … }`. Reason: the analyzer +/// double-prefixes qualified proc names declared inside a matching +/// namespace-eval block (`proc ::T::from` inside +/// `namespace eval { … }` becomes `::::T::from`), so +/// external references never resolve. Keeping the newtype block at +/// top level with fully-qualified names avoids that. Legal thanks +/// to this slice's `validate::reject_nested_qualified` update. +fn emit_family_prelude(out: &mut String, ip_name: &str, f: &IndexedFamily<'_>) { + let stem_props = stem_props_name(ip_name, &f.stem); + let stem_lower = lowercase_ident(&f.stem); + writeln!( + out, + "## Typed configuration slot for one [{ip_name}::{stem_lower}] on \ + [{ip_name}::create].", + ) + .unwrap(); + // Tcl requires the target namespace to exist before qualified + // proc names like `::::from` are legal. Nested + // `namespace eval` calls establish the whole chain. The outer + // `namespace eval {}` shape is idempotent — the proc-block + // emitted below re-enters the same namespace. + writeln!(out, "namespace eval {ip_name} {{}}").unwrap(); + writeln!(out, "namespace eval {stem_props} {{}}").unwrap(); + writeln!(out, "type {stem_props} = Properties").unwrap(); + writeln!( + out, + "proc {stem_props}::repr {{ v: {stem_props} }} string \ + {{ return [Properties::repr -v [{stem_props}::to -v $v]] }}" + ) + .unwrap(); + writeln!( + out, + "proc {stem_props}::from {{ v: Properties }} {stem_props} \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {stem_props}::to {{ v: {stem_props} }} Properties \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {stem_props}::empty {{}} {stem_props} \ + {{ return [{stem_props}::from -v [Properties::empty]] }}" + ) + .unwrap(); +} + +/// Emit the family constructor `::` — a pure +/// value-builder that takes the same typed args the collapsed +/// per-N procs took, packs the supplied ones into a `Properties` +/// dict, and returns it wrapped in the newtype. NO Vivado side +/// effects; the atomic materialization happens later in +/// `::create`. +fn emit_family_constructor( + out: &mut String, + ip_name: &str, + component: &Component, + presets: &crate::presets::PresetMap, + opts: &GenerateOptions, + f: &IndexedFamily<'_>, +) { + // Emitted inside `namespace eval { … }` — use a bare + // proc name; Tcl resolves it to `::` at load + // time via the enclosing namespace. + let stem_lower = lowercase_ident(&f.stem); + let ret_ty = stem_props_name(ip_name, &f.stem); + + let mut doc = Doc::new(); + doc.push(Item::DocComment(format!( + "Configuration value for one of [{ip_name}::create]'s \ + `-{stem_lower}` slots. Composes into the top proc so \ + every provided slot's fields land in ONE atomic \ + `set_property -dict` call.", + ))); + doc.push(Item::Blank); + // Each field's `@enum(...)` values are the UNION of that + // field's per-member enum choices — the constructor has to + // accept any value any port can take. See ShapeSlot's docs + // for why choice_ref differences don't block the family. + for p in &f.shape { + let field_short = strip_prefix(&p.name, &f.shape_member_label); + let per_member: Vec<&Parameter> = f + .members_direct + .iter() + .zip(&f.member_labels) + .filter_map(|(direct, label)| { + direct + .iter() + .copied() + .find(|q| strip_prefix(&q.name, label) == field_short) + }) + .collect(); + emit_arg_decl_family( + &mut doc, + component, + presets, + p, + &per_member, + &f.shape_member_label, + opts, + ); + } + + // Body: build a dict from the supplied kwargs and wrap it. + let mut body = String::new(); + writeln!(body, "set _vw_d [dict create]").unwrap(); + for p in &f.shape { + let arg = lowercase_ident(strip_prefix(&p.name, &f.shape_member_label)); + // Post-strip key becomes the CONFIG._ suffix at + // top-proc merge time. Store un-prefixed field name here. + let field_key = strip_prefix(&p.name, &f.shape_member_label); + // Properties-typed sub-slots arrive as raw paired-list + // dicts; scalar sub-slots are plain strings. Both cases + // reduce to `$arg`. The old `Properties::to_raw` step + // required Property::Scalar/Nested tags that our new + // configure-based value flow no longer produces. + // + // Bool-typed sub-slots get the same marshaling as + // top-level bool args — the composed-family paired list + // ends up merged into the top proc's `_vw_d` via a + // `foreach`, so the canonical wire form (`true`/`false` + // for pure bool, `1`/`0` for pseudo bool per BoolKind) + // needs to be in place before that step. + let value_expr = if let Some(kind) = bool_kind(component, p) { + bool_value_expr(&arg, kind) + } else { + format!("${arg}") + }; + writeln!( + body, + "if {{${{__vw_kw_{arg}_set}}}} \ + {{ dict set _vw_d {field_key} {value_expr} }}" + ) + .unwrap(); + } + writeln!( + body, + "return [{ret_ty}::from -v [Properties::from -v $_vw_d]]" + ) + .unwrap(); + emit_proc(out, &stem_lower, &doc, Some(&ret_ty), &body); +} + +/// Reconstruct a family member's tree-node label from stem + index. +/// The `` naming (no separator) matches how +/// `tree::build_tree` produces `MAC_PORT0`, `MAC_PORT1`, …. +fn stem_index_label(stem: &str, idx: u32) -> String { + format!("{stem}{idx}") +} + +/// Fully-qualified newtype name from IP-name + uppercase-with- +/// underscores stem. `("dcmac", "MAC_PORT")` → `"dcmac::MacPortProps"`. +/// +/// Namespaced under the IP so `dcmac::` completion surfaces the +/// type alongside the constructor and top proc. Requires +/// vw-htcl's validator to accept qualified newtype names (landed +/// as part of this slice — see `validate::reject_nested_qualified`). +fn stem_props_name(ip_name: &str, stem: &str) -> String { + format!("{ip_name}::{}", stem_props_local(stem)) +} + +/// Local (unqualified) newtype segment — the PascalCase stem + +/// `Props`. Used inside `namespace eval { … }` where bare +/// names are preferred; the outer emission builds the qualified +/// form via [`stem_props_name`]. +fn stem_props_local(stem: &str) -> String { + let mut out = String::new(); + for seg in stem.split('_').filter(|s| !s.is_empty()) { + out.push_str(&pascal_case(seg)); + } + out.push_str("Props"); + out +} + +/// Fully-qualified newtype name for a dict-schema parameter's +/// composed value. `("versal_cips", "PS_PMC_CONFIG")` → +/// `"versal_cips::PsPmcConfig"`. Same compositional-value pattern +/// as the family-based [`stem_props_name`], applied to Xilinx's +/// `structured_tcldict` parameter surface so top-proc kwargs like +/// `-ps_pmc_config` get a typed newtype instead of raw Properties. +/// Fully-qualified newtype name for a dict-schema sub-proc. +/// +/// `("gtwiz_versal", ["intf", "gt_settings"], "LR0_SETTINGS")` → +/// `"gtwiz_versal::intf::gt_settings::Lr0Settings"`. The IP name +/// keeps its snake_case; namespace-prefix segments keep their +/// lowercase form; the leaf param name becomes PascalCase. +fn dict_props_name( + ip_name: &str, + namespace_prefix: &[&str], + param_name: &str, +) -> String { + let scope = proc_scope(ip_name, namespace_prefix); + format!("{scope}::{}", dict_props_local(param_name)) +} + +/// Local (unqualified) form of the dict-schema newtype name — +/// PascalCase of the param name. +fn dict_props_local(param_name: &str) -> String { + let mut out = String::new(); + for seg in param_name.split('_').filter(|s| !s.is_empty()) { + out.push_str(&pascal_case(seg)); + } + out +} + +/// Build the `param_name → qualified_newtype_name` map that +/// [`write_set_property_dict`] consults to inject the `::to` +/// unwrap step. Empty when the IP has no dict schemas. +fn build_dict_schema_newtypes( + ip_name: &str, + dict_schemas: &std::collections::HashMap, +) -> std::collections::HashMap { + dict_schemas + .keys() + // Top-level dict-schema newtype lookup for the top-proc + // arg-decl (`-ps_pmc_config: versal_cips::PsPmcConfig`). + // Nested newtypes referenced by sub-schemas aren't part of + // this map — they're referenced by the sub-procs directly. + .map(|k| (k.clone(), dict_props_name(ip_name, &[], k))) + .collect() +} + +/// Convert an identifier segment to PascalCase — first char upper, +/// rest lower. Non-ASCII passes through unchanged. +fn pascal_case(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + if let Some(c) = chars.next() { + out.push(c.to_ascii_uppercase()); + for c in chars { + out.push(c.to_ascii_lowercase()); + } + } + out +} + +/// Bundle of names needed to emit one family's per-index merge +/// block inside [`write_set_property_dict`]. Built at the top-proc +/// call site from an [`IndexedFamily`] + the IP's namespace. +struct FamilyMerge<'a> { + /// Uppercase stem (`"MAC_PORT"`). + stem: String, + /// Lowercase-ident stem for arg names (`"mac_port"`). + stem_lower: String, + /// Present indices (`[0, 1, 2, 3, 4, 5]`). + indices: Vec, + /// Fully-qualified newtype path for the `::to` unwrap + /// (`"dcmac::MacPortProps"`). + newtype_qualified: String, + #[allow(dead_code)] + marker: std::marker::PhantomData<&'a ()>, +} + +#[allow(clippy::too_many_arguments)] +/// Normalize an IP-XACT `spirit:format="bool"` default value into +/// the canonical HTCL literal `true` / `false`. XML defaults for +/// bool params come in a mix of shapes: quick-xml emits `true` / +/// `false` for boolean-typed values, but plenty of IPs still +/// stamp `1` / `0` even when `spirit:format="bool"` is declared, +/// and empty defaults mean "the IP has an internal default; if +/// the caller doesn't override, the wrapper's guard block skips +/// the assignment entirely." +/// +/// The `false` fallback for empty and unrecognized values is safe +/// because it flows into `@default(false)` — the runtime +/// `__vw_kw__set` guard only merges the value into the +/// property dict when the caller passed the flag explicitly, so +/// the default is inert unless referenced. +fn normalize_bool_default(default: &str) -> &'static str { + match default.trim() { + "1" | "true" | "TRUE" | "True" => "true", + // Empty defaults / unrecognized shapes get the safer + // `false` fallback — the `__vw_kw__set` guard makes + // this inert unless the caller sets the flag explicitly. + _ => "false", + } +} + +/// Which lexical form a bool-typed parameter takes on the wire to +/// Vivado's `set_property`. HTCL surface is the same either way +/// (bare `true` / `false`); the wrapper's marshaling picks the +/// lexical based on how the IP-XACT declares the underlying +/// parameter. +/// +/// See `bool_kind` for the classification rule. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BoolKind { + /// IP-XACT `spirit:format="bool"`. Vivado's parameter engine + /// normalizes this family and accepts the canonical `true` / + /// `false` string. Emit `[if $arg {list true} {list false}]`. + Pure, + /// Pseudo-boolean: IP-XACT `spirit:format="long"` bound to a + /// two-entry choice pair `(text="false" → 0, text="true" → + /// 1)`. Vivado STORES the numeric `0` / `1` — passing + /// `true` / `false` errors at `set_property` or fails later + /// at `validate_ip`. Emit `[if $arg {list 1} {list 0}]`. + /// + /// Extremely common on CoreGen-lineage IPs — cpm5's + /// component.xml alone has ~3,500 params of this shape. See + /// the audit in the pseudo-bool implementation planning. + Pseudo, +} + +/// Classify a parameter as bool-typed at the HTCL surface, and +/// return the wire form the wrapper should marshal to. `None` +/// when the parameter isn't bool-typed (a scalar string / int / +/// float / dict / enum / newtype-composed slot). +/// +/// The classification is deliberately schema-driven, not +/// heuristic: `Pure` requires the IP-XACT `spirit:format="bool"` +/// tag; `Pseudo` requires `format="long"` PLUS a `choiceRef` +/// resolving to a choice with EXACTLY two enumerations valued +/// `0` and `1` with labels `false` / `true` (case-insensitive). +/// The 8-choice audit of cpm5 confirmed this signature has no +/// false positives — string enums like `{REFCLK0=0, REFCLK1=1}` +/// or size enums like `{16KB=0, 32KB=1}` all miss the label +/// match and correctly stay untyped. +fn bool_kind(component: &Component, p: &Parameter) -> Option { + if p.is_bool() { + return Some(BoolKind::Pure); + } + if param_is_pseudo_bool(component, p) { + return Some(BoolKind::Pseudo); + } + None +} + +/// Detect pseudo-boolean parameters: `spirit:format="long"` with +/// a `choiceRef` pointing at a `{(text="false" → 0), (text="true" +/// → 1)}` choice. See `BoolKind::Pseudo` for the motivation. +fn param_is_pseudo_bool(component: &Component, p: &Parameter) -> bool { + if p.value.format.as_deref() != Some("long") { + return false; + } + let Some(choice_ref) = p.value.choice_ref.as_deref() else { + return false; + }; + let Some(choice) = component.find_choice(choice_ref) else { + return false; + }; + if choice.enumerations.len() != 2 { + return false; + } + let mut saw_true = false; + let mut saw_false = false; + for e in &choice.enumerations { + let label = e.label.as_deref().unwrap_or("").trim(); + match (label.to_ascii_lowercase().as_str(), e.value.trim()) { + ("true", "1") => saw_true = true, + ("false", "0") => saw_false = true, + _ => return false, + } + } + saw_true && saw_false +} + +/// The Tcl `value_expr` fragment for a bool-typed param's +/// marshaling site. Callers write it into their +/// `if {${__vw_kw__set}} { … }` template. +/// +/// `if`-form rather than `expr`-form so `$arg` appears as a +/// whole-word VarRef the analyzer's usage walker can see — +/// otherwise every bool arg produces a spurious "unused proc +/// arg" warning. +fn bool_value_expr(arg: &str, kind: BoolKind) -> String { + let (t, f) = match kind { + BoolKind::Pure => ("true", "false"), + BoolKind::Pseudo => ("1", "0"), + }; + format!("[if ${arg} {{list {t}}} {{list {f}}}]") +} + +// The arg count is intentionally high — this fn threads every +// input the analyzer needs to type-annotate and default a single +// proc arg (doc, component, presets, param, opts, prefix strip, +// ip name, dict schema map, dict namespace). Bundling them into +// a struct would just push the same coupling into the caller. +#[allow(clippy::too_many_arguments)] +fn emit_arg_decl( + doc: &mut Doc, + component: &Component, + presets: &crate::presets::PresetMap, + p: &Parameter, + opts: &GenerateOptions, + prefix_to_strip: &str, + ip_name: &str, + dict_schemas: &std::collections::HashMap, + // Namespace prefix under which any dict-schema newtype for `p` + // has been (or will be) emitted. Empty for top-proc args (the + // classic PS_PMC_CONFIG rail); non-empty for split-node procs + // that emit their dict-schema sub-procs under + // `::::`. Ignored when the param isn't + // dict-schema-backed. + dict_ns: &[&str], +) { + if opts.include_descriptions { + if let Some(desc) = p.description.as_deref().filter(|s| !s.is_empty()) { + for line in desc.lines() { + doc.push(Item::DocComment(line.trim_end().into())); + } + } + } + // Dict-schema-backed params get their composed newtype form + // instead of raw Properties. The IP-XACT default (a paired-list + // string) is unrepresentable as a newtype value, so we omit the + // `@default(...)` — the top-proc body's `__vw_kw__set` + // guard skips unset args, and callers explicitly compose the + // slot via the constructor when they want to override. + // + // Dict-schema map keys: split-node schemas are keyed by the + // STRIPPED slot name (`CHANNEL_MAP`) so the emitted newtype is + // `::intf0::ChannelMap` not `::intf0::Intf0ChannelMap`. + // Top-proc schemas use the raw param name (`PS_PMC_CONFIG`) as + // key with an empty prefix_to_strip → strip is a no-op. + let schema_key = strip_prefix(&p.name, prefix_to_strip); + let dict_schema = dict_schemas.get(schema_key); + if let Some(_schema) = dict_schema { + let ctor_scope = proc_scope(ip_name, dict_ns); + let ctor = format!("{ctor_scope}::{}", schema_key.to_ascii_lowercase()); + doc.push(Item::DocComment(format!("Composed via [{ctor}]."))); + } + let mut words = Vec::new(); + if dict_schema.is_none() { + // Bool-typed params (pure IP-XACT `format="bool"` OR + // pseudo-bool `format="long"` + `{false=0,true=1}` + // choice) get the HTCL `bool` type. Skip `@enum` + // emission — bool accepts exactly `true` / `false` and + // the analyzer's literal check enforces it structurally; + // an `@enum(true, false)` annotation would be redundant. + // Normalize the XML default to the canonical `true` / + // `false` lexical form so `@default(...)` matches the + // HTCL surface (the wire-form marshaling to `1`/`0` for + // pseudo-bool happens later at the set_property emit + // site). + if bool_kind(component, p).is_some() { + let default = normalize_bool_default(p.value.default_value()); + words.push(Word::Raw(format!("@default({default})"))); + } else { + let enum_values = enum_values_for(component, presets, p); + if !enum_values.is_empty() { + let formatted: Vec = enum_values + .iter() + .map(|v| format_attribute_value(v)) + .collect(); + words.push(Word::Raw(format!( + "@enum({})", + formatted.join(", ") + ))); + } + // Always emit `@default(...)` — a missing default would + // make the param required at the analyzer level, but every + // IP-XACT parameter is optional in Vivado's semantics (the + // IP uses its own internal default when the user doesn't + // override). Empty defaults happen when the XML's + // `` is whitespace-only (BOARD_PARAMETER, + // ANLT_PARAMETERS in gtwiz_versal, etc.) — quick-xml's + // `$text` strips leading/trailing whitespace, so we see + // `""`. Emit `@default("")` as the placeholder and let the + // runtime `__vw_kw__set` guard skip the merge loop + // when the caller didn't pass a value — same pattern as + // the family / split-node kwarg placeholders. + let default = p.value.default_value(); + if !default.is_empty() { + words.push(Word::Raw(format!( + "@default({})", + format_attribute_value(default) + ))); + } else { + words.push(Word::Raw("@default(\"\")".into())); + } + } + } else { + // Dict-schema-backed param: empty-string placeholder + // default. The runtime guard (`__vw_kw__set`) + // prevents the placeholder from ever being dereferenced + // through the newtype machinery. + words.push(Word::Raw("@default(\"\")".into())); + } + let lowered = lowercase_ident(strip_prefix(&p.name, prefix_to_strip)); + // Typing rules for the param: + // - Dict-schema-backed: use its typed newtype + // (`versal_cips::PsPmcConfig`) so `dcmac::` / `versal_cips::` + // completion surfaces the type alongside the constructor, + // and misuse (passing e.g. `CpmConfig` to `-ps_pmc_config`) + // is a compile-time error. + // - Otherwise Properties-shaped (paired-dict default without a + // registered schema): keep raw `Properties` — callers still + // compose these by hand, and the top-proc body unwraps via + // `Properties::to_raw` at the `set_property -dict` boundary + // (see [`write_set_property_dict`]). + // - Plain scalar: no type annotation. + let typed_name = if dict_schema.is_some() { + // Dict-schema slot — reference the newtype at whatever + // namespace it was emitted under. Top-proc args pass + // `dict_ns=&[]`; split-node args pass `dict_ns=&[node_local]`. + // Use the stripped key so the PascalCase newtype name isn't + // prefixed with the split-node's label (see schema_key). + format!( + "{lowered}: {}", + dict_props_name(ip_name, dict_ns, schema_key) + ) + } else if is_properties_shaped_param(p) { + format!("{lowered}: Properties") + } else if bool_kind(component, p).is_some() { + // HTCL `bool` primitive — same surface for pure + // (`format="bool"`) and pseudo (`format="long"` + + // false/true choice) bools. The analyzer's bool literal + // check gates values through this slot; wire-form + // marshaling picks true/false vs 1/0 at the + // `lappend _vw_d` site based on the same `bool_kind`. + format!("{lowered}: bool") + } else { + lowered + }; + words.push(Word::Bare(typed_name)); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words, + body: None, + })); +} + +/// Union of the parameter's IP-XACT `` values and any +/// presets, in insertion order. Order is *IP-XACT first* (preserving +/// the vendor's intended ordering when both sources agree) followed +/// by preset-only values; duplicates are filtered. +fn enum_values_for( + component: &Component, + presets: &crate::presets::PresetMap, + p: &Parameter, +) -> Vec { + let mut seen = std::collections::HashSet::new(); + let mut out: Vec = Vec::new(); + if let Some(choice) = p + .value + .choice_ref + .as_deref() + .and_then(|name| component.find_choice(name)) + { + for e in &choice.enumerations { + if seen.insert(e.value.clone()) { + out.push(e.value.clone()); + } + } + } + if let Some(extra) = presets.get(&p.name) { + for v in extra { + if seen.insert(v.clone()) { + out.push(v.clone()); + } + } + } + out +} + +/// True when the default value parses as a paired-dict Tcl-list +/// shape — i.e. has an even number of whitespace-separated tokens +/// (≥ 2) with identifier-shaped keys at every even index. Used by +/// [`emit_arg_decl`] / [`write_set_property_dict`] to decide which +/// wrapper args get the typed `Properties` annotation + automatic +/// `Properties::to_raw` unwrap at the extern boundary. +/// +/// Vivado IP-XACT defaults for CONFIG.* dict slots typically look +/// like `"KEY1 VAL1 KEY2 VAL2"` (e.g. `CPM_PCIE0_MODES None`, +/// `SMON_ALARMS Set_Alarms_On SMON_ENABLE_TEMP_AVERAGING 0`), +/// while scalar params look like `Custom` or `0` or +/// `versal_cips_v3_4`. Two-pair-shaped strings whose keys happen +/// to be bare-identifier-shaped slip through as Properties even +/// when the IP author meant them as a scalar — unlikely enough +/// to be acceptable noise; the wrapper still works when the +/// caller passes a string-shaped raw value (it round-trips through +/// `Properties::to_raw` returning the same paired list). +/// Parameter-level Properties-shape decision. Prefers Xilinx's +/// authoritative vendor tag (`tcldict` +/// on ``) when present, then falls back to the +/// structural default-value heuristic below. +/// +/// Xilinx writes the default of a tcldict param as the bare +/// sentinel `0` (see `INTF0_GT_SETTINGS`, `INTF0_GT_INTERNAL`, +/// `INTF1_*` in the `gtwiz_versal_v1_0` component.xml), which the +/// structural heuristic reads as scalar and gets wrong. The +/// vendor tag captures the array-indexed compound-property +/// intent — Vivado's Tcl uses `CONFIG.INTF0_GT_SETTINGS(LR0_SETTINGS) +/// {…}` on those params — so params carrying it must be exposed +/// as Properties-typed args regardless of their default shape. +fn is_properties_shaped_param(p: &Parameter) -> bool { + p.has_parameter_type("tcldict") + || is_properties_shaped(p.value.default_value()) +} + +/// Look up the sibling `` whose name matches +/// `param_name` and return its default. Vivado sometimes hides the +/// paired-list schema for a user-facing scalar param in an +/// internally-flagged model parameter with the same name — the +/// `INTF_PARENT_PIN_LIST` case (top-level default `0`, model-param +/// default `QUAD0_RX0 undef QUAD0_RX1 undef …`). `None` when no +/// matching model param exists or when the two names diverge. +fn model_param_anchor_default( + component: &Component, + param_name: &str, +) -> Option { + component + .model_parameters() + .find(|mp| mp.name == param_name) + .map(|mp| mp.value.default_value().to_string()) +} + +/// Post-process an anchor-derived schema: if its keys follow the +/// `QUAD_(RX|TX)` pattern (Vivado's per-quad-per-channel +/// pin-map convention), replicate the slot set across every quad +/// the IP declares and attach an `@enum(…)` value list of valid +/// pin paths. +/// +/// **Key extrapolation.** Model-param anchors typically only +/// enumerate ONE quad's worth of slots as a template. Detect the +/// `QUAD_` prefix and, for each other quad `q` up to the IP's +/// max, clone the `QUAD0_*` slots as `QUAD_*`. The max is +/// derived from the parameter tree — every top-level param whose +/// name starts `QUAD_` counts, so gtwiz-versal's 5-quad layout +/// materializes 40 slots (5 × 8) from the 8-slot QUAD0 template. +/// +/// **Value enum.** For each slot whose stripped name matches +/// `RX` or `TX`, build the enum +/// `[undef, /INTF0__GT_IP_Interface_0, /INTF1__…, +/// … /INTF__…]`. `N` = number of interface split +/// nodes on the IP (8 for gtwiz-versal, detected by scanning the +/// parameter set for the `INTF_` prefix). `n` ranges over the +/// channel count implied by the source slot name — same convention +/// Vivado's BD builder uses when materializing GT interface pins. +fn extrapolate_quad_schema( + schema: &mut crate::DictSchema, + component: &Component, + shape_path: &str, + opts: &GenerateOptions, +) { + // Detect the quad-prefix pattern on any existing field. + let has_quad_pattern = schema + .fields + .iter() + .any(|f| parse_quad_slot(&f.name).is_some()); + if !has_quad_pattern { + return; + } + let max_quads = count_indexed_prefix(component, "QUAD"); + let max_intfs = count_indexed_prefix(component, "INTF"); + // Number of RX/TX channels per interface. Look at any INTF_RXn + // or INTF_TXn model params — same shape as the QUAD channels. + // Fall back to 4 (Xilinx's fixed per-interface channel count on + // Versal GT wizards) if we can't count. + let n_channels = detect_channel_count(schema).max(1); + + // Build a fresh field list: one per (quad, orig_local_slot) + // pair, ordered quad-major then original-order. + let template: Vec = std::mem::take(&mut schema.fields); + let mut fields = Vec::with_capacity(template.len() * max_quads.max(1)); + for q in 0..max_quads.max(1) { + for f in &template { + let Some((_orig_q, local)) = parse_quad_slot(&f.name) else { + // Non-QUAD-shaped keys stay as-is on the first quad + // iteration only, so we don't duplicate them. + if q == 0 { + fields.push(f.clone()); + } + continue; + }; + let name = format!("QUAD{q}_{local}"); + let field_lookup = lowercase_ident(&name); + let field_override = + opts.overrides.field(shape_path, &field_lookup); + let mut enum_values = f.enum_values.clone(); + // Auto-derived pin-path enum. Only overwrite when the + // slot name has a recognizable direction — leaves + // non-RX/TX slots (uncommon on this pattern but + // possible) alone. + if let Some(dir_ch) = parse_dir_channel(&local) { + let derived = + derive_pin_enum(dir_ch.0, dir_ch.1, max_intfs, n_channels); + if !derived.is_empty() { + enum_values = derived; + } + } + let mut default = f.default.clone(); + if let Some(fo) = field_override { + if let Some(d) = &fo.default { + default = d.clone(); + } + if let Some(ev) = &fo.enum_values { + enum_values = ev.iter().cloned().collect(); + } + } + fields.push(crate::DictField { + name, + default, + description: f.description.clone(), + enum_values, + }); + } + } + schema.fields = fields; +} + +/// Split a `QUAD_` slot name into `(n, REST)`. Returns +/// `None` when the prefix doesn't match. +fn parse_quad_slot(name: &str) -> Option<(usize, String)> { + let rest = name.strip_prefix("QUAD")?; + let digit_end = rest.find(|c: char| !c.is_ascii_digit())?; + let n: usize = rest[..digit_end].parse().ok()?; + let after = rest[digit_end..].strip_prefix('_')?; + Some((n, after.to_string())) +} + +/// Split an `RX` / `TX` slot local name into +/// `(direction, m)`. Direction is `"RX"` or `"TX"`. +fn parse_dir_channel(local: &str) -> Option<(&'static str, usize)> { + let (dir, rest) = if let Some(r) = local.strip_prefix("RX") { + ("RX", r) + } else if let Some(r) = local.strip_prefix("TX") { + ("TX", r) + } else { + return None; + }; + // Channel index has to be a run of digits terminating the local + // name (`RX0`, `TX3`). Anything else disqualifies. + if rest.is_empty() || !rest.chars().all(|c| c.is_ascii_digit()) { + return None; + } + let m: usize = rest.parse().ok()?; + Some((dir, m)) +} + +/// Count how many top-level params share the `_` shape, +/// which gives us the max index for that family. Used to figure +/// out the IP's max quad count and interface count without +/// hard-coding. +fn count_indexed_prefix(component: &Component, prefix: &str) -> usize { + let mut indices = std::collections::BTreeSet::new(); + for p in component.component_parameters() { + let Some(rest) = p.name.strip_prefix(prefix) else { + continue; + }; + let Some(digit_end) = rest.find(|c: char| !c.is_ascii_digit()) else { + continue; + }; + if rest[digit_end..].starts_with('_') { + if let Ok(n) = rest[..digit_end].parse::() { + indices.insert(n); + } + } + } + indices.iter().max().map(|n| n + 1).unwrap_or(0) +} + +/// Peek at an anchor's fields to figure out how many RX/TX +/// channels the shape carries. `QUAD0_RX0..RX3` → 4. Returns 0 +/// when no direction-shaped slots are present. +fn detect_channel_count(schema: &crate::DictSchema) -> usize { + let mut max_ch: usize = 0; + for f in &schema.fields { + let Some((_q, local)) = parse_quad_slot(&f.name) else { + continue; + }; + if let Some((_dir, ch)) = parse_dir_channel(&local) { + max_ch = max_ch.max(ch + 1); + } + } + max_ch +} + +/// Build the `@enum(…)` value list for a `QUAD_` slot. +/// Enum contents: the `undef` sentinel + every valid interface +/// pin path (`/INTF__GT_IP_Interface_0`) for `i` in +/// `0..n_intfs` and `m` in `0..n_channels`. Empty when either +/// axis is zero (defensive — caller falls back to whatever the +/// XML default declares). +fn derive_pin_enum( + dir: &'static str, + _slot_channel: usize, + n_intfs: usize, + n_channels: usize, +) -> std::collections::BTreeSet { + let mut out = std::collections::BTreeSet::new(); + if n_intfs == 0 || n_channels == 0 { + return out; + } + out.insert("undef".to_string()); + for i in 0..n_intfs { + for m in 0..n_channels { + out.insert(format!("/INTF{i}_{dir}{m}_GT_IP_Interface_0")); + } + } + out +} + +fn is_properties_shaped(default: &str) -> bool { + // Fast path via the Tcl-aware paired-list tokenizer — catches + // defaults with `{…}`-grouped values (INTF*_TXRX_OPTIONAL_PORTS + // and other tcldict params carry braces in their inner + // `INTF_LR_SETTINGS {LR0_SETTINGS {…}}` payload). The naive + // `split_whitespace` heuristic below tokenizes those braces as + // separate items and misclassifies the default as non-paired. + if !crate::paired_list::parse_paired_list(default).is_empty() { + return true; + } + let tokens: Vec<&str> = default.split_whitespace().collect(); + if tokens.len() < 2 || !tokens.len().is_multiple_of(2) { + return false; + } + for (i, t) in tokens.iter().enumerate() { + if i % 2 != 0 { + continue; + } + let mut chars = t.chars(); + let first = match chars.next() { + Some(c) => c, + None => return false, + }; + if !first.is_ascii_alphabetic() && first != '_' { + return false; + } + for c in chars { + if !(c.is_ascii_alphanumeric() || c == '_' || c == '.') { + return false; + } + } + } + true +} + +/// Lowercase an IP-XACT parameter name into a valid htcl argument +/// name. The htcl proc-arg grammar is `/[a-zA-Z_][a-zA-Z0-9_]*/`, so +/// an empty result or a digit-leading result (which prefix-stripping +/// can produce — e.g. `64BIT` after stripping `CPM_PCIE1_PF0_BAR0_`) +/// gets a leading underscore to land back in the grammar. +fn lowercase_ident(name: &str) -> String { + let mut out = String::with_capacity(name.len() + 1); + for c in name.chars() { + if c.is_ascii_alphanumeric() || c == '_' { + out.push(c.to_ascii_lowercase()); + } else { + out.push('_'); + } + } + let needs_leading_underscore = out + .as_bytes() + .first() + .map(|b| b.is_ascii_digit()) + .unwrap_or(true); + if needs_leading_underscore { + out.insert(0, '_'); + } + out +} + +/// Sanitize an arbitrary string for use as an htcl identifier. +fn sanitize_ident(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if c.is_ascii_alphanumeric() || c == '_' { + out.push(c); + } else { + out.push('_'); + } + } + out +} + +/// Render an IP-XACT default value as it should appear inside an +/// `@default(...)` attribute. The htcl proc-args grammar accepts only +/// three attribute-value forms — `integer_literal`, `attribute_value_ident` +/// (`[a-zA-Z_][a-zA-Z0-9_]*`), and double-quoted strings. Anything that +/// isn't a clean ident or integer is double-quoted (with `"` escaped). +fn format_attribute_value(s: &str) -> String { + if is_integer_literal(s) || is_attribute_ident(s) { + s.to_string() + } else { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } +} + +fn is_integer_literal(s: &str) -> bool { + let body = s.strip_prefix('-').unwrap_or(s); + !body.is_empty() && body.bytes().all(|b| b.is_ascii_digit()) +} + +fn is_attribute_ident(s: &str) -> bool { + let mut chars = s.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !(first.is_ascii_alphabetic() || first == '_') { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +#[cfg(test)] +mod tests { + use super::*; + use ipxact::{ + Choice, Choices, Component, Enumeration, ParamValue, Parameter, + Parameters, + }; + + fn mk_component() -> Component { + Component { + vendor: "acme".into(), + library: "ip".into(), + name: "demo".into(), + version: "1.0".into(), + description: Some("A demo IP.".into()), + parameters: Some(Parameters { + entries: vec![ + Parameter { + name: "BUS_WIDTH".into(), + description: Some("Bus width in bits.".into()), + value: ParamValue { + text: "32".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }, + Parameter { + name: "MODE".into(), + value: ParamValue { + text: "FAST".into(), + resolve: Some("user".into()), + choice_ref: Some("mode_choices".into()), + ..Default::default() + }, + ..Default::default() + }, + ], + }), + choices: Some(Choices { + entries: vec![Choice { + name: "mode_choices".into(), + enumerations: vec![ + Enumeration { + value: "FAST".into(), + ..Default::default() + }, + Enumeration { + value: "SLOW".into(), + ..Default::default() + }, + ], + }], + }), + ..Default::default() + } + } + + fn mk_split_component(n_per_group: usize) -> Component { + // Build a component with two big groups and a smattering of + // small ones, all above the split threshold. + let mut entries = Vec::new(); + for i in 0..n_per_group { + entries.push(Parameter { + name: format!("BIG_ONE_FIELD{i}"), + value: ParamValue { + text: "0".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }); + entries.push(Parameter { + name: format!("BIG_TWO_FIELD{i}"), + value: ParamValue { + text: "1".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }); + } + // A pair of tiny groups that should be collapsed into _misc. + for name in ["TINY_A_ONE", "TINY_B_ONE", "TINY_C_ONE", "STRAY_THING"] { + entries.push(Parameter { + name: name.into(), + value: ParamValue { + text: "x".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }); + } + Component { + vendor: "acme".into(), + library: "ip".into(), + name: "wide".into(), + version: "1.0".into(), + parameters: Some(Parameters { entries }), + ..Default::default() + } + } + + #[test] + fn single_mode_below_threshold() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + // Procs live inside `namespace eval { … }` and get + // the 2-space indent from `write_namespace_block`. Two + // procs: `configure` (typed value constructor) and + // `create` (cell instantiator + config applier). + let n_procs = out.matches("\n proc ").count(); + assert_eq!(n_procs, 2, "{out}"); + assert!(out.contains("proc configure")); + assert!(out.contains("proc create")); + assert!(out.contains("namespace eval demo {")); + // Top-level Config newtype ships outside the namespace + // block (see emit_config_prelude for why). + assert!(out.contains("type demo::Config = Properties")); + } + + #[test] + fn split_mode_emits_top_and_sub_procs() { + let component = mk_split_component(60); // 60 * 2 + 4 = 124 params > 100 + let out = generate( + &component, + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + eprintln!("--- generated ---\n{out}\n--- end ---"); + assert!(out.contains("proc create")); + // Split-node procs use fully-qualified names now — they live + // outside the namespace-block so they can be extracted into + // sibling `.htcl` files by `split_into_files`. + assert!(out.contains("proc wide::big_one")); + assert!(out.contains("proc wide::big_two")); + let parsed = vw_htcl::parse(&out); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + let diags = vw_htcl::validate(&parsed.document, &out); + let errors: Vec<_> = diags + .iter() + .filter(|d| d.severity == vw_htcl::Severity::Error) + // The generator emits calls into vivado-cmd + // (`ip::check`, `create_bd_cell`, `set_property`); + // those resolve when the wrapper is sourced through + // the loader, but this unit test runs the validator + // on the bare generated text. The unknown-call + // diagnostic is *expected* in that mode; we filter it + // out so the test catches real structural breakage. + .filter(|d| !d.message.starts_with("undefined proc")) + .collect(); + assert!(errors.is_empty(), "{errors:#?}"); + } + + #[test] + fn split_sub_procs_take_cell_as_first_arg() { + let component = mk_split_component(60); + let out = generate( + &component, + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + // Sub-procs are pure value-constructors — no `cell:` + // first arg, no bd switch — and return the node's typed + // newtype instead of `bd_cell`. Assert the shape. + // Fully-qualified name shape after the split-file refactor. + assert!( + out.contains("proc wide::big_one {\n ## Configuration value"), + "{out}" + ); + assert!(out.contains("} wide::BigOneProps {"), "{out}"); + // The old `cell: bd_cell` mutator arg must NOT appear. + assert!(!out.contains("cell: bd_cell"), "{out}"); + } + + #[test] + fn tiny_groups_land_on_the_parent_proc() { + let component = mk_split_component(60); + let out = generate( + &component, + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + // None of the tiny prefix groups becomes its own proc... + for name in ["proc tiny_a ", "proc tiny_b ", "proc stray "] { + assert!(!out.contains(name), "unexpected {name} in:\n{out}"); + } + // ...and the params instead appear as args on the top + // proc's configure counterpart. Post-refactor `create` has + // just `-name`/`-bd`/`-config`; the full documented kwarg + // pile lives on `configure`. Slice the configure range so + // we can search its args without accidentally matching arg + // names embedded in one of the value-constructor sub-procs' + // bodies. + let cfg_start = + out.find("proc configure {").expect("no proc configure"); + let cfg_body = &out[cfg_start..]; + let cfg_end = cfg_body + .find("\n }\n") + .map(|e| e + 5) + .unwrap_or(cfg_body.len()); + let cfg_range = &cfg_body[..cfg_end]; + for arg in ["tiny_a_one", "tiny_b_one", "tiny_c_one", "stray_thing"] { + assert!( + cfg_range.contains(arg), + "{arg} missing from configure proc: {cfg_range}" + ); + } + } + + #[test] + fn arg_name_strips_node_prefix() { + // Two big groups whose internal arg names should be the + // segments *after* the group prefix, not the full name. + let entries = (0..10) + .flat_map(|i| { + [ + Parameter { + name: format!("GROUP_A_FIELD{i}"), + value: ParamValue { + text: "0".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }, + Parameter { + name: format!("GROUP_B_FIELD{i}"), + value: ParamValue { + text: "0".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }, + ] + }) + .collect(); + let component = Component { + vendor: "acme".into(), + library: "ip".into(), + name: "demo".into(), + version: "1.0".into(), + parameters: Some(Parameters { entries }), + ..Default::default() + }; + let opts = GenerateOptions { + split_threshold: 5, + ..GenerateOptions::default() + }; + let out = generate( + &component, + &Default::default(), + &::std::collections::HashMap::new(), + &opts, + ) + .into_single(); + // Inside the GROUP_A proc, the arg names should be `field0`, + // not `group_a_field0`. + assert!(out.contains("@default(0) field0\n"), "{out}"); + assert!(!out.contains("group_a_field0"), "{out}"); + // The constructor stores the index-stripped key in its + // Properties dict… + assert!(out.contains("dict set _vw_d FIELD0 $field0"), "{out}"); + // …and the top proc's merge loop prefixes with + // `CONFIG.GROUP_A_` when composing atomically. The literal + // format uses `$_vw_f` at runtime, so we assert on the + // prefix pattern. + assert!(out.contains("CONFIG.GROUP_A_$_vw_f"), "{out}"); + } + + #[test] + fn generated_output_parses_back() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + let parsed = vw_htcl::parse(&out); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors + ); + } + + #[test] + fn bd_switch_arg_toggles_construction() { + // Every generated wrapper carries the typed `-bd` arg + // (real `bool`, defaulting to `false` = project-IP mode), + // and the body branches on it: `-bd true` calls + // `create_bd_cell`, `-bd false` calls `create_ip`. Both + // paths produce a plain-string handle so the wrapper's + // declared return type (`string`) covers either branch. + for out in [ + generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(), + generate( + &mk_split_component(6), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions { + split_threshold: 5, + ..GenerateOptions::default() + }, + ) + .into_single(), + ] { + assert!(out.contains("@default(false) bd: bool"), "{out}"); + assert!(out.contains("if {$bd} {"), "{out}"); + assert!(out.contains("create_bd_cell"), "{out}"); + assert!(out.contains("create_ip -vlnv"), "{out}"); + let parsed = vw_htcl::parse(&out); + assert!( + parsed.errors.is_empty(), + "wrapped output should parse cleanly: {:?}", + parsed.errors + ); + } + } + + #[test] + fn includes_description_as_doc_comment() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + assert!(out.contains("## A demo IP."), "{out}"); + assert!(out.contains("## Bus width in bits."), "{out}"); + } + + #[test] + fn emits_default_and_enum_attributes() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + assert!(out.contains("@default(32) bus_width"), "{out}"); + assert!(out.contains("@enum(FAST, SLOW)"), "{out}"); + assert!(out.contains("@default(FAST) mode"), "{out}"); + } + + #[test] + fn emits_set_property_for_each_param() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + // Post-refactor these lappend lines live inside `configure`, + // not `create`. `create`'s body just unwraps `-config` and + // splats the resulting dict via `set_property -dict`. + assert!(out.contains("CONFIG.BUS_WIDTH $bus_width"), "{out}"); + assert!(out.contains("CONFIG.MODE $mode"), "{out}"); + } + + /// `configure`'s body is pure dict assembly + a Config wrap. + /// Any of the side-effecting Vivado calls appearing inside its + /// body would mean the seam wasn't cleanly cut. + #[test] + fn configure_returns_typed_config_no_side_effects() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + let cfg_start = out.find("proc configure {").expect("no configure"); + let cfg_body = &out[cfg_start..]; + let cfg_end = cfg_body + .find("\n }\n") + .map(|e| e + 5) + .unwrap_or(cfg_body.len()); + let cfg_range = &cfg_body[..cfg_end]; + for forbidden in ["create_bd_cell", "create_ip", "set_property"] { + assert!( + !cfg_range.contains(forbidden), + "configure body should not contain `{forbidden}`:\n{cfg_range}" + ); + } + // But it SHOULD wrap the assembled dict as a Config value. + assert!(cfg_range.contains("demo::Config::from"), "{cfg_range}"); + } + + /// `create`'s arg surface is exactly `-name`, `-bd`, `-config` + /// under the post-refactor design. No documented-kwarg pile. + #[test] + fn create_takes_only_name_bd_config() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + let create_start = out.find("proc create {").expect("no create"); + // Argspec ends at `} string {`. + let create_body = &out[create_start..]; + let argspec_end = create_body + .find("} string {") + .expect("create should return string"); + let argspec = &create_body[..argspec_end]; + for expected in ["name", "bd: bool", "config: demo::Config"] { + assert!( + argspec.contains(expected), + "expected `{expected}` in create argspec:\n{argspec}" + ); + } + // No leftover typed IP-param kwargs on create. + for forbidden in ["bus_width: int", "@enum(FAST, SLOW)"] { + assert!( + !argspec.contains(forbidden), + "IP-param kwarg `{forbidden}` leaked onto create:\n{argspec}" + ); + } + } + + /// `create`'s body unwraps `$config` through the Config newtype + /// and applies via `set_property -dict` in both `-bd 1` and + /// `-bd 0` branches. + #[test] + fn create_body_unwraps_config_and_applies() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + let create_start = out.find("proc create {").expect("no create"); + let create_range = &out[create_start..]; + // Unwrap step. + assert!( + create_range.contains("demo::Config::to -v $config"), + "create should unwrap $config via Config::to:\n{create_range}" + ); + // Guard for the empty-Config default (bracket-expr @default + // fallback pattern documented in build_single_create_body). + assert!( + create_range.contains("demo::Config::empty"), + "create should coerce empty-string default to Config::empty:\n{create_range}" + ); + // Both bd branches: `-bd true` targets `$handle` (the + // bd_cell path directly); `-bd false` fetches the IP + // object via `[get_ips $handle]`. + assert!( + create_range.contains("set_property -dict $_dict -objects $handle") + ); + assert!(create_range + .contains("set_property -dict $_dict -objects [get_ips $handle]")); + } + + // ------------------------------------------------------------------ + // is_properties_shaped_param — Xilinx vendor-tag routing. + // ------------------------------------------------------------------ + + use ipxact::{ParameterInfo, VendorExtensions}; + + fn mk_param(default: &str, tcldict: bool) -> Parameter { + Parameter { + name: "X".into(), + value: ParamValue { + text: default.into(), + ..Default::default() + }, + vendor_extensions: if tcldict { + Some(VendorExtensions { + xilinx_parameter_info: Some(ParameterInfo { + parameter_type: vec!["tcldict".into()], + }), + }) + } else { + None + }, + ..Default::default() + } + } + + #[test] + fn tcldict_tag_overrides_scalar_default() { + // The `INTF0_GT_SETTINGS` shape — vendor tag says + // `tcldict`, default is Xilinx's `0` sentinel that reads + // as scalar. The vendor tag must win. + let p = mk_param("0", true); + assert!(is_properties_shaped_param(&p)); + } + + #[test] + fn structural_shape_still_wins_without_tag() { + // The `INTF0_LR0_SETTINGS` shape — no vendor tag but the + // default's paired-list shape gives it away. + let p = mk_param("NA NA", false); + assert!(is_properties_shaped_param(&p)); + } + + #[test] + fn neither_tag_nor_shape_stays_scalar() { + // Plain scalar param. Was scalar before this change, + // stays scalar after. + let p = mk_param("0", false); + assert!(!is_properties_shaped_param(&p)); + } +} diff --git a/vw-ip/src/group.rs b/vw-ip/src/group.rs new file mode 100644 index 0000000..eaf3952 --- /dev/null +++ b/vw-ip/src/group.rs @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Derive parameter groups from naming conventions. +//! +//! IP-XACT components published by Xilinx carry no machine-readable +//! grouping for their configuration parameters. The xgui Tcl scripts +//! that drive the GUI grouping are encrypted, so they're not a source +//! we can use. What we *can* use is the strong prefix structure of the +//! parameter names themselves: in CPM5, 4200 parameters start with +//! `CPM_PCIE0_`, another 4200 with `CPM_PCIE1_`, 136 with `CPM_CCIX_`, +//! and so on. That structure is the right grain for a sub-proc. +//! +//! The grouping strategy: +//! +//! 1. Split each parameter name on `_`. +//! 2. Take the first N segments as the group key. We pick N to balance +//! group cardinality vs. group size — small enough that there are +//! few groups (so each becomes a manageable proc), big enough that +//! no single group is so huge it's just a flat dump. +//! 3. Parameters with no underscore, or whose only group would be a +//! singleton, fall into a catch-all `_misc` group. + +use std::collections::BTreeMap; + +use ipxact::Parameter; + +#[derive(Clone, Debug)] +pub struct ParameterGroup<'a> { + /// Key used as the group name (e.g. `CPM_PCIE0`). + pub key: String, + /// Parameters in this group, in input order. + pub parameters: Vec<&'a Parameter>, +} + +/// Group parameters by their leading underscore-separated segments. +/// `prefix_segments` controls how many leading segments form the key: +/// 1 = `CPM`, 2 = `CPM_PCIE0`, etc. 2 is the right default for Xilinx's +/// big IPs; their first segment is a coarse domain (`CPM`, `PS`, `PMC`) +/// and the second names the controller / subsystem. +pub fn group_parameters<'a, I>( + parameters: I, + prefix_segments: usize, +) -> Vec> +where + I: IntoIterator, +{ + // BTreeMap keeps groups in a stable, readable order. + let mut groups: BTreeMap> = BTreeMap::new(); + for p in parameters { + let key = prefix_key(&p.name, prefix_segments); + groups.entry(key).or_default().push(p); + } + groups + .into_iter() + .map(|(key, parameters)| ParameterGroup { key, parameters }) + .collect() +} + +/// First `n` underscore-separated segments of `name`. If `name` has +/// fewer than `n` segments (or no underscores), returns the whole name. +/// Empty names map to the literal `_misc`. +fn prefix_key(name: &str, n: usize) -> String { + if name.is_empty() { + return "_misc".into(); + } + let mut out = String::new(); + for (i, seg) in name.split('_').enumerate().take(n) { + if i > 0 { + out.push('_'); + } + out.push_str(seg); + } + // If the name has fewer than `n` segments, we end up with the full + // name as the key — that's fine; it just means the group is named + // after the parameter itself. Singletons coalesce later if we want. + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(name: &str) -> Parameter { + Parameter { + name: name.into(), + ..Default::default() + } + } + + #[test] + fn groups_by_two_prefix_segments() { + let params = [ + p("CPM_PCIE0_FOO"), + p("CPM_PCIE0_BAR"), + p("CPM_PCIE1_BAZ"), + p("CPM_CCIX_QUX"), + ]; + let groups = group_parameters(¶ms, 2); + let by_key: Vec<_> = groups + .iter() + .map(|g| (g.key.clone(), g.parameters.len())) + .collect(); + assert_eq!( + by_key, + vec![ + ("CPM_CCIX".to_string(), 1), + ("CPM_PCIE0".to_string(), 2), + ("CPM_PCIE1".to_string(), 1), + ] + ); + } + + #[test] + fn names_with_fewer_segments_become_their_own_key() { + let params = [p("FOO"), p("FOO_BAR_BAZ")]; + let groups = group_parameters(¶ms, 2); + let keys: Vec<_> = groups.iter().map(|g| g.key.as_str()).collect(); + // "FOO" stays "FOO" (only one segment), "FOO_BAR_BAZ" becomes "FOO_BAR". + assert!(keys.contains(&"FOO")); + assert!(keys.contains(&"FOO_BAR")); + } +} diff --git a/vw-ip/src/lib.rs b/vw-ip/src/lib.rs new file mode 100644 index 0000000..1438502 --- /dev/null +++ b/vw-ip/src/lib.rs @@ -0,0 +1,59 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! IP-XACT → htcl wrapper generation. +//! +//! Reads an IP-XACT component description (via the `ipxact` crate) and +//! emits an htcl instantiation proc for it — the "configuration +//! interface" layer described in the project plan: one top-level proc +//! per IP, with sub-procs for parameter groups when an IP's surface is +//! too large for a single proc to be tractable (CPM5 has ~8700 +//! parameters). +//! +//! Group recovery: IP-XACT itself carries no grouping metadata for +//! large Xilinx IPs (no `` etc., and the `xgui/*.tcl` +//! files that *do* carry the UI grouping are encrypted). Instead, we +//! derive groups from the convention Xilinx uses in parameter naming — +//! `CPM_PCIE0_*`, `CPM_PCIE1_*`, `PS_PMC_*` and so on are clear +//! prefix clusters. See [`group_parameters`]. + +pub mod cips_dict; +pub mod family; +pub mod generate; +pub mod group; +pub mod overrides; +pub mod paired_list; +pub mod presets; +pub mod summary; +pub mod targets; +pub mod tree; + +pub use cips_dict::{ + load_schemas as load_cips_dict_schemas, DictField, DictSchema, +}; +pub use family::{detect_families, DetectOptions, IndexedFamily}; +pub use generate::{generate, GenerateOptions}; +pub use group::{group_parameters, ParameterGroup}; +pub use presets::{ + discover_for as discover_presets, load_files as load_presets, PresetMap, +}; +pub use summary::Summary; +pub use tree::{build_tree, Node, TreeOptions}; + +use std::path::Path; + +use ipxact::Component; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("loading IP-XACT component: {0}")] + Ipxact(#[from] ipxact::Error), +} + +pub type Result = std::result::Result; + +/// Load an IP-XACT component from disk. +pub fn load(path: impl AsRef) -> Result { + Ok(Component::from_file(path)?) +} diff --git a/vw-ip/src/overrides.rs b/vw-ip/src/overrides.rs new file mode 100644 index 0000000..f33afc1 --- /dev/null +++ b/vw-ip/src/overrides.rs @@ -0,0 +1,234 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Per-IP TOML overrides for typed-constructor field emission. +//! +//! The generator's default source of field metadata is the IP-XACT +//! `` paired-list default — good for populating +//! `@default(...)` but silent on bounded vocabularies (Vivado wants +//! `RX_PAM_SEL` to be one of `NRZ` / `PAM4`, but the XML default is +//! just a bare string). An `overrides.toml` file colocated with each +//! IP's `regenerate.sh` refines the emitted surface where the XML +//! is silent: attach `@enum(…)` restrictions to specific fields, +//! override the XML default, etc. +//! +//! Discovery: the CLI accepts `--overrides ` and threads the +//! parsed [`OverridesFile`] through `GenerateOptions`. When no +//! override file exists, the generator falls back to XML-only +//! defaults (schema shape derived entirely from ``). +//! +//! File shape: +//! +//! ```toml +//! [shapes."intf::gt_settings::lr0_settings"] +//! fields.rx_pam_sel = { enum = ["NRZ", "PAM4"] } +//! fields.rx_refclk_source = { enum = ["R0", "R1", "R2", "R3", "R4", "R5", "ERR"] } +//! fields.rx_line_rate = { default = "10.3125" } +//! ``` +//! +//! `shape_path` uses `::`-separated segments matching the emitted +//! proc's namespace path *below* the IP name — i.e. how a caller +//! writes `gtwiz_versal::intf::gt_settings::lr0_settings` refers to +//! shape `"intf::gt_settings::lr0_settings"`. Per-N sub-newtypes +//! (`intf0` vs `intf1`) share their shape's overrides — the emitter +//! matches on the stem (`intf`), not the indexed instance. + +use std::collections::HashMap; +use std::path::Path; + +use serde::Deserialize; + +/// Errors produced by [`OverridesFile::load_from`]. +#[derive(Debug, thiserror::Error)] +pub enum OverridesError { + /// The path exists but couldn't be read (permissions, IO error). + #[error("reading overrides file {path}: {source}")] + Read { + path: String, + #[source] + source: std::io::Error, + }, + /// The file exists but doesn't parse as valid TOML matching the + /// override schema. + #[error("parsing overrides file {path}: {source}")] + Parse { + path: String, + #[source] + source: toml::de::Error, + }, +} + +/// The parsed overrides file. `shapes["intf::gt_settings::lr0_settings"]` +/// looks up refinements for one emitted proc's field list. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct OverridesFile { + #[serde(default)] + pub shapes: HashMap, +} + +impl OverridesFile { + /// Load an overrides file from `path`. Missing file is not an + /// error — returns an empty overrides object so callers can pass + /// the result through unconditionally. + pub fn load_from(path: impl AsRef) -> Result { + let path = path.as_ref(); + if !path.exists() { + return Ok(Self::default()); + } + let text = std::fs::read_to_string(path).map_err(|source| { + OverridesError::Read { + path: path.display().to_string(), + source, + } + })?; + toml::from_str(&text).map_err(|source| OverridesError::Parse { + path: path.display().to_string(), + source, + }) + } + + /// Look up a field's override in `shape_path`. `None` when the + /// shape isn't listed, or the shape is listed but the field + /// isn't. Callers merge with the XML-derived default when present. + /// + /// Field names are matched exactly as the emitter writes them + /// (lowercase form of the XML key — e.g. XML `RX_PAM_SEL` + /// becomes field key `rx_pam_sel`). + pub fn field( + &self, + shape_path: &str, + field_name: &str, + ) -> Option<&FieldOverride> { + self.shapes.get(shape_path)?.fields.get(field_name) + } + + /// True when this override set is empty — no shapes registered. + /// Callers use this to skip the whole override-application pass + /// when there's nothing to do (fast path for IPs without an + /// override file). + pub fn is_empty(&self) -> bool { + self.shapes.is_empty() + } +} + +/// Overrides scoped to one emitted proc / shape. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ShapeOverrides { + #[serde(default)] + pub fields: HashMap, +} + +/// Per-field refinement applied on top of the XML-derived schema. +/// +/// - `enum_values`: attach `@enum(v1, v2, …)` to the emitted arg, +/// restricting valid callsite values. Overrides absent → no +/// enum annotation. +/// - `default`: replace the XML default. Rare — the XML is usually +/// authoritative, but Xilinx sometimes ships defaults that are +/// sentinels rather than useful values (e.g. `NA NA`). Overrides +/// absent → use XML value. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct FieldOverride { + #[serde(default, rename = "enum")] + pub enum_values: Option>, + #[serde(default)] + pub default: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + use std::io::Write; + let mut f = tempfile::NamedTempFile::new().unwrap(); + f.write_all(contents.as_bytes()).unwrap(); + f + } + + #[test] + fn missing_file_yields_empty() { + let path = + std::path::Path::new("/nonexistent-vw-ip-test-overrides.toml"); + assert!(!path.exists()); + let ov = OverridesFile::load_from(path).unwrap(); + assert!(ov.is_empty()); + } + + #[test] + fn empty_file_yields_empty_shapes_map() { + let f = write_tmp(""); + let ov = OverridesFile::load_from(f.path()).unwrap(); + assert!(ov.is_empty()); + } + + #[test] + fn single_shape_single_field_enum() { + let f = write_tmp( + r#" +[shapes."intf::gt_settings::lr0_settings"] +fields.rx_pam_sel = { enum = ["NRZ", "PAM4"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + let field = ov + .field("intf::gt_settings::lr0_settings", "rx_pam_sel") + .expect("field present"); + assert_eq!( + field.enum_values.as_deref(), + Some(&["NRZ".to_string(), "PAM4".to_string()][..]) + ); + assert!(field.default.is_none()); + } + + #[test] + fn field_default_override() { + let f = write_tmp( + r#" +[shapes."intf::lr0_settings"] +fields.rx_line_rate = { default = "10.3125" } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + let field = ov + .field("intf::lr0_settings", "rx_line_rate") + .expect("field present"); + assert_eq!(field.default.as_deref(), Some("10.3125")); + assert!(field.enum_values.is_none()); + } + + #[test] + fn missing_shape_returns_none() { + let f = write_tmp( + r#"[shapes."intf::gt_settings"] +fields.dummy = { enum = ["A", "B"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + assert!(ov.field("intf::channel_map", "dummy").is_none()); + } + + #[test] + fn missing_field_within_present_shape_returns_none() { + let f = write_tmp( + r#"[shapes."intf::gt_settings"] +fields.dummy = { enum = ["A", "B"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + assert!(ov.field("intf::gt_settings", "not_dummy").is_none()); + } + + #[test] + fn malformed_toml_errors_with_path_context() { + let f = write_tmp("this is not [ valid toml"); + let err = OverridesFile::load_from(f.path()).unwrap_err(); + // The error message should mention the path so a user seeing + // it in `vw` output can find the file that needs fixing. + assert!( + err.to_string().contains(&f.path().display().to_string()), + "err: {err}" + ); + } +} diff --git a/vw-ip/src/paired_list.rs b/vw-ip/src/paired_list.rs new file mode 100644 index 0000000..8bb86e6 --- /dev/null +++ b/vw-ip/src/paired_list.rs @@ -0,0 +1,346 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Parse Xilinx IP-XACT paired-list default values into a nested tree. +//! +//! Many Xilinx IP-XACT `` defaults are Tcl-list-shaped +//! paired dicts — `KEY VAL KEY VAL …`, where a `VAL` can itself be a +//! braced Tcl list holding another paired dict. The gtwiz-versal +//! `INTF0_TXRX_OPTIONAL_PORTS` param is the canonical example: ~300 +//! flat fields at the top level, terminating in +//! `INTF_LR_SETTINGS {LR0_SETTINGS {~80 fields} LR1_SETTINGS { } … }`. +//! +//! That structure IS the field schema. The generator uses it to emit +//! typed constructor procs with named args — no more asking the caller +//! to hand-populate `Properties::from -v {…}` from memory. See +//! `cips_dict::DictSchema::from_paired_default` for the extractor that +//! turns a [`PairedValue`] tree into a `DictSchema`. +//! +//! Semantics match Tcl list tokenization: whitespace separates tokens; +//! `{…}` groups a token, stripping only the outermost braces; nested +//! `{…}` are preserved verbatim inside the outer token. Nothing else is +//! interpreted — `$var` / `[cmd]` substitution isn't touched, since +//! IP-XACT defaults arrive with literal text only. + +use std::fmt; + +/// A parsed value inside a paired-list dict. A `Scalar` is a single +/// bare token (or a braced token whose interior isn't itself a +/// paired list). A `Nested` value is a braced token whose interior +/// parses as another paired list — recursively. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PairedValue { + Scalar(String), + Nested(Vec<(String, PairedValue)>), +} + +impl PairedValue { + /// True when this value is a `Scalar`. Convenience for callers + /// that iterate paired lists and treat scalar / nested slots + /// differently. + pub fn is_scalar(&self) -> bool { + matches!(self, Self::Scalar(_)) + } + + /// The underlying string for a `Scalar`, or `None` for `Nested`. + pub fn as_scalar(&self) -> Option<&str> { + match self { + Self::Scalar(s) => Some(s), + _ => None, + } + } +} + +impl fmt::Display for PairedValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Scalar(s) => write!(f, "{s}"), + Self::Nested(pairs) => { + write!(f, "{{")?; + for (i, (k, v)) in pairs.iter().enumerate() { + if i > 0 { + write!(f, " ")?; + } + write!(f, "{k} {v}")?; + } + write!(f, "}}") + } + } + } +} + +/// Parse a Tcl-list-shaped paired-dict default string into pairs. +/// +/// Returns an empty list when the input doesn't tokenize as an even- +/// count sequence — that's the "not really a paired dict" signal +/// callers use to fall back to treating the whole default as scalar. +/// Empty input returns empty pairs (a valid empty dict). +pub fn parse_paired_list(input: &str) -> Vec<(String, PairedValue)> { + let cleaned = fix_wrapped_tokens(input); + let tokens = tokenize(&cleaned); + if tokens.is_empty() || !tokens.len().is_multiple_of(2) { + return Vec::new(); + } + // Bail on non-ident-shaped keys at even indices. A dict whose + // "keys" don't look like idents is almost certainly a random + // scalar default that happened to have an even token count — + // treating it as pairs would produce garbage schema fields. + for (i, tok) in tokens.iter().enumerate() { + if i % 2 == 0 && !is_ident_shaped(tok) { + return Vec::new(); + } + } + let mut pairs = Vec::with_capacity(tokens.len() / 2); + let mut iter = tokens.into_iter(); + while let (Some(k), Some(v)) = (iter.next(), iter.next()) { + pairs.push((k, classify_value(&v))); + } + pairs +} + +/// Classify a token as `Scalar` or `Nested` by attempting to re-parse +/// its content as another paired list. If the re-parse yields at +/// least one valid pair, it's `Nested`; otherwise it's a `Scalar` +/// carrying the original text verbatim. +/// +/// An empty string classifies as `Nested([])` — an empty inner dict, +/// which is exactly the shape `LR1_SETTINGS { }` … `LR15_SETTINGS { }` +/// take in the `INTF_LR_SETTINGS` payload. Preserving the nested +/// classification for empties matters so the generator sees a +/// consistent tree shape across every LRn slot even when the anchor +/// only populates LR0. +fn classify_value(text: &str) -> PairedValue { + if text.is_empty() || text.chars().all(char::is_whitespace) { + return PairedValue::Nested(Vec::new()); + } + let inner = parse_paired_list(text); + if inner.is_empty() { + PairedValue::Scalar(text.to_string()) + } else { + PairedValue::Nested(inner) + } +} + +/// True when `s` is shaped like a bare identifier — leading letter or +/// underscore, then letters / digits / underscore / dot. Matches the +/// key form Xilinx uses everywhere in paired-list defaults +/// (`RX_REFCLK_FREQUENCY`, `ch_txdata`, `CONFIG.CPM_PCIE0_MODES`). +fn is_ident_shaped(s: &str) -> bool { + let mut chars = s.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !first.is_ascii_alphabetic() && first != '_' { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.') +} + +/// Repair Xilinx-style mid-identifier line wraps in IP-XACT +/// `` payloads. Their XML formatter wraps long lines +/// by inserting `\n` INSIDE a token (`ch_txpr\necursor3` instead of +/// `ch_txprecursor3`) — the standard Tcl-list tokenizer then splits +/// the token in half, blowing the paired-list even-count invariant. +/// +/// The repair: when `\n` sits between two identifier characters, +/// drop it — reconstructs the original token. Newlines that are +/// legitimate whitespace between tokens (adjacent to other +/// whitespace or non-identifier chars) survive unchanged. +fn fix_wrapped_tokens(input: &str) -> String { + let bytes = input.as_bytes(); + let mut out = String::with_capacity(input.len()); + for i in 0..bytes.len() { + let c = bytes[i] as char; + if c == '\n' { + let prev_ok = i > 0 && is_ident_byte(bytes[i - 1]); + let next_ok = i + 1 < bytes.len() && is_ident_byte(bytes[i + 1]); + if prev_ok && next_ok { + // Wrapped mid-token — drop. + continue; + } + } + out.push(c); + } + out +} + +fn is_ident_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || b == b'_' || b == b'.' +} + +/// Tokenize a Tcl list. Whitespace separates tokens; `{` opens a +/// braced token that swallows characters up to the matching `}`, +/// with nested `{…}` preserved verbatim (only the OUTER braces are +/// stripped from the emitted token). +/// +/// Unbalanced braces silently truncate — sufficient for well-formed +/// IP-XACT defaults and simpler than a full error type. Callers that +/// need to detect malformed input can compare tokenized length to +/// expected pair count. +fn tokenize(input: &str) -> Vec { + let mut tokens = Vec::new(); + let mut chars = input.chars().peekable(); + loop { + while chars.next_if(|c| c.is_whitespace()).is_some() {} + let Some(&c) = chars.peek() else { break }; + let mut tok = String::new(); + if c == '{' { + chars.next(); + let mut depth = 1_usize; + for ch in chars.by_ref() { + if ch == '{' { + depth += 1; + tok.push(ch); + } else if ch == '}' { + depth -= 1; + if depth == 0 { + break; + } + tok.push(ch); + } else { + tok.push(ch); + } + } + } else { + while let Some(&ch) = chars.peek() { + if ch.is_whitespace() { + break; + } + tok.push(ch); + chars.next(); + } + } + tokens.push(tok); + } + tokens +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scalar(s: &str) -> PairedValue { + PairedValue::Scalar(s.into()) + } + + fn nested(pairs: Vec<(&str, PairedValue)>) -> PairedValue { + PairedValue::Nested( + pairs.into_iter().map(|(k, v)| (k.into(), v)).collect(), + ) + } + + #[test] + fn flat_paired_list() { + // The `intf::channel_map` shape — 4 pairs, all scalar values. + let src = "INTF0_RX0 QUAD0_RX0 INTF0_TX0 QUAD0_TX0"; + let pairs = parse_paired_list(src); + assert_eq!(pairs.len(), 2); + assert_eq!(pairs[0].0, "INTF0_RX0"); + assert_eq!(pairs[0].1, scalar("QUAD0_RX0")); + assert_eq!(pairs[1].0, "INTF0_TX0"); + assert_eq!(pairs[1].1, scalar("QUAD0_TX0")); + } + + #[test] + fn one_level_nested_lr_settings() { + // The `INTF_LR_SETTINGS` payload shape — one nested value. + let src = "LR0_SETTINGS {RX_REFCLK_FREQUENCY 156.25 TX_REFCLK_FREQUENCY 156.25}"; + let pairs = parse_paired_list(src); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].0, "LR0_SETTINGS"); + assert_eq!( + pairs[0].1, + nested(vec![ + ("RX_REFCLK_FREQUENCY", scalar("156.25")), + ("TX_REFCLK_FREQUENCY", scalar("156.25")), + ]), + ); + } + + #[test] + fn two_level_nesting_matches_txrx_optional_ports_shape() { + // The `INTF0_TXRX_OPTIONAL_PORTS` anchor shape — flat outer + // pairs terminating in a nested INTF_LR_SETTINGS block whose + // inner values are themselves paired dicts. + let src = "GT_TYPE GTY GT_DIRECTION DUPLEX INTF_LR_SETTINGS \ + {LR0_SETTINGS {RX_HD_EN 0 TX_HD_EN 0} LR1_SETTINGS { }}"; + let pairs = parse_paired_list(src); + assert_eq!(pairs.len(), 3); + assert_eq!(pairs[0], ("GT_TYPE".into(), scalar("GTY"))); + assert_eq!(pairs[1], ("GT_DIRECTION".into(), scalar("DUPLEX"))); + let PairedValue::Nested(intf_lr) = &pairs[2].1 else { + panic!("expected nested INTF_LR_SETTINGS payload"); + }; + assert_eq!(intf_lr.len(), 2); + assert_eq!(intf_lr[0].0, "LR0_SETTINGS"); + assert_eq!( + intf_lr[0].1, + nested(vec![("RX_HD_EN", scalar("0")), ("TX_HD_EN", scalar("0")),]), + ); + // LR1_SETTINGS { } — empty braces classify as an empty + // Nested dict, not a Scalar with empty content. Critical: + // otherwise the schema for LR1 would come out as "one field + // named ''" instead of "nested slot, presently empty". + assert_eq!(intf_lr[1].0, "LR1_SETTINGS"); + assert_eq!(intf_lr[1].1, PairedValue::Nested(Vec::new())); + } + + #[test] + fn odd_token_count_yields_empty_pairs() { + // Signal to the caller: "this isn't a paired dict, treat the + // whole default as scalar." The `intf::gt_settings` param has + // default `0` (a single token) — must NOT parse as a pair. + assert!(parse_paired_list("0").is_empty()); + assert!(parse_paired_list("Custom").is_empty()); + assert!(parse_paired_list("A B C").is_empty()); + } + + #[test] + fn empty_input_yields_empty_pairs() { + assert!(parse_paired_list("").is_empty()); + assert!(parse_paired_list(" ").is_empty()); + } + + #[test] + fn non_ident_keys_reject_pair_interpretation() { + // `NA NA` — the current-generator misfire that would parse as + // one pair with key=NA (ident-shaped) value=NA. That IS + // ident-shaped so it DOES parse. Test the reject path with + // something that shouldn't. + assert!(parse_paired_list("32.0 GT/s").is_empty(), "leading digit"); + assert!(parse_paired_list("--foo bar").is_empty(), "leading punct"); + } + + #[test] + fn scalar_value_with_dot() { + // Frequencies (`156.25`, `10.3125`) tokenize as ident-shaped + // under our rules (dot allowed). Match Xilinx's usage — a + // pair like `RX_LINE_RATE 10.3125` should classify the value + // as scalar even though `10.3125` passes `is_ident_shaped`. + // classify_value tries to re-parse; a single token doesn't + // form a pair; falls back to Scalar. Belt-and-suspenders test. + let pairs = parse_paired_list("RX_LINE_RATE 10.3125"); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].1, scalar("10.3125")); + } + + #[test] + fn deeply_nested_braces_preserved() { + // Tokenizer strips ONLY the outer braces — inner `{…}` are + // kept verbatim so downstream re-parses see the same shape. + let toks = tokenize("A {B {C D} E} F G"); + assert_eq!(toks, vec!["A", "B {C D} E", "F", "G"]); + } + + #[test] + fn empty_braces_tokenize_as_empty_string() { + // `LR1_SETTINGS { }` — the value tokenizes to `""` (or `" "` + // then trimmed by classify_value). Either way, classifies as + // an empty nested dict, not a lone Scalar. + let toks = tokenize("K { }"); + assert_eq!(toks.len(), 2); + assert_eq!(toks[0], "K"); + assert!(toks[1].chars().all(char::is_whitespace) || toks[1].is_empty()); + } +} diff --git a/vw-ip/src/presets.rs b/vw-ip/src/presets.rs new file mode 100644 index 0000000..710abdb --- /dev/null +++ b/vw-ip/src/presets.rs @@ -0,0 +1,268 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Out-of-band parameter value sources for IP-XACT components. +//! +//! Some Xilinx IPs (notably CIPS / CPM5) ship the bulk of their +//! parameter enumerations *outside* the IP-XACT XML, in +//! `cpm_preset*.xml` files Vivado bundles under `data/versal/ps_pmc/`. +//! Without them, parameters like `CPM_PCIE1_PF0_BASE_CLASS_MENU` would +//! only carry their declared default in the generated `@enum(...)`, +//! and there's no other principled signal to recover the legal values +//! from. This module reads those files into a flat map the generator +//! can merge against the IP-XACT `` lists. +//! +//! The XML shape is uniform across the files I've seen: +//! +//! ```xml +//! +//! +//! +//! ... +//! +//! ``` + +use std::collections::{BTreeSet, HashMap}; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("reading preset file {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("parsing preset file {path}: {source}")] + Xml { + path: PathBuf, + #[source] + source: quick_xml::DeError, + }, +} + +/// `param_name → set of valid values`. `BTreeSet` keeps the iteration +/// order stable so generated `@enum(...)` lists are deterministic. +pub type PresetMap = HashMap>; + +#[derive(Debug, Default, Deserialize)] +struct Root { + #[serde(default, rename = "preset")] + entries: Vec, +} + +#[derive(Debug, Deserialize)] +struct Entry { + #[serde(rename = "@param")] + param: String, + #[serde(rename = "@name")] + name: String, +} + +/// Load one preset XML file into a fresh map. +pub fn load_file(path: &Path) -> Result { + let xml = fs::read_to_string(path).map_err(|source| Error::Io { + path: path.to_path_buf(), + source, + })?; + let root: Root = + quick_xml::de::from_str(&xml).map_err(|source| Error::Xml { + path: path.to_path_buf(), + source, + })?; + let mut map = PresetMap::new(); + for e in root.entries { + map.entry(e.param).or_default().insert(e.name); + } + Ok(map) +} + +/// Load several preset XML files and merge their entries into one map. +pub fn load_files(paths: I) -> Result +where + I: IntoIterator, + I::Item: AsRef, +{ + let mut merged = PresetMap::new(); + for p in paths { + let map = load_file(p.as_ref())?; + for (param, values) in map { + merged.entry(param).or_default().extend(values); + } + } + Ok(merged) +} + +/// Try to find sibling preset files for the IP whose +/// `component.xml` lives at `component_path`. +/// +/// Walks up from the component file looking for a Vivado-style +/// `data/` ancestor directory and then peeks at +/// `data/versal/ps_pmc//`. Any `*preset*.xml` found there +/// (recursively) is returned. Returns an empty vector — not an error — +/// when the layout doesn't match; the caller should treat it as a +/// best-effort hint. +pub fn discover_for(component_path: &Path) -> Vec { + let Some(data_root) = data_root_of(component_path) else { + return Vec::new(); + }; + let ip_name = ip_name_from(component_path); + let Some(ip_name) = ip_name else { + return Vec::new(); + }; + let ip_dir = data_root.join("versal").join("ps_pmc").join(&ip_name); + if !ip_dir.is_dir() { + return Vec::new(); + } + let mut out = Vec::new(); + collect_preset_files(&ip_dir, &mut out); + out.sort(); + out +} + +/// Recurse through `dir` collecting any `*preset*.xml` file paths. +fn collect_preset_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_preset_files(&path, out); + continue; + } + let Some(name) = path.file_name().and_then(|s| s.to_str()) else { + continue; + }; + if name.contains("preset") && name.ends_with(".xml") { + out.push(path); + } + } +} + +/// Walk up `component_path`'s ancestors looking for a directory +/// literally named `data` (Vivado's install root convention). +fn data_root_of(component_path: &Path) -> Option { + for ancestor in component_path.ancestors() { + if ancestor.file_name().and_then(|s| s.to_str()) == Some("data") { + return Some(ancestor.to_path_buf()); + } + } + None +} + +/// Recover an IP's short name from a Vivado-style versioned directory +/// (`cpm5_v1_0` → `cpm5`, `axi_dma_v7_1` → `axi_dma`). The trailing +/// `_v_` suffix is the convention Xilinx uses across IPs. +fn ip_name_from(component_path: &Path) -> Option { + let ip_dir = component_path.parent()?; + let name = ip_dir.file_name()?.to_str()?; + Some(strip_version_suffix(name).to_string()) +} + +fn strip_version_suffix(name: &str) -> &str { + // Find a trailing `_v_` and trim it. + let bytes = name.as_bytes(); + let mut end = bytes.len(); + // Trailing digits (minor) + while end > 0 && bytes[end - 1].is_ascii_digit() { + end -= 1; + } + if end == 0 || bytes[end - 1] != b'_' { + return name; + } + let after_minor = end; + end -= 1; // skip the `_` + while end > 0 && bytes[end - 1].is_ascii_digit() { + end -= 1; + } + let after_major_digits = end; + if end < 1 || &bytes[end.saturating_sub(2)..end] != b"_v" { + // Doesn't end in `_v_` — leave as-is. + return name; + } + let _ = after_minor; + let _ = after_major_digits; + &name[..end - 2] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_preset_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("p.xml"); + fs::write( + &path, + r#" + + + + "#, + ) + .unwrap(); + let m = load_file(&path).unwrap(); + let a: Vec<&str> = m["A"].iter().map(String::as_str).collect(); + assert_eq!(a, vec!["x", "y"]); + assert!(m["B"].contains("z")); + } + + #[test] + fn merges_multiple_files() { + let dir = tempfile::tempdir().unwrap(); + let p1 = dir.path().join("a.xml"); + fs::write(&p1, r#""#) + .unwrap(); + let p2 = dir.path().join("b.xml"); + fs::write(&p2, r#""#) + .unwrap(); + let m = load_files(&[p1, p2]).unwrap(); + let v: Vec<&str> = m["K"].iter().map(String::as_str).collect(); + assert_eq!(v, vec!["1", "2"]); + } + + #[test] + fn strips_xilinx_version_suffix() { + assert_eq!(strip_version_suffix("cpm5_v1_0"), "cpm5"); + assert_eq!(strip_version_suffix("axi_dma_v7_1"), "axi_dma"); + // No version → unchanged. + assert_eq!(strip_version_suffix("foo_bar"), "foo_bar"); + // Almost-but-not version → unchanged. + assert_eq!(strip_version_suffix("foo_v1"), "foo_v1"); + } + + #[test] + fn discovers_under_data_layout() { + let dir = tempfile::tempdir().unwrap(); + // Mimic Xilinx layout: data/ip/xilinx/_v1_0/component.xml + let data = dir.path().join("data"); + let ip = data.join("ip").join("xilinx").join("widget_v2_3"); + fs::create_dir_all(&ip).unwrap(); + let component = ip.join("component.xml"); + fs::write(&component, "").unwrap(); + // And sibling: data/versal/ps_pmc/widget/p.xml + let preset_dir = data.join("versal").join("ps_pmc").join("widget"); + fs::create_dir_all(&preset_dir).unwrap(); + let preset = preset_dir.join("my_preset.xml"); + fs::write(&preset, "").unwrap(); + // Unrelated file shouldn't be picked up. + fs::write(preset_dir.join("README.md"), "ignored").unwrap(); + + let found = discover_for(&component); + assert_eq!(found, vec![preset]); + } + + #[test] + fn discovery_empty_when_layout_doesnt_match() { + let dir = tempfile::tempdir().unwrap(); + let component = dir.path().join("loose.xml"); + fs::write(&component, "").unwrap(); + assert!(discover_for(&component).is_empty()); + } +} diff --git a/vw-ip/src/summary.rs b/vw-ip/src/summary.rs new file mode 100644 index 0000000..665eabd --- /dev/null +++ b/vw-ip/src/summary.rs @@ -0,0 +1,38 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! A small, human-friendly summary of an IP-XACT component — used by +//! `vw ip generate` to print what it's processing before emitting code. + +use ipxact::Component; + +#[derive(Clone, Debug)] +pub struct Summary { + pub vlnv: String, + pub description: Option, + pub parameter_count: usize, + pub user_parameter_count: usize, + pub model_parameter_count: usize, + pub port_count: usize, + pub choice_count: usize, +} + +impl Summary { + pub fn of(c: &Component) -> Self { + let parameters: Vec<_> = c.component_parameters().collect(); + let user_parameter_count = parameters + .iter() + .filter(|p| p.value.is_user_configurable()) + .count(); + Self { + vlnv: c.vlnv(), + description: c.description.clone(), + parameter_count: parameters.len(), + user_parameter_count, + model_parameter_count: c.model_parameters().count(), + port_count: c.ports().count(), + choice_count: c.choices().count(), + } + } +} diff --git a/vw-ip/src/targets.rs b/vw-ip/src/targets.rs new file mode 100644 index 0000000..b01ba8d --- /dev/null +++ b/vw-ip/src/targets.rs @@ -0,0 +1,312 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `[targets]` extraction for the generated `vw.toml`. +//! +//! Reads `` out of an IP's `component.xml`, +//! CATEGORIZES each entry by its `xilinx:lifeCycle` attribute, and +//! returns two brace-form pattern lists ready to be written into a +//! workspace `vw.toml` under `[targets]`: +//! +//! - `supported = [...]` — entries whose lifeCycle is `Production`, +//! `Beta`, or `Pre-Production`. Xilinx has blessed the IP for the +//! listed parts. +//! - `not-supported = [...]` — entries with `lifeCycle="Not-Supported"`. +//! Xilinx has attested the IP is NOT usable on those parts. +//! +//! The split matters because Vivado's IP catalog is not gated by +//! `` — `get_ipdefs` returns an IP even +//! for families not in the list. So a static "reject if the target +//! isn't listed" rule produces false positives (e.g. clk_wizard_v1_0 +//! has EVERY entry marked Not-Supported yet works fine on `xcvp1202` +//! in real projects). The list is a lifeCycle-tagged compatibility +//! matrix, not a filter — `vw check` uses it to distinguish +//! definitively-forbidden combinations (error) from merely-unblessed +//! ones (warning). See `vw_lib::TargetMismatchKind` for the check +//! side. +//! +//! Normalization rules per entry: +//! - Entries already in brace form (`versal{xcvm3(.*)}`) pass through +//! verbatim. +//! - Bare-family entries (`artix7`) get widened to `{.+}` — a +//! permissive placeholder that says "we support any part in this +//! family, but we haven't narrowed the pattern here." A future +//! extension can query Vivado at `vw ip generate` time to derive +//! precise regexes for each legacy family. +//! +//! The extractor is regex-driven rather than XML-parsed to avoid +//! adding an XML dep to `vw-ip`. The `` elements have a +//! tightly constrained shape in every component.xml we've observed; +//! the regex covers each cleanly. + +use std::path::Path; + +/// The two `[targets]` lists produced by [`extract_targets`], each +/// already normalized to brace form and ready for TOML upsert. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ExtractedTargets { + /// `Production` / `Beta` / `Pre-Production` entries. + pub supported: Vec, + /// `Not-Supported` entries. + pub not_supported: Vec, +} + +/// Read the `` entries from `component_path` and +/// split them by lifeCycle. Returns empty lists when the file has +/// no `` section (older or +/// non-family-aware component.xml). I/O errors and malformed XML +/// both surface as an empty result — the generator wraps this +/// with a warning when both lists are empty on an IP that clearly +/// should have families. +pub fn extract_targets(component_path: &Path) -> ExtractedTargets { + let Ok(xml) = std::fs::read_to_string(component_path) else { + return ExtractedTargets::default(); + }; + extract_targets_from_xml(&xml) +} + +/// Same as [`extract_targets`] but takes the XML text directly — +/// used by unit tests to feed known snippets without disk I/O. +pub fn extract_targets_from_xml(xml: &str) -> ExtractedTargets { + // `versal{xcvm3(.*)}` + // (with optional attributes on the opening tag). We care both + // about the attribute cluster (to sniff out `xilinx:lifeCycle`) + // and the interior text. + let entry_re = regex::Regex::new( + r#"(?s)]*)>\s*([^<]*?)\s*"#, + ) + .expect("family entry regex must compile"); + let lifecycle_re = + regex::Regex::new(r#"(?i)xilinx:lifeCycle\s*=\s*"([^"]*)""#) + .expect("lifecycle regex must compile"); + let mut out = ExtractedTargets::default(); + for cap in entry_re.captures_iter(xml) { + let attrs = &cap[1]; + let raw = cap[2].trim(); + if raw.is_empty() { + continue; + } + let normalized = normalize_family_entry(raw); + let lifecycle = lifecycle_re + .captures(attrs) + .map(|c| c[1].to_string()) + .unwrap_or_default(); + // Anything explicitly "Not-Supported" goes into the ban + // list. Every other value — Production, Beta, + // Pre-Production, or missing — is treated as blessed. + // Missing lifeCycle is the common case for older / + // non-annotated component.xml files; blessing is the safer + // default since a NON-match on the blessed list produces + // a warning, not an error. + if lifecycle.eq_ignore_ascii_case("Not-Supported") { + out.not_supported.push(normalized); + } else { + out.supported.push(normalized); + } + } + // Lift `ARCHITECTURE=` clauses out of the IP's + // `` and add a family-wide + // `{.+}` pattern per unique architecture. This + // captures the intent of IPs like `clk_wizard_v1_0` whose + // filter is `((ARCHITECTURE=versal)&&(MMCM>0))` — the IP is + // blessed for the entire versal architecture; the `MMCM > 0` + // and per-family `not-supported` entries then prune specific + // parts out. Without this lift, an IP whose supportedFamilies + // list is empty (or entirely Not-Supported) looks like it has + // no blessed patterns at all, which fires spurious "not + // blessed" warnings on parts Vivado will happily instantiate. + // + // We don't try to evaluate the boolean expression as a whole + // (capability clauses like `MMCM > 0` need per-part device + // properties we don't have statically). Extracting the pure + // architecture predicates is enough for the "blessed vs. not" + // question — capabilities and per-part bans still narrow at + // check time. + for arch in extract_architectures_from_filter(xml) { + let pat = format!("{arch}{{.+}}"); + if !out.supported.contains(&pat) { + out.supported.push(pat); + } + } + out +} + +/// Pull every `ARCHITECTURE=` clause out of any +/// `` block in `xml`. The +/// filter is a boolean expression written in a Xilinx-specific +/// mini-language, with entity-encoded operators (`&&`, +/// `>`). We only care about the architecture predicates; +/// capability clauses (`MMCM > 0`, `CPM5 > 0`, …) are ignored. +fn extract_architectures_from_filter(xml: &str) -> Vec { + let block_re = regex::Regex::new( + r#"(?s)\s*([^<]*?)\s*"#, + ) + .expect("autoDevicePropertiesFilter block regex must compile"); + let arch_re = + regex::Regex::new(r#"(?i)ARCHITECTURE\s*=\s*([A-Za-z0-9_]+)"#) + .expect("architecture predicate regex must compile"); + let mut out = Vec::new(); + for cap in block_re.captures_iter(xml) { + for a in arch_re.captures_iter(&cap[1]) { + let name = a[1].to_string(); + if !out.contains(&name) { + out.push(name); + } + } + } + out +} + +/// Normalize one raw `` payload into +/// `{}` form. Entries already in brace form pass +/// through verbatim. +fn normalize_family_entry(raw: &str) -> String { + if raw.contains('{') && raw.ends_with('}') { + return raw.to_string(); + } + // Bare family — widen to `family{.+}`. Any part name matches; + // the check effectively becomes "does the consumer target + // string exist at all," which is a strict subset of what + // Vivado would allow but never rejects a genuinely-supported + // part. + format!("{raw}{{.+}}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn brace_form_without_lifecycle_is_blessed() { + // No `xilinx:lifeCycle` attribute → treat as blessed + // (`supported`), since a missing lifecycle is the common + // shape in older component.xml files. + let out = extract_targets_from_xml( + r#"versal{xcvm3(.*)}"#, + ); + assert_eq!(out.supported, vec!["versal{xcvm3(.*)}"]); + assert!(out.not_supported.is_empty()); + } + + #[test] + fn production_lifecycle_lands_in_supported() { + let out = extract_targets_from_xml( + r#"versal{xcvm3(.*)}"#, + ); + assert_eq!(out.supported, vec!["versal{xcvm3(.*)}"]); + assert!(out.not_supported.is_empty()); + } + + #[test] + fn not_supported_lifecycle_lands_in_ban_list() { + let out = extract_targets_from_xml( + r#"versal{xcvm3(.*)}"#, + ); + assert_eq!(out.not_supported, vec!["versal{xcvm3(.*)}"]); + assert!(out.supported.is_empty()); + } + + #[test] + fn bare_family_widens_to_placeholder_regex() { + let out = extract_targets_from_xml( + r#"artix7"#, + ); + assert_eq!(out.supported, vec!["artix7{.+}"]); + } + + #[test] + fn mixed_lifecycles_are_split() { + let out = extract_targets_from_xml( + r#" + + versal{xcvm3(.*)} + artix7{xc7a35t(.*)} + versal{xcvp1202(.*)} + + "#, + ); + assert_eq!( + out.supported, + vec!["versal{xcvm3(.*)}", "versal{xcvp1202(.*)}"], + ); + assert_eq!(out.not_supported, vec!["artix7{xc7a35t(.*)}"]); + } + + #[test] + fn missing_section_returns_empty() { + let out = + extract_targets_from_xml("no families here"); + assert!(out.supported.is_empty()); + assert!(out.not_supported.is_empty()); + } + + #[test] + fn architecture_filter_widens_blessed_list() { + // clk_wizard_v1_0's exact shape: every family entry is + // Not-Supported, but the filter blesses the whole versal + // architecture. Result: `versal{.+}` in supported; + // specific parts still in not_supported. + let out = extract_targets_from_xml( + r#" + + versal{xa2ve3288(.*)} + versal{xc2ve3(.*)} + + ((ARCHITECTURE=versal)&&(MMCM>0)) + "#, + ); + assert_eq!(out.supported, vec!["versal{.+}"]); + assert_eq!( + out.not_supported, + vec!["versal{xa2ve3288(.*)}", "versal{xc2ve3(.*)}"], + ); + } + + #[test] + fn architecture_filter_dedupes_against_existing_supported() { + // If the supported list already carries a versal entry, + // adding another family-wide one would be redundant. But + // `versal{.+}` and `versal{xcvm3(.*)}` are DIFFERENT + // patterns; both belong. We only dedupe on exact-string + // equality so the same regex isn't repeated. + let out = extract_targets_from_xml( + r#" + + versal{xcvm3(.*)} + + (ARCHITECTURE=versal) + "#, + ); + assert_eq!(out.supported, vec!["versal{xcvm3(.*)}", "versal{.+}"]); + } + + #[test] + fn capability_only_filter_adds_nothing_to_supported() { + // dcmac_v3_0 / cpm5_v1_0 have capability-only filters — + // no ARCHITECTURE clause. Nothing to lift; the family + // list alone drives the blessed set. + let out = extract_targets_from_xml( + r#" + + versal{xcvp1202(.*)} + + (CPM5 > 0) + "#, + ); + assert_eq!(out.supported, vec!["versal{xcvp1202(.*)}"]); + } + + #[test] + fn multiple_architecture_clauses_each_lift() { + // Rare but possible: an IP that supports several + // architectures. Each named `ARCHITECTURE=` clause + // becomes its own family-wide pattern. + let out = extract_targets_from_xml( + r#" + ((ARCHITECTURE=versal) || (ARCHITECTURE=zynquplus)) + "#, + ); + assert_eq!(out.supported, vec!["versal{.+}", "zynquplus{.+}"]); + } +} diff --git a/vw-ip/src/tree.rs b/vw-ip/src/tree.rs new file mode 100644 index 0000000..98dc90c --- /dev/null +++ b/vw-ip/src/tree.rs @@ -0,0 +1,268 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Recursive prefix tree over parameter names. +//! +//! Big Xilinx IPs encode their configuration hierarchy in parameter +//! names, not in IP-XACT structure: `CPM_PCIE1_PF0_BAR0_64BIT` lives +//! under PCIE1 → PF0 → BAR0, but that's all conveyed by underscores. +//! A flat depth-1 grouping leaves PCIE1 with ~4200 args, which is +//! useless in an LSP. We recurse: at each depth, partition by the next +//! segment; subgroups bigger than `min_split_size` become children +//! that recurse again; everything smaller absorbs into the current +//! node as direct parameters. Generation walks the tree and emits one +//! proc per node, which keeps every proc small enough to navigate by +//! flag completion. + +use std::collections::BTreeMap; + +use ipxact::Parameter; + +#[derive(Clone, Debug)] +pub struct TreeOptions { + /// Don't split a subgroup into its own child node unless it has at + /// least this many parameters. Smaller subgroups stay as direct + /// args of the parent — keeps singleton segments from becoming + /// their own procs. + pub min_split_size: usize, +} + +impl Default for TreeOptions { + fn default() -> Self { + Self { min_split_size: 8 } + } +} + +#[derive(Clone, Debug)] +pub struct Node<'a> { + /// Full underscore-joined prefix that names this node + /// (e.g. `CPM_PCIE1_PF0`). Empty for the root. + pub label: String, + /// Underscore-separated depth: 0 at root, 1 for `CPM`, 2 for + /// `CPM_PCIE1`, 3 for `CPM_PCIE1_PF0`, ... + pub depth: usize, + /// Parameters whose proc-level args belong on *this* node. Their + /// arg names are derived by stripping the node's prefix. + pub direct: Vec<&'a Parameter>, + /// Child nodes, keyed by their additional segment. + pub children: Vec>, +} + +impl<'a> Node<'a> { + /// Total parameters reachable from this node, including children. + pub fn total_params(&self) -> usize { + self.direct.len() + + self.children.iter().map(Node::total_params).sum::() + } + + /// Number of nodes in this subtree, including self. + pub fn node_count(&self) -> usize { + 1 + self.children.iter().map(Node::node_count).sum::() + } + + /// Pre-order walk: visit self, then each child recursively. + pub fn walk(&self, f: &mut impl FnMut(&Node<'a>)) { + f(self); + for c in &self.children { + c.walk(f); + } + } + + /// True when this node has no sub-children — its whole + /// configuration surface lives in `direct`. Used by the + /// indexed-family collapse pass to gate whether a set of + /// sibling nodes can flatten into one constructor: nodes with + /// their own sub-trees carry more shape than a flat + /// `` value can capture. + pub fn is_leaf_only(&self) -> bool { + self.children.is_empty() + } + + /// Collect references to every node in this subtree in pre-order. + /// Used by code-gen, which needs to iterate the tree twice (once + /// for the header summary, once to emit procs) without re-walking + /// through a closure that can't escape `&Node` references. + pub fn collect<'t>(&'t self) -> Vec<&'t Node<'a>> { + let mut out = Vec::new(); + self.collect_into(&mut out); + out + } + + fn collect_into<'t>(&'t self, out: &mut Vec<&'t Node<'a>>) { + out.push(self); + for c in &self.children { + c.collect_into(out); + } + } +} + +/// Build the prefix tree from a flat parameter list. +pub fn build_tree<'a, I>(params: I, opts: &TreeOptions) -> Node<'a> +where + I: IntoIterator, +{ + build_node(0, String::new(), params.into_iter().collect(), opts) +} + +fn build_node<'a>( + depth: usize, + label: String, + params: Vec<&'a Parameter>, + opts: &TreeOptions, +) -> Node<'a> { + let mut direct: Vec<&'a Parameter> = Vec::new(); + let mut subgroups: BTreeMap> = BTreeMap::new(); + + for p in params { + let segs: Vec<&str> = p.name.split('_').collect(); + if depth + 1 >= segs.len() { + // No further segments to split on — this parameter belongs + // to the current node directly. + direct.push(p); + } else { + // Group by the segment at position `depth` — the next one + // not yet absorbed into the label. + subgroups + .entry(segs[depth].to_string()) + .or_default() + .push(p); + } + } + + let mut children = Vec::new(); + for (seg, group) in subgroups { + // A subgroup that's smaller than the split threshold isn't + // worth its own proc — keep its parameters at this level. + if group.len() < opts.min_split_size { + direct.extend(group); + continue; + } + let child_label = if label.is_empty() { + seg.clone() + } else { + format!("{label}_{seg}") + }; + children.push(build_node(depth + 1, child_label, group, opts)); + } + + Node { + label, + depth, + direct, + children, + } +} + +/// Return the portion of `param_name` after the node's `label_prefix` +/// (and the underscore separating them). Used so arg names inside a +/// node's proc don't redundantly repeat the prefix. +pub fn strip_prefix<'a>(param_name: &'a str, label_prefix: &str) -> &'a str { + if label_prefix.is_empty() { + return param_name; + } + if let Some(rest) = param_name.strip_prefix(label_prefix) { + rest.strip_prefix('_').unwrap_or(rest) + } else { + param_name + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(name: &str) -> Parameter { + Parameter { + name: name.into(), + ..Default::default() + } + } + + #[test] + fn empty_input_returns_empty_root() { + let tree = + build_tree(Vec::<&Parameter>::new(), &TreeOptions::default()); + assert_eq!(tree.label, ""); + assert_eq!(tree.direct.len(), 0); + assert_eq!(tree.children.len(), 0); + } + + #[test] + fn singletons_stay_at_root() { + let params = [p("A"), p("B"), p("C")]; + let opts = TreeOptions::default(); + let tree = build_tree(params.iter(), &opts); + // Each is one segment, no subgroups; all direct at root. + assert_eq!(tree.direct.len(), 3); + assert!(tree.children.is_empty()); + } + + #[test] + fn splits_when_subgroup_exceeds_threshold() { + let mut params: Vec = + (0..10).map(|i| p(&format!("CPM_PCIE1_FIELD{i}"))).collect(); + params.extend((0..10).map(|i| p(&format!("CPM_PCIE0_FIELD{i}")))); + let opts = TreeOptions { min_split_size: 5 }; + let tree = build_tree(params.iter(), &opts); + // Root has one child `CPM`; under `CPM`, children `PCIE0` and + // `PCIE1`, each with 10 direct params. + assert_eq!(tree.children.len(), 1); + let cpm = &tree.children[0]; + assert_eq!(cpm.label, "CPM"); + assert_eq!(cpm.children.len(), 2); + for c in &cpm.children { + assert_eq!(c.direct.len(), 10); + } + } + + #[test] + fn nested_hierarchy_splits_recursively() { + // Mimic PCIE1's structure: a bunch of PF0/PF1/PF2 sub-trees, + // each with BARs and CAPs. + let mut params: Vec = Vec::new(); + for pf in 0..3 { + for bar in 0..6 { + for f in 0..10 { + params.push(p(&format!( + "CPM_PCIE1_PF{pf}_BAR{bar}_FIELD{f}" + ))); + } + } + for cap in 0..3 { + for f in 0..5 { + params.push(p(&format!( + "CPM_PCIE1_PF{pf}_CAP{cap}_FIELD{f}" + ))); + } + } + } + let opts = TreeOptions { min_split_size: 5 }; + let tree = build_tree(params.iter(), &opts); + // Drill into the tree: root → CPM → PCIE1 → PF0/PF1/PF2. + let cpm = &tree.children[0]; + let pcie1 = &cpm.children[0]; + assert_eq!(pcie1.label, "CPM_PCIE1"); + assert_eq!(pcie1.children.len(), 3); // PF0, PF1, PF2 + let pf0 = &pcie1.children[0]; + // PF0 should have BAR0..BAR5 + CAP0..CAP2 as children. + let bar_count = pf0 + .children + .iter() + .filter(|c| c.label.contains("BAR")) + .count(); + assert_eq!(bar_count, 6, "{pf0:#?}"); + } + + #[test] + fn strip_prefix_returns_local_name() { + assert_eq!( + strip_prefix("CPM_PCIE1_PF0_BAR0_ENABLED", "CPM_PCIE1_PF0_BAR0"), + "ENABLED" + ); + // No prefix: returns the name unchanged. + assert_eq!(strip_prefix("FOO", ""), "FOO"); + // Prefix doesn't match: returns unchanged (defensive). + assert_eq!(strip_prefix("FOO_BAR", "BAZ"), "FOO_BAR"); + } +} diff --git a/vw-ip/tests/load_real_files.rs b/vw-ip/tests/load_real_files.rs new file mode 100644 index 0000000..243c3ef --- /dev/null +++ b/vw-ip/tests/load_real_files.rs @@ -0,0 +1,239 @@ +// Smoke tests that load the actual Xilinx IP-XACT files from the local +// Vivado install. Skipped automatically when the files aren't present +// so this still passes in CI without a Vivado install. + +use std::path::Path; + +use vw_ip::{generate, group_parameters, load, GenerateOptions, Summary}; + +const CIPS: &str = + "/home/ry/Xilinx/2025.1/data/rsb/iprepos/versal_cips_v3_4/component.xml"; +const CPM5: &str = + "/home/ry/Xilinx/2025.1/data/ip/xilinx/cpm5_v1_0/component.xml"; + +fn skip_if_missing(p: &str) -> bool { + if Path::new(p).exists() { + false + } else { + eprintln!("skipping: {p} not present"); + true + } +} + +#[test] +fn loads_cips_component() { + if skip_if_missing(CIPS) { + return; + } + let component = load(CIPS).expect("load CIPS"); + let summary = Summary::of(&component); + eprintln!("CIPS summary: {summary:#?}"); + assert!(summary.vlnv.contains("versal_cips")); +} + +#[test] +fn loads_cpm5_component() { + if skip_if_missing(CPM5) { + return; + } + let component = load(CPM5).expect("load CPM5"); + let summary = Summary::of(&component); + eprintln!("CPM5 summary: {summary:#?}"); + assert!(summary.vlnv.contains("cpm5")); + // CPM5 should be huge. + assert!( + summary.parameter_count > 1000, + "expected many parameters, got {}", + summary.parameter_count + ); +} + +#[test] +fn generates_cips_wrapper_that_reparses() { + if skip_if_missing(CIPS) { + return; + } + let component = load(CIPS).expect("load CIPS"); + let out = generate( + &component, + &Default::default(), + &Default::default(), + &GenerateOptions::default(), + ) + .into_single(); + eprintln!("--- generated CIPS wrapper (first 60 lines) ---"); + for line in out.lines().take(60) { + eprintln!("{line}"); + } + eprintln!("--- ({} lines total) ---", out.lines().count()); + + let parsed = vw_htcl::parse(&out); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors + ); + + // Validate the generated wrapper against its own signature using + // the same validator the LSP runs. + let diags = vw_htcl::validate(&parsed.document, &out); + let errors: Vec<_> = diags + .iter() + .filter(|d| d.severity == vw_htcl::Severity::Error) + // The generator emits calls into vivado-cmd (`ip::check`, + // `create_bd_cell`, `set_property`); those resolve when + // the wrapper is sourced through the loader, but this + // integration test validates the bare generated text. + // The unknown-call diagnostic is expected in that mode. + .filter(|d| !d.message.starts_with("undefined proc")) + .collect(); + assert!(errors.is_empty(), "validator errors: {errors:#?}"); +} + +#[test] +fn generates_cpm5_wrapper_in_split_mode() { + if skip_if_missing(CPM5) { + return; + } + let component = load(CPM5).expect("load CPM5"); + let out = generate( + &component, + &Default::default(), + &Default::default(), + &GenerateOptions::default(), + ) + .into_single(); + + // Walk the generated source and measure per-proc arg counts so we + // can assert nothing is anywhere near the 4200-arg PCIE1 disaster + // we started with. + let mut proc_sizes: Vec<(String, usize)> = Vec::new(); + let mut current: Option<(String, usize)> = None; + let mut in_args = false; + // Track the indent of the current proc — procs live inside + // `namespace eval { … }` and get a 2-space prefix. + let mut proc_indent = 0usize; + for line in out.lines() { + let trimmed = line.trim_start(); + let indent = line.len() - trimmed.len(); + if let Some(name) = trimmed + .strip_prefix("proc ") + .and_then(|s| s.split_once(' ').map(|(n, _)| n)) + { + current = Some((name.to_string(), 0)); + in_args = true; + proc_indent = indent; + } else if trimmed == "} {" + || (trimmed.starts_with("} ") && trimmed.ends_with(" {")) + { + // `} {` is the old (untyped) body opener; `} TYPE {` + // (e.g. `} bd_cell {`, `} unit {`) is the new + // type-annotated form. Match on the exact indent so + // an inner `} {` in the body doesn't close the outer. + if indent == proc_indent { + if let Some(c) = current.take() { + proc_sizes.push(c); + } + in_args = false; + } + } else if in_args + && indent > proc_indent + && !trimmed.starts_with("##") + && !trimmed.is_empty() + { + if let Some(c) = current.as_mut() { + c.1 += 1; + } + } + } + proc_sizes.sort_by_key(|(_, n)| std::cmp::Reverse(*n)); + let (max_name, max_size) = proc_sizes[0].clone(); + let total_procs = proc_sizes.len(); + eprintln!( + "CPM5 wrapper: {} procs, {} lines, biggest is {} ({} args)", + total_procs, + out.lines().count(), + max_name, + max_size + ); + for (n, s) in proc_sizes.iter().take(8) { + eprintln!(" {n:>40} = {s} args"); + } + + // Under the flat compositional model the top-level `create` + // proc IS the single composition point — one kwarg per split + // sub-tree constructor. For a giant IP like CPM5 that's a lot + // of kwargs but each is typed, LSP-navigable, and users + // typically only pass a handful. Cap at 1000 to guard against + // runaway growth without pretending we're back to the + // 200-arg hierarchical shape. + assert!( + max_size <= 1000, + "biggest proc {max_name} has {max_size} args; \ + even the compositional top proc shouldn't exceed 1000" + ); + // And the overall proc count should reflect that we *are* splitting. + assert!( + total_procs > 50, + "only {total_procs} procs — hierarchy isn't being built" + ); + + // Under the compositional model + namespace-eval wrapping: + // - Newtype preludes emit at file top level (outside the block). + // - Inside `namespace eval cpm5 { … }`: value constructors, + // the top-level `create` proc, all with bare names and + // 2-space indent. + // The exact ordering of constructors depends on tree traversal; + // just assert the wrapping block exists and expected procs + // appear inside. + assert!( + out.contains("namespace eval cpm5 {"), + "{}", + &out[..out.len().min(1200)] + ); + assert!(out.contains(" proc create {")); + // Split-node procs use fully-qualified names now — they live + // OUTSIDE the namespace-block (in sibling `.htcl` files after + // the split pass) so they can be individually loaded without + // Tcl doubling the namespace prefix. + assert!(out.contains("proc cpm5::cpm_pcie0 ")); + assert!(out.contains("proc cpm5::cpm_pcie1 ")); + + let parsed = vw_htcl::parse(&out); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors + ); + let diags = vw_htcl::validate(&parsed.document, &out); + let errors: Vec<_> = diags + .iter() + .filter(|d| d.severity == vw_htcl::Severity::Error) + // The generator emits calls into vivado-cmd (`ip::check`, + // `create_bd_cell`, `set_property`); those resolve when + // the wrapper is sourced through the loader, but this + // integration test validates the bare generated text. + // The unknown-call diagnostic is expected in that mode. + .filter(|d| !d.message.starts_with("undefined proc")) + .collect(); + assert!(errors.is_empty(), "validator errors: {errors:#?}"); +} + +#[test] +fn groups_cpm5_parameters_into_handful_of_buckets() { + if skip_if_missing(CPM5) { + return; + } + let component = load(CPM5).expect("load CPM5"); + let params: Vec<_> = component.component_parameters().collect(); + let groups = group_parameters(params.iter().copied(), 2); + eprintln!("CPM5 has {} groups at prefix=2:", groups.len()); + for g in groups.iter().take(20) { + eprintln!(" {:>32} = {} params", g.key, g.parameters.len()); + } + eprintln!(" ... ({} total)", groups.len()); + // We expect a manageable number of groups (not one giant flat list, + // not thousands of singletons). + assert!(groups.len() < 200, "too many groups: {}", groups.len()); + assert!(groups.len() > 2, "too few groups: {}", groups.len()); +} diff --git a/vw-lib/src/lib.rs b/vw-lib/src/lib.rs index 7e3ab38..bd8a76b 100644 --- a/vw-lib/src/lib.rs +++ b/vw-lib/src/lib.rs @@ -189,7 +189,26 @@ impl fmt::Display for VhdlStandard { pub struct WorkspaceConfig { #[allow(dead_code)] pub workspace: WorkspaceInfo, + #[serde(default)] pub dependencies: HashMap, + /// Test-only dependencies. Only the ENTRY workspace's + /// `[test-dependencies]` are honored — a transitive dep's + /// test-deps are private to itself. Cargo-parity semantic for + /// `dev-dependencies`. Consumed by `vw test` via + /// [`transitive_dep_cache_paths_with_test`]. + #[serde(default, rename = "test-dependencies")] + pub test_dependencies: HashMap, + /// Library-scope: the set of Vivado device families/parts the + /// files in this workspace support. Populated by `vw ip + /// generate` from the underlying IP's ``, normalized so every entry has a brace-form + /// regex against the raw part name. + /// + /// Project workspaces (those that declare `[workspace] + /// target-part`) omit this — projects consume libraries, they + /// don't publish their own supported-parts list. + #[serde(default)] + pub targets: Option, #[serde(default)] pub tools: Option, } @@ -200,15 +219,581 @@ pub struct WorkspaceInfo { pub name: String, #[allow(dead_code)] pub version: String, + /// Project-scope: every Vivado device part this workspace can + /// target. Each entry is a full Vivado specifier (e.g. + /// `xcvp1202-vsva2785-2MHP-e-S`) — package and speed grade + /// matter for downstream implementation / timing analysis + /// even if the family alone is enough for IP-availability + /// checks. + /// + /// One entry must be marked `default = true` when the list + /// has more than one; the default drives the auto-project + /// on `vw run` / `vw repl` / `vw test`. The CLI's + /// `--part ` flag selects a non-default + /// entry. + /// + /// Empty for library workspaces (they publish `[targets]` + /// instead — see [`TargetsConfig`]). + /// + /// Mutually exclusive with [`variants`](Self::variants) — a + /// workspace declares one shape or the other, never both. + /// Variants own their parts inline; `[[target-parts]]` is for + /// projects whose parts are truly interchangeable and don't + /// change what source files compile. + #[serde(default, rename = "target-parts")] + pub target_parts: Vec, + /// Project-scope: named feature-flag-style variants. Each + /// variant declares its own part inline and an optional list + /// of `exclusive` file paths (workspace-relative globs) that + /// are ONLY compiled when that variant is active. + /// + /// Mutually exclusive with [`target_parts`](Self::target_parts) + /// — see the docstring there. Empty for the common + /// "no variants" case, in which case part selection is + /// driven purely by `[[target-parts]]`. + /// + /// One entry must be marked `default = true` when the list + /// has more than one; the default drives the auto-project on + /// `vw run` / `vw repl` / `vw test`. The CLI's + /// `--variant ` flag selects a non-default entry. + #[serde(default)] + pub variants: Vec, } -#[derive(Debug, Deserialize, Serialize)] -pub struct Dependency { - pub repo: String, +/// One entry in a workspace's `[[target-parts]]` list. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TargetPart { + /// Full Vivado part identifier, e.g. + /// `xcvp1202-vsva2785-2MHP-e-S`. + pub part: String, + /// `true` marks this entry as the default target part. Exactly + /// one entry must set this when the list has more than one + /// entry. A single-entry list may omit the flag — the sole + /// entry is implicitly the default. + #[serde(default)] + pub default: bool, +} + +/// Errors surfaced when validating or selecting a target part +/// from a workspace's `[[target-parts]]` list. Wrapped by the +/// `WorkspaceInfo` accessors so callers can render precise +/// diagnostics. +#[derive(Debug, thiserror::Error)] +pub enum TargetSelectError { + #[error( + "workspace has {count} `[[target-parts]]` entries but none are marked \ + `default = true`; add `default = true` to exactly one entry" + )] + NoDefault { count: usize }, + #[error( + "workspace has multiple `[[target-parts]]` entries marked \ + `default = true` ({defaults:?}); only one may be default" + )] + MultipleDefaults { defaults: Vec }, + #[error("no `[[target-parts]]` entry matches `{query}`")] + NoMatch { query: String }, + #[error( + "`{query}` matches multiple `[[target-parts]]` entries ({matches:?}); \ + disambiguate with a longer substring or the full part ID" + )] + Ambiguous { query: String, matches: Vec }, +} + +/// One entry in a workspace's `[[workspace.variants]]` list. +/// +/// A variant is a feature-flag-style selection that: +/// - pins a specific Vivado part (inline, not via cross-reference) +/// - optionally names an `exclusive` list of source-file globs +/// (workspace-relative) that ONLY compile when this variant +/// is the active one. +/// +/// Files NOT listed in any variant's `exclusive` set are shared — +/// they always contribute to `vhdl_design_sources` regardless +/// of the active variant. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Variant { + /// Human-facing selector, e.g. `vpk120` or `metro`. Matched + /// exactly by `--variant `. Must be unique within a + /// workspace's variants list. + pub name: String, + /// Full Vivado part identifier, e.g. + /// `xcvp1202-vsva2785-2MHP-e-S`. Drives the auto-project + /// when this variant is active. + pub part: String, + /// `true` marks this entry as the default variant. Exactly + /// one entry must set this when the list has more than one. + /// A single-entry list may omit the flag — the sole entry + /// is implicitly the default. #[serde(default)] - pub branch: Option, + pub default: bool, + /// Workspace-relative globs (e.g. `"hdl/ethernet-vpk120.vhd"` + /// or `"hdl/board-vpk120/**/*.vhd"`) matching files that + /// ONLY compile when this variant is active. Files that + /// don't match any variant's exclusive set are always shared. + #[serde(default)] + pub exclusive: Vec, +} + +/// Errors surfaced when validating or selecting a variant from a +/// workspace's `[[variants]]` list. Same shape as +/// [`TargetSelectError`] with variant-flavored messages. +#[derive(Debug, thiserror::Error)] +pub enum VariantSelectError { + #[error( + "workspace declares both `[[target-parts]]` and \ + `[[workspace.variants]]` — they're mutually exclusive; \ + variants own their parts inline, so remove `[[target-parts]]`" + )] + BothPartsAndVariants, + #[error( + "workspace has {count} `[[workspace.variants]]` entries but \ + none are marked `default = true`; add `default = true` to \ + exactly one entry" + )] + NoDefault { count: usize }, + #[error( + "workspace has multiple `[[workspace.variants]]` entries \ + marked `default = true` ({defaults:?}); only one may be default" + )] + MultipleDefaults { defaults: Vec }, + #[error("no `[[workspace.variants]]` entry named `{query}`")] + NoMatch { query: String }, + #[error( + "duplicate variant name `{name}` in \ + `[[workspace.variants]]` — variant names must be unique" + )] + DuplicateName { name: String }, +} + +impl WorkspaceInfo { + /// Return the default target part, if any. Empty list yields + /// `Ok(None)`. Single-entry list yields that entry as the + /// implicit default (regardless of the `default` flag). + /// Multi-entry list requires exactly one `default = true`. + pub fn default_target_part( + &self, + ) -> std::result::Result, TargetSelectError> { + match self.target_parts.len() { + 0 => Ok(None), + 1 => Ok(Some(self.target_parts[0].part.as_str())), + _ => { + let defaults: Vec<&TargetPart> = + self.target_parts.iter().filter(|p| p.default).collect(); + match defaults.len() { + 1 => Ok(Some(defaults[0].part.as_str())), + 0 => Err(TargetSelectError::NoDefault { + count: self.target_parts.len(), + }), + _ => Err(TargetSelectError::MultipleDefaults { + defaults: defaults + .iter() + .map(|p| p.part.clone()) + .collect(), + }), + } + } + } + } + + /// Resolve a CLI `--part ` selector against the + /// workspace's target parts. `None` returns the default (via + /// [`default_target_part`](Self::default_target_part)). `Some` + /// matches by exact part ID first; failing that, by unique + /// substring. Multiple substring matches, or zero matches, + /// error out. + pub fn select_target_part( + &self, + query: Option<&str>, + ) -> std::result::Result, TargetSelectError> { + let Some(q) = query else { + return self.default_target_part(); + }; + if let Some(exact) = self.target_parts.iter().find(|p| p.part == q) { + return Ok(Some(exact.part.as_str())); + } + let matches: Vec<&TargetPart> = self + .target_parts + .iter() + .filter(|p| p.part.contains(q)) + .collect(); + match matches.len() { + 1 => Ok(Some(matches[0].part.as_str())), + 0 => Err(TargetSelectError::NoMatch { + query: q.to_string(), + }), + _ => Err(TargetSelectError::Ambiguous { + query: q.to_string(), + matches: matches.iter().map(|p| p.part.clone()).collect(), + }), + } + } + + /// Return the default variant, if any. Empty list yields + /// `Ok(None)` (workspaces without variants). Single-entry + /// list yields that entry as the implicit default + /// (regardless of the `default` flag). Multi-entry list + /// requires exactly one `default = true`. + pub fn default_variant( + &self, + ) -> std::result::Result, VariantSelectError> { + match self.variants.len() { + 0 => Ok(None), + 1 => Ok(Some(&self.variants[0])), + _ => { + let defaults: Vec<&Variant> = + self.variants.iter().filter(|v| v.default).collect(); + match defaults.len() { + 1 => Ok(Some(defaults[0])), + 0 => Err(VariantSelectError::NoDefault { + count: self.variants.len(), + }), + _ => Err(VariantSelectError::MultipleDefaults { + defaults: defaults + .iter() + .map(|v| v.name.clone()) + .collect(), + }), + } + } + } + } + + /// Resolve a CLI `--variant ` selector against the + /// workspace's variants. `None` returns the default (via + /// [`default_variant`](Self::default_variant)). `Some` + /// matches by exact name only — unlike `--part`, no + /// substring fallback: variant names are short enough that + /// substring matching would be more confusing than useful. + pub fn select_variant( + &self, + query: Option<&str>, + ) -> std::result::Result, VariantSelectError> { + let Some(q) = query else { + return self.default_variant(); + }; + self.variants + .iter() + .find(|v| v.name == q) + .map(Some) + .ok_or_else(|| VariantSelectError::NoMatch { + query: q.to_string(), + }) + } +} + +/// Library-scope target metadata. Populated by `vw ip generate` +/// so downstream `vw check` can verify a consumer's target-part +/// is supported by every transitive dep without ever touching +/// Vivado at check time. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TargetsConfig { + /// Blessed patterns — `` entries whose + /// `lifeCycle` is `Production`, `Beta`, or `Pre-Production`. + /// A target-part that matches any of these is a clean pass; + /// the check emits no diagnostic for that dep. + /// + /// Every string is in the form `{}` + /// — see [`parse_target_pattern`] for the parse rules. Bare + /// family names are rejected; `vw ip generate` normalizes them + /// before writing. #[serde(default)] - pub commit: Option, + pub supported: Vec, + /// Explicitly-unsupported patterns — `` entries + /// with `lifeCycle="Not-Supported"`. Xilinx has attested that + /// the IP is NOT usable on parts matching these patterns; if + /// the target-part matches one, `vw check` fires an ERROR. + /// + /// Note: an entire component.xml with only Not-Supported + /// entries (e.g. an experimental IP still in incubation) + /// leaves `supported = []` and populates only this list. In + /// that case the check treats a NON-match here as "unblessed + /// but not forbidden" — a warning, not an error. + #[serde(default, rename = "not-supported")] + pub not_supported: Vec, +} + +/// A parsed `family{regex}` target pattern. The regex is +/// pre-compiled at parse time so hot paths (validator, LSP +/// diagnostics) don't re-compile per-check. +#[derive(Debug, Clone)] +pub struct TargetPattern { + /// The family word out front — `versal`, `artix7`, etc. Kept + /// verbatim for diagnostics: "clk-wizard supports the versal + /// family; your target `xcvm3358…` isn't in that family." + pub family: String, + /// The compiled regex from inside the braces. Applied against + /// the consumer's target-part string; a match means the + /// library supports the target. + pub regex: regex::Regex, + /// Original source text (`versal{xcvm3(.*)}`) — used to + /// point diagnostics back at the exact vw.toml line. + pub raw: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum TargetParseError { + #[error("target pattern `{raw}` is missing the `{{regex}}` part")] + MissingBraces { raw: String }, + #[error("target pattern `{raw}`: {source}")] + BadRegex { + raw: String, + #[source] + source: regex::Error, + }, +} + +/// Parse one entry of `[targets].supported` — the form +/// `{}`. Bare-family entries (no braces) are +/// rejected with [`TargetParseError::MissingBraces`]; the +/// generator's job is to normalize them into brace form before +/// they reach downstream consumers. +/// Snapshot of per-dep target-support metadata used by the +/// project-vs-dep compatibility check. Populated from each dep's +/// `[targets] supported` list at the entry workspace's transitive +/// walk time; used later by [`check_target_compatibility`] to +/// verify a given target-part is supported. +/// +/// Deps with no `[targets]` at all are `Vec::new()` — the check +/// treats an empty list as "universal / no constraint," so +/// non-IP libraries (@vw, @test) don't need to declare anything. +/// +/// Parse errors during pattern compile are turned into +/// `(dep_name, error)` entries in the second field so callers +/// can surface them as diagnostics without dropping the whole +/// dep's info. +#[derive(Debug, Default)] +pub struct DepTargets { + /// Blessed (`Production`/`Beta`/`Pre-Production`) patterns + /// per dep. A target-part matching any of these clears the + /// check clean. + pub per_dep: HashMap>, + /// Explicitly `Not-Supported` patterns per dep. A target-part + /// matching any of these is an ERROR — Xilinx has attested + /// the IP is not usable there. + pub per_dep_not_supported: HashMap>, + pub errors: Vec<(String, TargetParseError)>, +} + +/// Walk the entry workspace's transitive deps and collect each +/// dep's `[targets].supported` patterns. Returns a +/// [`DepTargets`] snapshot ready to feed into +/// [`check_target_compatibility`]. +/// +/// Skips deps whose `vw.toml` doesn't load — they contribute +/// nothing (treated as universal). This matches the "empty = +/// universal" policy the compat check applies. +pub fn collect_dep_targets(entry_workspace_dir: &Utf8Path) -> DepTargets { + let mut out = DepTargets::default(); + let Ok(paths) = transitive_dep_cache_paths(entry_workspace_dir) else { + return out; + }; + for (name, path) in paths { + let Ok(path_utf8) = Utf8PathBuf::from_path_buf(path) else { + continue; + }; + let Ok(cfg) = load_workspace_config(&path_utf8) else { + continue; + }; + let Some(targets) = cfg.targets else { + out.per_dep.insert(name.clone(), Vec::new()); + out.per_dep_not_supported.insert(name, Vec::new()); + continue; + }; + let mut compiled: Vec = Vec::new(); + for raw in &targets.supported { + match parse_target_pattern(raw) { + Ok(p) => compiled.push(p), + Err(e) => out.errors.push((name.clone(), e)), + } + } + let mut compiled_not_supported: Vec = Vec::new(); + for raw in &targets.not_supported { + match parse_target_pattern(raw) { + Ok(p) => compiled_not_supported.push(p), + Err(e) => out.errors.push((name.clone(), e)), + } + } + out.per_dep.insert(name.clone(), compiled); + out.per_dep_not_supported + .insert(name, compiled_not_supported); + } + out +} + +/// Severity of a target-compatibility mismatch. The two-tier +/// treatment reflects what `` actually +/// means in Vivado's IP catalog: the family list is a lifeCycle- +/// tagged compatibility MATRIX, not a hard availability filter. +/// Vivado exposes IPs even for families that aren't listed. So: +/// +/// - `NotSupported` — target-part matched a Not-Supported entry. +/// Xilinx explicitly attests the IP won't work here. Error. +/// - `Unblessed` — target-part matched no entry at all (neither +/// supported nor not-supported). Warning: the IP MAY work but +/// Xilinx hasn't blessed the combination. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetMismatchKind { + NotSupported, + Unblessed, +} + +/// One target-compatibility observation. Emitted by +/// [`check_target_compatibility`]; the caller uses `kind` to +/// choose diagnostic severity (error vs warning). +#[derive(Debug, Clone)] +pub struct TargetMismatch { + /// Dep name (e.g. `clk-wizard`). + pub dep: String, + /// The target part string that was checked. + pub target_part: String, + /// Family cues gathered from the dep's BLESSED patterns — + /// its `[targets] supported` list. Empty when the dep has no + /// blessed patterns (e.g. clk-wizard v1.0, which has only + /// `not-supported` entries). The diagnostic message uses this + /// to say "clk-wizard blesses parts in the following families; + /// yours isn't among them." + pub supported_families: Vec, + /// Family cues gathered from the dep's BAN LIST — its + /// `[targets] not-supported` list. Reported separately from + /// `supported_families` because the semantics differ: + /// families named in `supported` are blessed for use, families + /// named ONLY in `not-supported` are ones Xilinx has attested + /// don't work (at least for specific parts). Reporting them + /// as "declared families" without qualification is misleading + /// (see #vw-check clarity). + pub not_supported_families: Vec, + /// Whether Xilinx explicitly forbids this combination or + /// simply hasn't blessed it. + pub kind: TargetMismatchKind, +} + +/// Return each dep's target-compatibility observation against +/// `target_part`. Deps with no `[targets]` at all (`supported` +/// AND `not_supported` both empty) contribute nothing — treated +/// as universal. When `target_part` is `None` (library workspace, +/// no project target) the check is a no-op. +/// +/// Decision matrix per dep: +/// - matches a Not-Supported pattern → `NotSupported` (error). +/// Trumps a supported match — an explicit ban wins. +/// - matches a Supported pattern → clean pass, no observation. +/// - matches neither, but the dep has SOME patterns declared → +/// `Unblessed` (warning). +pub fn check_target_compatibility( + target_part: Option<&str>, + dep_targets: &DepTargets, +) -> Vec { + let Some(part) = target_part else { + return Vec::new(); + }; + let mut out = Vec::new(); + for (dep, supported) in &dep_targets.per_dep { + let not_supported = dep_targets + .per_dep_not_supported + .get(dep) + .map(|v| v.as_slice()) + .unwrap_or(&[]); + if supported.is_empty() && not_supported.is_empty() { + continue; + } + let sup_families = dedup_families(supported); + let ns_families = dedup_families(not_supported); + let hit_not_supported = + not_supported.iter().any(|p| p.regex.is_match(part)); + if hit_not_supported { + out.push(TargetMismatch { + dep: dep.clone(), + target_part: part.to_string(), + supported_families: sup_families, + not_supported_families: ns_families, + kind: TargetMismatchKind::NotSupported, + }); + continue; + } + let hit_supported = supported.iter().any(|p| p.regex.is_match(part)); + if hit_supported { + continue; + } + out.push(TargetMismatch { + dep: dep.clone(), + target_part: part.to_string(), + supported_families: sup_families, + not_supported_families: ns_families, + kind: TargetMismatchKind::Unblessed, + }); + } + out +} + +fn dedup_families(patterns: &[TargetPattern]) -> Vec { + let mut families: Vec = + patterns.iter().map(|p| p.family.clone()).collect(); + families.sort(); + families.dedup(); + families +} + +pub fn parse_target_pattern( + raw: &str, +) -> std::result::Result { + let Some((family, rest)) = raw.split_once('{') else { + return Err(TargetParseError::MissingBraces { + raw: raw.to_string(), + }); + }; + let Some(regex_src) = rest.strip_suffix('}') else { + return Err(TargetParseError::MissingBraces { + raw: raw.to_string(), + }); + }; + // Anchor at start: `xcvm3(.*)` should match ONLY parts that + // begin with `xcvm3`, not any string containing `xcvm3` mid- + // way. End anchor is optional because the source patterns + // themselves use `(.*)` to slop up the suffix. + let anchored = format!("^{regex_src}"); + let regex = regex::Regex::new(&anchored).map_err(|e| { + TargetParseError::BadRegex { + raw: raw.to_string(), + source: e, + } + })?; + Ok(TargetPattern { + family: family.to_string(), + regex, + raw: raw.to_string(), + }) +} + +/// How a workspace dependency identifies its source. Currently a git +/// repo or a local filesystem path; the natural future addition is a +/// registry-resolved variant (`Registry { name, version }`) once a +/// crates.io-like index exists. +/// +/// `#[serde(untagged)]` keeps the `vw.toml` ergonomics that came +/// before: an entry with `repo = "..."` reads as `Git`, an entry with +/// `path = "..."` reads as `Path`. New variants need new +/// non-ambiguous required keys for serde to discriminate cleanly. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum DependencySource { + Git { + repo: String, + #[serde(default)] + branch: Option, + #[serde(default)] + commit: Option, + #[serde(default)] + submodules: bool, + }, + Path { + path: PathBuf, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Dependency { + #[serde(flatten)] + pub source: DependencySource, #[serde(default)] pub src: Vec, #[serde(default)] @@ -216,11 +801,51 @@ pub struct Dependency { #[serde(default)] pub sim_only: bool, #[serde(default)] - pub submodules: bool, - #[serde(default)] pub exclude: Vec, } +impl Dependency { + pub fn is_local(&self) -> bool { + matches!(self.source, DependencySource::Path { .. }) + } + + /// Git-only accessor: the upstream repo URL. + pub fn repo(&self) -> Option<&str> { + match &self.source { + DependencySource::Git { repo, .. } => Some(repo), + DependencySource::Path { .. } => None, + } + } + + pub fn branch(&self) -> Option<&str> { + match &self.source { + DependencySource::Git { branch, .. } => branch.as_deref(), + DependencySource::Path { .. } => None, + } + } + + pub fn commit(&self) -> Option<&str> { + match &self.source { + DependencySource::Git { commit, .. } => commit.as_deref(), + DependencySource::Path { .. } => None, + } + } + + pub fn submodules(&self) -> bool { + match &self.source { + DependencySource::Git { submodules, .. } => *submodules, + DependencySource::Path { .. } => false, + } + } + + pub fn local_path(&self) -> Option<&Path> { + match &self.source { + DependencySource::Path { path } => Some(path.as_path()), + DependencySource::Git { .. } => None, + } + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct LockFile { pub dependencies: HashMap, @@ -391,6 +1016,48 @@ pub fn get_access_credentials_from_netrc( Ok(None) } +/// Look up netrc credentials for a git-repository URL. Returns +/// `None` when the URL has no parseable host, or the host has no +/// entry in `~/.netrc`. Never errors — a missing / malformed +/// netrc is treated as "no credentials," matching the "unauthenticated +/// clone" path. +/// +/// Callers: `vw-cli::Commands::Update`, `vw-vivado`'s +/// auto-update RPC handler. +pub fn get_access_credentials_for_repo(repo_url: &str) -> Option { + let hostname = extract_hostname_from_repo_url(repo_url).ok()?; + get_access_credentials_from_netrc(&hostname).ok().flatten() +} + +/// Scan the workspace's declared git dependencies for the first +/// one that has netrc credentials. Cheap and pragmatic: one set +/// of credentials feeds the whole `update_workspace_with_token` +/// pass (all deps to the same host share the same login), and +/// most workspaces target a single provider (github, gitea, …) +/// so the first match is usually the right one. +/// +/// `include_test = true` also scans `[test-dependencies]`, +/// matching the same shape [`vhdl_dependency_sources_with_test`] +/// uses. Returns `None` when no git dep has creds — e.g. when +/// every git URL is a public repo. +pub fn get_access_credentials_for_workspace( + workspace_dir: &Utf8Path, + include_test: bool, +) -> Option { + let cfg = load_workspace_config(workspace_dir).ok()?; + for dep in cfg + .dependencies + .values() + .chain(cfg.test_dependencies.values().filter(|_| include_test)) + { + let Some(repo) = dep.repo() else { continue }; + if let Some(creds) = get_access_credentials_for_repo(repo) { + return Some(creds); + } + } + None +} + /// Get access token for a given host from the netrc file. /// /// This function reads the user's .netrc file and looks for credentials @@ -450,8 +1117,12 @@ pub fn init_workspace(workspace_dir: &Utf8Path, name: String) -> Result<()> { workspace: WorkspaceInfo { name, version: "0.1.0".to_string(), + target_parts: Vec::new(), + variants: Vec::new(), }, dependencies: HashMap::new(), + test_dependencies: HashMap::new(), + targets: None, tools: None, }; @@ -502,76 +1173,113 @@ pub async fn update_workspace_with_token( let mut update_info = Vec::new(); - for (name, dep) in &config.dependencies { - // Use credentials passed from caller + // Fetch both regular and test dependencies. Same lockfile + // section (locks are name → commit; there's no downstream code + // that cares whether a locked entry came from `[dependencies]` + // vs `[test-dependencies]`). VHDL libraries still get indexed + // for both, matching Cargo's model where dev-deps are visible + // to test builds. + for (name, dep) in config + .test_dependencies + .iter() + .chain(config.dependencies.iter()) + { let creds = credentials .as_ref() .map(|c| (c.username.as_str(), c.password.as_str())); - let commit_sha = resolve_dependency_commit( - &dep.repo, - &dep.branch, - &dep.commit, - creds, - ) - .await - .map_err(|e| VwError::Dependency { - message: format!( - "Failed to resolve commit for dependency '{name}': {e}" - ), - })?; - - let dep_path = deps_dir.join(format!("{name}-{commit_sha}")); - - let was_cached = dep_path.exists(); - - if !was_cached { - download_dependency( - &dep.repo, - &commit_sha, - &dep.src, - &dep_path, - dep.recursive, - &dep.exclude, - dep.submodules, - creds, - ) - .await - .map_err(|e| VwError::Dependency { - message: format!("Failed to download dependency '{name}': {e}"), - })?; - } - - update_info.push(DependencyUpdateInfo { - name: name.clone(), - commit: commit_sha.clone(), - was_cached, - }); - - lock_file.dependencies.insert( - name.clone(), - LockedDependency { - repo: dep.repo.clone(), - commit: commit_sha.clone(), - src: dep.src.clone(), - path: PathBuf::from(format!("{name}-{commit_sha}")), - recursive: dep.recursive, - sim_only: dep.sim_only, - submodules: dep.submodules, - exclude: dep.exclude.clone(), - }, - ); + // Decide the on-disk location for this dep: cache directory + // for git deps (downloaded if missing), or the declared + // filesystem path for local deps. Local deps don't get a lock + // entry — there's no commit to pin. + let dep_path = match &dep.source { + DependencySource::Git { + repo, + branch, + commit, + submodules, + } => { + let commit_sha = + resolve_dependency_commit(repo, branch, commit, creds) + .await + .map_err(|e| VwError::Dependency { + message: format!( + "Failed to resolve commit for dependency '{name}': {e}" + ), + })?; + let dep_path = deps_dir.join(format!("{name}-{commit_sha}")); + let was_cached = dep_path.exists(); + if !was_cached { + download_dependency( + repo, + &commit_sha, + &dep.src, + &dep_path, + dep.recursive, + &dep.exclude, + *submodules, + creds, + ) + .await + .map_err(|e| VwError::Dependency { + message: format!( + "Failed to download dependency '{name}': {e}" + ), + })?; + } + update_info.push(DependencyUpdateInfo { + name: name.clone(), + commit: commit_sha.clone(), + was_cached, + }); + lock_file.dependencies.insert( + name.clone(), + LockedDependency { + repo: repo.clone(), + commit: commit_sha.clone(), + src: dep.src.clone(), + path: PathBuf::from(format!("{name}-{commit_sha}")), + recursive: dep.recursive, + sim_only: dep.sim_only, + submodules: *submodules, + exclude: dep.exclude.clone(), + }, + ); + dep_path + } + DependencySource::Path { path } => { + if !path.exists() { + return Err(VwError::Dependency { + message: format!( + "Local dependency '{name}' path does not exist: {}", + path.display() + ), + }); + } + update_info.push(DependencyUpdateInfo { + name: name.clone(), + commit: "local".into(), + was_cached: true, + }); + path.clone() + } + }; - // Find VHDL files in the cached dependency directory + // VHDL libraries: same treatment regardless of source. Local + // deps' file paths are kept absolute (they aren't relative to + // the per-user cache); git deps stay portable. let vhdl_files = find_vhdl_files(&dep_path, dep.recursive, &dep.exclude)?; if !vhdl_files.is_empty() { - let portable_files = - vhdl_files.into_iter().map(make_path_portable).collect(); + let files = if dep.is_local() { + vhdl_files + } else { + vhdl_files.into_iter().map(make_path_portable).collect() + }; vhdl_ls_config.libraries.insert( name.clone(), VhdlLsLibrary { - files: portable_files, + files, exclude: None, is_third_party: None, }, @@ -643,8 +1351,12 @@ pub async fn add_dependency_with_token( workspace: WorkspaceInfo { name: "workspace".to_string(), version: "0.1.0".to_string(), + target_parts: Vec::new(), + variants: Vec::new(), }, dependencies: HashMap::new(), + test_dependencies: HashMap::new(), + targets: None, tools: None, } }); @@ -660,13 +1372,15 @@ pub async fn add_dependency_with_token( let src_paths = vec![src.unwrap_or_else(|| ".".to_string())]; let dependency = Dependency { - repo: repo.clone(), - branch, - commit, + source: DependencySource::Git { + repo: repo.clone(), + branch, + commit, + submodules: false, + }, src: src_paths, recursive, sim_only, - submodules: false, exclude: Vec::new(), }; @@ -721,12 +1435,14 @@ pub fn clear_cache(workspace_dir: &Utf8Path) -> Result> { Ok(cleared) } -/// List all dependencies in the workspace. +/// List all dependencies in the workspace (both regular and +/// test-dependencies). Callers that want to render them in +/// separate sections can filter on [`DependencyInfo::is_test`]. pub fn list_dependencies( workspace_dir: &Utf8Path, ) -> Result> { let config = load_workspace_config(workspace_dir)?; - if config.dependencies.is_empty() { + if config.dependencies.is_empty() && config.test_dependencies.is_empty() { return Ok(Vec::new()); } @@ -734,44 +1450,50 @@ pub fn list_dependencies( let lock_file = load_lock_file(workspace_dir).ok(); let mut deps = Vec::new(); - for (name, dep) in &config.dependencies { - let version_info = match &lock_file { - Some(lock) => { - if let Some(locked_dep) = lock.dependencies.get(name) { - VersionInfo::Locked { - commit: locked_dep.commit.clone(), - } - } else { - // Not yet resolved, show branch/commit from config - match (&dep.branch, &dep.commit) { - (Some(branch), None) => VersionInfo::Branch { - branch: branch.clone(), - }, - (None, Some(commit)) => VersionInfo::Commit { - commit: commit.clone(), - }, - _ => VersionInfo::Unknown, - } - } + for (name, dep, is_test) in config + .dependencies + .iter() + .map(|(n, d)| (n, d, false)) + .chain(config.test_dependencies.iter().map(|(n, d)| (n, d, true))) + { + let (source_label, version_info) = match &dep.source { + DependencySource::Path { path } => { + (path.display().to_string(), VersionInfo::Local) } - None => { - // No lock file, show branch/commit from config - match (&dep.branch, &dep.commit) { - (Some(branch), None) => VersionInfo::Branch { - branch: branch.clone(), - }, - (None, Some(commit)) => VersionInfo::Commit { - commit: commit.clone(), + DependencySource::Git { + repo, + branch, + commit, + .. + } => { + let from_config = + || match (branch.as_deref(), commit.as_deref()) { + (Some(b), None) => { + VersionInfo::Branch { branch: b.into() } + } + (None, Some(c)) => { + VersionInfo::Commit { commit: c.into() } + } + _ => VersionInfo::Unknown, + }; + let version = match &lock_file { + Some(lock) => match lock.dependencies.get(name) { + Some(locked_dep) => VersionInfo::Locked { + commit: locked_dep.commit.clone(), + }, + None => from_config(), }, - _ => VersionInfo::Unknown, - } + None => from_config(), + }; + (repo.clone(), version) } }; deps.push(DependencyInfo { name: name.clone(), - repo: dep.repo.clone(), + source: source_label, version: version_info, + is_test, }); } @@ -781,15 +1503,29 @@ pub fn list_dependencies( #[derive(Debug, Clone)] pub struct DependencyInfo { pub name: String, - pub repo: String, + /// User-facing source description: the repo URL for git deps, + /// the local path for path deps. + pub source: String, pub version: VersionInfo, + /// True when this entry came from `[test-dependencies]` rather + /// than `[dependencies]`. Test-deps only affect `vw test`; other + /// commands see them but don't act on them. + pub is_test: bool, } #[derive(Debug, Clone)] pub enum VersionInfo { - Branch { branch: String }, - Commit { commit: String }, - Locked { commit: String }, + Branch { + branch: String, + }, + Commit { + commit: String, + }, + Locked { + commit: String, + }, + /// Local filesystem dependency — no commit to pin. + Local, Unknown, } @@ -984,53 +1720,1276 @@ pub struct TestbenchInfo { pub path: PathBuf, } -pub struct RecordProcessor { - pub vhdl_std: VhdlStandard, - pub symbols: HashMap, - pub symbol_to_file: HashMap, - pub tagged_names: HashSet, - pub file_info: HashMap, - pub target_attr: String, +/// Recursively enumerate every `*.htcl` file under +/// `/test/`. Skips hidden directories (`.git`, +/// `.vscode`, etc.) and any directory literally named `target` +/// (the vw-standard build-output location, matches the shape used +/// by `vw::make_wrapper`). Returns file paths sorted +/// lexicographically for deterministic test order. +/// +/// Returns an empty vec when `/test/` doesn't exist +/// — matches `vw test`'s expected "no tests found" UX rather than +/// erroring. +/// One VHDL source shipped by a dep: the target VHDL library name +/// it should compile into, plus the absolute on-disk path. +/// +/// Library name is derived from the dep name with hyphens replaced +/// by underscores — the same rule the `vhdl_ls.toml` generator +/// already uses, matching NVC/Vivado convention. This will move to +/// a dep-controlled override later; for now the rule is uniform. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VhdlDepSource { + pub library: String, + pub path: PathBuf, } -const RECORD_PARSE_ATTRIBUTE: &str = "serialize_rust"; -impl RecordProcessor { - pub fn new(std: VhdlStandard) -> Self { - Self { - vhdl_std: std, - symbols: HashMap::new(), - symbol_to_file: HashMap::new(), - tagged_names: HashSet::new(), - file_info: HashMap::new(), - target_attr: RECORD_PARSE_ATTRIBUTE.to_string(), - } - } +/// Enumerate every VHDL source published by any transitive dep +/// of `workspace_dir`. Files are absolute paths pointing at the +/// dep's materialized cache (or its local path, for +/// `path = "..."` deps). +/// +/// A dep only contributes when it explicitly declares a `src` +/// field in its vw.toml entry. Path deps for htcl-only libraries +/// (e.g. `[dependencies.vw] path = "..."`) omit `src` and +/// therefore publish no VHDL — otherwise a recursive scan would +/// happily pick up the library's OWN `target/`, `test/`, and +/// other non-shipped subtrees. +/// +/// For each `src` entry we honor the dep's `recursive` / +/// `exclude` config, matching what `vw update` uses when it +/// populates the cache from a git dep. For path deps the same +/// filtering runs at read time (no copy step) so both dep kinds +/// present identical surfaces. +/// +/// Depends on the deps being present on disk — call after +/// `vw update`. Missing dep caches are silently skipped so a +/// half-updated workspace still gives a partial result rather +/// than erroring mid-enumeration. +/// +/// Sort order: library name, then path within library. Callers +/// that need topological order do their own downstream analysis. +pub fn vhdl_dependency_sources( + workspace_dir: &Utf8Path, +) -> Result> { + vhdl_dependency_sources_ext(workspace_dir, false, false) } -// ============================================================================ -// File Cache - Reduces redundant file reads during build -// ============================================================================ +/// Detect whether the workspace has any git deps declared in +/// vw.toml but missing from vw.lock — the state where the user +/// hasn't yet run `vw update`, or the lockfile has been +/// truncated / wiped. Cheap check (loads the config + lockfile +/// once, no network); consumers use it to decide whether to +/// auto-invoke [`update_workspace`]. +/// +/// `include_test` mirrors the same flag on the enumeration side +/// so a test-deps-only unlocked entry is caught when the caller +/// intends to enumerate test-deps too. +pub fn workspace_has_unlocked_git_deps( + workspace_dir: &Utf8Path, + include_test: bool, +) -> Result { + let cfg = load_workspace_config(workspace_dir)?; + let git_names: Vec<&str> = cfg + .dependencies + .iter() + .chain(cfg.test_dependencies.iter().filter(|_| include_test)) + .filter(|(_, dep)| matches!(dep.source, DependencySource::Git { .. })) + .map(|(name, _)| name.as_str()) + .collect(); + if git_names.is_empty() { + return Ok(false); + } + match load_lock_file(workspace_dir) { + Ok(lock) => Ok(git_names + .iter() + .any(|n| !lock.dependencies.contains_key(*n))), + // No lockfile at all: every git dep is unlocked. + Err(_) => Ok(true), + } +} -/// Cache for parsed file data to avoid redundant parsing during builds. -/// Only caches parsed results, not raw file contents. -pub struct FileCache { - dependencies: HashMap>, - provided_symbols: HashMap>, - entities: HashMap>, +/// Same as [`vhdl_dependency_sources`] but optionally includes +/// the entry workspace's `[test-dependencies]`. Cargo-parity +/// semantic for `dev-dependencies`: test-deps are private to the +/// workspace that declares them. Recursed-into workspaces are +/// walked with `include_test = false` so a dep's own test-deps +/// aren't pulled into your consumer. +/// +/// The test runner uses `include_test = true` so htcl tests +/// under `test/` can enumerate `[test-dependencies]` VHDL +/// alongside regular deps. Production `vw run` uses `false` so +/// test-only VHDL doesn't sneak into a synth flow. +pub fn vhdl_dependency_sources_with_test( + workspace_dir: &Utf8Path, + include_test: bool, +) -> Result> { + vhdl_dependency_sources_ext(workspace_dir, include_test, false) } -impl FileCache { - pub fn new() -> Self { - Self { - dependencies: HashMap::new(), - provided_symbols: HashMap::new(), - entities: HashMap::new(), +/// Full-shape enumeration primitive. `exclude_sim_only = true` +/// drops every dep whose vw.toml sets `sim_only = true` (unisim, +/// xpm, etc.) — used by synth flows that need the design-only +/// surface. `include_test` mirrors the same knob on the sibling +/// wrapper. +pub fn vhdl_dependency_sources_ext( + workspace_dir: &Utf8Path, + include_test: bool, + exclude_sim_only: bool, +) -> Result> { + // Walk the entry workspace's deps + transitive deps. We need + // each dep's Dependency config (for src/recursive/exclude), + // so `transitive_dep_cache_paths` (name → path only) isn't + // enough — walk the graph ourselves. + let mut out = Vec::new(); + let mut visited: std::collections::HashSet = + std::collections::HashSet::new(); + let mut queue: Vec<(Utf8PathBuf, bool)> = + vec![(workspace_dir.to_path_buf(), include_test)]; + while let Some((ws, want_test)) = queue.pop() { + if !visited.insert(ws.as_std_path().to_path_buf()) { + continue; + } + let Ok(cfg) = load_workspace_config(&ws) else { + continue; + }; + // Combine regular + test deps for this level. + let deps: Vec<(String, Dependency)> = cfg + .dependencies + .into_iter() + .chain(cfg.test_dependencies.into_iter().filter(|_| want_test)) + .collect(); + for (name, dep) in deps { + // Skip sim-only deps when the caller wants a + // synth-clean surface. Filter happens BEFORE the + // transitive-workspace push, so if a dep is a + // workspace whose only purpose is sim glue, we + // don't descend and pick up its transitive deps + // either. + if exclude_sim_only && dep.sim_only { + continue; + } + let Some(dep_path) = + resolve_dep_source_path(workspace_dir, &ws, &name, &dep)? + else { + continue; + }; + // If the dep is itself a workspace, follow it too so + // we pick up its own deps' VHDL. Recursed workspaces + // never see their own test-deps — Cargo parity. + if dep_path.join("vw.toml").is_file() { + if let Ok(u) = Utf8PathBuf::from_path_buf(dep_path.clone()) { + queue.push((u, false)); + } + } + let files = enumerate_dep_vhdl_files(&dep_path, &dep)?; + if files.is_empty() { + continue; + } + let library = library_name_for_dep(&name); + for path in files { + out.push(VhdlDepSource { + library: library.clone(), + path, + }); + } } } + out.sort_by(|a, b| a.library.cmp(&b.library).then(a.path.cmp(&b.path))); + Ok(out) +} - /// Get cached file dependencies, reading and parsing file if not cached. - pub fn get_dependencies(&mut self, path: &Path) -> Result<&Vec> { - match self.dependencies.entry(path.to_path_buf()) { +/// Resolve one dep's on-disk root. Local (`path = "..."`) deps +/// point at the user's tree; git deps resolve through the +/// workspace's lockfile. Returns `Ok(None)` when the dep is a +/// git dep the caller hasn't `vw update`-d yet — the enumeration +/// treats that as "no VHDL" rather than erroring, since a half- +/// updated workspace shouldn't gate every downstream call. +fn resolve_dep_source_path( + entry_workspace_dir: &Utf8Path, + parent_workspace_dir: &Utf8Path, + name: &str, + dep: &Dependency, +) -> Result> { + if let Some(p) = dep.local_path() { + // Relative path deps resolve against the workspace that + // DECLARES them (same rule Cargo uses). Absolute paths + // pass through unchanged. This lets a workspace ship a + // portable path-dep like `path = "test/fixtures/foo"` + // without hard-coding a machine-specific prefix. + if p.is_absolute() { + return Ok(Some(p.to_path_buf())); + } + return Ok(Some(parent_workspace_dir.as_std_path().join(p))); + } + // Git dep — look up the resolved cache path in the lockfile + // of the ENTRY workspace (only the entry has a meaningful + // lockfile; transitive walks reuse the entry's pins for + // Cargo-parity). + let Ok(lock) = load_lock_file(entry_workspace_dir) else { + return Ok(None); + }; + let Some(locked) = lock.dependencies.get(name) else { + return Ok(None); + }; + Ok(Some(resolve_dep_path(&locked.path)?)) +} + +/// Enumerate every VHDL file a dep publishes, honoring its +/// declared `src` / `recursive` / `exclude` filters. Empty when +/// `dep.src` is empty (htcl-only dep, publishes no VHDL). +/// +/// Two dep kinds diverge here: +/// - **Git deps** cache into `~/.vw/deps/-/` with the +/// `src` prefix STRIPPED at copy time. `copy_vhdl_files_glob` +/// flattens away the source repo's directory structure. So +/// applying `src` here as a subdirectory path finds nothing — +/// we just walk the whole cache dir recursively (its contents +/// were already filtered by the update step). +/// - **Path deps** point at an unmodified checkout of the dep's +/// tree, so `src` still maps to a real subdirectory. +fn enumerate_dep_vhdl_files( + dep_root: &Path, + dep: &Dependency, +) -> Result> { + if dep.src.is_empty() { + return Ok(Vec::new()); + } + // Git-dep cache: flattened at copy time — walk everything and + // apply the exclude patterns (which are structure-relative, so + // they still work as-is over the flat tree). + if !dep.is_local() { + let mut files = + find_vhdl_files(dep_root, /*recursive=*/ true, &[])?; + if !dep.exclude.is_empty() { + let exclude_patterns: Vec = dep + .exclude + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect(); + files.retain(|f| { + let rel = f.strip_prefix(dep_root).unwrap_or(f); + let rel_str = rel.to_string_lossy(); + !exclude_patterns.iter().any(|p| p.matches(&rel_str)) + }); + } + files.sort(); + files.dedup(); + return Ok(files); + } + // Path dep: honor src patterns against the real tree. + let exclude_patterns: Vec = dep + .exclude + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect(); + let mut out = Vec::new(); + for src_pattern in &dep.src { + let src_path = dep_root.join(src_pattern); + let candidates = if src_path.is_dir() { + let base = + src_path.to_str().ok_or_else(|| VwError::FileSystem { + message: "Invalid UTF-8 in dep src path".to_string(), + })?; + let mut cands = Vec::new(); + let patterns = if dep.recursive { + vec![format!("{base}/**/*.vhd"), format!("{base}/**/*.vhdl")] + } else { + vec![format!("{base}/*.vhd"), format!("{base}/*.vhdl")] + }; + for p in patterns { + let entries = + glob::glob(&p).map_err(|e| VwError::FileSystem { + message: format!("Invalid glob pattern '{p}': {e}"), + })?; + for entry in entries.flatten() { + cands.push((src_path.clone(), entry)); + } + } + cands + } else if src_path.is_file() { + vec![( + src_path + .parent() + .ok_or_else(|| VwError::FileSystem { + message: "dep src file has no parent".to_string(), + })? + .to_path_buf(), + src_path.clone(), + )] + } else { + // Glob pattern rooted at the dep root — exclude + // patterns match relative to the dep root here. + let base = dep_root.to_path_buf(); + let pat = src_path + .to_str() + .ok_or_else(|| VwError::FileSystem { + message: "Invalid UTF-8 in dep src glob".to_string(), + })? + .to_string(); + let entries = + glob::glob(&pat).map_err(|e| VwError::FileSystem { + message: format!("Invalid glob pattern '{pat}': {e}"), + })?; + entries.flatten().map(|e| (base.clone(), e)).collect() + }; + for (strip_prefix, path) in candidates { + if !path.is_file() { + continue; + } + let ext = path.extension().and_then(|e| e.to_str()); + if ext != Some("vhd") && ext != Some("vhdl") { + continue; + } + if !exclude_patterns.is_empty() { + let rel = path.strip_prefix(&strip_prefix).unwrap_or(&path); + let rel_str = rel.to_string_lossy(); + if exclude_patterns.iter().any(|p| p.matches(&rel_str)) { + continue; + } + } + out.push(path); + } + } + out.sort(); + out.dedup(); + Ok(out) +} + +/// Enumerate every VHDL source under `/hdl/` +/// (recursively). These are the workspace's own design sources, +/// as distinct from IP wrappers (which live under `target/ip/` and +/// are enumerated by a separate helper) and testbenches (which +/// live under `bench/`). +/// +/// Returns an empty vec when `/hdl/` doesn't exist +/// — a freshly-scaffolded workspace hasn't checked anything in +/// yet, and that's not an error. +pub fn vhdl_design_sources(workspace_dir: &Utf8Path) -> Result> { + vhdl_design_sources_for_variant(workspace_dir, None) +} + +/// Enumerate every VHDL source under `/hdl/**` and +/// filter by the `exclusive` file lists on the workspace's +/// variants: +/// +/// - A file is "variant-owned" if it matches any `exclusive` +/// glob on ANY variant (across the whole list). +/// - Variant-owned files contribute ONLY when their owning +/// variant is the active one. +/// - Files not in any `exclusive` set are shared and always +/// contribute. +/// +/// `active_variant` is the CURRENTLY active variant's name; when +/// `None`, no variant is active (used by the analyzer's default +/// path and by tools that don't yet flow a variant selection +/// through). Under `None`, variant-owned files are still +/// excluded — otherwise a variant-mode workspace would drag +/// every other variant's exclusive sources into the surface. +/// +/// Empty vec when `/hdl/` doesn't exist. +pub fn vhdl_design_sources_for_variant( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, +) -> Result> { + let hdl_dir = workspace_dir.join("hdl"); + if !hdl_dir.exists() { + return Ok(Vec::new()); + } + let mut files = + find_vhdl_files(hdl_dir.as_std_path(), /*recursive=*/ true, &[])?; + files.sort(); + // Compile each variant's `exclusive` globs relative to the + // workspace root. Empty variants list → nothing to filter, + // early-return keeps the common no-variants path cheap. + let cfg = match load_workspace_config(workspace_dir) { + Ok(c) => c, + // No workspace config → can't know about variants, skip filter. + Err(_) => return Ok(files), + }; + if cfg.workspace.variants.is_empty() { + return Ok(files); + } + let owner = build_variant_ownership(workspace_dir, &cfg.workspace)?; + files.retain(|path| { + match owner.owner_of(path) { + // Shared file — always keep. + None => true, + // Variant-owned file — keep iff active variant owns it. + Some(name) => Some(name) == active_variant, + } + }); + Ok(files) +} + +/// Precomputed variant-ownership index. Each entry maps a +/// canonicalized absolute path to the variant that "owns" it +/// (the first variant whose `exclusive` glob matched during the +/// build). Files not present in the map are shared. +struct VariantOwnership { + /// Absolute path → owning variant name. + owners: std::collections::HashMap, +} + +impl VariantOwnership { + fn owner_of(&self, path: &Path) -> Option<&str> { + self.owners.get(path).map(|s| s.as_str()) + } +} + +fn build_variant_ownership( + workspace_dir: &Utf8Path, + ws: &WorkspaceInfo, +) -> Result { + let mut owners: std::collections::HashMap = + std::collections::HashMap::new(); + for variant in &ws.variants { + for pattern in &variant.exclusive { + // Absolutize relative-to-workspace patterns so the + // glob crate walks the right filesystem tree. + let abs_pattern = workspace_dir.as_std_path().join(pattern); + let pattern_str = + abs_pattern.to_str().ok_or_else(|| VwError::FileSystem { + message: format!( + "variant `{}` exclusive pattern is not valid UTF-8: {}", + variant.name, + abs_pattern.display(), + ), + })?; + let entries = + glob::glob(pattern_str).map_err(|e| VwError::FileSystem { + message: format!( + "variant `{}` invalid glob `{pattern}`: {e}", + variant.name, + ), + })?; + for entry in entries.flatten() { + if !entry.is_file() { + continue; + } + // First-writer wins: if two variants claim the + // same file exclusively, the first entry in the + // list owns it. That's a config bug the user + // should fix; we don't error to keep the surface + // predictable in the interim. + owners.entry(entry).or_insert_with(|| variant.name.clone()); + } + } + } + Ok(VariantOwnership { owners }) +} + +/// Enumerate every Vivado design-constraint file under +/// `/constraints/**/*.{xdc,sdc}`. Handles both +/// physical (`.xdc`) and Synopsys-style (`.sdc`) constraints — +/// both are accepted by `read_xdc` in Vivado. +/// +/// Returned separately from [`vhdl_design_sources`] because +/// constraints have their own file kind and a different +/// consumption command (`read_xdc` vs. `read_vhdl`). +/// +/// Empty vec when `constraints/` doesn't exist yet. +pub fn design_constraints(workspace_dir: &Utf8Path) -> Result> { + design_constraints_in(workspace_dir, None) +} + +/// Enumerate only the `synth`-scoped constraints — +/// `/constraints/synth/**/*.{xdc,sdc}`. Used to +/// hand synthesis-only constraints to `read_xdc -used_in +/// synthesis` (or the equivalent set_property USED_IN) so +/// route/place-only constraints don't spuriously apply during +/// synth. Empty vec when the subdir doesn't exist. +pub fn design_synth_constraints( + workspace_dir: &Utf8Path, +) -> Result> { + design_constraints_in(workspace_dir, Some("synth")) +} + +/// Enumerate only the `place`-scoped constraints under +/// `/constraints/place/**/*.{xdc,sdc}`. Mirrors +/// [`design_synth_constraints`] for the placement flow. Empty +/// vec when the subdir doesn't exist. +pub fn design_place_constraints( + workspace_dir: &Utf8Path, +) -> Result> { + design_constraints_in(workspace_dir, Some("place")) +} + +/// Enumerate only the `route`-scoped constraints under +/// `/constraints/route/**/*.{xdc,sdc}`. Mirrors +/// [`design_synth_constraints`] for the routing flow. Empty +/// vec when the subdir doesn't exist. +pub fn design_route_constraints( + workspace_dir: &Utf8Path, +) -> Result> { + design_constraints_in(workspace_dir, Some("route")) +} + +/// Enumeration primitive shared by [`design_constraints`] and +/// the phase-scoped variants. `subdir = None` walks +/// `/constraints/`; `subdir = Some(name)` walks +/// `/constraints//`. +fn design_constraints_in( + workspace_dir: &Utf8Path, + subdir: Option<&str>, +) -> Result> { + let mut dir = workspace_dir.join("constraints"); + if let Some(sub) = subdir { + dir = dir.join(sub); + } + if !dir.exists() { + return Ok(Vec::new()); + } + let mut files = Vec::new(); + find_constraint_files_impl( + dir.as_std_path(), + &mut files, + /*recursive=*/ true, + )?; + files.sort(); + Ok(files) +} + +/// Mirror of `find_vhdl_files_impl` but for the `.xdc` / `.sdc` +/// extension set. Kept separate rather than parameterizing the +/// existing walker because the extension list is small and +/// domain-specific — a generic "find by extensions" helper would +/// obscure the intent at the call site. +fn find_constraint_files_impl( + dir: &Path, + files: &mut Vec, + recursive: bool, +) -> Result<()> { + for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory: {e}"), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + let path = entry.path(); + if path.is_dir() { + if recursive { + find_constraint_files_impl(&path, files, recursive)?; + } + } else if let Some(ext) = path.extension().and_then(|s| s.to_str()) { + if ext == "xdc" || ext == "sdc" { + files.push(path); + } + } + } + Ok(()) +} + +/// Enumerate every generated IP wrapper under +/// `/target/ip/**/*.{vhd,vhdl}`. Populated by +/// `vw::make_wrapper` — see `~/src/htcl/vw/module.htcl` — which +/// drops one `wrapper.vhd` per IP into `target/ip//`. +/// +/// Returned separately from [`vhdl_design_sources`] because IP +/// wrappers have a different lifecycle: they're TOOL-generated +/// (regen on IP config change), not human-authored, and typically +/// compile into their own VHDL library (`ip` by convention). The +/// caller decides the library assignment. +/// +/// **Excludes**: +/// - `target/ip/bd/**` and `target/ip/xci/**` — legacy caches +/// from the old custom BD/XCI save-restore path (retained as +/// an exclusion for now to survive workspaces that still have +/// those dirs on disk; the caches themselves are deleted at +/// session start — see the migration cleanup in `vw run` / +/// `vw repl`). +/// - `target/vw-project/**` — defensive. The on-disk Vivado +/// project (see `vw_project_dir`) sits SIBLING to `target/ip/`, +/// so this walker's root at `target/ip/` shouldn't ever +/// traverse it — but keep the filter as an invariant guard +/// for future walker refactors and to document intent. +/// +/// The excluded content lands in the Vivado project via +/// `read_bd`/`read_ip`/`synth_ip` — re-adding through +/// `read_vhdl` would trigger `[filemgmt 20-1440] already exists +/// in the project as a part of sub-design file` CRITICAL WARNINGs. +/// +/// Empty vec when `target/ip/` doesn't exist yet — a fresh +/// workspace hasn't run `vw::make_wrapper` for anything. +pub fn vhdl_ip_sources(workspace_dir: &Utf8Path) -> Result> { + let ip_dir = workspace_dir.join("target").join("ip"); + if !ip_dir.exists() { + return Ok(Vec::new()); + } + let bd_cache = ip_dir.join("bd"); + let xci_cache = ip_dir.join("xci"); + let vw_project = workspace_dir.join("target").join("vw-project"); + let mut files = + find_vhdl_files(ip_dir.as_std_path(), /*recursive=*/ true, &[])?; + // `starts_with` on each canonical prefix filters every + // nested path (`bd/cips/synth/cips.vhd`, `xci/primary_clock/ + // primary_clock.vhd`, and defensively any hypothetical + // `vw-project/...` VHDL that a future walker refactor might + // reach). + files.retain(|p| { + !p.starts_with(bd_cache.as_std_path()) + && !p.starts_with(xci_cache.as_std_path()) + && !p.starts_with(vw_project.as_std_path()) + }); + files.sort(); + Ok(files) +} + +/// Derive the VHDL library name a dep's sources compile into. +/// Hyphens become underscores (Vivado's `xelab` and NVC both +/// reject library names containing hyphens). Same rule +/// `vhdl_ls.toml` generation uses so the analyzer and the +/// synthesizer see identical library assignments. +fn library_name_for_dep(name: &str) -> String { + name.replace('-', "_") +} + +pub fn list_htcl_tests(workspace_dir: &Utf8Path) -> Result> { + let test_dir = workspace_dir.join("test"); + if !test_dir.exists() { + return Ok(Vec::new()); + } + let mut out = Vec::new(); + walk_htcl_tests(test_dir.as_std_path(), &mut out)?; + out.sort(); + Ok(out) +} + +fn walk_htcl_tests(dir: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { + message: format!( + "Failed to read test directory {}: {e}", + dir.display() + ), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + let path = entry.path(); + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with('.') || name_str == "target" { + continue; + } + if path.is_file() { + if path.extension().and_then(|s| s.to_str()) == Some("htcl") { + out.push(path); + } + } else if path.is_dir() { + walk_htcl_tests(&path, out)?; + } + } + Ok(()) +} + +/// Enumerate every `.htcl` file under `` recursively, +/// excluding hidden dirs (`.git`, `.vw`), the build-artifact `target/` +/// dir, and vendored deps (`~/.vw/deps/` isn't under a workspace but +/// callers should never point us there anyway). +/// +/// Broader than [`list_htcl_tests`] (which is `test/**/*.htcl` only) +/// — this walks the whole workspace so `synth_needs_update` can +/// invalidate a checkpoint when ANY authored htcl changes (the +/// entry-file `design.htcl`, per-IP `ip/*.htcl`, whatever the user +/// writes). Sorted for determinism. +pub fn list_workspace_htcl_files( + workspace_dir: &Utf8Path, +) -> Result> { + let mut out = Vec::new(); + walk_workspace_htcl(workspace_dir.as_std_path(), &mut out)?; + out.sort(); + Ok(out) +} + +fn walk_workspace_htcl(dir: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { + message: format!( + "Failed to read workspace directory {}: {e}", + dir.display() + ), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + let path = entry.path(); + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + // Skip hidden dirs (.git, .vw, .vscode, etc.) and the + // build-artifact `target/` tree. A user's checked-in + // sources never live in either. + if name_str.starts_with('.') || name_str == "target" { + continue; + } + if path.is_file() { + if path.extension().and_then(|s| s.to_str()) == Some("htcl") { + out.push(path); + } + } else if path.is_dir() { + walk_workspace_htcl(&path, out)?; + } + } + Ok(()) +} + +/// Enumerate every tracked source file for the synth checkpoint, +/// in the fixed order fingerprint computation depends on. +/// +/// Tracked source scope — matches what [`vw::synth`] actually +/// reads, plus a few coarse-grained triggers: +/// - VHDL design under `/hdl/**` (variant-filtered). +/// - IP wrappers under `/target/ip/**`. +/// - Synth-scoped XDCs under `/constraints/synth/**`. +/// - Every `.htcl` under `` (excludes hidden dirs + `target/`) +/// — captures edits to `design.htcl`, `ip/*.htcl`, and any +/// local htcl libs. +/// - Every dep-published VHDL file (from +/// `vhdl_dependency_sources_ext(exclude_sim_only=true)`, same +/// surface `vw::synth` feeds `read_vhdl`). Includes git deps +/// (materialized in `~/.vw/deps/-`, content locked +/// by commit) and path deps (files change in place). +/// - `/vw.toml` — variant / target-part / deps-list changes +/// should re-trigger synth. +/// - `/vw.lock` — captures dep-version bumps that alter +/// which cache-dir a name resolves to. +/// +/// Missing files pass through as-is; the fingerprint hasher +/// distinguishes "file present with content X" from "file +/// absent" by only folding present files into the digest and +/// including the sorted list of paths as part of the mix. +fn synth_source_paths( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, +) -> Result> { + let mut sources: Vec = Vec::new(); + sources.extend(vhdl_design_sources_for_variant( + workspace_dir, + active_variant, + )?); + sources.extend(vhdl_ip_sources(workspace_dir)?); + sources.extend(design_synth_constraints(workspace_dir)?); + sources.extend(list_workspace_htcl_files(workspace_dir)?); + // Dependency VHDL — same surface `vw::synth` feeds into + // `read_vhdl`. Matters for path deps (whose files change + // in place, invisible to vw.lock) and belt-and-braces for + // git deps (content is locked by commit, but hashing the + // materialized files means a torn `.vw/deps` extraction + // or a manual edit invalidates too). + if let Ok(dep_sources) = + vhdl_dependency_sources_ext(workspace_dir, false, true) + { + sources.extend(dep_sources.into_iter().map(|s| s.path)); + } + sources.push(workspace_dir.join("vw.toml").into_std_path_buf()); + sources.push(workspace_dir.join("vw.lock").into_std_path_buf()); + sources.sort(); + sources.dedup(); + Ok(sources) +} + +/// FNV-1a 64-bit hash. Stable across Rust versions (unlike +/// `std::hash::DefaultHasher`, which the language reserves the +/// right to change), fast, and good enough for cache-invalidation +/// use. Not cryptographic — collisions here just mean a false +/// "up-to-date" and a stale checkpoint, but the search space is +/// dozens to hundreds of files. +fn fnv1a_64(bytes: &[u8]) -> u64 { + fnv1a_64_extend(0xcbf2_9ce4_8422_2325, bytes) +} + +fn fnv1a_64_extend(mut h: u64, bytes: &[u8]) -> u64 { + for &b in bytes { + h ^= b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// Combined fingerprint over the contents + relative paths of a +/// list of source files. Mtime-independent, so tool-regenerated +/// files (IP wrappers rewritten identically each run) don't +/// spuriously invalidate a fresh checkpoint. Missing files are +/// folded in with a marker byte so a source appearing / +/// disappearing changes the digest. Shared engine for every +/// checkpoint kind — synth, IP configure, and future +/// checkpoints just supply their own path list. +fn fingerprint_paths(workspace_dir: &Utf8Path, paths: &[PathBuf]) -> u64 { + let mut digest: u64 = 0xcbf2_9ce4_8422_2325; + for path in paths { + // Fold the path (relative to workspace root when possible, + // else absolute) so renames register as changes even when + // content is identical. + let rel = path + .strip_prefix(workspace_dir.as_std_path()) + .unwrap_or(path); + digest = fnv1a_64_extend(digest, rel.to_string_lossy().as_bytes()); + match fs::read(path) { + Ok(content) => { + // Marker byte `0x01` for "file present"; then the + // file's own content hash mixed into the running + // digest. Hashing the per-file hash (rather than + // the full content stream) keeps this branch small. + digest = fnv1a_64_extend(digest, &[0x01]); + let file_hash = fnv1a_64(&content); + digest = fnv1a_64_extend(digest, &file_hash.to_le_bytes()); + } + Err(_) => { + // Marker byte `0x00` for "file absent". Consistent + // with the "missing = separate state from empty" + // rule so `vw.lock` present-and-empty ≠ absent. + digest = fnv1a_64_extend(digest, &[0x00]); + } + } + } + digest +} + +/// Combined fingerprint over the synth source set. Backs +/// [`synth_needs_update`] and [`write_synth_checkpoint_manifest`]. +pub fn synth_source_fingerprint( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, +) -> Result { + let paths = synth_source_paths(workspace_dir, active_variant)?; + Ok(fingerprint_paths(workspace_dir, &paths)) +} + +/// Manifest sidecar path — sits next to the checkpoint under +/// `.manifest`. Small plain-text file (single u64 in +/// decimal). Kept alongside the checkpoint so `rm -rf target/` +/// wipes both together and there's never a stale manifest +/// pointing at a deleted checkpoint. +fn checkpoint_manifest_path(checkpoint: &Path) -> PathBuf { + let mut name = checkpoint.file_name().unwrap_or_default().to_os_string(); + name.push(".manifest"); + checkpoint.with_file_name(name) +} + +/// Write a manifest sidecar recording `fingerprint` next to +/// `checkpoint`. Shared by `write_synth_checkpoint_manifest`, +/// `write_place_checkpoint_manifest`, and `write_project_manifest` +/// so every checkpoint kind uses the same on-disk format. +fn write_checkpoint_manifest_with_fingerprint( + checkpoint: &Path, + fingerprint: u64, +) -> Result<()> { + let manifest = checkpoint_manifest_path(checkpoint); + fs::write(&manifest, format!("{fingerprint}\n")).map_err(|e| { + VwError::FileSystem { + message: format!( + "Failed to write checkpoint manifest {}: {e}", + manifest.display() + ), + } + }) +} + +/// Compare `current_fingerprint` against the manifest sidecar +/// next to `checkpoint`. Returns `true` when the checkpoint is +/// missing, the manifest is missing / unparseable, or the +/// fingerprints disagree. Shared by every `*_needs_update` fn. +fn checkpoint_needs_update_with_fingerprint( + checkpoint: &Path, + current_fingerprint: u64, +) -> bool { + if !checkpoint.exists() { + return true; + } + let manifest = checkpoint_manifest_path(checkpoint); + let Ok(stored) = fs::read_to_string(&manifest) else { + return true; + }; + let Ok(stored_fp) = stored.trim().parse::() else { + return true; + }; + stored_fp != current_fingerprint +} + +/// Write the manifest sidecar for a freshly-produced synth +/// checkpoint. Records the current source fingerprint so +/// [`synth_needs_update`] can decide freshness by content +/// comparison rather than mtime. +/// +/// Called from `vw::synth` immediately after +/// `vivado_cmd::write_checkpoint` completes. +pub fn write_synth_checkpoint_manifest( + workspace_dir: &Utf8Path, + checkpoint: &Path, + active_variant: Option<&str>, +) -> Result<()> { + let fp = synth_source_fingerprint(workspace_dir, active_variant)?; + write_checkpoint_manifest_with_fingerprint(checkpoint, fp) +} + +/// Returns `true` when either +/// - the checkpoint file is missing, or +/// - its manifest sidecar is missing / unreadable / stores a +/// fingerprint different from the tracked source set's current +/// fingerprint. +/// +/// Content-hash based (not mtime): identical `make_wrapper` +/// output with a shifted mtime does NOT invalidate the checkpoint, +/// which is the whole point — the wrapper is regenerated on every +/// design.htcl run but its stripped-header body is stable when +/// the source `.bd` hasn't changed. +pub fn synth_needs_update( + workspace_dir: &Utf8Path, + checkpoint: &Path, + active_variant: Option<&str>, +) -> Result { + let current_fp = synth_source_fingerprint(workspace_dir, active_variant)?; + Ok(checkpoint_needs_update_with_fingerprint( + checkpoint, current_fp, + )) +} + +// --------------------------------------------------------------------- +// Place checkpoint — a per-workspace cache scoped to the place stage. +// Backs `vw::place` in the htcl vw module. +// +// Source scope is narrower than synth's: place XDCs under +// `/constraints/place/**` PLUS the upstream synth checkpoint +// file itself. The synth DCP acts as a proxy for "everything synth +// depended on" — if any synth-scope source changed, synth re-ran +// and produced a fresh DCP, invalidating the place fingerprint. +// This avoids duplicating the synth source enumeration here. +// --------------------------------------------------------------------- + +/// Path list feeding [`place_source_fingerprint`]. Sorted + +/// deduped. Missing files pass through — the fingerprint's +/// present/absent marker byte covers the "checkpoint doesn't +/// exist yet" case correctly. +fn place_source_paths( + workspace_dir: &Utf8Path, + synth_checkpoint: &Path, +) -> Result> { + let mut sources: Vec = Vec::new(); + sources.extend(design_place_constraints(workspace_dir)?); + sources.push(synth_checkpoint.to_path_buf()); + sources.sort(); + sources.dedup(); + Ok(sources) +} + +/// Combined fingerprint over the place source set (place XDCs + +/// the upstream synth checkpoint file). Backs +/// [`place_needs_update`] and [`write_place_checkpoint_manifest`]. +pub fn place_source_fingerprint( + workspace_dir: &Utf8Path, + synth_checkpoint: &Path, +) -> Result { + let paths = place_source_paths(workspace_dir, synth_checkpoint)?; + Ok(fingerprint_paths(workspace_dir, &paths)) +} + +/// Write the manifest sidecar for a freshly-produced place +/// checkpoint. Called from `vw::place` after +/// `vivado_cmd::write_checkpoint` completes. +pub fn write_place_checkpoint_manifest( + workspace_dir: &Utf8Path, + place_checkpoint: &Path, + synth_checkpoint: &Path, +) -> Result<()> { + let fp = place_source_fingerprint(workspace_dir, synth_checkpoint)?; + write_checkpoint_manifest_with_fingerprint(place_checkpoint, fp) +} + +/// Returns `true` when the place checkpoint OR its manifest is +/// missing, OR the fingerprint stored in the manifest differs +/// from the current one. Mirrors [`synth_needs_update`] with a +/// tighter source scope (place XDCs + the synth DCP proxy). +pub fn place_needs_update( + workspace_dir: &Utf8Path, + place_checkpoint: &Path, + synth_checkpoint: &Path, +) -> Result { + let current_fp = place_source_fingerprint(workspace_dir, synth_checkpoint)?; + Ok(checkpoint_needs_update_with_fingerprint( + place_checkpoint, + current_fp, + )) +} + +// --------------------------------------------------------------------- +// IP configuration checkpoint — a per-workspace cache scoped to the +// entry `ip/module.htcl` (and everything it srcs). Backs +// `vw::configure_ip` in the htcl vw module. +// +// Source scope: every `.htcl` under `/ip/`. This is intentionally +// tighter than the synth scope — `ip::configure` produces BDs / XCI +// IPs / wrappers whose input surface is defined entirely by the ip/ +// tree. If the user's ip/ htcl changes (adding an IP, tweaking +// a configure_* parameter), the checkpoint invalidates. If the +// user's design.htcl changes, it doesn't — that only matters for +// the downstream synth step, which has its own checkpoint. +// --------------------------------------------------------------------- + +/// Enumerate every `.htcl` file under `/ip/`, +/// recursively, sorted. Empty vec when `ip/` doesn't exist yet. +/// Skips hidden dirs (`.git`, `.vw`) and `target/` for the same +/// reason [`list_workspace_htcl_files`] does — those aren't +/// authored sources. +pub fn list_ip_htcl_files(workspace_dir: &Utf8Path) -> Result> { + let ip_dir = workspace_dir.join("ip"); + if !ip_dir.exists() { + return Ok(Vec::new()); + } + let mut out = Vec::new(); + walk_workspace_htcl(ip_dir.as_std_path(), &mut out)?; + out.sort(); + Ok(out) +} + +// --------------------------------------------------------------------- +// On-disk Vivado project — vw manages a persistent Vivado project at +// `/target/vw-project//` so BD/IP state survives across +// sessions via Vivado's native `save_project`/`open_project` +// machinery (instead of hand-serializing BD .bd files, XCI dirs, +// ipshared/, etc.). See `~/.claude/plans/abundant-petting-lecun.md`. +// +// Fingerprint scope: `/ip/**/*.htcl` + `/vw.toml`. Any +// substantive edit to those invalidates the on-disk project — +// wiped + recreated on the next spawn, then `ip::configure` re-runs +// and `vw::mark_project_configured` writes a fresh manifest. +// --------------------------------------------------------------------- + +/// Absolute path of the on-disk Vivado project directory vw +/// manages. The `.xpr` itself lives at +/// `//.xpr` (Vivado's +/// `create_project -dir` convention — it always nests one level +/// deep under the given dir). +pub fn vw_project_dir(workspace_dir: &Utf8Path) -> Utf8PathBuf { + workspace_dir.join("target").join("vw-project") +} + +/// Path list feeding [`project_source_fingerprint`]. Sorted + +/// deduped. Every `.htcl` under `/ip/` plus `/vw.toml`. +/// A missing `vw.toml` folds in as the "absent" marker via +/// [`fingerprint_paths`]'s present/absent handling, so an empty +/// or missing workspace still hashes deterministically. +fn project_source_paths(workspace_dir: &Utf8Path) -> Result> { + let mut sources = list_ip_htcl_files(workspace_dir)?; + sources.push(workspace_dir.join("vw.toml").into_std_path_buf()); + sources.sort(); + sources.dedup(); + Ok(sources) +} + +/// Combined fingerprint over the on-disk-project source set. +/// Backs [`project_needs_wipe`] and [`write_project_manifest`]. +pub fn project_source_fingerprint(workspace_dir: &Utf8Path) -> Result { + let paths = project_source_paths(workspace_dir)?; + Ok(fingerprint_paths(workspace_dir, &paths)) +} + +/// Absolute path of the `.xpr` inside the on-disk project dir. +/// Vivado's `create_project -dir -name ` produces +/// `//.xpr`. All manifest / checkpoint plumbing uses +/// this path as the "checkpoint" argument to the shared helpers +/// (`write_checkpoint_manifest_with_fingerprint`, +/// `checkpoint_needs_update_with_fingerprint`), which append +/// `.manifest` to derive the sidecar path +/// (`//.xpr.manifest`). +pub fn vw_project_xpr(project_dir: &Path, name: &str) -> PathBuf { + project_dir.join(name).join(format!("{name}.xpr")) +} + +/// Write the manifest sidecar for a freshly-configured on-disk +/// project. Called from `vw::configure_ip` (via the +/// `mark_project_configured` RPC) after `save_project` completes. +/// +/// Invariant: manifest presence means "the project at this dir +/// was successfully configured with these sources' fingerprint." +/// `project_needs_wipe` relies on this — never write the manifest +/// before `save_project` succeeds. +pub fn write_project_manifest( + workspace_dir: &Utf8Path, + project_dir: &Path, + name: &str, +) -> Result<()> { + let fp = project_source_fingerprint(workspace_dir)?; + write_checkpoint_manifest_with_fingerprint( + &vw_project_xpr(project_dir, name), + fp, + ) +} + +/// Returns `true` when the on-disk project at `//` +/// should be wiped + recreated: +/// - the `.xpr` is missing, OR +/// - the manifest sidecar is missing / unreadable, OR +/// - the stored fingerprint differs from the current one. +/// +/// Called by `vw run` / `vw repl` before spawning Vivado (see +/// `vw-cli/src/main.rs` / `vw-repl/src/app.rs`); when it returns +/// true, the caller `remove_dir_all(project_dir)` before passing +/// `persist_dir: Some(project_dir)` to `AutoProject`. +pub fn project_needs_wipe( + workspace_dir: &Utf8Path, + project_dir: &Path, + name: &str, +) -> Result { + let current_fp = project_source_fingerprint(workspace_dir)?; + Ok(checkpoint_needs_update_with_fingerprint( + &vw_project_xpr(project_dir, name), + current_fp, + )) +} + +/// Idempotent one-shot cleanup of the legacy IP-cache artifacts +/// that lived under `/target/ip/` before the on-disk Vivado +/// project migration: +/// `/target/ip/bd/` +/// `/target/ip/xci/` +/// `/target/ip/.ip-cache` +/// `/target/ip/.ip-cache.manifest` +/// +/// Called by `vw run` / `vw repl` at bootstrap; on first +/// on-disk-mode session it wipes stale bytes and returns the +/// count. On subsequent sessions it's a no-op (returns 0). +/// +/// Deliberately does NOT touch `/target/ip//wrapper.vhd` +/// — those are `vw::make_wrapper` outputs still consumed by +/// `vw::synth`, and their `` sibling dirs may still be +/// populated by the on-disk project's `generate_target` outputs. +pub fn cleanup_legacy_ip_cache(workspace_dir: &Utf8Path) -> usize { + let ip_dir = workspace_dir.join("target").join("ip"); + let targets = [ + ip_dir.join("bd"), + ip_dir.join("xci"), + ip_dir.join(".ip-cache"), + ip_dir.join(".ip-cache.manifest"), + ]; + let mut removed = 0; + for t in targets { + let p = t.as_std_path(); + if !p.exists() { + continue; + } + let ok = if p.is_dir() { + fs::remove_dir_all(p).is_ok() + } else { + fs::remove_file(p).is_ok() + }; + if ok { + removed += 1; + } + } + removed +} + +/// Outcome of [`prepare_vw_project_dir`]. The caller decides how +/// to surface the messages (`tracing::info!`, REPL banner, etc.); +/// vw-lib itself intentionally doesn't do user-facing IO. +#[derive(Debug, Clone)] +pub struct PreparedProjectDir { + /// Absolute path of `/target/vw-project/`, ready to pass + /// as `AutoProject::persist_dir = Some(...)`. + pub project_dir: Utf8PathBuf, + /// Number of legacy `/target/ip/{bd,xci,.ip-cache*}` + /// entries removed by the one-shot Phase 6 migration cleanup. + pub legacy_cache_removed: usize, + /// `Some(path)` iff the persist dir existed AND + /// [`project_needs_wipe`] returned true, so we wiped it before + /// returning. The caller can log this as the reason for the + /// fresh recreate that Vivado is about to do. + /// + /// We wipe `/target/vw-project/` in its entirety (not + /// just the per-`name` subdir): historically bugs have + /// caused Vivado to scatter flat siblings like + /// `/.xpr`, `/.srcs` alongside + /// the nested `//` — a full-dir wipe reliably + /// sweeps any layout drift instead of leaving cross-version + /// clutter behind. + pub wiped_project: Option, +} + +/// Bootstrap the on-disk Vivado project dir for a workspace and +/// return its absolute path, ready to pass as +/// `AutoProject::persist_dir = Some(...)`. +/// +/// Does three things in order (all idempotent): +/// +/// 1. Silently clean up legacy `target/ip/{bd,xci,.ip-cache*}` +/// artifacts from the pre-migration IP cache +/// ([`cleanup_legacy_ip_cache`]). +/// 2. Consult [`project_needs_wipe`]; if the `.xpr` is missing, +/// the manifest sidecar is missing, or the fingerprint is +/// stale, `remove_dir_all(/)` so the worker +/// takes the fresh-create branch instead of `open_project` on +/// stale bytes. +/// 3. `create_dir_all()` so Vivado's later +/// `create_project -dir` has a real parent. +/// +/// Everything worth reporting to the user (legacy-cleanup counts, +/// wipe reason) lands in the returned [`PreparedProjectDir`] for +/// the caller to log. +pub fn prepare_vw_project_dir( + workspace_dir: &Utf8Path, + name: &str, +) -> Result { + let project_dir = vw_project_dir(workspace_dir); + let legacy_cache_removed = cleanup_legacy_ip_cache(workspace_dir); + let mut wiped_project = None; + if project_needs_wipe(workspace_dir, project_dir.as_std_path(), name)? + && project_dir.exists() + { + fs::remove_dir_all(project_dir.as_std_path())?; + wiped_project = Some(project_dir.clone()); + } + fs::create_dir_all(project_dir.as_std_path())?; + Ok(PreparedProjectDir { + project_dir, + legacy_cache_removed, + wiped_project, + }) +} + +pub struct RecordProcessor { + pub vhdl_std: VhdlStandard, + pub symbols: HashMap, + pub symbol_to_file: HashMap, + pub tagged_names: HashSet, + pub file_info: HashMap, + pub target_attr: String, +} + +const RECORD_PARSE_ATTRIBUTE: &str = "serialize_rust"; +impl RecordProcessor { + pub fn new(std: VhdlStandard) -> Self { + Self { + vhdl_std: std, + symbols: HashMap::new(), + symbol_to_file: HashMap::new(), + tagged_names: HashSet::new(), + file_info: HashMap::new(), + target_attr: RECORD_PARSE_ATTRIBUTE.to_string(), + } + } +} + +// ============================================================================ +// File Cache - Reduces redundant file reads during build +// ============================================================================ + +/// Cache for parsed file data to avoid redundant parsing during builds. +/// Only caches parsed results, not raw file contents. +pub struct FileCache { + dependencies: HashMap>, + provided_symbols: HashMap>, + entities: HashMap>, +} + +impl FileCache { + pub fn new() -> Self { + Self { + dependencies: HashMap::new(), + provided_symbols: HashMap::new(), + entities: HashMap::new(), + } + } + + /// Get cached file dependencies, reading and parsing file if not cached. + pub fn get_dependencies(&mut self, path: &Path) -> Result<&Vec> { + match self.dependencies.entry(path.to_path_buf()) { Entry::Occupied(e) => Ok(e.into_mut()), Entry::Vacant(e) => { let content = fs::read_to_string(path).map_err(|e| { @@ -1890,6 +3849,68 @@ fn save_workspace_config( Ok(()) } +/// Walk up from `start` (typically a directory) looking for the +/// first `vw.toml` file in the ancestor chain. Returns the +/// containing directory, or `None` if none is found. +/// +/// Callers that hold a FILE path — say `entry.htcl` — should pass +/// `entry.parent()` since a file itself cannot contain a +/// `vw.toml`. The check uses `is_file()` so a stray directory +/// named `vw.toml` doesn't trip a false positive. +/// +/// Consolidates three near-duplicate helpers that previously +/// lived in `vw-cli::find_workspace_dir`, `vw-repl::app:: +/// find_vw_toml_ancestor`, and `vw-repl::lower:: +/// find_workspace_dir`. The `vw-analyzer` multi-root discovery +/// (LSP `initialize`-supplied roots) is a different concept and +/// stays in the analyzer. +/// Return `/design.htcl` when it exists on disk, +/// else `None`. Mirrors the `module.htcl` convention for library +/// workspaces — `design.htcl` is the project workspace's entry +/// script, auto-discovered by `vw run` / `vw repl` / `vw check` +/// when the user invokes them with no file argument. +pub fn find_design_file(workspace_dir: &Utf8Path) -> Option { + let p = workspace_dir.join("design.htcl"); + p.is_file().then_some(p) +} + +pub fn find_workspace_dir(start: &Path) -> Option { + // Canonicalize UP FRONT so the walk-up starts from an + // absolute path. Callers hand us relative or empty paths in + // real workflows: `vw run prime.htcl` derives + // `Path("prime.htcl").parent() == ""`, which used to yield an + // empty-string workspace root — served over RPC to htcl, + // that caused `[file join $root target ip $ip]` to compute a + // RELATIVE path (`target/ip/cips`), which Vivado created + // inside its auto-cleaned tempdir cwd. The wrapper was + // silently written and immediately deleted on process exit. + // Canonicalizing here fixes every downstream consumer + // (workspace_root RPC, LSP compat check, htcl-test) in one + // place. Fallback to the raw start when canonicalize fails + // (e.g. path doesn't exist yet) so we don't regress the "no + // workspace" branch for freshly-created files. + // Empty path is a special case — `Path("").canonicalize()` + // errors with ENOENT, and `parent()` on it yields None + // immediately, so we'd terminate the walk before ever + // checking the cwd. Fold empty → cwd first, then canonicalize. + let start_pb = if start.as_os_str().is_empty() { + std::env::current_dir().ok()? + } else { + start.to_path_buf() + }; + let canon = start_pb.canonicalize().unwrap_or(start_pb); + let mut cur = Utf8PathBuf::from_path_buf(canon).ok()?; + loop { + if cur.join("vw.toml").is_file() { + return Some(cur); + } + // `Utf8Path::parent()` returns `None` at the filesystem + // root; the loop naturally terminates without needing a + // manual `parent == cur` guard. + cur = cur.parent()?.to_path_buf(); + } +} + pub fn load_workspace_config( workspace_dir: &Utf8Path, ) -> Result { @@ -1906,10 +3927,36 @@ pub fn load_workspace_config( })?; let config: WorkspaceConfig = toml::from_str(&config_content)?; - + validate_variant_shape(&config.workspace)?; Ok(config) } +/// Post-deserialize validation for the `[[target-parts]]` / +/// `[[workspace.variants]]` mutual exclusion + variant-name +/// uniqueness. Returns [`VwError::Config`] with the same +/// user-facing message the [`VariantSelectError`] carries so +/// the loader surfaces the specific failure verbatim. +fn validate_variant_shape(ws: &WorkspaceInfo) -> Result<()> { + if !ws.variants.is_empty() && !ws.target_parts.is_empty() { + return Err(VwError::Config { + message: VariantSelectError::BothPartsAndVariants.to_string(), + }); + } + let mut seen: std::collections::HashSet<&str> = + std::collections::HashSet::new(); + for v in &ws.variants { + if !seen.insert(v.name.as_str()) { + return Err(VwError::Config { + message: VariantSelectError::DuplicateName { + name: v.name.clone(), + } + .to_string(), + }); + } + } + Ok(()) +} + fn load_lock_file(workspace_dir: &Utf8Path) -> Result { let lock_path = workspace_dir.join("vw.lock"); if !lock_path.exists() { @@ -1959,12 +4006,147 @@ pub fn deps_directory() -> Result { /// per-user `$HOME/.vw/deps` directory) so the file is identical across /// machines. Absolute paths are returned unchanged to remain compatible /// with lock files written by older versions of vw. +/// Build a `name → absolute cache path` map for every dependency in +/// the workspace's `vw.lock`. Used by htcl's `src @name/...` resolver +/// in `vw-htcl::src_path::Resolver` so the language-layer crate stays +/// free of workspace / lockfile concerns. +/// +/// Returns an empty map (not an error) if the workspace has no +/// `vw.lock` yet — relative and absolute `src` imports still work +/// against an empty resolver, only `@name/` lookups fail. +pub fn dep_cache_paths( + workspace_dir: &Utf8Path, +) -> Result> { + dep_cache_paths_with_test(workspace_dir, false) +} + +/// Same as [`dep_cache_paths`] but optionally includes +/// `[test-dependencies]`. Only `vw test` should pass +/// `include_test = true`; other callers see the same map they +/// always did, so `vw run`/`vw check`/`vw update`'s dep behavior +/// is unchanged. +pub fn dep_cache_paths_with_test( + workspace_dir: &Utf8Path, + include_test: bool, +) -> Result> { + let mut out = HashMap::new(); + + // Local deps live wherever `vw.toml` says; they don't need a + // lockfile (nothing to pin). Read them straight from the workspace + // config so they work before — or without — a `vw update`. + if let Ok(config) = load_workspace_config(workspace_dir) { + for (name, dep) in config.dependencies { + if let Some(path) = dep.local_path() { + out.insert(name, path.to_path_buf()); + } + } + if include_test { + for (name, dep) in config.test_dependencies { + if let Some(path) = dep.local_path() { + out.insert(name, path.to_path_buf()); + } + } + } + } + + // Git deps are resolved through the lockfile and the per-user + // cache. A missing lock isn't an error here — just skip git entries. + // The lockfile stores test-dep and normal-dep entries in the same + // `dependencies` section (no separate section for locks — see the + // rationale on `update_workspace_with_token`). Non-test callers + // want to filter out entries that came exclusively from + // `[test-dependencies]`; simplest safe rule for now: everything in + // the lockfile is exposed regardless of section. A subsequent PR + // can add a `test = true` marker per lock entry if this becomes a + // real concern. + match load_lock_file(workspace_dir) { + Ok(lock) => { + for (name, locked) in lock.dependencies { + let abs = resolve_dep_path(&locked.path)?; + out.insert(name, abs); + } + } + Err(VwError::Config { .. }) => {} + Err(e) => return Err(e), + } + + Ok(out) +} + fn resolve_dep_path(path: &Path) -> Result { if path.is_absolute() { return Ok(path.to_path_buf()); } - let deps_dir = deps_directory()?; - Ok(deps_dir.join(path)) + let deps_dir = deps_directory()?; + Ok(deps_dir.join(path)) +} + +/// Like [`dep_cache_paths`], but walks the dependency graph +/// transitively: for every dep whose cached root is itself a +/// workspace (i.e. has its own `vw.toml`), pull in *its* deps too, +/// and so on. The result is a flat `name → root` map covering every +/// dep any file in this workspace's transitive closure might +/// `src @/...`-import. +/// +/// First-seen-wins on name conflicts so the entry workspace's +/// declarations take precedence over a dep's choice of the same +/// name (matching Cargo's resolution: the top-level `Cargo.toml` +/// pins the version for the whole graph). +/// +/// Returns an empty map (not an error) if the entry workspace has +/// no deps. Per-dep failures (missing `vw.toml`, malformed config) +/// are skipped: a dep may not be its own htcl workspace, and that's +/// fine — we just won't see *its* deps. +pub fn transitive_dep_cache_paths( + entry_workspace_dir: &Utf8Path, +) -> Result> { + transitive_dep_cache_paths_with_test(entry_workspace_dir, false) +} + +/// Like [`transitive_dep_cache_paths`] but optionally includes +/// the ENTRY workspace's `[test-dependencies]`. Cargo-parity +/// semantic for `dev-dependencies`: test-deps are private to the +/// workspace that declares them. Recursed-into workspaces are +/// walked with `include_test = false` so a dep's own test-deps +/// aren't pulled into your consumer. +pub fn transitive_dep_cache_paths_with_test( + entry_workspace_dir: &Utf8Path, + include_test: bool, +) -> Result> { + let mut out: HashMap = HashMap::new(); + let mut visited: std::collections::HashSet = + std::collections::HashSet::new(); + // Only the entry workspace's paths are gathered with + // `include_test`; everything queued after gets the normal + // treatment. + let mut first_iter = true; + let mut queue: Vec = + vec![entry_workspace_dir.as_std_path().to_path_buf()]; + + while let Some(ws) = queue.pop() { + if !visited.insert(ws.clone()) { + continue; + } + let Ok(ws_utf8) = Utf8PathBuf::from_path_buf(ws) else { + continue; + }; + let want_test = include_test && first_iter; + first_iter = false; + let Ok(paths) = dep_cache_paths_with_test(&ws_utf8, want_test) else { + continue; + }; + for (name, dep_path) in paths { + // First-seen wins — don't let a transitive dep override + // the entry workspace's choice. + out.entry(name).or_insert_with(|| dep_path.clone()); + // If the dep is itself a workspace, recurse into it. A + // dep without a `vw.toml` is a leaf (just files). + if dep_path.join("vw.toml").exists() { + queue.push(dep_path); + } + } + } + Ok(out) } async fn resolve_dependency_commit( @@ -2618,3 +4800,1817 @@ async fn build_rust_library( Ok(lib_path.into()) } + +#[cfg(test)] +mod dependency_source_tests { + use super::*; + + /// A `vw.toml` entry with `repo = "..."` parses as a git source — + /// the historical behaviour that pre-dates path deps. + #[test] + fn git_dep_parses_from_repo_key() { + let toml = r#" + [workspace] + name = "demo" + version = "0.1.0" + + [dependencies.quartz] + repo = "https://github.com/oxidecomputer/quartz" + branch = "main" + src = ["hdl/ip/vhd"] + recursive = true + "#; + let config: WorkspaceConfig = toml::from_str(toml).unwrap(); + let dep = &config.dependencies["quartz"]; + assert!(!dep.is_local()); + assert_eq!(dep.repo(), Some("https://github.com/oxidecomputer/quartz")); + assert_eq!(dep.branch(), Some("main")); + assert!(dep.recursive); + assert_eq!(dep.src, vec!["hdl/ip/vhd".to_string()]); + } + + /// The metroid layout: `path = "..."` and nothing else. + #[test] + fn path_dep_parses_from_path_key() { + let toml = r#" + [workspace] + name = "metroid" + version = "0.1.0" + + [dependencies.amd-htcl] + path = "/home/ry/src/amd-htcl" + "#; + let config: WorkspaceConfig = toml::from_str(toml).unwrap(); + let dep = &config.dependencies["amd-htcl"]; + assert!(dep.is_local()); + assert_eq!(dep.local_path(), Some(Path::new("/home/ry/src/amd-htcl"))); + assert_eq!(dep.repo(), None); + assert_eq!(dep.branch(), None); + } + + #[test] + fn transitive_dep_resolution_pulls_in_lib_of_lib() { + // metroid → cips → vivado-cmd. Asking for metroid's deps + // transitively should return cips AND vivado-cmd, even though + // metroid only declares cips. + let dir = tempfile::tempdir().unwrap(); + let metroid = dir.path().join("metroid"); + let cips = dir.path().join("cips"); + let vivado_cmd = dir.path().join("vivado-cmd"); + std::fs::create_dir_all(&metroid).unwrap(); + std::fs::create_dir_all(&cips).unwrap(); + std::fs::create_dir_all(&vivado_cmd).unwrap(); + std::fs::write( + metroid.join("vw.toml"), + format!( + "[workspace]\nname=\"metroid\"\nversion=\"0.1.0\"\n\n\ + [dependencies.cips]\npath = \"{}\"\n", + cips.display() + ), + ) + .unwrap(); + std::fs::write( + cips.join("vw.toml"), + format!( + "[workspace]\nname=\"cips\"\nversion=\"0.1.0\"\n\n\ + [dependencies.vivado-cmd]\npath = \"{}\"\n", + vivado_cmd.display() + ), + ) + .unwrap(); + // vivado-cmd is a leaf — has a vw.toml but no deps of its own. + std::fs::write( + vivado_cmd.join("vw.toml"), + "[workspace]\nname=\"vivado-cmd\"\nversion=\"0.1.0\"\n", + ) + .unwrap(); + + let metroid_utf8 = Utf8PathBuf::from_path_buf(metroid.clone()).unwrap(); + let resolved = transitive_dep_cache_paths(&metroid_utf8).unwrap(); + assert_eq!(resolved.get("cips"), Some(&cips)); + assert_eq!(resolved.get("vivado-cmd"), Some(&vivado_cmd)); + assert_eq!(resolved.len(), 2, "{resolved:?}"); + } + + #[test] + fn transitive_dep_resolution_first_seen_wins() { + // entry → A and entry → B, both A and B declare a dep + // `shared` pointing at different paths. Entry's view of + // `shared` is whichever was inserted first; entry itself + // doesn't declare `shared`, so the test just asserts we got + // *one* deterministic answer rather than a panic / duplicate. + let dir = tempfile::tempdir().unwrap(); + let entry = dir.path().join("entry"); + let a = dir.path().join("a"); + let b = dir.path().join("b"); + let shared_v1 = dir.path().join("shared-v1"); + let shared_v2 = dir.path().join("shared-v2"); + for d in [&entry, &a, &b, &shared_v1, &shared_v2] { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::write( + entry.join("vw.toml"), + format!( + "[workspace]\nname=\"entry\"\nversion=\"0.1.0\"\n\n\ + [dependencies.a]\npath = \"{}\"\n\ + [dependencies.b]\npath = \"{}\"\n", + a.display(), + b.display() + ), + ) + .unwrap(); + std::fs::write( + a.join("vw.toml"), + format!( + "[workspace]\nname=\"a\"\nversion=\"0.1.0\"\n\n\ + [dependencies.shared]\npath = \"{}\"\n", + shared_v1.display() + ), + ) + .unwrap(); + std::fs::write( + b.join("vw.toml"), + format!( + "[workspace]\nname=\"b\"\nversion=\"0.1.0\"\n\n\ + [dependencies.shared]\npath = \"{}\"\n", + shared_v2.display() + ), + ) + .unwrap(); + + let entry_utf8 = Utf8PathBuf::from_path_buf(entry).unwrap(); + let resolved = transitive_dep_cache_paths(&entry_utf8).unwrap(); + // `shared` is present exactly once and points at one of the + // two candidates; we don't pin which (HashMap iter order). + let shared = resolved.get("shared").unwrap(); + assert!(*shared == shared_v1 || *shared == shared_v2, "{shared:?}"); + } + + /// Local deps round-trip through serialize/deserialize. + #[test] + fn path_dep_roundtrips() { + let dep = Dependency { + source: DependencySource::Path { + path: PathBuf::from("/some/where"), + }, + src: Vec::new(), + recursive: false, + sim_only: false, + exclude: Vec::new(), + }; + let serialized = toml::to_string(&dep).unwrap(); + let deserialized: Dependency = toml::from_str(&serialized).unwrap(); + assert!(deserialized.is_local()); + assert_eq!(deserialized.local_path(), Some(Path::new("/some/where"))); + } + + #[test] + fn test_dependencies_parse_from_test_dependencies_section() { + let toml = r#" + [workspace] + name = "demo" + version = "0.1.0" + + [dependencies.vivado-cmd] + path = "/home/x/vivado-cmd" + + [test-dependencies.test] + path = "/home/x/test" + "#; + let config: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!(config.dependencies.len(), 1); + assert_eq!(config.test_dependencies.len(), 1); + assert!(config.test_dependencies["test"].is_local()); + } + + #[test] + fn list_htcl_tests_walks_test_directory() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + std::fs::write(ws.join("vw.toml"), "").unwrap(); + std::fs::create_dir_all(ws.join("test/nested")).unwrap(); + std::fs::create_dir_all(ws.join("test/.hidden")).unwrap(); + std::fs::create_dir_all(ws.join("test/target")).unwrap(); + std::fs::write(ws.join("test/a.htcl"), "").unwrap(); + std::fs::write(ws.join("test/b.htcl"), "").unwrap(); + std::fs::write(ws.join("test/skip.vhd"), "").unwrap(); + std::fs::write(ws.join("test/nested/c.htcl"), "").unwrap(); + std::fs::write(ws.join("test/.hidden/z.htcl"), "").unwrap(); + std::fs::write(ws.join("test/target/z.htcl"), "").unwrap(); + let tests = list_htcl_tests(&ws).unwrap(); + assert_eq!(tests.len(), 3, "{:?}", tests); + assert!(tests[0].ends_with("test/a.htcl")); + assert!(tests[1].ends_with("test/b.htcl")); + assert!(tests[2].ends_with("test/nested/c.htcl")); + } + + #[test] + fn list_htcl_tests_returns_empty_when_test_dir_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + std::fs::write(ws.join("vw.toml"), "").unwrap(); + let tests = list_htcl_tests(&ws).unwrap(); + assert!(tests.is_empty()); + } + + #[test] + fn target_pattern_parses_brace_form() { + let p = parse_target_pattern("versal{xcvm3(.*)}").unwrap(); + assert_eq!(p.family, "versal"); + assert!(p.regex.is_match("xcvm3358-vsvh1747-2M-e-S")); + assert!(!p.regex.is_match("xc7z020clg484-1")); + } + + #[test] + fn target_pattern_rejects_bare_family() { + // Bare `artix7` (no braces) shouldn't reach downstream vw. + // toml; `vw ip generate` normalizes into brace form. + let e = parse_target_pattern("artix7").unwrap_err(); + assert!(matches!(e, TargetParseError::MissingBraces { .. })); + } + + #[test] + fn target_pattern_anchors_at_start() { + // `xcvm3(.*)` should NOT match "xxx-xcvm3358" (regex would + // otherwise be "contains xcvm3"). Anchoring at start + // prevents that. + let p = parse_target_pattern("versal{xcvm3(.*)}").unwrap(); + assert!(p.regex.is_match("xcvm3358")); + assert!(!p.regex.is_match("blah-xcvm3358")); + } + + #[test] + fn workspace_config_parses_target_part_and_targets() { + let toml = r#" + [workspace] + name = "clk-wizard" + version = "0.1.0" + + [targets] + supported = [ + "versal{xcvm3(.*)}", + "versal{xc2ve3(.*)}", + ] + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!(cfg.workspace.name, "clk-wizard"); + assert!(cfg.workspace.target_parts.is_empty()); + let t = cfg.targets.expect("expected [targets]"); + assert_eq!(t.supported.len(), 2); + } + + #[test] + fn workspace_config_parses_multi_target_parts() { + let toml = r#" + [workspace] + name = "metroid" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!(cfg.workspace.target_parts.len(), 2); + assert_eq!( + cfg.workspace.default_target_part().unwrap(), + Some("xcvp1202-vsva2785-2MHP-e-S"), + ); + // Substring selector picks the non-default. + assert_eq!( + cfg.workspace.select_target_part(Some("3HP")).unwrap(), + Some("xcvp1202-vsva2785-3HP-e-S"), + ); + } + + #[test] + fn multi_parts_without_default_flag_errors() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-2MHP-e-S" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(matches!( + cfg.workspace.default_target_part(), + Err(TargetSelectError::NoDefault { count: 2 }), + )); + } + + #[test] + fn ambiguous_substring_errors() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + // "xcvp1202" matches both entries. + assert!(matches!( + cfg.workspace.select_target_part(Some("xcvp1202")), + Err(TargetSelectError::Ambiguous { .. }), + )); + } + + #[test] + fn single_part_is_implicit_default() { + let toml = r#" + [workspace] + name = "vw" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!( + cfg.workspace.default_target_part().unwrap(), + Some("xcvp1202-vsva2785-3HP-e-S"), + ); + } + + #[test] + fn workspace_config_parses_variants_block() { + let toml = r#" + [workspace] + name = "metroid" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + exclusive = ["hdl/ethernet-vpk120.vhd"] + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + exclusive = ["hdl/ethernet-metro.vhd"] + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!(cfg.workspace.variants.len(), 2); + assert_eq!(cfg.workspace.variants[0].name, "vpk120"); + assert_eq!( + cfg.workspace.variants[0].part, + "xcvp1202-vsva2785-2MHP-e-S" + ); + assert!(cfg.workspace.variants[0].default); + assert_eq!( + cfg.workspace.variants[0].exclusive, + vec!["hdl/ethernet-vpk120.vhd"], + ); + assert_eq!(cfg.workspace.variants[1].name, "metro"); + assert!(!cfg.workspace.variants[1].default); + // Empty target_parts — variants own their parts inline. + assert!(cfg.workspace.target_parts.is_empty()); + } + + #[test] + fn variants_and_target_parts_are_mutually_exclusive() { + // Deserialization allows both (unknown-field serde is + // lenient), but `load_workspace_config` refuses to + // return a config that has both — variants own parts. + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-2MHP-e-S" + + [[workspace.variants]] + name = "v" + part = "xcvp1202-vsva2785-2MHP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + let err = validate_variant_shape(&cfg.workspace).unwrap_err(); + assert!( + err.to_string().contains("mutually exclusive"), + "expected mutual-exclusion error: {err}", + ); + } + + #[test] + fn duplicate_variant_names_error() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + let err = validate_variant_shape(&cfg.workspace).unwrap_err(); + assert!( + err.to_string().contains("duplicate variant name"), + "expected duplicate-name error: {err}", + ); + } + + #[test] + fn default_variant_no_variants_yields_none() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(cfg.workspace.default_variant().unwrap().is_none()); + } + + #[test] + fn default_variant_multi_with_default_flag() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + let v = cfg.workspace.default_variant().unwrap().unwrap(); + assert_eq!(v.name, "vpk120"); + } + + #[test] + fn default_variant_multi_without_default_errors() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(matches!( + cfg.workspace.default_variant(), + Err(VariantSelectError::NoDefault { count: 2 }), + )); + } + + #[test] + fn select_variant_exact_match_only() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + // Exact match returns the entry. + assert_eq!( + cfg.workspace + .select_variant(Some("metro")) + .unwrap() + .unwrap() + .name, + "metro", + ); + // Substring is NOT accepted — variant names are the + // whole selector. + assert!(matches!( + cfg.workspace.select_variant(Some("vpk")), + Err(VariantSelectError::NoMatch { .. }), + )); + } + + #[test] + fn single_variant_no_default_flag_still_parses() { + // A single-variant list is legal without `default = true` + // (the default becomes implicit — same rule as + // `[[target-parts]]` single-entry). + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(validate_variant_shape(&cfg.workspace).is_ok()); + assert_eq!(cfg.workspace.variants.len(), 1); + } + + fn make_dep(name: &str, patterns: &[&str]) -> (String, Vec) { + let compiled: Vec = patterns + .iter() + .map(|s| parse_target_pattern(s).unwrap()) + .collect(); + (name.to_string(), compiled) + } + + #[test] + fn target_compat_matches_when_pattern_covers_part() { + let mut dt = DepTargets::default(); + let (name, patterns) = make_dep("clk-wizard", &["versal{xcvm3(.*)}"]); + dt.per_dep.insert(name, patterns); + let mismatches = + check_target_compatibility(Some("xcvm3358-vsvh1747-2M-e-S"), &dt); + assert!(mismatches.is_empty(), "{mismatches:?}"); + } + + #[test] + fn target_compat_reports_unblessed_when_no_pattern_matches() { + let mut dt = DepTargets::default(); + let (name, patterns) = make_dep("clk-wizard", &["versal{xcvm3(.*)}"]); + dt.per_dep.insert(name, patterns); + let mismatches = + check_target_compatibility(Some("xc7z020clg484-1"), &dt); + assert_eq!(mismatches.len(), 1); + assert_eq!(mismatches[0].dep, "clk-wizard"); + assert_eq!(mismatches[0].supported_families, vec!["versal"]); + assert_eq!(mismatches[0].kind, TargetMismatchKind::Unblessed); + } + + #[test] + fn target_compat_reports_not_supported_wins_over_supported() { + // If a target matches both a supported and a not-supported + // pattern, the explicit ban must win — Xilinx has attested + // the combination doesn't work. + let mut dt = DepTargets::default(); + let (name, sup) = make_dep("gadget", &["versal{xcv(.*)}"]); + dt.per_dep.insert(name.clone(), sup); + let (_, ns) = make_dep("gadget", &["versal{xcvp1202.*}"]); + dt.per_dep_not_supported.insert(name, ns); + let mismatches = + check_target_compatibility(Some("xcvp1202-vsva2785-3HP-e-S"), &dt); + assert_eq!(mismatches.len(), 1); + assert_eq!(mismatches[0].kind, TargetMismatchKind::NotSupported); + } + + #[test] + fn target_compat_supported_match_clears_when_no_ban() { + let mut dt = DepTargets::default(); + let (name, sup) = make_dep("gadget", &["versal{xcvp1202.*}"]); + dt.per_dep.insert(name, sup); + let mismatches = + check_target_compatibility(Some("xcvp1202-vsva2785-3HP-e-S"), &dt); + assert!(mismatches.is_empty(), "{mismatches:?}"); + } + + #[test] + fn target_compat_treats_empty_patterns_as_universal() { + // @vw / @test — no [targets] means "we support anything." + let mut dt = DepTargets::default(); + dt.per_dep.insert("vw".into(), Vec::new()); + let mismatches = + check_target_compatibility(Some("xc7z020clg484-1"), &dt); + assert!(mismatches.is_empty()); + } + + #[test] + fn target_compat_no_op_when_no_target_declared() { + // Library workspaces have no target-part — check should + // silently accept. + let mut dt = DepTargets::default(); + let (name, patterns) = make_dep("clk-wizard", &["versal{xcvm3(.*)}"]); + dt.per_dep.insert(name, patterns); + let mismatches = check_target_compatibility(None, &dt); + assert!(mismatches.is_empty()); + } + + #[test] + fn library_name_hyphens_become_underscores() { + assert_eq!(library_name_for_dep("clk-wizard"), "clk_wizard"); + assert_eq!(library_name_for_dep("gtwiz-versal"), "gtwiz_versal"); + assert_eq!(library_name_for_dep("cpm5"), "cpm5"); + } + + #[test] + fn vhdl_design_sources_empty_when_no_hdl_dir() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + // No `hdl/` yet — return empty, not error. + let sources = vhdl_design_sources(&ws).unwrap(); + assert!(sources.is_empty()); + } + + #[test] + fn vhdl_dependency_sources_skips_deps_without_src() { + // Regression guard: a path dep with no `src` field is + // htcl-only and shouldn't contribute VHDL. Previously the + // enumeration walked each dep's whole tree recursively, + // scooping up e.g. `target/ip/*/wrapper.vhd` from an htcl + // library's own generated artifacts. + let tmp = tempfile::tempdir().unwrap(); + // Fake htcl-only dep: has a stray .vhd (like a generated + // wrapper) but declares no `src`. + let htcl_dep = tmp.path().join("htcl-only-dep"); + std::fs::create_dir_all(htcl_dep.join("target/ip/foo")).unwrap(); + std::fs::write( + htcl_dep.join("target/ip/foo/wrapper.vhd"), + "-- generated", + ) + .unwrap(); + // Fake VHDL dep: declares `src = ["hdl"]`. + let vhdl_dep = tmp.path().join("vhdl-dep"); + std::fs::create_dir_all(vhdl_dep.join("hdl")).unwrap(); + std::fs::write(vhdl_dep.join("hdl/mod.vhd"), "-- source").unwrap(); + + // Entry workspace vw.toml referencing both. + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + format!( + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.htcl-only-dep] +path = "{}" + +[dependencies.vhdl-dep] +path = "{}" +src = ["hdl"] +recursive = true +"#, + htcl_dep.display(), + vhdl_dep.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + // Only the vhdl-dep contributes. htcl-only-dep is skipped + // even though its tree contains a .vhd file. + assert_eq!(sources.len(), 1, "{sources:?}"); + assert_eq!(sources[0].library, "vhdl_dep"); + assert!( + sources[0].path.ends_with("hdl/mod.vhd"), + "unexpected path {}", + sources[0].path.display(), + ); + } + + #[test] + fn vhdl_dependency_sources_git_cache_is_flattened() { + // Regression: git-dep caches under `~/.vw/deps/-/` + // are FLATTENED at copy time — `copy_vhdl_files_glob` strips + // the source repo's `hdl/ip/vhd/synchronizers/` prefix off, + // so files land directly at the cache root. Enumeration + // must NOT re-apply the `src` pattern as a subdir join + // (which would find nothing) — instead it walks the cache + // root recursively. + let tmp = tempfile::tempdir().unwrap(); + // Simulated cache — flat file layout. + let cache = tmp.path().join("cache/quartz_sync-abc"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(cache.join("meta_sync.vhd"), "").unwrap(); + std::fs::write(cache.join("bacd.vhd"), "").unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.quartz_sync] +repo = "https://example.invalid/quartz" +branch = "main" +src = ["hdl/ip/vhd/synchronizers"] +"#, + ) + .unwrap(); + std::fs::write( + ws.join("vw.lock"), + format!( + r#" +[dependencies.quartz_sync] +repo = "https://example.invalid/quartz" +commit = "abc" +path = "{}" +src = ["hdl/ip/vhd/synchronizers"] +recursive = false +sim_only = false +submodules = false +exclude = [] +"#, + cache.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + // Both files show up despite `src` pointing at a + // subdirectory that doesn't exist in the flat cache. + assert_eq!(sources.len(), 2, "{sources:?}"); + assert!(sources.iter().all(|s| s.library == "quartz_sync")); + } + + #[test] + fn vhdl_dependency_sources_finds_git_dep_via_lockfile() { + // Simulates the real workflow: a git dep declared in + // vw.toml, resolved to a cache dir via vw.lock. The + // lockfile's `path` is absolute so we don't need to + // override `VW_DEPS_DIR`. + let tmp = tempfile::tempdir().unwrap(); + let cache = tmp.path().join("cache/quartz_sync-abc123"); + std::fs::create_dir_all(cache.join("hdl/ip/vhd/synchronizers")) + .unwrap(); + std::fs::write( + cache.join("hdl/ip/vhd/synchronizers/sync.vhd"), + "-- synced", + ) + .unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.quartz-sync] +repo = "https://example.invalid/quartz" +branch = "main" +src = ["hdl/ip/vhd/synchronizers"] +recursive = false +"#, + ) + .unwrap(); + std::fs::write( + ws.join("vw.lock"), + format!( + r#" +[dependencies.quartz-sync] +repo = "https://example.invalid/quartz" +commit = "abc123" +path = "{}" +src = ["hdl/ip/vhd/synchronizers"] +recursive = false +sim_only = false +submodules = false +exclude = [] +"#, + cache.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + assert_eq!(sources.len(), 1, "{sources:?}"); + assert_eq!(sources[0].library, "quartz_sync"); + assert!(sources[0].path.ends_with("sync.vhd")); + } + + #[test] + fn vhdl_dependency_sources_resolves_relative_path_dep() { + // Portable fixture pattern: a path dep whose `path` + // is relative to the declaring workspace's vw.toml — + // Cargo-parity. Same fixture works from any machine. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(ws.join("fixtures/lib/hdl")).unwrap(); + std::fs::write(ws.join("fixtures/lib/hdl/a.vhd"), "").unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.lib] +path = "fixtures/lib" +src = ["hdl"] +recursive = true +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + assert_eq!(sources.len(), 1, "{sources:?}"); + assert!(sources[0].path.ends_with("hdl/a.vhd")); + } + + #[test] + fn get_access_credentials_for_workspace_only_scans_test_deps_when_asked() { + // No netrc → both variants return None regardless of + // dep-set — regression guard for the `include_test` + // dispatch path. The bigger scenario (netrc HIT for a + // git URL) is covered by the underlying + // `get_access_credentials_from_netrc` test. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.g] +repo = "https://example.invalid/x" +branch = "main" + +[test-dependencies.gt] +repo = "https://example.invalid/y" +branch = "main" +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + assert!( + get_access_credentials_for_workspace(&ws_utf8, false).is_none(), + ); + assert!(get_access_credentials_for_workspace(&ws_utf8, true).is_none(),); + } + + #[test] + fn unlocked_git_deps_detection() { + // No git deps → never unlocked, regardless of lockfile. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.local] +path = "/tmp/somewhere" +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws.clone()).unwrap(); + assert!(!workspace_has_unlocked_git_deps(&ws_utf8, false).unwrap()); + assert!(!workspace_has_unlocked_git_deps(&ws_utf8, true).unwrap()); + + // Git dep + missing lockfile → unlocked. + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.g] +repo = "https://example.invalid/x" +branch = "main" +src = ["hdl"] +"#, + ) + .unwrap(); + assert!(workspace_has_unlocked_git_deps(&ws_utf8, false).unwrap()); + + // Git dep + lockfile that has an entry for it → locked. + std::fs::write( + ws.join("vw.lock"), + r#" +[dependencies.g] +repo = "https://example.invalid/x" +commit = "abc" +path = "/tmp/g" +src = ["hdl"] +recursive = false +sim_only = false +submodules = false +exclude = [] +"#, + ) + .unwrap(); + assert!(!workspace_has_unlocked_git_deps(&ws_utf8, false).unwrap()); + + // Git test-dep, lockfile only has the regular dep → unlocked + // for the with-test caller, locked for the plain caller. + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.g] +repo = "https://example.invalid/x" +branch = "main" + +[test-dependencies.gt] +repo = "https://example.invalid/y" +branch = "main" +"#, + ) + .unwrap(); + assert!(!workspace_has_unlocked_git_deps(&ws_utf8, false).unwrap()); + assert!(workspace_has_unlocked_git_deps(&ws_utf8, true).unwrap()); + } + + #[test] + fn vhdl_dependency_sources_exclude_sim_only_flag() { + // Two path deps: one flagged `sim_only = true` (mirrors + // real deps like `unisim` / `xpm`) and one regular. + // With the flag off both contribute; with it on only + // the non-sim dep does. + let tmp = tempfile::tempdir().unwrap(); + let sim_dep = tmp.path().join("sim"); + std::fs::create_dir_all(sim_dep.join("hdl")).unwrap(); + std::fs::write(sim_dep.join("hdl/sim.vhd"), "").unwrap(); + let real_dep = tmp.path().join("real"); + std::fs::create_dir_all(real_dep.join("hdl")).unwrap(); + std::fs::write(real_dep.join("hdl/real.vhd"), "").unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + format!( + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.sim] +path = "{}" +src = ["hdl"] +recursive = true +sim_only = true + +[dependencies.real] +path = "{}" +src = ["hdl"] +recursive = true +"#, + sim_dep.display(), + real_dep.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + + // Default (flag = false): both deps contribute. + let all = vhdl_dependency_sources(&ws_utf8).unwrap(); + assert_eq!(all.len(), 2, "{all:?}"); + + // Flag on: only the non-sim dep survives. + let synth_clean = + vhdl_dependency_sources_ext(&ws_utf8, false, true).unwrap(); + assert_eq!(synth_clean.len(), 1, "{synth_clean:?}"); + assert_eq!(synth_clean[0].library, "real"); + } + + #[test] + fn vhdl_dependency_sources_include_test_flag() { + // A test-only dep contributes iff include_test is set. + let tmp = tempfile::tempdir().unwrap(); + let dep = tmp.path().join("dep"); + std::fs::create_dir_all(dep.join("hdl")).unwrap(); + std::fs::write(dep.join("hdl/tbutil.vhd"), "").unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + format!( + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[test-dependencies.tbutil] +path = "{}" +src = ["hdl"] +recursive = true +"#, + dep.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + // Production mode: test-dep hidden. + assert!(vhdl_dependency_sources(&ws_utf8).unwrap().is_empty()); + // Test mode: test-dep visible. + let with_test = + vhdl_dependency_sources_with_test(&ws_utf8, true).unwrap(); + assert_eq!(with_test.len(), 1); + assert_eq!(with_test[0].library, "tbutil"); + } + + #[test] + fn vhdl_dependency_sources_honors_exclude() { + let tmp = tempfile::tempdir().unwrap(); + let dep = tmp.path().join("dep"); + std::fs::create_dir_all(dep.join("hdl/sims")).unwrap(); + std::fs::write(dep.join("hdl/a.vhd"), "").unwrap(); + std::fs::write(dep.join("hdl/b_tb.vhd"), "").unwrap(); + std::fs::write(dep.join("hdl/sims/x.vhd"), "").unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + format!( + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.dep] +path = "{}" +src = ["hdl"] +recursive = true +exclude = ["**/sims/**", "**/*_tb.vhd"] +"#, + dep.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + assert_eq!(sources.len(), 1, "{sources:?}"); + assert!(sources[0].path.ends_with("a.vhd")); + } + + #[test] + fn design_constraints_empty_when_no_dir() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + assert!(design_constraints(&ws).unwrap().is_empty()); + } + + #[test] + fn phase_scoped_constraints_isolate_per_subdir() { + // Regression guard: `synth/` files must not leak into + // `place/` or `route/`, and vice versa. Also verifies + // the whole-tree `design_constraints` still returns + // everything. + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let c = ws.join("constraints"); + std::fs::create_dir_all(c.join("synth")).unwrap(); + std::fs::create_dir_all(c.join("place")).unwrap(); + std::fs::create_dir_all(c.join("route")).unwrap(); + std::fs::write(c.join("global.xdc"), "").unwrap(); + std::fs::write(c.join("synth/only.xdc"), "").unwrap(); + std::fs::write(c.join("place/only.xdc"), "").unwrap(); + std::fs::write(c.join("route/only.xdc"), "").unwrap(); + + let synth = design_synth_constraints(&ws).unwrap(); + assert_eq!(synth.len(), 1); + assert!(synth[0].ends_with("synth/only.xdc")); + + let place = design_place_constraints(&ws).unwrap(); + assert_eq!(place.len(), 1); + assert!(place[0].ends_with("place/only.xdc")); + + let route = design_route_constraints(&ws).unwrap(); + assert_eq!(route.len(), 1); + assert!(route[0].ends_with("route/only.xdc")); + + // Aggregate walk returns everything under constraints/ + // regardless of subdir. + let all = design_constraints(&ws).unwrap(); + assert_eq!(all.len(), 4); + } + + #[test] + fn phase_scoped_constraints_empty_when_subdir_missing() { + // `constraints/` exists (some other subdir) but + // `constraints/synth/` does not — expect empty, not error. + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let c = ws.join("constraints"); + std::fs::create_dir_all(c.join("place")).unwrap(); + std::fs::write(c.join("place/only.xdc"), "").unwrap(); + + assert!(design_synth_constraints(&ws).unwrap().is_empty()); + assert!(design_route_constraints(&ws).unwrap().is_empty()); + assert_eq!(design_place_constraints(&ws).unwrap().len(), 1); + } + + #[test] + fn design_constraints_finds_xdc_and_sdc_recursively() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let c = ws.join("constraints"); + std::fs::create_dir_all(c.join("sub")).unwrap(); + std::fs::write(c.join("timing.xdc"), "").unwrap(); + std::fs::write(c.join("sub/pins.xdc"), "").unwrap(); + std::fs::write(c.join("sub/synopsys.sdc"), "").unwrap(); + // Non-constraint sibling — should be skipped. + std::fs::write(c.join("readme.md"), "").unwrap(); + + let files = design_constraints(&ws).unwrap(); + let names: Vec = files + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert_eq!(files.len(), 3, "{files:?}"); + assert!(names.contains(&"timing.xdc".to_string())); + assert!(names.contains(&"pins.xdc".to_string())); + assert!(names.contains(&"synopsys.sdc".to_string())); + assert!(!names.contains(&"readme.md".to_string())); + } + + #[test] + fn vhdl_ip_sources_empty_when_no_target_ip_dir() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + assert!(vhdl_ip_sources(&ws).unwrap().is_empty()); + } + + #[test] + fn vhdl_ip_sources_walks_target_ip_recursively() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let ip = ws.join("target/ip"); + std::fs::create_dir_all(ip.join("clocky")).unwrap(); + std::fs::create_dir_all(ip.join("cips")).unwrap(); + std::fs::write(ip.join("clocky/wrapper.vhd"), "").unwrap(); + std::fs::write(ip.join("cips/wrapper.vhd"), "").unwrap(); + // Non-VHDL siblings shouldn't get pulled in. + std::fs::write(ip.join("clocky/notes.md"), "").unwrap(); + + let sources = vhdl_ip_sources(&ws).unwrap(); + let names: Vec = sources + .iter() + .map(|p| { + let ip_name = p + .parent() + .and_then(|d| d.file_name()) + .and_then(|s| s.to_str()) + .unwrap_or(""); + ip_name.to_string() + }) + .collect(); + assert_eq!(sources.len(), 2, "{sources:?}"); + assert!(names.contains(&"clocky".to_string())); + assert!(names.contains(&"cips".to_string())); + } + + /// Regression: `target/ip/bd/**`, `target/ip/xci/**`, and + /// `target/vw-project/**` are the (legacy + on-disk) Vivado + /// cache trees. The `.vhd` files under them are registered + /// with the Vivado project via `read_bd` / `read_ip` / + /// `synth_ip`, so `vhdl_ip_sources` must NOT list them + /// (otherwise `synth`'s `read_vhdl` conflicts with the + /// sub-design registration and Vivado emits `[filemgmt + /// 20-1440]` CRITICAL WARNINGs). + /// + /// `target/vw-project/` is a sibling of `target/ip/` so the + /// current walker rooted at `target/ip/` doesn't actually + /// descend into it — the assertion here documents the + /// invariant so a future walker refactor (rooting higher up, + /// at `target/` for example) doesn't silently regress. + #[test] + fn vhdl_ip_sources_excludes_vivado_cache_subtrees() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let ip = ws.join("target/ip"); + // Legit wrappers. + std::fs::create_dir_all(ip.join("cips")).unwrap(); + std::fs::create_dir_all(ip.join("dcmac")).unwrap(); + std::fs::write(ip.join("cips/wrapper.vhd"), "").unwrap(); + std::fs::write(ip.join("dcmac/wrapper.vhd"), "").unwrap(); + // Legacy BD cache outputs — filter out. + std::fs::create_dir_all(ip.join("bd/cips/synth")).unwrap(); + std::fs::create_dir_all(ip.join("bd/cips/sim")).unwrap(); + std::fs::create_dir_all(ip.join("bd/dcmac/synth")).unwrap(); + std::fs::write(ip.join("bd/cips/synth/cips.vhd"), "").unwrap(); + std::fs::write(ip.join("bd/cips/sim/cips.vhd"), "").unwrap(); + std::fs::write(ip.join("bd/dcmac/synth/dcmac.vhd"), "").unwrap(); + // Legacy XCI cache outputs — filter out. + std::fs::create_dir_all(ip.join("xci/primary_clock")).unwrap(); + std::fs::write(ip.join("xci/primary_clock/primary_clock.vhd"), "") + .unwrap(); + // On-disk Vivado project sibling — invariant guard. + let vw_proj_gen = ws.join( + "target/vw-project/metroid/metroid.gen/sources_1/bd/cips/synth", + ); + std::fs::create_dir_all(&vw_proj_gen).unwrap(); + std::fs::write(vw_proj_gen.join("cips.vhd"), "").unwrap(); + + let sources = vhdl_ip_sources(&ws).unwrap(); + assert_eq!(sources.len(), 2, "{sources:?}"); + for p in &sources { + let s = p.to_string_lossy(); + assert!( + !s.contains("/target/ip/bd/"), + "bd cache file leaked into ip_sources: {s}" + ); + assert!( + !s.contains("/target/ip/xci/"), + "xci cache file leaked into ip_sources: {s}" + ); + assert!( + !s.contains("/target/vw-project/"), + "vw-project file leaked into ip_sources: {s}" + ); + assert!(s.ends_with("wrapper.vhd"), "unexpected file: {s}"); + } + } + + #[test] + fn vhdl_design_sources_walks_recursive_and_sorts() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let hdl = ws.join("hdl"); + std::fs::create_dir_all(hdl.join("sub")).unwrap(); + // Files under both root and a subdir; one non-VHDL to + // prove the extension filter kicks in. + std::fs::write(hdl.join("b.vhd"), "").unwrap(); + std::fs::write(hdl.join("a.vhd"), "").unwrap(); + std::fs::write(hdl.join("sub").join("c.vhdl"), "").unwrap(); + std::fs::write(hdl.join("readme.md"), "").unwrap(); + + let sources = vhdl_design_sources(&ws).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + // Sorted absolute paths → `a.vhd` before `b.vhd`, and + // `sub/c.vhdl` lands where its full path sorts. Not + // asserting exact order across subdirs — just checking + // both extensions and the recursion picked up the sub. + assert!(names.contains(&"a.vhd".to_string())); + assert!(names.contains(&"b.vhd".to_string())); + assert!(names.contains(&"c.vhdl".to_string())); + assert!(!names.contains(&"readme.md".to_string())); + } + + fn make_variant_ws(tmp: &tempfile::TempDir) -> Utf8PathBuf { + // Layout: + // hdl/shared.vhd (in no variant's exclusive → always included) + // hdl/ethernet-vpk120.vhd (owned by vpk120) + // hdl/ethernet-metro.vhd (owned by metro) + let ws = tmp.path().to_path_buf(); + let hdl = ws.join("hdl"); + std::fs::create_dir_all(&hdl).unwrap(); + std::fs::write(hdl.join("shared.vhd"), "").unwrap(); + std::fs::write(hdl.join("ethernet-vpk120.vhd"), "").unwrap(); + std::fs::write(hdl.join("ethernet-metro.vhd"), "").unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[[workspace.variants]] +name = "vpk120" +part = "xcvp1202-vsva2785-2MHP-e-S" +default = true +exclusive = ["hdl/ethernet-vpk120.vhd"] + +[[workspace.variants]] +name = "metro" +part = "xcvp1202-vsva2785-3HP-e-S" +exclusive = ["hdl/ethernet-metro.vhd"] +"#, + ) + .unwrap(); + Utf8PathBuf::from_path_buf(ws).unwrap() + } + + #[test] + fn design_sources_filter_by_active_variant() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_variant_ws(&tmp); + + // Active variant vpk120 → keeps shared + vpk120 file, + // excludes metro's. + let sources = + vhdl_design_sources_for_variant(&ws, Some("vpk120")).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert!(names.contains(&"shared.vhd".to_string())); + assert!(names.contains(&"ethernet-vpk120.vhd".to_string())); + assert!(!names.contains(&"ethernet-metro.vhd".to_string())); + + // Flip to metro. + let sources = + vhdl_design_sources_for_variant(&ws, Some("metro")).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert!(names.contains(&"shared.vhd".to_string())); + assert!(names.contains(&"ethernet-metro.vhd".to_string())); + assert!(!names.contains(&"ethernet-vpk120.vhd".to_string())); + } + + #[test] + fn design_sources_no_active_variant_still_filters_out_owned_files() { + // When active_variant is None but the workspace declares + // variants, ALL exclusive files are dropped — otherwise + // we'd spuriously pull every variant's owned files into + // one giant surface (which is the exact bug variants + // exist to solve). + let tmp = tempfile::tempdir().unwrap(); + let ws = make_variant_ws(&tmp); + let sources = vhdl_design_sources_for_variant(&ws, None).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert_eq!(names, vec!["shared.vhd"]); + } + + #[test] + fn design_sources_no_variants_declared_returns_all() { + // A workspace without any variants keeps the pre-variants + // behavior — every `.vhd` under `hdl/` shows up. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().to_path_buf(); + std::fs::create_dir_all(ws.join("hdl")).unwrap(); + std::fs::write(ws.join("hdl/a.vhd"), "").unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_design_sources_for_variant(&ws_utf8, None).unwrap(); + assert_eq!(sources.len(), 1); + } + + #[test] + fn design_sources_variant_exclusive_supports_globs() { + // `exclusive = ["hdl/board-vpk120/**/*.vhd"]` scopes an + // entire subtree to a variant. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().to_path_buf(); + std::fs::create_dir_all(ws.join("hdl/board-vpk120/sub")).unwrap(); + std::fs::create_dir_all(ws.join("hdl/board-metro")).unwrap(); + std::fs::write(ws.join("hdl/shared.vhd"), "").unwrap(); + std::fs::write(ws.join("hdl/board-vpk120/top.vhd"), "").unwrap(); + std::fs::write(ws.join("hdl/board-vpk120/sub/x.vhd"), "").unwrap(); + std::fs::write(ws.join("hdl/board-metro/top.vhd"), "").unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[[workspace.variants]] +name = "vpk120" +part = "xcvp1202-vsva2785-2MHP-e-S" +default = true +exclusive = ["hdl/board-vpk120/**/*.vhd"] + +[[workspace.variants]] +name = "metro" +part = "xcvp1202-vsva2785-3HP-e-S" +exclusive = ["hdl/board-metro/**/*.vhd"] +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = + vhdl_design_sources_for_variant(&ws_utf8, Some("vpk120")).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + // shared + both vpk120 files, no metro files. + assert_eq!(sources.len(), 3, "{names:?}"); + assert!(names.iter().any(|n| n == "shared.vhd")); + assert!( + names.iter().filter(|n| n.as_str() == "top.vhd").count() == 1, + "expected exactly one top.vhd (vpk120's), got {names:?}", + ); + } + + /// Minimal workspace scaffold for `synth_needs_update` tests: + /// a `vw.toml`, empty `vw.lock`, one hdl file, one synth XDC, + /// and one workspace htcl file. Returned as `Utf8PathBuf` so + /// the caller can hand it to the enumerator directly. + fn make_synth_ws(tmp: &tempfile::TempDir) -> Utf8PathBuf { + let ws = tmp.path().to_path_buf(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"snws\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + std::fs::write(ws.join("vw.lock"), "{}\n").unwrap(); + let hdl = ws.join("hdl"); + std::fs::create_dir_all(&hdl).unwrap(); + std::fs::write(hdl.join("top.vhd"), "-- vhdl\n").unwrap(); + let xdc = ws.join("constraints").join("synth"); + std::fs::create_dir_all(&xdc).unwrap(); + std::fs::write(xdc.join("timing.xdc"), "# xdc\n").unwrap(); + std::fs::write(ws.join("design.htcl"), "# htcl\n").unwrap(); + Utf8PathBuf::from_path_buf(ws).unwrap() + } + + /// A missing checkpoint file is always stale — the whole point + /// of the cache-check is to gate a first-time synth on this + /// condition. + #[test] + fn synth_needs_update_true_when_checkpoint_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + assert!(synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } + + /// Checkpoint present but no manifest → stale (either + /// pre-manifest era or a manually-copied checkpoint from + /// another workspace). Forces the next synth to write both, + /// which is the safe recovery path. + #[test] + fn synth_needs_update_true_when_manifest_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + assert!(synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } + + /// Manifest matches current fingerprint → fresh. Simulates + /// the post-`vw::synth` state on an unchanged tree — no + /// resynthesis required. + #[test] + fn synth_needs_update_false_when_manifest_matches() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + write_synth_checkpoint_manifest(&ws, cp.as_std_path(), None).unwrap(); + assert!(!synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } + + /// After a checkpoint+manifest pair, rewriting a source with + /// NEW bytes invalidates the fingerprint. This is the primary + /// invalidation path — a real edit to a tracked file. + #[test] + fn synth_needs_update_true_when_source_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + write_synth_checkpoint_manifest(&ws, cp.as_std_path(), None).unwrap(); + std::fs::write(ws.join("hdl/top.vhd"), "-- vhdl updated\n").unwrap(); + assert!(synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } + + /// The regression this whole switch to content hashing fixes: + /// rewriting a tracked file with IDENTICAL bytes must NOT + /// invalidate the manifest. Simulates `make_wrapper` + /// regenerating `target/ip/*/wrapper.vhd` on every design.htcl + /// run — same stripped-header body, fresh mtime. + #[test] + fn synth_needs_update_false_after_identical_rewrite() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + write_synth_checkpoint_manifest(&ws, cp.as_std_path(), None).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(20)); + // Overwrite with the same bytes the fixture wrote. Fresh + // mtime, unchanged content. Under the old mtime-based + // check this returned `true` (stale); under content + // hashing it stays `false`. + std::fs::write(ws.join("hdl/top.vhd"), "-- vhdl\n").unwrap(); + assert!(!synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } + + /// Editing a workspace `.htcl` invalidates the manifest. + /// Exercises `list_workspace_htcl_files` — a source + /// enumerator distinct from the VHDL / XDC paths. + #[test] + fn synth_needs_update_true_when_htcl_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + write_synth_checkpoint_manifest(&ws, cp.as_std_path(), None).unwrap(); + std::fs::write(ws.join("design.htcl"), "# htcl updated\n").unwrap(); + assert!(synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } + + /// Minimal workspace scaffold for `project_needs_wipe` tests: + /// a `vw.toml` and an `ip/module.htcl` (+ one submodule so + /// the recursive walk has something to cover). + fn make_project_ws(tmp: &tempfile::TempDir) -> Utf8PathBuf { + let ws = tmp.path().to_path_buf(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"prws\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + let ip = ws.join("ip"); + std::fs::create_dir_all(&ip).unwrap(); + std::fs::write( + ip.join("module.htcl"), + "namespace eval ip { proc configure {} unit {} }\n", + ) + .unwrap(); + std::fs::write(ip.join("cips.htcl"), "# cips\n").unwrap(); + Utf8PathBuf::from_path_buf(ws).unwrap() + } + + /// Materialize `//.xpr` as an empty + /// placeholder — `project_needs_wipe` short-circuits when + /// the `.xpr` is missing, so tests that want to exercise the + /// manifest branch need one present. + fn touch_placeholder_xpr(ws: &Utf8Path, name: &str) -> PathBuf { + let project_dir = vw_project_dir(ws); + let inner = project_dir.join(name); + std::fs::create_dir_all(inner.as_std_path()).unwrap(); + let xpr = inner.join(format!("{name}.xpr")); + std::fs::write(xpr.as_std_path(), "").unwrap(); + project_dir.into_std_path_buf() + } + + /// Missing `.xpr` → always needs wipe (first-time-project + /// bootstrap). The manifest presence is irrelevant when the + /// `.xpr` isn't there — nothing to open. + #[test] + fn project_needs_wipe_true_when_xpr_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = vw_project_dir(&ws); + assert!( + project_needs_wipe(&ws, project_dir.as_std_path(), "prws").unwrap() + ); + } + + /// `.xpr` present but manifest missing → wipe. This is the + /// "someone deleted the sidecar" or "pre-manifest-era + /// project" recovery path. + #[test] + fn project_needs_wipe_true_when_manifest_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + assert!(project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } + + /// Fresh manifest matching current fingerprint → do NOT + /// wipe. Simulates the post-`vw::configure_ip` state on an + /// unchanged tree. + #[test] + fn project_needs_wipe_false_when_manifest_matches() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + assert!(!project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } + + /// Editing any `.htcl` under `/ip/` invalidates. Key + /// test — the invalidation trigger the user actually + /// controls (adding an IP, tweaking a configure_* parameter). + #[test] + fn project_needs_wipe_true_when_ip_htcl_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + std::fs::write(ws.join("ip/cips.htcl"), "# cips updated\n").unwrap(); + assert!(project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } + + /// Editing `vw.toml` (target-part, deps list, etc.) also + /// invalidates — those changes usually mean the project + /// itself needs a fresh `create_project -part ...`. + #[test] + fn project_needs_wipe_true_when_vw_toml_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"prws\"\nversion = \"0.2.0\"\n", + ) + .unwrap(); + assert!(project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } + + /// Editing a workspace htcl OUTSIDE ip/ (e.g. design.htcl) + /// must NOT invalidate the on-disk project. Design-level + /// changes are the synth checkpoint's concern; the project + /// scope stays narrow so we don't nuke the expensive BD/IP + /// state on every design edit. + #[test] + fn project_needs_wipe_false_when_non_ip_htcl_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + std::fs::write(ws.join("design.htcl"), "# design\n").unwrap(); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + std::fs::write(ws.join("design.htcl"), "# design updated\n").unwrap(); + assert!(!project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } + + /// The regression parallel to the synth case: rewriting an + /// `ip/*.htcl` with IDENTICAL bytes must not invalidate. + /// Content-hash based, not mtime. + #[test] + fn project_needs_wipe_false_after_identical_rewrite() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + std::thread::sleep(std::time::Duration::from_millis(20)); + std::fs::write(ws.join("ip/cips.htcl"), "# cips\n").unwrap(); + assert!(!project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } + + /// Minimal workspace scaffold for `place_needs_update` tests: + /// a `vw.toml`, one place-scoped XDC, and a stand-in synth DCP + /// that the fingerprint folds in as a proxy for "everything + /// synth depended on". + fn make_place_ws( + tmp: &tempfile::TempDir, + ) -> (Utf8PathBuf, PathBuf, PathBuf) { + let ws = tmp.path().to_path_buf(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"plws\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + let xdc = ws.join("constraints").join("place"); + std::fs::create_dir_all(&xdc).unwrap(); + std::fs::write(xdc.join("place.xdc"), "# place xdc\n").unwrap(); + let synth_dcp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(synth_dcp.parent().unwrap()).unwrap(); + std::fs::write(&synth_dcp, "").unwrap(); + let place_dcp = ws.join("target/place/top.dcp"); + ( + Utf8PathBuf::from_path_buf(ws).unwrap(), + place_dcp, + synth_dcp, + ) + } + + /// Missing checkpoint → stale. Same first-place rule the + /// synth/ip caches use. + #[test] + fn place_needs_update_true_when_checkpoint_missing() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + assert!(place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } + + /// Fresh manifest → not stale. Verifies the write+check + /// round-trip on the place scope (place XDCs + synth DCP). + #[test] + fn place_needs_update_false_when_manifest_matches() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + write_place_checkpoint_manifest( + &ws, + place_dcp.as_path(), + synth_dcp.as_path(), + ) + .unwrap(); + assert!(!place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } + + /// Editing a place XDC invalidates. Trigger the user + /// controls most directly (tweak a place constraint, + /// re-place should fire). + #[test] + fn place_needs_update_true_when_place_xdc_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + write_place_checkpoint_manifest( + &ws, + place_dcp.as_path(), + synth_dcp.as_path(), + ) + .unwrap(); + std::fs::write( + ws.join("constraints/place/place.xdc"), + "# place xdc updated\n", + ) + .unwrap(); + assert!(place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } + + /// Synth re-ran → synth DCP content changed → place + /// invalidates. The synth DCP is intentionally folded into + /// the place fingerprint as a proxy for "everything synth + /// depended on". + #[test] + fn place_needs_update_true_when_synth_checkpoint_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + write_place_checkpoint_manifest( + &ws, + place_dcp.as_path(), + synth_dcp.as_path(), + ) + .unwrap(); + std::fs::write(&synth_dcp, "different bytes").unwrap(); + assert!(place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } + + /// Non-place workspace file changes must NOT invalidate the + /// place cache (design.htcl, hdl/, etc. are captured by the + /// synth stage; place scope is narrower). + #[test] + fn place_needs_update_false_when_non_place_htcl_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + std::fs::write(ws.join("design.htcl"), "# design\n").unwrap(); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + write_place_checkpoint_manifest( + &ws, + place_dcp.as_path(), + synth_dcp.as_path(), + ) + .unwrap(); + std::fs::write(ws.join("design.htcl"), "# design updated\n").unwrap(); + assert!(!place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } +} diff --git a/vw-quote/Cargo.toml b/vw-quote/Cargo.toml new file mode 100644 index 0000000..f01e952 --- /dev/null +++ b/vw-quote/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "vw-quote" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +syn = { version = "2", features = ["full"] } +quote = "1" + +[dev-dependencies] +vw-htcl = { path = "../vw-htcl" } diff --git a/vw-quote/src/lib.rs b/vw-quote/src/lib.rs new file mode 100644 index 0000000..b724e21 --- /dev/null +++ b/vw-quote/src/lib.rs @@ -0,0 +1,249 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `quote_htcl!` — generate htcl source code with interpolation. +//! +//! Analogous to the `quote` crate, but for htcl. Takes a string +//! literal of htcl source containing `#(expr)` interpolation markers +//! and produces a `String` of well-formed htcl at runtime. Each +//! interpolated value is passed through [`vw_htcl::emit::ToHtcl`] to +//! choose the right word form (bare vs. quoted vs. …) so generated +//! code is always parseable. +//! +//! ## Why a string literal? +//! +//! Rust's `TokenStream` doesn't preserve newlines, and newlines +//! terminate htcl commands — so a token-walking macro that reads +//! `quote_htcl! { proc x { … } { … } }` directly can't tell where one +//! statement ends and the next begins. Taking the input as a string +//! literal keeps the source text exact at zero cost in ergonomics. +//! +//! ## Syntax +//! +//! - `#(expr)` — interpolation slot. `expr` is parsed as a Rust +//! expression at macro time and emitted via +//! `vw_htcl::emit::ToHtcl::to_htcl(&expr)`. +//! - Anything else is literal htcl, copied verbatim except that `{` +//! and `}` in the template don't need any escaping (the macro +//! handles `format!` quoting for you). +//! +//! ## Example +//! +//! ```ignore +//! use vw_quote::quote_htcl; +//! let name = "greet"; +//! let who = "world"; +//! let s = quote_htcl!("\ +//! proc #(name) { +//! #(who) +//! } { puts hi } +//! "); +//! // s == "proc greet {\n world\n} { puts hi }\n" +//! // (the interpolated values get `ToHtcl`-formatted; here both are +//! // bare identifiers, so they emit as-is.) +//! ``` + +use proc_macro::TokenStream; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::quote; +use syn::{parse_macro_input, Expr, LitStr}; + +#[proc_macro] +pub fn quote_htcl(input: TokenStream) -> TokenStream { + expand(input, Dialect::Htcl) +} + +/// Same template grammar as [`quote_htcl!`], but routes interpolated +/// values through [`vw_htcl::emit::ToTcl`] and produces pure Tcl +/// (no htcl-specific attribute handling). Use for compiler-emitted +/// runtime helpers — `repr` procs, `kwargs` shim glue, anything that +/// lives in the Tcl interpreter and should never look like htcl. +/// +/// The split exists so future Tcl-only behavior (typed `Tcl_Obj` +/// handle quoting, etc.) can land on `ToTcl` without changing +/// `quote_htcl!`'s contract. +#[proc_macro] +pub fn quote_tcl(input: TokenStream) -> TokenStream { + expand(input, Dialect::Tcl) +} + +/// Which interpolation trait the macro routes through. The template +/// parsing is shared verbatim — the only thing that differs is the +/// trait + method name used in the generated `format!` arguments. +#[derive(Clone, Copy)] +enum Dialect { + Htcl, + Tcl, +} + +fn expand(input: TokenStream, dialect: Dialect) -> TokenStream { + let lit = parse_macro_input!(input as LitStr); + let template_text = lit.value(); + let lit_span = lit.span(); + + let (template, exprs) = match split_template(&template_text, lit_span) { + Ok(parts) => parts, + Err(e) => return e.to_compile_error().into(), + }; + + // Escape literal `{`/`}` so the format string parser leaves them + // alone; replace each `#(…)` site with a positional placeholder. + let format_string = render_format_string(&template, exprs.len()); + let format_lit = LitStr::new(&format_string, Span::call_site()); + + let exprs: Vec = + exprs.into_iter().map(|e| e.to_token_stream()).collect(); + + let out = match dialect { + Dialect::Htcl => quote! {{ + // Bring the trait into scope so `(&expr).to_htcl()` resolves + // without the caller needing to import it. + #[allow(unused_imports)] + use ::vw_htcl::emit::ToHtcl as _; + ::std::format!( + #format_lit, + #( (&{ #exprs }).to_htcl() ),* + ) + }}, + Dialect::Tcl => quote! {{ + #[allow(unused_imports)] + use ::vw_htcl::emit::ToTcl as _; + ::std::format!( + #format_lit, + #( (&{ #exprs }).to_tcl() ),* + ) + }}, + }; + out.into() +} + +// --- template parsing ------------------------------------------------------ + +/// One piece of the parsed template. +enum Piece { + /// Verbatim text from the template. + Text(String), + /// An interpolation site; index into the parallel `exprs` Vec. + Interp, +} + +/// Split the template string into alternating text / interpolation +/// pieces, parsing each `#(…)` body as a Rust expression. +fn split_template( + text: &str, + lit_span: Span, +) -> syn::Result<(Vec, Vec)> { + let mut pieces = Vec::new(); + let mut exprs = Vec::new(); + let mut buf = String::new(); + + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len() { + let c = bytes[i]; + // Allow `##` doc comments and `#` plain comments to pass + // through unmolested: interpolation is `#(...)`, so we only + // engage when `#` is immediately followed by `(`. + if c == b'#' && i + 1 < bytes.len() && bytes[i + 1] == b'(' { + if !buf.is_empty() { + pieces.push(Piece::Text(std::mem::take(&mut buf))); + } + let body_end = match find_matching_paren(bytes, i + 1) { + Some(end) => end, + None => { + return Err(syn::Error::new( + lit_span, + "unterminated `#(...)` in quote_htcl! template", + )); + } + }; + // bytes[i+2 .. body_end] is the inside of the parens. + let expr_src = &text[i + 2..body_end]; + let expr: Expr = syn::parse_str(expr_src).map_err(|e| { + syn::Error::new( + lit_span, + format!( + "could not parse interpolation `{expr_src}` as a \ + Rust expression: {e}" + ), + ) + })?; + exprs.push(expr); + pieces.push(Piece::Interp); + i = body_end + 1; + continue; + } + // Push this char (handle UTF-8 boundary by stepping a full + // char rather than a byte). + let ch_start = i; + // Safe because `i` is always at a UTF-8 boundary (we only + // advance by full chars or past ASCII chars we recognized). + let ch = text[ch_start..].chars().next().unwrap(); + buf.push(ch); + i += ch.len_utf8(); + } + if !buf.is_empty() { + pieces.push(Piece::Text(buf)); + } + Ok((pieces, exprs)) +} + +/// Find the byte index of the `)` that matches an opening `(` at +/// `bytes[open]`. Tracks nested parens. Returns `None` on unterminated. +fn find_matching_paren(bytes: &[u8], open: usize) -> Option { + debug_assert_eq!(bytes[open], b'('); + let mut depth = 1usize; + let mut i = open + 1; + while i < bytes.len() { + match bytes[i] { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Some(i); + } + } + _ => {} + } + i += 1; + } + None +} + +/// Build the format string passed to `std::format!`, with literal +/// `{`/`}` doubled and `{0}` / `{1}` / … placeholders inserted at each +/// interpolation site. +fn render_format_string(pieces: &[Piece], n_interps: usize) -> String { + let mut out = String::new(); + let mut next_interp = 0usize; + let _ = n_interps; + for piece in pieces { + match piece { + Piece::Text(s) => { + for c in s.chars() { + match c { + '{' => out.push_str("{{"), + '}' => out.push_str("}}"), + other => out.push(other), + } + } + } + Piece::Interp => { + use std::fmt::Write; + write!(out, "{{{}}}", next_interp).unwrap(); + next_interp += 1; + } + } + } + out +} + +trait ToTokenStream { + fn to_token_stream(&self) -> TokenStream2; +} +impl ToTokenStream for Expr { + fn to_token_stream(&self) -> TokenStream2 { + quote! { #self } + } +} diff --git a/vw-quote/tests/basic.rs b/vw-quote/tests/basic.rs new file mode 100644 index 0000000..1e0866f --- /dev/null +++ b/vw-quote/tests/basic.rs @@ -0,0 +1,140 @@ +// Integration tests for `quote_htcl!` and `quote_tcl!`. A proc-macro +// crate can't use its own macros from within `src/`, so these tests +// live in `tests/` and pull `vw-quote` and `vw-htcl` as dev-deps. + +use vw_quote::{quote_htcl, quote_tcl}; + +#[test] +fn literal_passthrough() { + let s = quote_htcl!("puts hi\n"); + assert_eq!(s, "puts hi\n"); +} + +#[test] +fn simple_ident_interpolation() { + let name = "greet"; + let s = quote_htcl!("proc #(name) {} { puts hi }\n"); + assert_eq!(s, "proc greet {} { puts hi }\n"); +} + +#[test] +fn expression_interpolation() { + let width = 16u32; + let s = quote_htcl!("set w #(width)\n"); + assert_eq!(s, "set w 16\n"); +} + +#[test] +fn values_needing_quoting_get_quoted() { + let msg = "hello world"; + let s = quote_htcl!("puts #(msg)\n"); + assert_eq!(s, "puts \"hello world\"\n"); +} + +#[test] +fn dollar_in_value_is_escaped() { + let s = "$x"; + let out = quote_htcl!("puts #(s)\n"); + // The value `$x` has special chars, so it gets quoted with + // `\$` escaped — preserving it as the literal text "$x" at runtime. + assert_eq!(out, "puts \"\\$x\"\n"); +} + +#[test] +fn braces_in_template_pass_through() { + let name = "f"; + let s = quote_htcl!("proc #(name) {} {\n puts hi\n}\n"); + assert_eq!(s, "proc f {} {\n puts hi\n}\n"); +} + +#[test] +fn doc_comment_passes_through() { + let s = quote_htcl!("## A doc comment.\nputs hi\n"); + assert_eq!(s, "## A doc comment.\nputs hi\n"); +} + +#[test] +fn multiple_interpolations() { + let name = "greet"; + let arg = "world"; + let s = quote_htcl!("proc #(name) { #(arg) } { puts hi }\n"); + assert_eq!(s, "proc greet { world } { puts hi }\n"); +} + +#[test] +fn method_call_in_interpolation() { + struct P { + name: &'static str, + } + let p = P { name: "greet" }; + let s = quote_htcl!("proc #(p.name) {} { }\n"); + assert_eq!(s, "proc greet {} { }\n"); +} + +#[test] +fn output_parses_as_valid_htcl() { + let name = "greet"; + let msg = "hi there"; + let s = quote_htcl!("proc #(name) {} {\n puts #(msg)\n}\n"); + let parsed = vw_htcl::parse(&s); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); +} + +// --- quote_tcl! ------------------------------------------------------------- +// +// Mirror the quote_htcl! shape tests. The Tcl-dialect macro shares +// the same template parser, so the same template should produce the +// same output for the cases that overlap (which is most of them +// today — the Tcl/htcl split exists so they can DIVERGE later, not +// because their current behavior differs). + +#[test] +fn tcl_literal_passthrough() { + let s = quote_tcl!("puts hi\n"); + assert_eq!(s, "puts hi\n"); +} + +#[test] +fn tcl_simple_ident_interpolation() { + let name = "greet"; + let s = quote_tcl!("proc #(name) {} { puts hi }\n"); + assert_eq!(s, "proc greet {} { puts hi }\n"); +} + +#[test] +fn tcl_expression_interpolation() { + let width = 16u32; + let s = quote_tcl!("set w #(width)\n"); + assert_eq!(s, "set w 16\n"); +} + +#[test] +fn tcl_values_needing_quoting_get_quoted() { + let msg = "hello world"; + let s = quote_tcl!("puts #(msg)\n"); + assert_eq!(s, "puts \"hello world\"\n"); +} + +#[test] +fn tcl_braces_in_template_pass_through() { + let name = "f"; + let s = quote_tcl!("proc #(name) {} {\n puts hi\n}\n"); + assert_eq!(s, "proc f {} {\n puts hi\n}\n"); +} + +#[test] +fn tcl_multiple_interpolations() { + let name = "greet"; + let arg = "world"; + let s = quote_tcl!("proc #(name) { #(arg) } { puts hi }\n"); + assert_eq!(s, "proc greet { world } { puts hi }\n"); +} + +#[test] +fn tcl_emits_repr_proc_shape() { + // A representative use case from step 2b: emit a per-type + // repr proc body via quote_tcl!. + let mangled = "string"; + let s = quote_tcl!("proc #(mangled)::repr {v} {\n return $v\n}\n"); + assert_eq!(s, "proc string::repr {v} {\n return $v\n}\n"); +} diff --git a/vw-repl/Cargo.toml b/vw-repl/Cargo.toml new file mode 100644 index 0000000..c8c85b7 --- /dev/null +++ b/vw-repl/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "vw-repl" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Interactive REPL for htcl scripts, driven by a long-lived Vivado worker" + +[dependencies] +vw-htcl = { path = "../vw-htcl" } +vw-eda = { path = "../vw-eda" } +vw-vivado = { path = "../vw-vivado" } +vw-lib = { path = "../vw-lib" } +camino.workspace = true +ratatui.workspace = true +crossterm.workspace = true +tui-textarea.workspace = true +tokio.workspace = true +thiserror.workspace = true +dirs.workspace = true +futures.workspace = true +tracing.workspace = true +arboard = "3" +base64 = "0.22" +winnow.workspace = true +nucleo-matcher.workspace = true +serde_json.workspace = true +serde.workspace = true +toml.workspace = true + +# Unix-only. Used by the Ctrl-C eval-interrupt path in `app.rs` — +# `libc::kill(pid, SIGINT)` sends SIGINT directly to the Vivado +# child (cached pid from `WorkerEvent::Started`) so cancellation +# fires immediately even while `worker_task` is blocked in +# `backend.eval().await` on the very eval we're trying to cancel. +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-repl/src/app.rs b/vw-repl/src/app.rs new file mode 100644 index 0000000..5d5efc2 --- /dev/null +++ b/vw-repl/src/app.rs @@ -0,0 +1,4736 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! REPL state machine + event loop. +//! +//! Single tokio task drives both the ratatui screen and the Vivado +//! worker. Inputs are crossterm key events; outputs are eval results +//! from the worker plus our own scrollback updates. A `tokio::select!` +//! arbitrates the two so neither side blocks the other. +//! +//! A Vivado eval can take seconds to minutes. The UI stays +//! responsive throughout: the input area locks (`eval_in_flight`) +//! but the screen still redraws, the worker's stdout still streams +//! into scrollback as it arrives, and Ctrl-C cancels the in-flight +//! eval (sent as a TCL interrupt to the worker). + +use std::time::Duration; + +use crossterm::event::{ + DisableBracketedPaste, DisableMouseCapture, EnableBracketedPaste, + EnableMouseCapture, Event, KeyCode, KeyEvent, KeyEventKind, KeyModifiers, + KeyboardEnhancementFlags, MouseButton, MouseEvent, MouseEventKind, + PopKeyboardEnhancementFlags, PushKeyboardEnhancementFlags, +}; +use crossterm::terminal::{ + disable_raw_mode, enable_raw_mode, EnterAlternateScreen, + LeaveAlternateScreen, +}; +use crossterm::ExecutableCommand; +use futures::StreamExt; +use ratatui::backend::CrosstermBackend; +use ratatui::layout::Rect; +use ratatui::Terminal; +use tokio::sync::mpsc; +use tui_textarea::{Input, TextArea}; +use vw_eda::EdaBackend; + +use crate::config::{self, CollapseMode}; +use crate::history::History; +use crate::lower::Origin; +use crate::session::{Session, SessionBatch}; +use crate::ui::{self, WorkerStatusView}; +use crate::{ReplError, ReplOptions}; + +/// A multi-line entry with more than this many source lines +/// auto-collapses into a `▶`-marked placeholder at push time, +/// Mathematica-notebook style. Shift-click still toggles the +/// state, so the raw content is one gesture away — the threshold +/// just keeps a wall of text from dominating scrollback when a +/// large `list` or `Properties` value comes back. Chosen at 100 +/// because that's roughly the transition point where a viewport- +/// filling block stops being scannable and starts being intrusive. +pub const COLLAPSE_AUTO_THRESHOLD: usize = 100; + +/// Decide the initial [`ScrollbackEntry::collapse_state`] for an +/// entry with `text`. Every multi-line entry gets a `Some(bool)` +/// so it's toggleable via Shift+click; single-line entries get +/// `None` because a placeholder around one row of content adds +/// affordance without meaningful eliding. Above the auto-threshold +/// the initial state is collapsed. +fn compute_collapse_state(text: &str, mode: CollapseMode) -> Option { + let lines = text.lines().count(); + if lines < 2 { + return None; + } + match mode { + // Aggressive: every multi-line entry starts collapsed — + // `▶`-marked placeholders that expand on demand. Turns + // scrollback into a compact index. See + // [`crate::config::CollapseMode`]. + CollapseMode::Aggressive => Some(true), + // Normal: only wall-of-text entries auto-collapse; smaller + // multi-line output stays inline for at-a-glance scanning. + CollapseMode::Normal => Some(lines > COLLAPSE_AUTO_THRESHOLD), + } +} + +/// Split a tagged-diagnostic message into `(leading, trailing)`. +/// +/// A "tagged diagnostic line" has the shape +/// `: [] ` — e.g. +/// `ERROR: [Common 17-107] Cannot change read-only property …` +/// or `CRITICAL WARNING: [Project 1-486] Could not resolve …`. +/// Vivado (and our own eval-error renderer) sometimes appends +/// wrapped continuations, a ` Resolution: …` hint, or a +/// backtrace on the following lines — all valuable, but bulky. +/// +/// `leading` is the tagged first line (returned even when the +/// message is a single line, in which case `trailing` is `None`). +/// `trailing` is everything after the first line, trailing +/// newlines stripped; `None` when the message is a single line +/// or the tail is whitespace-only. +/// +/// Downstream, `leading` pushes as its OWN scrollback entry so +/// it stays at full brightness — one-liner, non-collapsible, +/// eye-catching gutter — while `trailing` pushes separately and +/// can auto-collapse / dim like any other multi-line entry. +/// This is the fix for the "critical error line got dimmed and +/// blended into the surrounding chatter" bug. +fn split_leading_diagnostic(text: &str) -> (String, Option) { + match text.split_once('\n') { + Some((head, tail)) => { + let tail = tail.trim_end_matches('\n'); + if tail.trim().is_empty() { + (head.to_string(), None) + } else { + (head.to_string(), Some(tail.to_string())) + } + } + None => (text.to_string(), None), + } +} + +/// Split a diagnostic's rendered text into `(body, stack)` at the +/// first line that looks like a stack frame — ` at :`. +/// The body is everything before that line (message + any wrapped +/// continuations); the stack is that line and everything after, +/// trailing newline stripped. `None` for stack means the diagnostic +/// carried no frames (traceless INFOs, some plain WARNINGs). +/// +/// The two-space indent + literal `at ` prefix is the shape +/// [`install_proc_body_wrap`] attaches in `vivado-shim.tcl` — see +/// its `format_stack` helper. If that format ever drifts, both +/// sides must stay in sync. +fn split_body_and_stack(text: &str) -> (String, Option) { + let needle = "\n at "; + match text.find(needle) { + // The `\n` at `idx` closes the body line; the stack begins + // at `idx + 1` so the ` at ` prefix is preserved on the + // first stack frame. + Some(idx) => { + let body = text[..idx].to_string(); + let stack = text[idx + 1..].trim_end_matches('\n').to_string(); + if stack.is_empty() { + (body, None) + } else { + (body, Some(stack)) + } + } + None => (text.to_string(), None), + } +} + +/// What category an entry in the scrollback log belongs to. Drives +/// the per-line gutter prefix and color. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScrollbackKind { + /// Echo of an input the user submitted. + Input, + /// A return value from a successful eval. + Result, + /// Captured stdout from `puts` etc. during an eval. + Stdout, + /// An error — TCL-level or REPL-level. Also the visual bucket + /// for Vivado CRITICAL WARNINGs, which the block classifier + /// keeps semantically distinct for log-level filtering but + /// which render with the same red ✗ treatment (per the + /// stream handler's Severity match). Entries in this kind + /// that came from a CW are additionally flagged via + /// [`ScrollbackEntry::is_critical_warning`] so the + /// diagnostics finder can offer a separate `Critical` filter + /// checkbox on top of the Error filter. + Error, + /// A pre-flight warning the user should see before the + /// underlying eval result — e.g. "this call uses keyword args + /// but isn't a loaded htcl wrapper." Distinct color from + /// notices so it actually pulls the eye. + Warning, + /// Internal notice (`vivado: ready`, `:load`, `:restart`, etc.). + Notice, + /// Non-diagnostic Vivado chatter that landed as its own single- + /// line NONE block — the `----` divider after an INFO, + /// `Attempting to get a license…` status echoes, single-line + /// section banners. Rendered dimmed dark-gray with a plain + /// ` ` prefix so it reads as "background noise" without + /// pretending to be an INFO (which would carry `· `) or a + /// user-facing Stdout entry (which would be bright white). + /// Multi-line NONE blocks go through the collapsible path + /// instead — see `push_none_block`. + Chatter, +} + +/// Tracks where each echoed top-level statement's Input entry +/// lives in scrollback AND which lowered command-index in the +/// batch is its last. When that command finishes evaluating, the +/// Input entry's timer freezes — giving accurate per-statement +/// durations in multi-statement load batches instead of all +/// entries sharing the whole-batch wall time. +#[derive(Clone, Debug)] +struct InputBoundary { + /// Position in `scrollback` where this boundary's echo lives, + /// or `None` when the echo is still queued. Non-first entries + /// start `None` and get pushed by `advance_input_timers` when + /// the prior boundary closes — so a `:load` batch prints + /// linearly: command, its output, then the *next* command. + scrollback_idx: Option, + /// Snippet captured up front so the deferred push has the + /// exact text `dispatch_eval_with_echo` would have used + /// eagerly. + snippet: String, + /// Eval-index in `pending_origins` of the last lowered + /// command that originated from this top-level statement. + /// When `pending_eval_index` reaches this value (i.e. the + /// command at this index has just finished), the entry's + /// timer should freeze. `None` when no lowered command was + /// attributed to this boundary — e.g. a `src` whose target + /// file lowered to zero Tcl commands. Such boundaries are + /// skipped when the prior boundary closes: nothing to wait + /// for and nothing to echo. + last_command_idx: Option, + /// Set to true once we've stamped this entry's `completed_at`, + /// so we don't re-stamp on subsequent EvalDones. + completed: bool, +} + +#[derive(Clone, Debug)] +pub struct ScrollbackEntry { + pub kind: ScrollbackKind, + pub text: String, + /// When this entry was pushed. Only set for `Input` entries + /// — used by the renderer to right-justify a `Ns` / + /// `M:SS` / `H:MM:SS` elapsed-time marker on the first + /// line. Non-input entries don't get timed and leave this + /// `None`. + pub started_at: Option, + /// When the corresponding eval finished. `None` while the + /// eval is still running (renderer shows live-updating + /// elapsed time from `started_at`); `Some(t)` freezes the + /// timer at the final duration once the batch completes. + pub completed_at: Option, + /// Non-`None` marks this entry as a collapsible NONE-severity + /// block (Vivado's non-diagnostic output: tables, banners, + /// section headers). `Some(true)` = collapsed (renderer shows + /// a single `▶` placeholder with a preview + hidden-line + /// count); `Some(false)` = expanded (all lines render dimmed + /// with a `▼` marker on the first line). `None` = normal + /// entry, no collapse handling. + /// + /// Only NONE blocks get this treatment — diagnostics are + /// always shown at full fidelity so a user scanning + /// scrollback for a WARNING never has to expand anything to + /// find it. + pub collapse_state: Option, + /// True when this entry originated from a Vivado CRITICAL + /// WARNING (not a plain ERROR). The two share + /// [`ScrollbackKind::Error`] for rendering — same red gutter + /// — but the diagnostics finder uses this flag to let the + /// user filter Critical warnings independently of plain + /// Errors. Default `false` for entries pushed by paths that + /// don't know severity (Input echoes, Result returns, + /// synthetic Notices). + pub is_critical_warning: bool, +} + +/// Drag-selection over scrollback rows. Coordinates are `(row, col)` +/// indices into the post-wrap line list (see +/// [`crate::render::wrap_lines`]) — i.e. the same indexing the +/// renderer uses for `Paragraph::scroll`. `anchor` is where the user +/// pressed; `cursor` updates while dragging. The range may be +/// inverted (cursor before anchor) — callers normalize via +/// [`Selection::ordered`] before applying. +#[derive(Clone, Copy, Debug)] +pub struct Selection { + pub anchor: (usize, usize), + pub cursor: (usize, usize), +} + +impl Selection { + /// Return `(start, end)` with `start <= end` so callers don't + /// have to special-case backwards drags. + pub fn ordered(&self) -> ((usize, usize), (usize, usize)) { + if self.anchor <= self.cursor { + (self.anchor, self.cursor) + } else { + (self.cursor, self.anchor) + } + } +} + +/// REPL meta-command catalog. Each entry is `(label, hint)` where +/// `label` is the full `:command` token (including the leading +/// colon, since that's what the user types and what Tab-completion +/// replaces) and `hint` is a one-line description shown in the +/// completion popup. +/// +/// Keep in sync with the `match` in [`App::run_meta_command`] and +/// with the cheat-sheet rows in [`crate::popup::HELP_ROWS`]. +pub const META_COMMANDS: &[(&str, &str)] = &[ + (":load", "evaluate a file's contents in this session"), + (":libs", "list loaded libraries + symbol counts"), + (":quit", "exit the REPL"), + (":exit", "exit the REPL (alias of :quit)"), + (":q", "exit the REPL (alias of :quit)"), + ( + ":restart", + "restart the Vivado worker (not yet implemented)", + ), +]; + +#[derive(Clone, Debug)] +pub struct ReverseSearch { + /// The substring the user is searching for. + pub query: String, + /// Index in [`History::entries`] of the current match. + pub match_index: Option, + /// The matched entry's text, cloned for the UI's static lifetime. + pub match_text: String, +} + +pub struct App { + opts: ReplOptions, + input: TextArea<'static>, + history: History, + /// Where the up/down (Ctrl-P/Ctrl-N) history walk is currently + /// positioned. `None` means "composing a fresh entry" — the + /// state any new keypress (other than Ctrl-P/N themselves) + /// drops back into so editing after walking history doesn't + /// keep recalling stale entries. `Some(i)` indexes + /// `History::entries()` directly. + history_cursor: Option, + /// In-progress draft saved when the user first walks back into + /// history with Ctrl-P. Restored on Ctrl-N past the newest + /// entry, so the draft they were typing isn't lost. + history_draft: String, + /// Session state, shareable across threads so background + /// prepare tasks (`dispatch_eval_with_echo`) can hold a read + /// guard for the ~seconds-to-minutes of a large `src` import + /// while the main event loop's frequent lookups (Ctrl-P + /// history walk, tab-completion, signature-help refresh, + /// input-completeness check) grab their own concurrent read + /// guards without contention. + /// + /// `RwLock` — not `Mutex` — because the main thread's typing- + /// time reads MUST run in parallel with a long-running + /// background prepare's read. A single writer (`commit` on a + /// successful prepare) briefly acquires the write lock; that + /// happens on the main task after the prepare returns, so + /// there's no active reader to wait for. + session: std::sync::Arc>, + /// Shared "already loaded in Vivado" map. Handed to the RPC + /// handler at spawn time; refreshed from + /// `session.loaded_paths()` after every `commit` so + /// `compile_htcl_module` skips re-shipping files whose + /// procs are already installed. Correctness invariant lives + /// with [`vw_vivado::SharedPreload`]. + preload: vw_vivado::SharedPreload, + scrollback: Vec, + /// Segments Vivado's classified stream into per-chunk Diagnostic + /// entries and grouped NONE-block collapsibles before pushing + /// into scrollback. Same accumulator vw-cli uses, but here it + /// lives on `App` because Stream events flow through the main + /// task's single event loop — no cross-thread sharing needed. + /// Diagnostic blocks and NONE blocks land in scrollback via + /// [`Self::push`] and [`Self::push_none_block`] respectively. + block_acc: vw_vivado::BlockAccumulator, + scrollback_scroll: u16, + /// Whether the terminal is currently capturing mouse events. + /// Default ON since we implement our own drag-to-select + + /// clipboard copy (see [`Selection`]). F2 toggles it off for + /// users who'd rather use terminal-native selection (which + /// requires capture to be disabled because the protocol is + /// all-or-nothing). + mouse_capture: bool, + /// Last scrollback render area, captured by `ui::draw_scrollback` + /// each frame. Lets mouse handlers map screen coords back to + /// scrollback-local cells without round-tripping through the UI. + scrollback_area: Option, + /// Active drag-selection in scrollback. Coordinates are + /// `(row, col)` indices into the post-wrap scrollback line list + /// — see `render::wrap_lines`. `None` outside of an active drag + /// or after a copy completes. + selection: Option, + /// Tail-follow mode. When `true`, the renderer pins the + /// effective scroll offset to the bottom of the wrapped-row + /// list — same model as `tail -f` or a fresh terminal. Manual + /// scroll-up flips this off so the user can read older content + /// without the view jumping out from under them; scrolling + /// back down to the bottom flips it back on. Submitting a new + /// command resets to `true`. + scrollback_follow: bool, + /// The effective scroll offset the renderer used on the most + /// recent frame. Written by `ui::draw_scrollback`, read by the + /// mouse / keyboard scroll handlers so a manual move from + /// tail-follow mode "takes over" the rendered position rather + /// than jumping back to whatever stale value is in + /// `scrollback_scroll`. + last_rendered_scroll: u16, + /// The `max_scroll` (= wrapped rows − viewport height) the + /// renderer computed on the most recent frame. Written by + /// `ui::draw_scrollback`; consulted by [`Self::scroll_by`] so a + /// downward scroll that lands at (or past) the bottom re-engages + /// tail-follow. This is safe now that the wrapped-row count is + /// exact — the old auto-re-engage misfired on large output + /// because it compared raw `text.lines().count()` against a + /// heuristic threshold. + last_max_scroll: u16, + /// Scrollback entry index the user last jumped to via the + /// diagnostic-finder popup (Ctrl-F → Enter). While `Some`, the + /// renderer paints a persistent left-gutter marker on every + /// wrapped row of that entry so the user can spot it in a + /// busy log. Alt-C clears it. `None` on startup and after + /// clear; not affected by scrolling or new entries appending. + marker_entry: Option, + /// One-shot request from the popup layer to scroll the + /// specified scrollback entry into view on the next frame. + /// Consumed by `ui::draw_scrollback` — it computes the + /// wrapped-row offset of that entry (from the same per-entry + /// count pass it already does) and writes it into + /// `scrollback_scroll` + disengages `scrollback_follow`. + /// Kept as a `usize` scrollback-idx rather than a pre-computed + /// row offset because the offset depends on area.width, which + /// the popup key handler doesn't know. + pending_jump: Option, + reverse_search: Option, + /// Active LSP-style popup over the input editor (completion, + /// signature help, hover). When `Some`, the key handler routes + /// navigation / dismissal keys to the popup BEFORE the catch-all + /// editor handoff. See [`crate::popup`]. + popup: Option, + worker_state: WorkerState, + /// Vivado child's OS pid, cached from `WorkerEvent::Started`. + /// Consulted by the Ctrl-C handler to send SIGINT for eval + /// cancellation without going through `worker_tx` (which is + /// blocked by the in-flight `backend.eval` we'd be trying to + /// cancel). `None` before Started, or when the PTY layer + /// couldn't report a pid — in either case Ctrl-C falls back + /// to its non-eval behavior (clear the current input). + child_pid: Option, + worker_tx: mpsc::Sender, + eval_rx: mpsc::UnboundedReceiver, + /// Sender-side of the worker-event channel, kept on App so + /// spawned background tasks (currently: `prepare` on a + /// blocking thread) can post their completion back to the + /// event loop without a separate channel. The receiver + /// (`eval_rx`) is drained inside the main select loop. + event_tx: mpsc::UnboundedSender, + /// Origins of every command we shipped to the worker in the + /// current batch, in eval order, paired with an index for the + /// next not-yet-acknowledged command. Lets the stream handler + /// tag a mid-eval Vivado warning with `at : + /// in ` even when Vivado bypasses our Tcl-level + /// stack capture (the IP_Flow C++ property validators don't + /// call `::common::send_msg_id`, so neither shim override + /// fires — without this fallback those warnings would arrive + /// stack-less). + pending_origins: Vec, + /// Parallel to `pending_origins`: the expected return type of + /// each shipped command, when statically resolvable. Used by + /// the `EvalDone` handler to (a) skip the heuristic formatter + /// when the value is already type-formatted by the wrapped + /// Tcl, and (b) suppress the Result push entirely for + /// `unit`-typed expressions. + pending_return_types: Vec>, + /// Parallel to `pending_return_types`, one entry per lowered + /// command. True when the command is a top-level `set VAR + /// ` — the app suppresses the Result echo for those: + /// binding is a plumbing operation, not a display, and + /// echoing the raw value can leak the internal tagged Tcl + /// list form (`{Scalar x}` rather than the parens repr). + pending_is_set_binding: Vec, + /// For per-Input-entry timer freezing: one entry per + /// echoed top-level statement in the current batch, in + /// source order. Each carries the scrollback index of its + /// Input entry and the eval-index of the LAST lowered + /// command that came from that statement. When EvalDone + /// fires for that command index, we freeze the entry's + /// timer AND start the next entry's timer (so per-statement + /// durations in a multi-statement load batch are accurate + /// instead of all reading the whole-batch wall time). + pending_input_boundaries: Vec, + pending_eval_index: usize, + /// The batch we shipped to the worker but haven't yet seen a + /// result for. Held aside so a successful eval (and only a + /// successful one) commits to the session — and so the error + /// renderer can look up procs declared in this in-flight + /// batch (which aren't yet in `session`) when drilling into a + /// Tcl stack frame. + pending_batch: Option, + /// Set when `:quit` (or Ctrl-D on an empty buffer) fires, so the + /// outer loop bails out after the current frame. + exit: bool, + /// Auto-collapse policy for multi-line scrollback entries. + /// Loaded from `/.vw/repl.toml` at startup and + /// consulted by [`compute_collapse_state`] on every `push`. + collapse_mode: CollapseMode, +} + +enum WorkerState { + Starting, + Ready, + Running, + Down, +} + +/// Commands sent from the UI to the worker task. A batch ships one +/// or more lowered htcl statements; the worker fires `eval` per +/// item, sends one [`WorkerEvent::EvalDone`] per item, and stops at +/// the first failure so we don't keep running a script after it's +/// hit an error. +enum WorkerCmd { + EvalBatch(Vec), + Shutdown, +} + +/// Events sent from the worker task back to the UI. +enum WorkerEvent { + /// Vivado spawn succeeded. Carries the child's OS pid so the + /// UI can send SIGINT for eval cancellation without going + /// through the worker channel (which is blocked by the + /// in-flight eval it would need to cancel). `None` when the + /// PTY layer couldn't report a pid — cancellation degrades + /// to a no-op in that case. + Started { + child_pid: Option, + }, + /// One streaming chunk from the worker, with its source-of- + /// origin tag so the UI can render Vivado WARNING/ERROR lines + /// distinctly from user `puts` output. + Stream { + kind: vw_vivado::StreamKind, + data: String, + }, + /// One item of a batch completed. `origin` is the htcl source + /// location the lowered Tcl came from so the renderer can show + /// `file:line` rather than a Tcl stack trace pointing at the + /// shim. `last_in_batch` lets the UI know when to commit to + /// the session document. + EvalDone { + origin: crate::lower::Origin, + result: Result, + last_in_batch: bool, + }, + StartFailed(vw_eda::BackendError), + /// The background `prepare` (parse + validate + lower) + /// finished. `dispatch_eval_with_echo` spawns prepare on a + /// blocking thread so the event loop can render the input + /// echo + tick the timer while the (potentially minute-scale) + /// work runs. When this arrives, `handle_prepare_done` takes + /// over: surfaces warnings, commits or ships to the worker. + /// + /// `text` and `echo` are threaded through so the completion + /// handler has everything it needs without re-consulting the + /// input state (which may have moved on). + PrepareDone { + text: String, + echo: bool, + result: Result, + }, +} + +pub async fn run(opts: ReplOptions) -> Result<(), ReplError> { + enable_raw_mode()?; + let mut stdout = std::io::stdout(); + stdout.execute(EnterAlternateScreen)?; + // Mouse capture ON by default — the app implements its own + // drag-to-select + clipboard copy so users get text selection + // back (helix-style: app-level highlight rendered with + // `Modifier::REVERSED`, copied to the OS clipboard on mouse + // release). F2 toggles capture off if a user would rather use + // their terminal's native selection. + stdout.execute(EnableMouseCapture)?; + // Bracketed paste — the terminal wraps pasted content in + // sentinel byte sequences so we can distinguish it from + // manually-typed input. Without this, a pasted multi-line + // block delivers embedded `\n`s as raw Enter events, each of + // which the app treats as "submit" — the exact bug this + // enables us to fix. + stdout.execute(EnableBracketedPaste)?; + // Kitty keyboard protocol (minimal set) — asks the terminal + // to disambiguate keys that would otherwise collide (Ctrl+I + // vs. Tab, etc.). Doesn't help with Shift+Enter — this + // terminal (and many others) sends Shift+Enter as Ctrl+J + // instead of a distinct CSI-u sequence, so the outer key + // handler binds Ctrl+J to `insert_newline`. Unsupported + // flags are ignored, so this is safe everywhere. + let _ = stdout.execute(PushKeyboardEnhancementFlags( + KeyboardEnhancementFlags::DISAMBIGUATE_ESCAPE_CODES, + )); + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + + let result = run_inner(&mut terminal, opts).await; + + disable_raw_mode()?; + let mut stdout = std::io::stdout(); + // No-op if capture was already disabled via F2. + let _ = stdout.execute(DisableMouseCapture); + let _ = stdout.execute(DisableBracketedPaste); + let _ = stdout.execute(PopKeyboardEnhancementFlags); + stdout.execute(LeaveAlternateScreen)?; + terminal.show_cursor()?; + result +} + +async fn run_inner( + terminal: &mut Terminal>, + opts: ReplOptions, +) -> Result<(), ReplError> { + let (worker_tx, worker_rx) = mpsc::channel::(8); + let (event_tx, eval_rx) = mpsc::unbounded_channel::(); + let info_with_stack = opts.info_with_stack; + // At `--log-level=debug` the user wants the unclassified PTY + // firehose (banners, source echo, idle chatter) too; anything + // less filters it out. The firehose can't go to stderr under + // the TUI's alternate screen — that fd is the render surface — + // so route it to a per-process tempfile the user can `tail -f` + // from another terminal. Distinct from the raw byte-log wired + // downstream: this file holds only the leftover unclassified + // lines, whereas the raw log holds every byte. + let verbose = matches!(opts.log_level, vw_vivado::LogLevel::Debug); + let verbose_log_path = if verbose { + Some( + std::env::temp_dir() + .join(format!("vw-repl-vivado-{}.log", std::process::id())), + ) + } else { + None + }; + // Clone before moving into worker_task — App keeps its own + // handle so background prepare tasks can post PrepareDone + // events back onto the same channel the worker uses. + let event_tx_for_app = event_tx.clone(); + // Workspace-root discovery mirrors `vw run` / `vw check`: + // walk up from the initial-load file if provided, else from + // the current cwd, looking for the nearest `vw.toml`. Used + // to answer the htcl `vw::workspace_root` RPC — served + // through the Vivado shim's rpc_call primitive at eval time. + let rpc_workspace_root: Option = { + let start_dir = opts + .initial_load + .as_ref() + .and_then(|p| { + p.as_std_path().parent().map(std::path::Path::to_path_buf) + }) + .or_else(|| std::env::current_dir().ok()); + start_dir + .and_then(|d| vw_lib::find_workspace_dir(&d)) + .map(|p| p.into_std_path_buf()) + }; + // Load `/.vw/repl.toml` — currently just the `[ui] collapse` + // knob. Absent / malformed / no-workspace all fall back to + // defaults; a config file is optional infrastructure, not a + // startup dependency. + let repl_config = config::load(rpc_workspace_root.as_deref()); + // Shared preload map — grows as batches commit. Both the RPC + // handler (inside worker_task) and App hold clones of the same + // Arc so `compile_htcl_module` sees loaded-file updates as + // soon as `App::sync_preload_from_session` publishes them. + // See `vw_vivado::SharedPreload` for the correctness invariant. + let preload: vw_vivado::SharedPreload = std::sync::Arc::new( + std::sync::RwLock::new(std::collections::HashMap::new()), + ); + tokio::spawn(worker_task( + worker_rx, + event_tx, + verbose, + verbose_log_path.clone(), + info_with_stack, + opts.part.clone(), + opts.variant.clone(), + rpc_workspace_root, + preload.clone(), + )); + + let mut app = App::new( + opts, + worker_tx, + eval_rx, + event_tx_for_app, + repl_config.ui.collapse, + preload, + ); + if let Some(p) = verbose_log_path { + app.push( + ScrollbackKind::Notice, + format!( + "verbose output streaming to {} — `tail -f` from \ + another terminal", + p.display() + ), + ); + } + let mut crossterm_events = crossterm::event::EventStream::new(); + + loop { + terminal.draw(|f| ui::draw(f, &mut app))?; + if app.exit { + let _ = app.worker_tx.send(WorkerCmd::Shutdown).await; + return Ok(()); + } + + tokio::select! { + // Bias toward user input over worker events. Without + // this, a streaming eval (hundreds of stream chunks per + // second from Vivado) drowns out individual keystrokes: + // tokio::select! picks randomly when multiple branches + // are ready, and the worker channel is ready far more + // often. Result: Tab during eval looks dead because the + // keypress queues behind a long run of worker events. + // Biased order guarantees a ready crossterm event always + // wins, then the drain phase below catches up on worker + // events before the next draw. + biased; + maybe_event = crossterm_events.next() => { + match maybe_event { + Some(Ok(ev)) => app.handle_terminal_event(ev).await, + Some(Err(e)) => { + app.push(ScrollbackKind::Error, format!("terminal: {e}")); + } + None => { + app.exit = true; + } + } + } + Some(event) = app.eval_rx.recv() => { + app.handle_worker_event(event).await; + } + _ = tokio::time::sleep(Duration::from_millis(250)) => { + // Periodic wake: lets the spinner / "starting" status + // animate even when nothing else is happening. + } + } + // Drain additional pending events from BOTH streams + // before the next draw. Without this, a burst of N + // mouse-wheel events or worker stream chunks each + // triggers its own draw — even though only the final + // state matters visually — and the queue bloats faster + // than draws can keep up. + // + // The biased `select!` plus a wildcard always-ready + // branch acts as a non-blocking "is anything pending?" + // check: if neither real branch is immediately ready, + // the wildcard wins and we break out. Capped at 256 + // events per cycle so a sustained event firehose still + // yields back to drawing periodically (the user sees + // forward progress instead of "frozen until the whole + // burst is processed"). + for _ in 0..256 { + let made_progress = tokio::select! { + biased; + Some(maybe_event) = crossterm_events.next() => { + match maybe_event { + Ok(ev) => app.handle_terminal_event(ev).await, + Err(e) => app.push( + ScrollbackKind::Error, + format!("terminal: {e}"), + ), + } + true + } + Some(event) = app.eval_rx.recv() => { + app.handle_worker_event(event).await; + true + } + _ = std::future::ready(()) => false, + }; + if !made_progress { + break; + } + } + } +} + +impl App { + fn new( + opts: ReplOptions, + worker_tx: mpsc::Sender, + eval_rx: mpsc::UnboundedReceiver, + event_tx: mpsc::UnboundedSender, + collapse_mode: CollapseMode, + preload: vw_vivado::SharedPreload, + ) -> Self { + let mut input = TextArea::default(); + input.set_cursor_line_style(ratatui::style::Style::default()); + Self { + opts, + input, + history: History::load_default(), + history_cursor: None, + history_draft: String::new(), + session: std::sync::Arc::new( + std::sync::RwLock::new(Session::new()), + ), + preload, + scrollback: Vec::new(), + block_acc: vw_vivado::BlockAccumulator::new(), + scrollback_scroll: 0, + mouse_capture: true, + scrollback_area: None, + selection: None, + scrollback_follow: true, + last_rendered_scroll: 0, + last_max_scroll: 0, + marker_entry: None, + pending_jump: None, + reverse_search: None, + popup: None, + worker_state: WorkerState::Starting, + child_pid: None, + worker_tx, + eval_rx, + event_tx, + pending_batch: None, + pending_origins: Vec::new(), + pending_return_types: Vec::new(), + pending_is_set_binding: Vec::new(), + pending_input_boundaries: Vec::new(), + pending_eval_index: 0, + exit: false, + collapse_mode, + } + } + + /// Refresh the shared preload map from the current session's + /// `loaded_paths()`. Called immediately after every + /// `session.commit(...)` so the next `compile_htcl_module` + /// RPC sees the latest set of files installed in the Vivado + /// interpreter. + /// + /// Called AFTER commit (not before / concurrently) so the map + /// only names files whose lowered Tcl has been eval'd by + /// Vivado — the safety rule spelled out on + /// `vw_vivado::SharedPreload`. Wholesale replace (not merge) + /// so a file removed from the session — e.g. via a hot-edit + /// path in the future — drops out of the preload set too. + fn sync_preload_from_session(&self) { + let paths = { + let s = self.session.read().unwrap(); + s.loaded_paths() + }; + if let Ok(mut g) = self.preload.write() { + *g = paths; + } + } + + // --- queries used by ui.rs --------------------------------------- + + pub fn scrollback(&self) -> &[ScrollbackEntry] { + &self.scrollback + } + pub fn scrollback_scroll(&self) -> u16 { + self.scrollback_scroll + } + pub fn input_mut(&mut self) -> &mut TextArea<'static> { + &mut self.input + } + pub fn input_line_count(&self) -> usize { + self.input.lines().len() + } + pub fn reverse_search(&self) -> Option<&ReverseSearch> { + self.reverse_search.as_ref() + } + pub fn mouse_capture(&self) -> bool { + self.mouse_capture + } + pub fn selection(&self) -> Option { + self.selection + } + /// Called by `ui::draw_scrollback` each frame so subsequent + /// mouse events can translate absolute screen coords into + /// scrollback-local rows/cols. + pub fn set_scrollback_area(&mut self, area: Rect) { + self.scrollback_area = Some(area); + } + + pub fn scrollback_follow(&self) -> bool { + self.scrollback_follow + } + + /// Renderer-side writeback: records the scroll offset that was + /// actually used to paint the current frame. Mouse / keyboard + /// scroll handlers anchor their deltas off this so transitioning + /// out of tail-follow doesn't jump back to a stale + /// `scrollback_scroll` value. + pub fn set_last_rendered_scroll(&mut self, offset: u16) { + self.last_rendered_scroll = offset; + } + + /// Renderer-side writeback for the current frame's `max_scroll` + /// (wrapped rows − viewport height). Consulted by + /// [`Self::scroll_by`] so a downward wheel/PageDown that lands + /// at the bottom re-engages tail-follow. + pub fn set_last_max_scroll(&mut self, max_scroll: u16) { + self.last_max_scroll = max_scroll; + } + + /// Renderer-invoked writeback used by the pending-jump path — + /// the popup handler doesn't know area.width so it can't + /// compute the target scroll offset itself, and instead + /// stashes a `pending_jump` scrollback index that the + /// renderer translates into an offset and writes back here. + /// General-purpose scroll changes go through + /// [`Self::scroll_by`]; this setter is not a substitute. + pub fn set_scrollback_scroll(&mut self, offset: u16) { + self.scrollback_scroll = offset; + } + + /// Scrollback entry index the user last jumped to (or `None` + /// if the marker has been cleared or was never set). Consulted + /// by `ui::draw_scrollback` to paint the persistent gutter + /// marker on that entry's wrapped rows. + pub fn marker_entry(&self) -> Option { + self.marker_entry + } + + /// Consume any pending "scroll this entry into view" request + /// from the popup layer. Returns `Some(idx)` exactly once per + /// jump request; subsequent frames return `None`. The renderer + /// uses the per-entry wrapped-row count it computes anyway to + /// translate `idx` into an absolute scroll offset — that + /// translation needs area.width, which is why the popup can't + /// pre-compute it. + pub fn take_pending_jump(&mut self) -> Option { + self.pending_jump.take() + } + + /// Toggle terminal mouse capture. Writes the enable/disable + /// sequence directly to stdout — the alternate-screen / raw-mode + /// context that `run()` set up is still active. + fn toggle_mouse_capture(&mut self) { + let mut stdout = std::io::stdout(); + let _ = if self.mouse_capture { + stdout.execute(DisableMouseCapture) + } else { + stdout.execute(EnableMouseCapture) + }; + self.mouse_capture = !self.mouse_capture; + } + pub fn worker_state(&self) -> WorkerStatusView { + match self.worker_state { + WorkerState::Starting => WorkerStatusView::Starting, + WorkerState::Ready => WorkerStatusView::Ready, + WorkerState::Running => WorkerStatusView::Running, + WorkerState::Down => WorkerStatusView::Down, + } + } + pub fn eval_in_flight(&self) -> bool { + matches!(self.worker_state, WorkerState::Running) + } + + /// Whether the parser considers the current input buffer ready + /// to ship. Drives the input-area title and Enter behavior. + pub fn input_is_complete(&self) -> bool { + let buf = self.current_input_text(); + let session = self.session.read().unwrap(); + is_buffer_complete(&buf, &session.signature_table()) + } + + fn current_input_text(&self) -> String { + self.input.lines().join("\n") + } + + /// Translate the input editor's `(row, col)` cursor into a byte + /// offset within `current_input_text()`. Returns `None` when the + /// cursor is past EOF (shouldn't happen — TextArea keeps it in + /// bounds — but defensive). + fn cursor_byte_offset(&self) -> Option { + let (row, col) = self.input.cursor(); + let buffer = self.current_input_text(); + let line_idx = vw_htcl::line_index::LineIndex::new(&buffer); + Some(line_idx.offset_of(vw_htcl::line_index::LineCol { + line: row as u32, + character: col as u32, + })) + } + + /// Map an input-buffer (row, col) to a screen cell within the + /// input editor's rendered area. Used to anchor popups (slice 4+) + /// just below the cursor. Returns `None` when no scrollback area + /// has been captured yet (shouldn't happen post-first-render but + /// defensive against early key events). + fn cursor_screen_cell(&self) -> Option<(u16, u16)> { + // We don't have direct access to the input area Rect here + // (only the scrollback's). The popup anchor instead uses a + // best-effort approximation: assume the input area starts + // just below the scrollback and the popup will clamp to the + // frame in the renderer. Concretely: row = bottom of the + // visible scrollback (where the input border sits) + cursor + // row in the editor, col = cursor col + the input area's + // left edge (we use scrollback's x which they share). + let area = self.scrollback_area?; + let (row, col) = self.input.cursor(); + // +1 to step past the input box's top border, +area.y to land + // inside the input region. The renderer's popup positioning + // does additional clamping so over- and under-shoots are safe. + let screen_y = area.y + area.height + 1 + row as u16; + let screen_x = area.x + 1 + col as u16; + Some((screen_x, screen_y)) + } + + /// Trigger a completion popup at the current cursor position. No-op + /// when there are no completions to show. Called from the Tab key + /// handler. + fn trigger_completion(&mut self) { + let input = self.current_input_text(); + let Some(offset) = self.cursor_byte_offset() else { + return; + }; + // Parse ONLY the in-flight input. Earlier we tried merging + // session.merged_source() (~6MB after `src @vivado-cmd`) + // into the analysis source so `util::` would see + // session-known procs. That had two fatal problems: + // + // 1. Per-Tab cost was a multi-MB parse + a multi-MB + // `cmdline::analyze` walk-back. The UI froze for + // seconds; queued keypresses (ctrl-D, backspace) + // drained after the parse finished. + // 2. `cmdline::analyze` balances `[` / `]` but doesn't + // know about `#` comments. The auto-generated Vivado + // docs contain `[get_hw_sysmons]`, `[Common 17-39]`, + // etc. inside `## doc-comment` blocks; an unmatched + // bracket in those docs put the analyzer in + // "inside-a-substitution" state forever, blowing past + // every newline and never finding the command + // boundary. End result: `partial="util::"` but + // `head_words` was the entire 6 MB session. + // + // Cheaper, correct approach: analyze only the in-flight + // input (small, fast, no rogue brackets), then pull + // candidate proc names directly out of + // `Session::signature_table()` — that's a HashMap built + // from already-parsed batches; O(N) iteration over + // existing data instead of an MB-scale reparse + walk. + let parsed = vw_htcl::parser::parse(&input); + let cmd_line = vw_htcl::cmdline::analyze(&input, offset); + let session_guard = self.session.read().unwrap(); + let session_sigs = session_guard.signature_table(); + let input_sigs = vw_htcl::signature_table(&parsed.document); + // The currently-shipping batch hasn't committed yet — session + // commit only happens on EvalDone(last_in_batch=true). During + // the prime.htcl load (e.g. `src @vivado-cmd` taking 50+ + // seconds), every proc the user wants to complete on (util::*, + // create_*, …) lives in `pending_batch.document` but NOT in + // session.signature_table(). Surface those too — they're + // already parsed; cost is one `signature_table` walk over the + // in-flight document's stmts. + let pending_sigs: std::collections::HashMap< + String, + &vw_htcl::ProcSignature, + > = self + .pending_batch + .as_ref() + .map(|b| vw_htcl::signature_table(&b.document)) + .unwrap_or_default(); + let mut items: Vec = Vec::new(); + // Meta-command branch: `:load`, `:quit`, etc. Detected by + // a leading `:` on the partial — these are App-side + // commands, not htcl, so they live above the cmdline + // analyzer's notion of command position. + if cmd_line.partial.starts_with(':') { + for (label, hint) in META_COMMANDS { + if label.starts_with(cmd_line.partial) { + items.push(vw_htcl::complete::Completion { + label: label.to_string(), + kind: vw_htcl::complete::CompletionKind::Proc, + detail: Some(hint.to_string()), + documentation: None, + replace: cmd_line.partial_span, + insert_text: None, + snippet: false, + }); + } + } + } else if cmd_line.in_command_position() { + // Proc-name completion: union of session + pending + + // in-flight proc names, filtered by the partial prefix. + let mut names: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + for name in session_sigs.keys() { + names.insert(name.clone()); + } + for name in pending_sigs.keys() { + names.insert(name.clone()); + } + for name in input_sigs.keys() { + names.insert(name.clone()); + } + for name in names { + if name.starts_with(cmd_line.partial) { + items.push(vw_htcl::complete::Completion { + label: name, + kind: vw_htcl::complete::CompletionKind::Proc, + detail: None, + documentation: None, + replace: cmd_line.partial_span, + insert_text: None, + snippet: false, + }); + } + } + } else if let Some(cmd_name) = cmd_line.command_name() { + // Flag completion: look up the called proc's signature + // in either source and emit its flag args. Matches the + // shape of `vw_htcl::complete::complete_at`'s flag path + // but uses our union of session+input signatures. + let sig = session_sigs + .get(cmd_name) + .copied() + .or_else(|| pending_sigs.get(cmd_name).copied()) + .or_else(|| input_sigs.get(cmd_name).copied()); + if let Some(sig) = sig { + let used: Vec<&str> = cmd_line.used_flags().collect(); + let needle_no_dash = cmd_line + .partial + .strip_prefix('-') + .unwrap_or(cmd_line.partial); + // Required (no @default) flags first, then optional, + // alphabetical within each group. Matches the + // signature-help popup's ordering so the user sees + // the same priority across surfaces. + for &i in &sorted_arg_indices(sig) { + let arg = &sig.args[i]; + let label = format!("-{}", arg.name); + if used.contains(&label.as_str()) { + continue; + } + if arg.name.starts_with(needle_no_dash) { + // Detail shows the type + default when + // available, so the completion popup row + // hints at what each flag expects without + // requiring the user to open hover. Format + // mirrors the sig-help line: `type = value`. + let detail = build_flag_detail(arg); + items.push(vw_htcl::complete::Completion { + label, + kind: vw_htcl::complete::CompletionKind::Flag, + detail, + documentation: None, + replace: cmd_line.partial_span, + insert_text: None, + snippet: false, + }); + } + } + } + } + let anchor = self.cursor_screen_cell().unwrap_or((0, 0)); + if let Some(popup) = crate::popup::CompletionPopup::new(items, anchor) { + self.popup = Some(crate::popup::PopupState::Completion(popup)); + } + } + + /// Route a key event to the active popup. Returns `true` when the + /// key was consumed (navigation / accept / dismiss); `false` lets + /// the key fall through to the rest of the handler. + fn handle_popup_key(&mut self, key: crossterm::event::KeyEvent) -> bool { + use crossterm::event::KeyCode; + let Some(popup) = self.popup.as_mut() else { + return false; + }; + match popup { + crate::popup::PopupState::Completion(comp) => { + match key.code { + KeyCode::Up => { + comp.move_up(); + true + } + KeyCode::Down => { + comp.move_down(); + true + } + KeyCode::Esc => { + self.popup = None; + true + } + KeyCode::Enter if !key.modifiers.is_empty() => { + // Any modifier on Enter (Shift, Alt, Ctrl, + // combos) is the "keep typing" escape + // hatch. Don't let the popup consume it — + // dismiss the popup so the outer handler + // can insert a newline. Terminals differ + // on which modifier they attach; accept + // any of them. + self.popup = None; + false + } + KeyCode::Char('j') + if key.modifiers.contains( + crossterm::event::KeyModifiers::CONTROL, + ) => + { + // Shift+Enter on legacy terminals arrives + // as Ctrl+J — see the outer handler's + // matching branch for the rationale. Let + // it through so the outer handler can + // insert a newline. + self.popup = None; + false + } + KeyCode::Enter => { + if let Some(item) = comp.current().cloned() { + self.apply_completion(&item); + } + self.popup = None; + true + } + KeyCode::Tab => { + // Tab re-triggers — just cycle for now. + comp.move_down(); + true + } + _ => { + // Any other key dismisses the popup and falls + // through to the editor — typical IDE + // behavior where you can keep typing past the + // popup to refine your input. + self.popup = None; + false + } + } + } + crate::popup::PopupState::Help(_) => { + // Any keystroke dismisses the help modal. We CONSUME + // the dismissing key (return true) so it doesn't + // also act on the input — pressing Ctrl-H to open + // then any other key to close shouldn't accidentally + // type the close key into the editor. + self.popup = None; + true + } + crate::popup::PopupState::SignatureHelp(sig) => { + // Signature help is the background auto-show; it + // doesn't consume keys, falls through to the editor. + // Esc lets users hide it without clearing input. + if key.code == KeyCode::Esc { + self.popup = None; + return true; + } + // Shift-↑ / Shift-↓: scroll through args when the + // signature is too tall to fit. Picked over + // Ctrl-↑/Ctrl-↓ because macOS reserves those for + // Mission Control. We consume these chords (return + // true) so they don't ALSO scroll the scrollback. + // Step by 1 — fine-grained because each arg is a + // self-contained row. + if key.modifiers.contains(KeyModifiers::SHIFT) { + match key.code { + KeyCode::Up => { + sig.scroll_offset = + sig.scroll_offset.saturating_sub(1); + return true; + } + KeyCode::Down => { + sig.scroll_offset = + sig.scroll_offset.saturating_add(1); + return true; + } + _ => {} + } + } + false + } + crate::popup::PopupState::Hover(_) => { + // Hover dismisses on any keystroke. We consume the + // key so the dismissing keystroke doesn't also act + // on the input — Ctrl-Y to open + any key to close + // shouldn't smuggle that key into the buffer. + self.popup = None; + true + } + crate::popup::PopupState::SymbolSearch(picker) => { + use crate::symbol_search::PickerView; + match key.code { + KeyCode::Esc => { + self.popup = None; + true + } + KeyCode::Up => { + picker.move_up(); + true + } + KeyCode::Down => { + picker.move_down(); + true + } + KeyCode::Tab => { + picker.toggle_view(); + true + } + KeyCode::Backspace => { + if picker.view == PickerView::Symbols { + picker.pop_char(); + } + true + } + KeyCode::Char(c) + if !key.modifiers.contains(KeyModifiers::CONTROL) + && picker.view == PickerView::Symbols => + { + picker.push_char(c); + true + } + KeyCode::Enter => { + match picker.view { + PickerView::Symbols => { + if let Some(sym) = + picker.current_symbol().cloned() + { + self.popup = None; + self.insert_at_cursor_replacing_word( + &sym.name, + ); + } else { + self.popup = None; + } + } + PickerView::Libraries => { + picker.apply_library_filter(); + } + } + true + } + _ => true, // swallow other keys; popup stays open + } + } + crate::popup::PopupState::DiagnosticSearch(picker) => { + match (key.code, key.modifiers) { + (KeyCode::Esc, _) => { + self.popup = None; + true + } + (KeyCode::Up, _) => { + picker.move_up(); + true + } + (KeyCode::Down, _) => { + picker.move_down(); + true + } + (KeyCode::Char('e'), m) + if m.contains(KeyModifiers::CONTROL) => + { + picker.toggle_kind(ScrollbackKind::Error); + true + } + (KeyCode::Char('w'), m) + if m.contains(KeyModifiers::CONTROL) => + { + picker.toggle_kind(ScrollbackKind::Warning); + true + } + (KeyCode::Char('n'), m) + if m.contains(KeyModifiers::CONTROL) => + { + picker.toggle_kind(ScrollbackKind::Notice); + true + } + (KeyCode::Char('k'), m) + if m.contains(KeyModifiers::CONTROL) => + { + // Ctrl-K toggles the Critical-warning + // subset filter. Only meaningful inside + // the popup — outside, Ctrl-K is + // scrollback-up, which the popup handler + // shadows while active. + picker.toggle_critical(); + true + } + (KeyCode::Backspace, _) => { + picker.pop_char(); + true + } + (KeyCode::Char(c), m) + if !m.contains(KeyModifiers::CONTROL) => + { + picker.push_char(c); + true + } + (KeyCode::Enter, _) => { + // Snapshot the target BEFORE dropping the + // popup — `picker` borrows through + // `self.popup`; setting `self.popup = None` + // invalidates it. + let target = picker + .current() + .map(|it| (it.scrollback_idx, it.kind)); + self.popup = None; + if let Some((idx, kind)) = target { + self.jump_to_scrollback_entry(idx, kind); + } + true + } + _ => true, + } + } + } + } + + /// Insert / replace text from a chosen completion. Replaces the + /// byte range `item.replace` (the partial word under the cursor, + /// or a zero-width insertion point) with `item.label`. + fn apply_completion(&mut self, item: &vw_htcl::complete::Completion) { + let buffer = self.current_input_text(); + let start = item.replace.start as usize; + let end = (item.replace.end as usize).min(buffer.len()); + if start > buffer.len() { + return; + } + let mut new_buffer = String::with_capacity( + buffer.len() - (end - start) + item.label.len(), + ); + new_buffer.push_str(&buffer[..start]); + new_buffer.push_str(&item.label); + new_buffer.push_str(&buffer[end..]); + // Place cursor just after the inserted label. + let new_cursor_byte = start + item.label.len(); + self.replace_input_with_cursor(new_buffer, new_cursor_byte); + } + + /// Replace the input buffer with `text` and move the cursor to + /// the byte offset `cursor_byte`. Cursor offset is translated to + /// (row, col) via `LineIndex`. Used by completion accept. + fn replace_input_with_cursor(&mut self, text: String, cursor_byte: usize) { + use tui_textarea::TextArea; + let line_idx = vw_htcl::line_index::LineIndex::new(&text); + // Build the textarea fresh from the new content (tui-textarea + // doesn't offer a "replace everything" API; recreating is the + // documented way per its issue tracker). + let lines: Vec = + text.split('\n').map(|s| s.to_string()).collect(); + let mut ta = TextArea::new(lines); + ta.set_cursor_line_style(ratatui::style::Style::default()); + // Position cursor. + let lc = line_idx.position(cursor_byte as u32); + let target_row = lc.line as usize; + let target_col = lc.character as usize; + // Use the textarea's `move_cursor` API. + while ta.cursor().0 < target_row { + ta.move_cursor(tui_textarea::CursorMove::Down); + } + while ta.cursor().0 > target_row { + ta.move_cursor(tui_textarea::CursorMove::Up); + } + while ta.cursor().1 < target_col { + ta.move_cursor(tui_textarea::CursorMove::Forward); + } + while ta.cursor().1 > target_col { + ta.move_cursor(tui_textarea::CursorMove::Back); + } + self.input = ta; + self.history_cursor = None; + } + + /// Whether a popup is currently open (renderer queries this to + /// decide whether to draw the popup overlay layer). + pub fn popup_state(&self) -> Option<&crate::popup::PopupState> { + self.popup.as_ref() + } + + /// Refresh signature help after a buffer-mutating keystroke. + /// Looks up the proc the cursor sits in by name across session, + /// pending batch, and in-flight input; populates a + /// `PopupState::SignatureHelp` with its args + active parameter. + /// Dismisses any existing sig-help popup when nothing matches. + /// + /// Coexistence rule: never displaces a Completion or Help popup + /// (the user explicitly opened those; sig-help is the background + /// auto-show). + fn refresh_signature_help(&mut self) { + // Don't fight explicit popups. + if matches!( + self.popup, + Some(crate::popup::PopupState::Completion(_)) + | Some(crate::popup::PopupState::Help(_)) + ) { + return; + } + let input = self.current_input_text(); + let Some(offset) = self.cursor_byte_offset() else { + self.dismiss_signature_help(); + return; + }; + let cmd_line = vw_htcl::cmdline::analyze(&input, offset); + let Some(name) = cmd_line.command_name() else { + self.dismiss_signature_help(); + return; + }; + // Find the proc's signature + doc comments. Signatures live + // in session.signature_table() / pending / in-flight; doc + // comments require walking the source document (kept on the + // Command, not the ProcSignature). + let parsed = vw_htcl::parser::parse(&input); + // Arc-clone the session handle first — this drops the + // borrow on `self` immediately, so we can call `self.push` / + // `self.dismiss_signature_help` etc. later without the + // read guard blocking the mutable borrow of self. + let session = std::sync::Arc::clone(&self.session); + let session_guard = session.read().unwrap(); + let session_sigs = session_guard.signature_table(); + let input_sigs = vw_htcl::signature_table(&parsed.document); + let pending_doc = self.pending_batch.as_ref().map(|b| &b.document); + let pending_sigs = pending_doc + .map(|d| vw_htcl::signature_table(d)) + .unwrap_or_default(); + let sig = session_sigs + .get(name) + .copied() + .or_else(|| pending_sigs.get(name).copied()) + .or_else(|| input_sigs.get(name).copied()); + let Some(sig) = sig else { + self.dismiss_signature_help(); + return; + }; + // Look up doc comments by walking docs in priority order + // (input → pending → session). Most recent wins, which + // matches Tcl's "later proc shadows earlier" semantics that + // the lowerer already uses. + let mut doc_comments: &[String] = &[]; + if let Some(d) = lookup_proc_doc_comments(&parsed.document, name) { + doc_comments = d; + } else if let Some(doc) = pending_doc { + if let Some(d) = lookup_proc_doc_comments(doc, name) { + doc_comments = d; + } + } else { + // Walk session batches in reverse (newest first). + for batch in session_guard.batches_for_doc_search() { + if let Some(d) = lookup_proc_doc_comments(&batch.document, name) + { + doc_comments = d; + break; + } + } + } + // Build the display permutation (required → optional, + // alphabetical within each group) and reorder args + the + // active-parameter index accordingly. + let display_order = sorted_arg_indices(sig); + let active_decl = compute_active_parameter(sig, &cmd_line); + let active = active_decl.and_then(|orig| { + display_order + .iter() + .position(|&i| i == orig as usize) + .map(|i| i as u32) + }); + let args: Vec = display_order + .iter() + .map(|&i| { + let a = &sig.args[i]; + crate::popup::SigHelpArg { + name: a.name.clone(), + type_str: a.type_annotation.as_ref().map(render_type), + default_str: format_default_value(a), + } + }) + .collect(); + let return_type = sig.return_type.as_ref().map(render_type); + let doc_brief = vw_htcl::doc::brief(doc_comments); + let anchor = self.cursor_screen_cell().unwrap_or((0, 0)); + // Preserve any user-set scroll offset across refreshes — + // manual Ctrl-↑/↓ scrolling shouldn't be reset by every + // keystroke. The renderer clamps the offset to a valid + // range, so an offset that's stale for the new arg list + // (e.g. switching to a smaller proc) silently snaps back. + let prev_scroll = match self.popup.as_ref() { + Some(crate::popup::PopupState::SignatureHelp(p)) => p.scroll_offset, + _ => 0, + }; + let popup = crate::popup::SignatureHelpPopup { + proc_name: name.to_string(), + args, + return_type, + doc_brief, + active: active.map(|a| a as usize), + anchor, + scroll_offset: prev_scroll, + }; + self.popup = Some(crate::popup::PopupState::SignatureHelp(popup)); + } + + /// Open the fuzzy symbol picker (Ctrl-T). Builds a fresh + /// `SymbolIndex` snapshot at open time — it stays stable while + /// the popup is alive so the result-row indices don't shift + /// out from under the user. The index is small to build + /// (walks already-parsed Documents) so per-open cost is fine. + fn trigger_symbol_search(&mut self) { + let input = self.current_input_text(); + let parsed = vw_htcl::parser::parse(&input); + let session_guard = self.session.read().unwrap(); + let index = + std::sync::Arc::new(crate::symbol_index::SymbolIndex::build( + &session_guard, + self.pending_batch.as_ref(), + Some(&parsed.document), + )); + let picker = crate::symbol_search::SymbolPicker::new(index); + self.popup = Some(crate::popup::PopupState::SymbolSearch(picker)); + } + + /// Ctrl-F opens the diagnostics finder. Snapshots the current + /// scrollback (Error/Warning/Notice only) and hands it to the + /// picker. Snapshot semantics: rows appended after open aren't + /// visible in this session of the picker; user reopens to see + /// them. Prevents result-index churn while typing a query + /// against a still-streaming scrollback. + fn trigger_diagnostic_search(&mut self) { + let picker = crate::diag_search::DiagnosticPicker::from_scrollback( + &self.scrollback, + ); + self.popup = Some(crate::popup::PopupState::DiagnosticSearch(picker)); + } + + /// Set the marker on `idx`, request the next render to scroll + /// that entry into view, and disengage tail-follow. Called + /// when the diagnostics-finder popup Accept fires; the + /// scrolling itself happens in the renderer next frame + /// (needs area.width to translate entry-idx → row offset). + /// `_kind` is captured for future use — right now the marker + /// styling is kind-agnostic (fixed color), but a per-kind + /// tint would use it. + fn jump_to_scrollback_entry(&mut self, idx: usize, _kind: ScrollbackKind) { + if idx >= self.scrollback.len() { + return; + } + self.marker_entry = Some(idx); + self.pending_jump = Some(idx); + self.scrollback_follow = false; + } + + /// Alt-C clears the persistent marker. No-op when no marker + /// is set; harmless to press repeatedly. + fn clear_marker(&mut self) { + self.marker_entry = None; + } + + /// Insert `text` at the current cursor position, replacing the + /// identifier-under-cursor (if any). Used by the symbol-picker + /// Enter handler to insert a chosen symbol name in place of the + /// partial word the user is typing. + fn insert_at_cursor_replacing_word(&mut self, text: &str) { + let buffer = self.current_input_text(); + let Some(offset) = self.cursor_byte_offset() else { + return; + }; + let off = offset as usize; + // Find word boundaries around the cursor (same rule as + // `ident_under_cursor`). + let bytes = buffer.as_bytes(); + let is_word_byte = |b: u8| -> bool { + b.is_ascii_alphanumeric() || b == b'_' || b == b':' + }; + let mut start = off; + while start > 0 && is_word_byte(bytes[start - 1]) { + start -= 1; + } + let mut end = off; + while end < bytes.len() && is_word_byte(bytes[end]) { + end += 1; + } + let mut new_buffer = + String::with_capacity(buffer.len() - (end - start) + text.len()); + new_buffer.push_str(&buffer[..start]); + new_buffer.push_str(text); + new_buffer.push_str(&buffer[end..]); + let new_cursor = start + text.len(); + self.replace_input_with_cursor(new_buffer, new_cursor); + } + + /// Hover-under-cursor: open a popup showing the proc / + /// variable / enum the cursor is on. Tries + /// [`vw_htcl::hover_at`] on the in-flight input first (catches + /// local vars, in-buffer proc decls, enum decls). Falls back to + /// a session-aware lookup: extracts the identifier under the + /// cursor and resolves it against + /// `Session::signature_table()` / pending / in-flight, so + /// `Ctrl-Y` on a `util::props` call surfaces the library's docs + /// even though the proc was defined in a separate session + /// batch. + fn trigger_hover(&mut self) { + let input = self.current_input_text(); + let Some(offset) = self.cursor_byte_offset() else { + return; + }; + let parsed = vw_htcl::parser::parse(&input); + let anchor = self.cursor_screen_cell().unwrap_or((0, 0)); + // Pass 1: in-document hover. Handles ProcDef, ProcArgDef, + // CallSite when the proc is in the buffer, CallArg, + // LocalVar, EnumDef. + if let Some(target) = + vw_htcl::hover_at(&parsed.document, &input, offset) + { + if let Some(popup) = hover_target_to_popup(target, anchor) { + self.popup = Some(crate::popup::PopupState::Hover(popup)); + return; + } + } + // Pass 2: session-aware proc lookup by identifier under + // cursor. Covers the common REPL case: cursor on a name + // referencing a session-loaded library proc. + let Some(name) = ident_under_cursor(&input, offset) else { + return; + }; + let session = std::sync::Arc::clone(&self.session); + let session_guard = session.read().unwrap(); + let session_sigs = session_guard.signature_table(); + let pending_doc = self.pending_batch.as_ref().map(|b| &b.document); + let pending_sigs = pending_doc + .map(|d| vw_htcl::signature_table(d)) + .unwrap_or_default(); + let input_sigs = vw_htcl::signature_table(&parsed.document); + let sig = session_sigs + .get(name) + .copied() + .or_else(|| pending_sigs.get(name).copied()) + .or_else(|| input_sigs.get(name).copied()); + let Some(sig) = sig else { return }; + // Doc comments: walk newest source first (input → pending → + // session newest-first). + let mut doc_comments: &[String] = &[]; + if let Some(d) = lookup_proc_doc_comments(&parsed.document, name) { + doc_comments = d; + } else if let Some(doc) = pending_doc { + if let Some(d) = lookup_proc_doc_comments(doc, name) { + doc_comments = d; + } + } else { + for batch in session_guard.batches_for_doc_search() { + if let Some(d) = lookup_proc_doc_comments(&batch.document, name) + { + doc_comments = d; + break; + } + } + } + let title = render_proc_title(name, sig); + let body = vw_htcl::doc::reflow_doc_comments(doc_comments); + self.popup = + Some(crate::popup::PopupState::Hover(crate::popup::HoverPopup { + title, + body, + anchor, + })); + } + + /// Dismiss the signature-help popup if one is active. Leaves + /// Completion / Help popups alone. + fn dismiss_signature_help(&mut self) { + if matches!( + self.popup, + Some(crate::popup::PopupState::SignatureHelp(_)) + ) { + self.popup = None; + } + } + + /// Walk the input history by `delta` (negative = older, + /// positive = newer). Readline-style: first step back from the + /// "composing" position saves the current draft; stepping + /// past the newest entry restores it. Empty history is a no-op. + fn history_step(&mut self, delta: i32) { + let entries = self.history.entries(); + if entries.is_empty() { + return; + } + let cursor = match (self.history_cursor, delta) { + (None, d) if d >= 0 => return, // already at draft, can't go newer + (None, _) => { + // Stepping back from the draft for the first time — + // capture the in-progress text so Ctrl-N past the + // newest entry can restore it. + self.history_draft = self.current_input_text(); + entries.len().saturating_sub(1) + } + (Some(i), d) => { + let new = i as i32 + d; + if new < 0 { + 0 + } else if new >= entries.len() as i32 { + // Past the newest entry — drop back to draft. + self.history_cursor = None; + let draft = std::mem::take(&mut self.history_draft); + self.replace_input_with(&draft); + return; + } else { + new as usize + } + } + }; + self.history_cursor = Some(cursor); + let text = entries[cursor].clone(); + self.replace_input_with(&text); + } + + /// Reset the input buffer to `text`, placing the cursor at the + /// end. Used by history navigation and reverse-search accept. + fn replace_input_with(&mut self, text: &str) { + self.input = TextArea::default(); + for (i, line) in text.lines().enumerate() { + if i > 0 { + self.input.insert_newline(); + } + self.input.insert_str(line); + } + // If `text` ended with a newline, `lines()` drops it; preserve. + if text.ends_with('\n') { + self.input.insert_newline(); + } + } + + // --- event handling --------------------------------------------- + + fn handle_mouse_event(&mut self, mouse: MouseEvent) { + // Wheel events scroll the scrollback buffer. 3 lines per + // notch is the de-facto terminal-emulator default and + // matches what feels natural when you've held the wheel + // for half a second. Keyboard scroll (Ctrl-J/K) still + // jumps 5 — the wheel is finer-grained because the user + // can keep spinning. + // + // Direction: `scrollback_scroll` is ratatui's `scroll.y`, + // which counts lines skipped from the TOP of the buffer. + // Wheel-up should reveal older content above the viewport + // (terminal convention), which means moving the viewport + // UP through the buffer — i.e. SUBTRACTING from + // `scrollback_scroll`. Wheel-down does the reverse. + match mouse.kind { + MouseEventKind::ScrollUp => { + self.scroll_by(-3); + return; + } + MouseEventKind::ScrollDown => { + self.scroll_by(3); + return; + } + _ => {} + } + + // Drag-selection lives within the scrollback area only. + // Outside it, mouse events are ignored — the input box has + // its own selection model via tui-textarea and we don't + // want a click on a status bar to start a scrollback drag. + let Some(area) = self.scrollback_area else { + return; + }; + let in_area = mouse.column >= area.x + && mouse.column < area.x + area.width + && mouse.row >= area.y + && mouse.row < area.y + area.height; + + match mouse.kind { + MouseEventKind::Down(MouseButton::Left) if in_area => { + self.selection = Some(Selection { + anchor: self.cell_to_buffer(mouse.column, mouse.row, area), + cursor: self.cell_to_buffer(mouse.column, mouse.row, area), + }); + } + MouseEventKind::Drag(MouseButton::Left) + if self.selection.is_some() => + { + // Auto-scroll when the drag wanders past the + // top or bottom edge of the scrollback area so + // selections can extend beyond the current + // viewport. Crossterm fires drag events per + // cell of mouse movement, so the user wiggles + // the mouse at the edge to keep scrolling; + // simpler than tracking a "held at edge" timer + // and good enough for selection-extension UX. + let bottom = area.y + area.height; + if mouse.row >= bottom { + self.scroll_by(3); + } else if mouse.row < area.y { + self.scroll_by(-3); + } + // Clamp to the area: dragging outside still + // updates the cursor to the edge so selection + // can extend through the visible viewport even + // when the mouse strays. + let col = mouse + .column + .clamp(area.x, area.x + area.width.saturating_sub(1)); + let row = mouse + .row + .clamp(area.y, area.y + area.height.saturating_sub(1)); + let cursor = self.cell_to_buffer(col, row, area); + if let Some(sel) = self.selection.as_mut() { + sel.cursor = cursor; + } + } + MouseEventKind::Up(MouseButton::Left) => { + if let Some(sel) = self.selection.take() { + // Shift + pure click (no drag) on a collapsible + // block toggles its expanded state. Detect "pure + // click" by comparing anchor == cursor: Drag + // events are the only path that moves `cursor` + // off `anchor`, so any drag at all falls through + // to the copy path. Drag-select inside an + // expanded block still works because the + // anchor/cursor pair diverges as soon as the + // first Drag event fires. + if mouse.modifiers.contains(KeyModifiers::SHIFT) + && sel.anchor == sel.cursor + && self.toggle_collapsible_at(sel.anchor.0) + { + return; + } + self.copy_selection_to_clipboard(sel); + } + } + _ => {} + } + } + + /// Map a wrapped-row index (0-based, spans all of `scrollback`) + /// to the entry that occupies it, and if that entry is a + /// collapsible NONE block, toggle its expand state. Returns + /// `true` when a toggle happened so the caller can suppress the + /// fallthrough "copy empty selection" path — a Shift-click on a + /// diagnostic line should still do nothing (not clobber the + /// clipboard with an empty string, not act on the diagnostic + /// entry), so a `false` here means "not our gesture, keep + /// falling through." + /// + /// Walks entries left-to-right, summing wrapped-row counts until + /// we find the entry the row lives in. O(N) per click — cheap at + /// scrollback sizes we care about. + fn toggle_collapsible_at(&mut self, wrapped_row: usize) -> bool { + let width = self.scrollback_area.map(|a| a.width).unwrap_or(0); + let mut cursor: usize = 0; + for entry in self.scrollback.iter_mut() { + let rows = crate::render::count_wrapped_rows(entry, width) as usize; + let end = cursor.saturating_add(rows); + if (cursor..end).contains(&wrapped_row) { + match entry.collapse_state { + Some(expanded) => { + entry.collapse_state = Some(!expanded); + return true; + } + None => return false, + } + } + cursor = end; + } + false + } + + /// Translate a screen cell `(col, row)` inside the scrollback + /// `area` into a `(row, col)` index into the post-wrap line list. + /// The row index is `effective_scroll + (row - area.y)` so the + /// caller doesn't have to know about scroll state. + /// + /// Anchors against `last_rendered_scroll` rather than + /// `scrollback_scroll`. While tail-follow is on the renderer + /// computes the pinned offset on the fly and never writes it + /// back to `scrollback_scroll` — using the stale field here + /// would map mouse clicks to the wrong buffer rows once any + /// real volume of output has scrolled the viewport. + fn cell_to_buffer(&self, col: u16, row: u16, area: Rect) -> (usize, usize) { + let local_row = row.saturating_sub(area.y) as usize; + let local_col = col.saturating_sub(area.x) as usize; + let buf_row = self.last_rendered_scroll as usize + local_row; + (buf_row, local_col) + } + + /// Build the same post-wrap line list the UI renders, extract + /// the cells inside `sel`, and write the resulting plain text to + /// the OS clipboard. Failure (no clipboard backend / Wayland + /// permissions denied / …) surfaces as a Notice line so the + /// user knows the copy didn't go through. + fn copy_selection_to_clipboard(&mut self, sel: Selection) { + let Some(area) = self.scrollback_area else { + return; + }; + let mut flat: Vec> = Vec::new(); + for entry in &self.scrollback { + for line in crate::render::entry_lines(entry, area.width) { + flat.push(line); + } + } + let wrapped = crate::render::wrap_lines(flat, area.width); + let (start, end) = sel.ordered(); + if start == end { + return; // pure click, nothing to copy + } + let mut out = String::new(); + let last_row = end.0.min(wrapped.len().saturating_sub(1)); + for (row_idx, line) in wrapped + .iter() + .enumerate() + .skip(start.0) + .take(last_row + 1 - start.0) + { + let plain = crate::render::line_plain_text(line); + let chars: Vec = plain.chars().collect(); + let row_start = if row_idx == start.0 { start.1 } else { 0 }; + let row_end = if row_idx == end.0 { end.1 } else { chars.len() }; + let row_end = row_end.min(chars.len()); + let row_start = row_start.min(row_end); + for c in &chars[row_start..row_end] { + out.push(*c); + } + if row_idx < end.0 { + out.push('\n'); + } + } + if out.is_empty() { + return; + } + // Primary path: OSC 52. The terminal itself puts the text on + // the system clipboard — no DISPLAY / Wayland socket / + // pbcopy dependency, and it works through SSH. Most modern + // terminals support it (kitty, ghostty, iTerm2, Alacritty, + // Wezterm, recent xterm). Some require an opt-in + // (`set -g set-clipboard on` in tmux, `Allow programs to use + // clipboard` in iTerm2's General → Selection prefs). + // + // Secondary path: arboard. When a real clipboard daemon is + // reachable, this also syncs into the X11/Wayland clipboard + // so other GUI apps see the text. Failures here are silent + // because OSC 52 above is already authoritative — the + // X11-unreachable / Wayland-without-perms case used to + // surface as a noisy "clipboard copy failed" Notice. + send_osc52(&out); + let _ = arboard::Clipboard::new().and_then(|mut c| c.set_text(out)); + } + + async fn handle_terminal_event(&mut self, ev: Event) { + if let Event::Mouse(mouse) = ev { + self.handle_mouse_event(mouse); + return; + } + if let Event::Paste(data) = ev { + self.handle_paste(data); + return; + } + let Event::Key(key) = ev else { return }; + if !matches!(key.kind, KeyEventKind::Press | KeyEventKind::Repeat) { + return; + } + + if self.reverse_search.is_some() { + self.handle_reverse_search_key(key).await; + return; + } + + // Popup navigation / dismissal takes precedence over both + // app-level chords AND the catch-all editor handoff, so + // Up/Down/Enter/Esc go to the popup when one is open. + if self.popup.is_some() && self.handle_popup_key(key) { + return; + } + + match (key.code, key.modifiers) { + (KeyCode::Char('d'), KeyModifiers::CONTROL) => { + // Exit unconditionally. The original behavior required + // the input to be empty (readline convention), but + // for a REPL it's strictly an annoyance — you can't + // exit a misformed in-progress command without + // clearing it first. Ctrl-C is the right key to + // discard the current input, and we already bind it + // to that. + self.push(ScrollbackKind::Notice, "exit".to_string()); + self.exit = true; + } + (KeyCode::Char('c'), KeyModifiers::CONTROL) => { + // Two modes for Ctrl-C: + // + // - **Eval in flight** (`WorkerState::Running`): + // send SIGINT to the Vivado child, which its Tcl + // runtime traps into `interp cancel` — the + // current eval aborts and returns a "interrupted" + // error through the shim protocol, without + // killing the Vivado session. Full mechanism + + // design assumptions on + // `vw_vivado::VivadoBackend::interrupt`. No fall- + // through to the input clear; the input line is + // probably empty during an eval anyway, and + // preserving whatever the user typed lets them + // resubmit / edit after the cancel lands. + // + // - **No eval running**: clear the current input + // (reedline / readline convention). + if matches!(self.worker_state, WorkerState::Running) { + if let Some(pid) = self.child_pid { + // Signal the process GROUP, not the process. + // Vivado's on-disk binary is a bash wrapper + // that forks loader+Vivado as children without + // `exec`, so `kill(pid, SIGINT)` would only + // hit bash and bash won't forward it. See + // `VivadoBackend::interrupt` for the full + // writeup — we mirror the same `-pid` + // negation here rather than round-tripping + // through the worker channel (which is + // blocked on the very eval we're cancelling). + // + // SAFETY: kill on a pgid we spawned. ESRCH + // (empty group / race with child exit) is a + // silent no-op. + unsafe { + libc::kill(-(pid as libc::pid_t), libc::SIGINT); + } + self.push( + ScrollbackKind::Notice, + "interrupt sent — Vivado will abort the \ + current eval and return" + .into(), + ); + } else { + self.push( + ScrollbackKind::Warning, + "no pid cached; can't interrupt eval — \ + restart the REPL to recover" + .into(), + ); + } + } else { + self.input = TextArea::default(); + self.history_cursor = None; + self.history_draft.clear(); + } + } + (KeyCode::F(2), _) => { + // Flip terminal mouse-capture mode. OFF (the default) + // lets the terminal handle text-selection drags + // natively; ON routes wheel events into the app for + // scrollback navigation, at the cost of text + // selection requiring Shift-drag / Option-drag. + self.toggle_mouse_capture(); + } + (KeyCode::Char('r'), KeyModifiers::CONTROL) => { + self.reverse_search = Some(ReverseSearch { + query: String::new(), + match_index: None, + match_text: String::new(), + }); + } + (KeyCode::Tab, _) => { + self.trigger_completion(); + } + (KeyCode::Char('h'), KeyModifiers::CONTROL) => { + self.popup = Some(crate::popup::PopupState::Help( + crate::popup::HelpPopup, + )); + } + (KeyCode::Char('y'), KeyModifiers::CONTROL) => { + // Hover under cursor. We use Ctrl-Y (not Ctrl-K, + // which is already scrollback-up) — picked because + // Ctrl-K is also commonly conflated with + // "kill-line" elsewhere. Y for "your symbol's docs." + self.trigger_hover(); + } + (KeyCode::Char('s'), KeyModifiers::CONTROL) => { + // Fuzzy symbol picker over the session + pending + + // in-flight input. Opens centered; Tab toggles to + // the libraries view. (Ctrl-S over Ctrl-T because + // the latter often gets eaten by terminal + // multiplexers, and "S" for "search" is the more + // discoverable mnemonic.) + self.trigger_symbol_search(); + } + (KeyCode::Char('f'), KeyModifiers::CONTROL) => { + // Fuzzy diagnostics finder. Snapshots the current + // scrollback's Error/Warning/Notice entries. Enter + // jumps the viewport to the chosen entry and drops + // a persistent left-gutter marker (Alt-C clears). + self.trigger_diagnostic_search(); + } + (KeyCode::Char('c'), KeyModifiers::ALT) => { + // Clear the diagnostics-finder jump marker. No-op + // when nothing is marked. Picked Alt-C over Ctrl-L + // (which many terminals eat for "clear screen") and + // Ctrl-K (already bound to scroll-up). + self.clear_marker(); + } + (KeyCode::Char('p'), KeyModifiers::CONTROL) => { + self.history_step(-1); + } + (KeyCode::Char('n'), KeyModifiers::CONTROL) => { + self.history_step(1); + } + // Scrollback nav. PageUp/PageDown for keyboards that + // have them; Ctrl-K (up) / Ctrl-J (down) for compact + // keyboards (Mac laptops, 60% boards) where PageUp + // doesn't exist physically. Vim-style direction + // mapping — `k` is up, `j` is down. Picked over + // Ctrl-↑/↓ because macOS intercepts those (Mission + // Control / app-switching). + // + // Direction: see `handle_mouse_event` — `k`/PageUp + // moves the viewport UP toward older content, which + // means decreasing the y-scroll offset. + // Vim-style scroll: Alt+K up, Alt+J down. Alt (not + // Ctrl) because Ctrl+J is claimed by Shift+Enter on + // legacy-encoding terminals (see the Ctrl+J-as- + // newline arm below), and single-modifier consistency + // beats splitting the pair across two modifiers. + (KeyCode::PageUp, _) | (KeyCode::Char('k'), KeyModifiers::ALT) => { + self.scroll_by(-5); + } + (KeyCode::PageDown, _) + | (KeyCode::Char('j'), KeyModifiers::ALT) => { + self.scroll_by(5); + } + // Snap to bottom + re-engage tail-follow. Use End + // (when available) or Ctrl-G as the compact-keyboard + // alternative. After scrolling up to inspect old + // output the user explicitly requests "back to live" + // here; we no longer auto-re-engage on every scroll + // (that auto-engage was firing spuriously due to a + // raw-vs-wrapped line-count mismatch, making scroll + // appear dead on large outputs). + (KeyCode::End, _) | (KeyCode::Char('g'), KeyModifiers::CONTROL) => { + self.scrollback_follow = true; + } + (KeyCode::Enter, KeyModifiers::NONE) => { + self.on_submit().await; + } + (KeyCode::Enter, m) if !m.is_empty() => { + // Enter with ANY modifier is a "keep typing" + // escape hatch — Ctrl+Enter and Alt+Enter both + // arrive here as `Enter + CTRL/ALT` under the + // kitty protocol. + self.insert_newline_preserving_indent(); + } + // Shift+Enter on legacy-encoding terminals (macOS + // Terminal.app, GNOME Terminal without kitty + // protocol, and many tmux configurations) arrives + // as Ctrl+J — because ASCII 0x0A (linefeed) IS what + // the shifted-Enter physically produces, and raw + // mode disables the `\n` → Enter auto-translation. + // Bind it to newline directly so the user gets the + // expected behavior everywhere. + (KeyCode::Char('j'), m) if m.contains(KeyModifiers::CONTROL) => { + self.insert_newline_preserving_indent(); + } + _ => { + // Forward everything else to the text editor. + // Once the user starts editing, drop the history + // cursor so subsequent Ctrl-P starts at "newest" + // again — readline behavior: an edited recall is + // a new entry, not a continued walk. + self.history_cursor = None; + let input: Input = key.into(); + let _consumed = self.input.input(input); + // Auto-trigger signature help on every buffer- + // mutating keystroke. Costs one parse of the (small) + // in-flight input + a HashMap lookup; bails when the + // cursor isn't in a known call. Respects existing + // Completion / Help popups (won't displace them). + self.refresh_signature_help(); + } + } + } + + async fn handle_reverse_search_key(&mut self, key: KeyEvent) { + let Some(rs) = self.reverse_search.as_mut() else { + return; + }; + match (key.code, key.modifiers) { + (KeyCode::Esc, _) => { + self.reverse_search = None; + } + (KeyCode::Enter, _) => { + let text = std::mem::take(&mut rs.match_text); + self.reverse_search = None; + if !text.is_empty() { + self.set_input_to(&text); + } + } + (KeyCode::Char('r'), KeyModifiers::CONTROL) => { + let start = rs.match_index; + if let Some((idx, hit)) = + self.history.search_back(&rs.query, start) + { + rs.match_index = Some(idx); + rs.match_text = hit.to_string(); + } + } + (KeyCode::Backspace, _) => { + rs.query.pop(); + self.rerun_reverse_search(); + } + (KeyCode::Char(c), m) + if !m.contains(KeyModifiers::CONTROL) + && !m.contains(KeyModifiers::ALT) => + { + rs.query.push(c); + self.rerun_reverse_search(); + } + _ => {} + } + } + + fn rerun_reverse_search(&mut self) { + let Some(rs) = self.reverse_search.as_mut() else { + return; + }; + match self.history.search_back(&rs.query, None) { + Some((idx, hit)) => { + rs.match_index = Some(idx); + rs.match_text = hit.to_string(); + } + None => { + rs.match_index = None; + rs.match_text.clear(); + } + } + } + + fn set_input_to(&mut self, text: &str) { + let mut ta = TextArea::default(); + ta.set_cursor_line_style(ratatui::style::Style::default()); + for (i, line) in text.split('\n').enumerate() { + if i > 0 { + ta.insert_newline(); + } + ta.insert_str(line); + } + self.input = ta; + } + + /// Insert bracketed-paste content into the input area, one + /// line at a time. Embedded newlines become real newlines in + /// the buffer, NOT Enter events — that's the whole reason + /// bracketed paste exists: without it, a pasted multi-line + /// block delivers each `\n` as a submit trigger and every + /// intermediate line runs as its own command. + /// + /// Also drops the history walk (any typed edit — paste + /// included — starts a fresh history search on the next + /// Ctrl-P) and turns off scrollback follow so the user can + /// still scroll up during the paste render. + /// Insert a newline and re-emit the CURRENT line's leading + /// whitespace on the new line. Matches how editors (and + /// Claude Code's TUI) behave on Shift+Enter — hitting it + /// inside a `-flag`-continued command keeps you at the same + /// column so `-foo\n -bar` becomes `-foo\n -bar\n |cursor` + /// instead of `-foo\n -bar\n|cursor`. Only whitespace is + /// copied (spaces + tabs) — never the actual line content. + /// + /// Applies at the CURRENT cursor row, not the top row: if the + /// user is mid-line and hits Ctrl+J, they see indent-copied + /// behavior on the CURRENT line's indent, matching every + /// other editor's rule. + fn insert_newline_preserving_indent(&mut self) { + let (row, _) = self.input.cursor(); + let indent: String = self.input.lines()[row] + .chars() + .take_while(|c| *c == ' ' || *c == '\t') + .collect(); + self.input.insert_newline(); + for ch in indent.chars() { + self.input.insert_char(ch); + } + } + + fn handle_paste(&mut self, data: String) { + self.history_cursor = None; + let mut first = true; + for line in data.split('\n') { + if !first { + self.input.insert_newline(); + } + first = false; + // Strip carriage returns some terminals prepend (CRLF + // sources on Windows / some remote sessions). + for ch in line.chars().filter(|&c| c != '\r') { + self.input.insert_char(ch); + } + } + } + + async fn on_submit(&mut self) { + let text = self.current_input_text(); + let trimmed = text.trim(); + if trimmed.is_empty() { + return; + } + if !is_buffer_complete( + &text, + &self.session.read().unwrap().signature_table(), + ) { + self.input.insert_newline(); + return; + } + + self.history.append(&text); + self.push(ScrollbackKind::Input, text.clone()); + + if let Some(cmd) = trimmed.strip_prefix(':') { + self.run_meta_command(cmd).await; + } else { + self.dispatch_eval(text).await; + } + + self.input = TextArea::default(); + // Reset history walk: the next Ctrl-P should start from the + // newest entry, not pick up where the previous walk left + // off across submits. + self.history_cursor = None; + self.history_draft.clear(); + // Re-engage tail-follow: every new submit shows the input + // echo + its output at the bottom of the viewport, even + // if the user had scrolled up to inspect earlier results. + // The actual pin happens in the renderer next frame. + self.scrollback_follow = true; + } + + fn resolve_stack_frames(&self, msg: &str) -> String { + let session_guard = self.session.read().unwrap(); + resolve_stack_frames( + msg, + &session_guard, + self.pending_batch.as_ref(), + self.input_file_for_resolve(), + ) + } + + /// Append a synthetic ` at :` frame to streamed + /// warnings/errors that already arrived without a stack. Vivado + /// emits some message classes (notably `[IP_Flow 19-7090]` + /// "Invalid parameter" warnings during `set_property`) from + /// C++ code paths that bypass the Tcl-level `send_msg_id` + /// override — so the shim never sees them and can't attach a + /// real Tcl call stack. The fallback is "which user command + /// was the worker chewing on when this byte stream arrived," + /// which `pending_origins[pending_eval_index]` gives us. Won't + /// add a frame if the message already has one (the + /// `\n at …` shape from `attach_stack_if_message`) or if it + /// isn't a warning/error severity. + fn tag_streamed_message( + &self, + kind: ScrollbackKind, + msg: String, + ) -> String { + if !matches!(kind, ScrollbackKind::Warning | ScrollbackKind::Error) { + return msg; + } + if msg.contains("\n at ") { + return msg; + } + let Some(origin) = self.pending_origins.get(self.pending_eval_index) + else { + return msg; + }; + let path = match origin.file.as_deref() { + Some(p) => display_path(p), + None => match self.input_file_for_resolve() { + Some(p) => display_path(p), + None => "".into(), + }, + }; + format!("{msg}\n at {path}:{}", origin.line) + } + + /// File to substitute for `` frames in stack traces. + /// Comes from `--load ` for the auto-loaded program — its + /// content was copied verbatim into the lowering scratch, so + /// scratch line N corresponds to load-file line N. For + /// REPL-typed input there's no source file, so callers leave + /// `` as-is. + fn input_file_for_resolve(&self) -> Option<&std::path::Path> { + self.opts.initial_load.as_ref().map(|p| p.as_std_path()) + } + + async fn dispatch_eval(&mut self, text: String) { + self.dispatch_eval_with_echo(text, false).await; + } + + /// Same as [`dispatch_eval`] but echoes each lowered top-level + /// statement as an Input entry first. Used by the `--load` + /// auto-run path so the user can see *which* commands ran when + /// reading the trace, the same way manual REPL input shows up + /// as `› ` for each submit. + async fn dispatch_eval_with_echo(&mut self, text: String, echo: bool) { + if matches!(self.worker_state, WorkerState::Down) { + self.push( + ScrollbackKind::Error, + "vivado worker is down — try :restart".into(), + ); + return; + } + if matches!(self.worker_state, WorkerState::Starting) { + self.push( + ScrollbackKind::Notice, + "queued — vivado still starting".into(), + ); + } + + // Lower htcl → Tcl on a blocking thread so the event + // loop can render the input echo + tick the per-input + // timer during the (potentially minute-scale) parse + + // validate + lower work. `prepare` is fully synchronous + // CPU + I/O; spawn_blocking is the right primitive. + // + // The main task returns immediately after spawning — + // `handle_worker_event` picks up `WorkerEvent::PrepareDone` + // when the background thread finishes, and continues the + // dispatch pipeline from there. + // Show a "preparing…" sentinel only when the input will + // actually pull new files through the loader — i.e., the + // top-level parse of `text` contains a `src` command. + // A one-liner like `bd::clobber -name txr0` doesn't touch + // disk and prepare finishes in low double-digit ms; a + // notice per submit for that case is just noise. + // + // The parse itself is cheap (input is usually a single + // line); prepare will re-parse the flat post-load source + // internally either way. False negatives are impossible + // — `src` is the only mechanism that adds files — and + // false positives are bounded to "user typed `src` + // without triggering big work" (already-preloaded + // target), where a brief notice does no harm. + let will_load = { + let parsed = vw_htcl::parse(&text); + parsed.document.stmts.iter().any(|s| { + matches!( + s, + vw_htcl::Stmt::Command(c) + if matches!(c.kind, vw_htcl::CommandKind::Src(_)) + ) + }) + }; + if will_load { + self.push( + ScrollbackKind::Notice, + "preparing… (parsing + validating imports)".into(), + ); + } + + let cwd = std::env::current_dir().unwrap_or_else(|_| ".".into()); + let session = std::sync::Arc::clone(&self.session); + let event_tx = self.event_tx.clone(); + let text_moved = text; + tokio::task::spawn_blocking(move || { + // Catch panics on the blocking thread so a bug in + // prepare surfaces as an ERROR row rather than a + // silent stall. Without this the JoinHandle we drop + // absorbs the panic and the UI sits waiting forever + // for a PrepareDone that will never come. + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let session_guard = session.read().unwrap(); + crate::lower::prepare(&text_moved, &cwd, &session_guard) + })); + match result { + Ok(r) => { + let _ = event_tx.send(WorkerEvent::PrepareDone { + text: text_moved, + echo, + result: r, + }); + } + Err(payload) => { + // Turn the panic message into a LowerError so + // the existing PrepareDone/handle_prepare_done + // path renders it as an ERROR row. + let msg = if let Some(s) = payload.downcast_ref::<&str>() { + (*s).to_string() + } else if let Some(s) = payload.downcast_ref::() { + s.clone() + } else { + "unknown panic in prepare".to_string() + }; + let _ = event_tx.send(WorkerEvent::PrepareDone { + text: text_moved, + echo, + result: Err(crate::lower::LowerError::Parse(format!( + "prepare panicked: {msg}", + ))), + }); + } + } + }); + } + + /// Continuation of `dispatch_eval_with_echo` — receives the + /// completed `Prepared` (or error) from the background prepare + /// task and finishes the dispatch pipeline: surfaces warnings, + /// commits pure-`src` imports directly, otherwise builds the + /// per-input timer boundaries and ships commands to the worker. + async fn handle_prepare_done( + &mut self, + _text: String, + echo: bool, + result: Result, + ) { + let lowered = match result { + Ok(l) => l, + Err(e) => { + self.push(ScrollbackKind::Error, format!("ERROR: {e}")); + // Prepare failed — no eval will run, so the Input + // entry's timer would otherwise tick forever. Freeze + // it at the "prepare failed" wall time. Same freeze + // path the empty-batch case (line ~2234) uses. + self.mark_inputs_completed(); + return; + } + }; + + // Surface any pre-flight warnings *before* shipping. If the + // eval then fails, the user already has the context they + // need to interpret the Vivado error. + for w in &lowered.warnings { + let where_ = + render_origin_path(w.origin.file.as_deref(), w.origin.line); + self.push( + ScrollbackKind::Warning, + format!("warning: {where_}: {}", w.message), + ); + } + if lowered.commands.is_empty() { + // Pure `src` import or comments-only input. Commit the + // parsed batch to the session anyway so future + // analyzer queries see the imported procs. + self.session.write().unwrap().commit(lowered.batch); + self.sync_preload_from_session(); + self.push(ScrollbackKind::Notice, "(no Tcl to evaluate)".into()); + // The per-input timer was ticking through prepare; + // freeze it now — this batch has nothing to eval. + self.mark_inputs_completed(); + return; + } + + // Build per-Input-entry timer boundaries. Empty when not + // in echo mode (the non-echo single-Input case uses the + // existing `mark_inputs_completed` end-of-batch path). + // + // Echo model: strictly linear. The batch's FIRST + // statement is echoed to scrollback now; every subsequent + // statement is registered as a boundary with + // `scrollback_idx: None` and echoed lazily by + // `advance_input_timers` when the prior boundary closes. + // That way a `:load prime.htcl` run reads like an + // interactive session — each command appears, its output + // and messages follow, then the next command appears. + let mut input_boundaries: Vec = Vec::new(); + if echo { + for origin in &lowered.entry_top_level { + input_boundaries.push(InputBoundary { + scrollback_idx: None, + snippet: origin.snippet.clone(), + last_command_idx: None, // filled below + completed: false, + }); + } + // For each entry-top-level Origin, find the LAST + // lowered command whose ultimate entry-file line + // matches it. A command's "entry line" is the line + // in the entry file it came from: directly when + // `origin.via` is empty (the command lives in the + // entry), or the bottom of the `via` chain (which + // lower.rs documents as "the last frame is the + // entry file / user input"). + for (cmd_idx, cmd) in lowered.commands.iter().enumerate() { + let entry_line = match cmd.origin.via.last() { + Some(f) => f.line, + None => cmd.origin.line, + }; + // Find which entry_top_level Origin this matches + // (linear scan — at most a handful of top-level + // statements per batch). + for (j, top) in lowered.entry_top_level.iter().enumerate() { + if top.line == entry_line { + if let Some(b) = input_boundaries.get_mut(j) { + b.last_command_idx = Some(cmd_idx); + } + break; + } + } + } + } + self.pending_input_boundaries = input_boundaries; + if echo { + // Push the first non-empty boundary's echo NOW so the + // user sees `› ` before the batch + // starts producing output. `activate_next_boundary` + // also handles the edge case of a leading empty + // boundary (a `src` whose target file lowered to zero + // commands): it echoes, freezes, and cascades. + self.activate_next_boundary(0); + } + + // Snapshot per-command origins + types for the stream- + // tagging + result-display paths. EvalBatch consumes + // `lowered.commands` below, so we grab both first. + self.pending_origins = + lowered.commands.iter().map(|c| c.origin.clone()).collect(); + self.pending_return_types = lowered + .commands + .iter() + .map(|c| c.expected_return_type.clone()) + .collect(); + self.pending_is_set_binding = + lowered.commands.iter().map(|c| c.is_set_binding).collect(); + self.pending_eval_index = 0; + + // Seed the shared preload map with this batch's file + // list BEFORE dispatching. Commands ship in document + // order, so any RPC that fires from a command MID-batch + // (currently only `compile_htcl_module` from + // `vw::configure_ip`) can trust that every proc from + // every file preceding it in the same batch is already + // installed in Vivado. Without this, the batch's own + // `src @vw` recursion doesn't reach the preload until + // AFTER the whole batch completes — which means the + // first `vw::configure_ip` call re-parses + re-lowers + // + re-ships @vw + @vivado-cmd (~10MB of Tcl, + // multiple minutes). Preload-then-dispatch turns that + // into "just workspace-local files" and shrinks the + // compile output by ~100×. + // + // Safety: the invariant on `SharedPreload` is "files + // whose Tcl has been eval'd by Vivado". Populating from + // this batch's file list before eval TECHNICALLY breaks + // the letter of that rule for the window between + // dispatch and completion — but for the specific caller + // that uses the preload (`compile_htcl_module`, invoked + // from mid-batch procs), the OR pending commands run + // strictly BEFORE the invocation, so the corresponding + // procs ARE installed by the time the RPC fires. + if let Ok(mut g) = self.preload.write() { + for f in &lowered.batch.program.files { + if let Some(t) = f.mtime { + g.insert(f.path.clone(), t); + } + } + } + // Commit to the session only after every command in the + // batch succeeds (see `handle_worker_event`); a failure + // mid-batch shouldn't pollute the analyzer's view. + let _ = self + .worker_tx + .send(WorkerCmd::EvalBatch(lowered.commands)) + .await; + self.pending_batch = Some(lowered.batch); + self.worker_state = WorkerState::Running; + } + + async fn run_meta_command(&mut self, cmd: &str) { + // Note for completion: keep [`META_COMMANDS`] in sync with + // the arms of this `match`. + let mut parts = cmd.splitn(2, char::is_whitespace); + let name = parts.next().unwrap_or(""); + let arg = parts.next().unwrap_or("").trim(); + match name { + "quit" | "q" | "exit" => { + self.exit = true; + } + "restart" => { + self.push( + ScrollbackKind::Notice, + "restart not yet implemented (stubbed for v1)".into(), + ); + } + "libs" => { + // List every library the session knows about + its + // symbol count, sorted by descending count. Built + // from the same SymbolIndex the Ctrl-T picker uses, + // so the totals stay consistent across surfaces. + let parsed_input = + vw_htcl::parser::parse(&self.current_input_text()); + let session = std::sync::Arc::clone(&self.session); + let session_guard = session.read().unwrap(); + let index = crate::symbol_index::SymbolIndex::build( + &session_guard, + self.pending_batch.as_ref(), + Some(&parsed_input.document), + ); + let libs = index.libraries(); + if libs.is_empty() { + self.push( + ScrollbackKind::Notice, + "no libraries loaded".to_string(), + ); + } else { + // Column widths: count gets 5 cells, library + // name takes the max of its actual lengths. + let max_name = libs + .iter() + .map(|l| l.library.display().chars().count()) + .max() + .unwrap_or(8); + let mut out = String::new(); + out.push_str(&format!( + "{:>5} {: { + "".to_string() + } + crate::symbol_index::LibraryRef::Import { + path, + .. + } => path.display().to_string(), + }; + out.push_str(&format!( + "{:>5} {: { + if arg.is_empty() { + self.push( + ScrollbackKind::Error, + ":load needs a path".into(), + ); + return; + } + match std::fs::read_to_string(arg) { + Ok(content) => { + self.push( + ScrollbackKind::Notice, + format!("loading {arg}"), + ); + self.dispatch_eval(content).await; + } + Err(e) => { + self.push( + ScrollbackKind::Error, + format!("could not read {arg}: {e}"), + ); + } + } + } + other => { + self.push( + ScrollbackKind::Error, + format!("unknown meta-command :{other}"), + ); + } + } + } + + async fn handle_worker_event(&mut self, event: WorkerEvent) { + match event { + WorkerEvent::Started { child_pid } => { + self.worker_state = WorkerState::Ready; + self.child_pid = child_pid; + self.push(ScrollbackKind::Notice, "vivado ready".into()); + if let Some(path) = self.opts.initial_load.clone() { + match std::fs::read_to_string(path.as_std_path()) { + Ok(content) => { + self.push( + ScrollbackKind::Notice, + format!("auto-loading {path}"), + ); + self.dispatch_eval_with_echo(content, true).await; + } + Err(e) => { + self.push( + ScrollbackKind::Error, + format!("could not read {path}: {e}"), + ); + } + } + } + } + WorkerEvent::StartFailed(e) => { + self.worker_state = WorkerState::Down; + self.push( + ScrollbackKind::Error, + format!("vivado failed to start: {e}"), + ); + } + WorkerEvent::PrepareDone { text, echo, result } => { + self.handle_prepare_done(text, echo, result).await; + } + WorkerEvent::Stream { kind, data } => { + // Feed the chunk into the block accumulator: NONE + // blocks (Vivado tables / banners / license chatter) + // group into one collapsible scrollback entry per + // run of Stdout chunks, while Diagnostic chunks + // (INFO / WARNING / CRITICAL / ERROR) flush any + // pending NONE and emit themselves at full fidelity. + for block in self.block_acc.push(kind, &data) { + match block { + vw_vivado::Block::None { lines } => { + self.push_none_block(lines); + } + vw_vivado::Block::Diagnostic { severity, lines } => { + let is_critical = matches!( + severity, + vw_vivado::Severity::CriticalWarning + ); + let scrollback_kind = match severity { + vw_vivado::Severity::Info => { + ScrollbackKind::Notice + } + vw_vivado::Severity::Warning => { + ScrollbackKind::Warning + } + // Critical warnings render with the + // same red ✗ treatment as ERROR in + // scrollback — the block classifier + // keeps them semantically distinct + // for log-level filtering, but the + // visual severity is the same. + // `is_critical` above carries the + // discriminator through to the + // scrollback entry for the finder. + vw_vivado::Severity::CriticalWarning + | vw_vivado::Severity::Error => { + ScrollbackKind::Error + } + // Segmenter guarantees Diagnostic + // blocks are never Severity::None, + // but the arm has to exist for the + // match to be exhaustive. + vw_vivado::Severity::None => { + ScrollbackKind::Stdout + } + }; + let joined = lines.join("\n"); + let resolved = self.resolve_stack_frames(&joined); + let tagged = self.tag_streamed_message( + scrollback_kind, + resolved, + ); + let (body, stack) = split_body_and_stack(&tagged); + self.push_diag(scrollback_kind, body, is_critical); + if let Some(stack) = stack { + self.push_stack_trace(stack); + } + } + } + } + } + WorkerEvent::EvalDone { + origin, + result, + last_in_batch, + } => { + // Flush any pending NONE-block content the segmenter + // has been holding. Pure `puts` output arrives as + // Stdout chunks (Severity::None) and never emits from + // the accumulator on its own — the accumulator only + // flushes when a *classified* chunk (INFO/WARNING/ + // ERROR) arrives after it. Without this flush, plain + // `puts "muffins"` output sits invisible in + // `pending_none` until the next diagnostic — which + // for a quiet REPL never comes. Flushing at + // EvalDone is the right boundary: every eval's own + // output should surface before the next input echo. + for block in self.block_acc.flush() { + match block { + vw_vivado::Block::None { lines } => { + self.push_none_block(lines); + } + vw_vivado::Block::Diagnostic { severity, lines } => { + // Diagnostic-in-flush is unexpected — + // the accumulator emits diagnostics + // immediately on push, never holds them + // pending. Route through the same + // scrollback-kind mapping as the + // Stream-event path so future accumulator + // changes don't silently lose messages. + let is_critical = matches!( + severity, + vw_vivado::Severity::CriticalWarning + ); + let scrollback_kind = match severity { + vw_vivado::Severity::Info => { + ScrollbackKind::Notice + } + vw_vivado::Severity::Warning => { + ScrollbackKind::Warning + } + vw_vivado::Severity::CriticalWarning + | vw_vivado::Severity::Error => { + ScrollbackKind::Error + } + vw_vivado::Severity::None => { + ScrollbackKind::Stdout + } + }; + let joined = lines.join("\n"); + let resolved = self.resolve_stack_frames(&joined); + let tagged = self.tag_streamed_message( + scrollback_kind, + resolved, + ); + let (body, stack) = split_body_and_stack(&tagged); + self.push_diag(scrollback_kind, body, is_critical); + if let Some(stack) = stack { + self.push_stack_trace(stack); + } + } + } + } + // Grab the return type + set-binding flag for + // THIS command (the one that just finished) + // before we advance the index and possibly clear + // the buffer. + let finished_is_set_binding = self + .pending_is_set_binding + .get(self.pending_eval_index) + .copied() + .unwrap_or(false); + let finished_return_type = self + .pending_return_types + .get(self.pending_eval_index) + .cloned() + .flatten(); + // Capture the just-finished command's eval-index + // before we advance — used to freeze any Input + // entry whose last-command boundary matches it. + let just_finished_idx = self.pending_eval_index; + // Advance past the command that just finished — the + // stream-tagging path uses `pending_origins[index]` + // to label warnings emitted by the *currently* + // executing command, so the index should always + // point at "in-flight," not "just done." + self.pending_eval_index = + self.pending_eval_index.saturating_add(1); + // Per-statement timer freezing: if any echoed + // Input entry's `last_command_idx` matches the + // just-finished command, stamp its + // `completed_at`. On success we cascade — activate + // (echo + stamp `started_at` for) the next + // uncompleted boundary. On failure we do NOT + // cascade: the batch aborts here, and echoing the + // NEXT statement's `› …` line before the error + // trace we're about to render would make the trace + // look like it belonged to that statement. The + // ordering guarantee we're preserving is: an + // eval's output — including its error trace when + // the eval fails — appears between its own echo + // and the next one, if any next one appears at all. + match &result { + Ok(_) => self.advance_input_timers(just_finished_idx), + Err(_) => { + self.freeze_input_boundary(just_finished_idx); + } + } + if last_in_batch { + self.pending_origins.clear(); + self.pending_return_types.clear(); + self.pending_input_boundaries.clear(); + self.pending_eval_index = 0; + } + match result { + Ok(out) => { + // Drop the per-statement chatter — only the + // last item's value lands in scrollback so a + // `src @vivado-cmd` that runs 851 wrappers + // doesn't drown the user in "ok" lines. The + // intermediate procs etc. are silent unless + // they `puts` something (already streamed). + if last_in_batch { + if !out.stdout.is_empty() { + self.push( + ScrollbackKind::Stdout, + out.stdout + .trim_end_matches('\n') + .to_string(), + ); + } + // Result-rendering policy: + // - `unit`-typed expressions push nothing + // (the value is meaningless by design). + // - Other typed expressions push verbatim + // — the wrapped Tcl already ran the + // type's `repr` proc, so `out.value` + // is the formatted display string. + // - Untyped expressions we can't repr + // through the type's proc: skip the + // push. `out.value` in this case + // is the raw Tcl representation + // (`{Scalar x}` tagged-list form + // for our nested Properties trees) + // — displaying that leaks the + // internal encoding rather than + // the compiler-emitted + // `Variant(payload)` repr shape. + // If a caller wants to inspect an + // untyped value, `puts` is the + // explicit form. + // - `set VAR ` is a binding, + // not a display. The value the + // user asked to name is now bound; + // showing it isn't part of what + // they wrote. Same suppression + // applies for consistency with + // the "no unrepr'd values leak" + // rule above. + let suppress_unit = matches!( + finished_return_type.as_ref(), + Some(vw_htcl::TypeExpr::Named { name, .. }) + if name == "unit" + ); + let suppress_untyped = + finished_return_type.is_none(); + let suppress = suppress_unit + || suppress_untyped + || finished_is_set_binding; + if !suppress && !out.value.is_empty() { + // finished_return_type is Some + // here (untyped is suppressed + // above), so the wrap_with_repr + // path has already rendered + // through the type's `repr` + // proc. Push verbatim. + let text = out.value.clone(); + self.push(ScrollbackKind::Result, text); + } + if let Some(batch) = self.pending_batch.take() { + self.session.write().unwrap().commit(batch); + self.sync_preload_from_session(); + } + self.worker_state = WorkerState::Ready; + // Freeze per-input timers at their + // final duration now that the batch + // has finished evaluating. + self.mark_inputs_completed(); + } + } + Err(err) => { + self.worker_state = WorkerState::Ready; + // Hold the pending batch for the renderer + // — drill-down lookups need its proc map. + // Cleared below once the trace is emitted. + render_eval_error(self, &origin, err); + // Commit the parsed batch to the session + // even though the eval failed. The batch's + // `Document` carries every proc, type, and + // enum decl that the parser saw — including + // whatever `src @vivado-cmd`, `src project`, + // `src ip/cips`, etc. brought in *before* + // the failing user command ran. Tab + // completion, hover, and signature help all + // query session-wide symbols, and dropping + // the batch strands them with an empty + // symbol table until the next successful + // eval. The runtime state in Vivado may be + // partial or wrong; that's a separate + // concern from what the analyzer sees. + if let Some(batch) = self.pending_batch.take() { + self.session.write().unwrap().commit(batch); + self.sync_preload_from_session(); + } + // Failed evals also freeze their per-input + // timer — otherwise the live counter would + // tick forever on an error result. + self.mark_inputs_completed(); + } + } + } + } + } + + pub(crate) fn push(&mut self, kind: ScrollbackKind, text: String) { + // O(1). The tail-follow pin happens in the renderer (which + // already knows the wrapped-row total for free), not here — + // doing it per-push was O(N) per call, making a long burst + // of Vivado stream chunks O(N²) and freezing the REPL for + // minutes during `src @vivado-cmd` style fan-outs. + // + // Input entries get a start timestamp so the renderer can + // show a per-input timer (live while running, frozen on + // batch completion). Other kinds leave timing unset. + let started_at = if matches!(kind, ScrollbackKind::Input) { + Some(std::time::Instant::now()) + } else { + None + }; + // Expand tabs to spaces at push time so BOTH `entry_lines_ + // windowed` (which builds rendered spans) and `count_wrapped_ + // rows` (which does the outer viewport math) see the same + // character content. If a `\t` survives into the render + // pipeline, ratatui's Buffer writes a `\t` cell that + // iTerm2 (and any real terminal) interprets as "move + // cursor to next tab stop" — a control code, not a + // printable glyph. Cells between the cursor's start + // column and the tab stop stay whatever they were in the + // previous frame, producing the `728_` / `FO:` / `.v:` + // leftover fragments Vivado's parameter-dump output was + // showing. Four spaces per tab matches typical editor + // defaults and keeps the expansion fixed-count so + // downstream width math (`chars().count()`) stays honest. + let text = if text.contains('\t') { + text.replace('\t', " ") + } else { + text + }; + // Uniform Mathematica-notebook-style collapsibility: every + // multi-line entry is toggleable (Shift+click), and + // anything larger than COLLAPSE_AUTO_THRESHOLD lines starts + // collapsed so a wall of text doesn't dominate the + // scrollback. Single-line entries get `None` — a placeholder + // for something that fits in one row is worse UX than just + // showing the row itself. + let collapse_state = compute_collapse_state(&text, self.collapse_mode); + self.scrollback.push(ScrollbackEntry { + kind, + text, + started_at, + completed_at: None, + collapse_state, + is_critical_warning: false, + }); + } + + /// Push a diagnostic that came from the Vivado stream, tagging + /// it as a CRITICAL WARNING when the source severity was + /// `Severity::CriticalWarning`. CW entries share + /// [`ScrollbackKind::Error`]'s red gutter but carry the + /// [`ScrollbackEntry::is_critical_warning`] flag so the + /// diagnostics-finder popup can offer a `Critical` filter + /// checkbox that surfaces just them. + pub(crate) fn push_diag( + &mut self, + kind: ScrollbackKind, + text: String, + is_critical_warning: bool, + ) { + // Split the tagged first line off. For any diagnostic — + // ERROR / CRITICAL WARNING / WARNING — the leading + // `: [] ` line MUST stay + // full-brightness and never dim-collapse, or a real + // problem in the middle of dozens of INFO lines + // disappears. Trailing content (Resolution hints, + // continuations) is fine to auto-collapse — that's what + // `split_leading_diagnostic` gives us. Notice / Stdout + // kinds don't tag their leading line, so the split just + // returns `(text, None)` and the entry pushes as before. + let (leading, trailing) = split_leading_diagnostic(&text); + let leading = if leading.contains('\t') { + leading.replace('\t', " ") + } else { + leading + }; + // The leading line pushes as its own entry with + // `collapse_state = None` (via the <2-lines branch of + // `compute_collapse_state`) so it renders at full + // brightness. Only CW entries carry the flag — this is + // what feeds the diagnostics-finder's `Critical` filter. + let leading_collapse = + compute_collapse_state(&leading, self.collapse_mode); + self.scrollback.push(ScrollbackEntry { + kind, + text: leading, + started_at: None, + completed_at: None, + collapse_state: leading_collapse, + is_critical_warning, + }); + if let Some(trailing) = trailing { + let trailing = if trailing.contains('\t') { + trailing.replace('\t', " ") + } else { + trailing + }; + let collapse_state = + compute_collapse_state(&trailing, self.collapse_mode); + self.scrollback.push(ScrollbackEntry { + kind: ScrollbackKind::Chatter, + text: trailing, + started_at: None, + completed_at: None, + collapse_state, + is_critical_warning: false, + }); + } + } + + /// Push a NONE-severity block: the accumulated non-diagnostic + /// chatter between two classified messages (Vivado tables, + /// section headers, banners, `VHDL Output written to …` lines). + /// Always uses [`ScrollbackKind::Chatter`] — the "background + /// noise" bucket that carries the dim dark-gray body style so + /// non-diagnostic output visually reads as elidable. Whether + /// the entry is collapsed / expanded / not-collapsible is + /// decided by [`Self::push`]'s threshold logic (multi-line + /// entries over the auto-collapse threshold start collapsed; + /// smaller ones expand). + pub(crate) fn push_none_block(&mut self, lines: Vec) { + if lines.is_empty() { + return; + } + self.push(ScrollbackKind::Chatter, lines.join("\n")); + } + + /// Push a stack trace as its own entry, always Chatter-styled + /// and always start-collapsed when it has 2+ lines. Used by the + /// diagnostic stream path to split the `at :` tail + /// off a WARNING/ERROR body so the human-readable message + /// stays fully visible while the stack becomes a + /// `▶ at ` placeholder — Shift+click to expand. + /// Bypasses [`Self::push`]'s auto-threshold: even a 3-frame + /// stack collapses, which is the whole point of the split. + /// Single-line stacks (one frame) fit in a row and stay + /// non-collapsible — a placeholder for a 40-char line reads + /// as noise. + pub(crate) fn push_stack_trace(&mut self, stack: String) { + let text = if stack.contains('\t') { + stack.replace('\t', " ") + } else { + stack + }; + let collapse_state = if text.lines().count() < 2 { + None + } else { + Some(true) + }; + self.scrollback.push(ScrollbackEntry { + kind: ScrollbackKind::Chatter, + text, + started_at: None, + completed_at: None, + collapse_state, + is_critical_warning: false, + }); + } + + /// Per-Input-entry timer advance triggered by an EvalDone. + /// If `just_finished_idx` matches any uncompleted boundary's + /// `last_command_idx`, freeze its scrollback entry's + /// `completed_at` and anchor the next uncompleted boundary's + /// `started_at` to NOW so its timer begins fresh rather than + /// inheriting the elapsed time from earlier statements' + /// commands. + fn advance_input_timers(&mut self, just_finished_idx: usize) { + if let Some(hit) = self.freeze_input_boundary(just_finished_idx) { + // Activate the next uncompleted boundary — pushes its + // echo and stamps `started_at`. Empty boundaries (no + // lowered commands attributed) get echoed + frozen + // instantly and we cascade to the next one; otherwise + // no EvalDone would ever close them and the batch would + // stall at that point in the visual trace. + self.activate_next_boundary(hit + 1); + } + } + + /// Freeze the boundary whose `last_command_idx` matches + /// `just_finished_idx` without cascading to the next + /// statement. Used by the failing-eval path so an error trace + /// isn't visually preceded by the next boundary's echo — the + /// batch is aborting, the next echo would be misleading, and + /// (worse) it would appear BEFORE the failure explanation the + /// user needs to read. Returns the boundary's index in + /// `pending_input_boundaries` when found, or `None` when this + /// EvalDone doesn't close any boundary (e.g. a synthetic + /// prelude command). + fn freeze_input_boundary( + &mut self, + just_finished_idx: usize, + ) -> Option { + let now = std::time::Instant::now(); + // Find the first uncompleted boundary whose + // last_command_idx matches. Multi-statement load + // batches process commands in order, so the matching + // boundary is always at the head of the uncompleted + // run. + let mut hit_position: Option = None; + for (i, b) in self.pending_input_boundaries.iter().enumerate() { + if b.completed { + continue; + } + if b.last_command_idx == Some(just_finished_idx) { + hit_position = Some(i); + } + break; + } + let hit = hit_position?; + self.pending_input_boundaries[hit].completed = true; + if let Some(idx) = self.pending_input_boundaries[hit].scrollback_idx { + if let Some(entry) = self.scrollback.get_mut(idx) { + if entry.completed_at.is_none() { + entry.completed_at = Some(now); + } + } + } + Some(hit) + } + + /// Push the echo for the first uncompleted boundary starting + /// at `start`, stamp its `started_at`, and freeze-and-cascade + /// past any empty boundaries encountered along the way. Used + /// both at batch dispatch (starting from index 0) and by + /// `advance_input_timers` when the prior boundary closes. + fn activate_next_boundary(&mut self, start: usize) { + let now = std::time::Instant::now(); + let mut i = start; + while i < self.pending_input_boundaries.len() { + if self.pending_input_boundaries[i].completed { + i += 1; + continue; + } + let has_commands = + self.pending_input_boundaries[i].last_command_idx.is_some(); + // Push echo lazily if we haven't already (the first + // boundary at batch dispatch may already have a + // scrollback_idx assigned). + let idx = match self.pending_input_boundaries[i].scrollback_idx { + Some(idx) => idx, + None => { + let snippet = + self.pending_input_boundaries[i].snippet.clone(); + let idx = self.scrollback.len(); + self.push(ScrollbackKind::Input, snippet); + self.pending_input_boundaries[i].scrollback_idx = Some(idx); + idx + } + }; + if let Some(entry) = self.scrollback.get_mut(idx) { + entry.started_at = Some(now); + } + if has_commands { + // Real boundary — wait for its EvalDone to close it. + return; + } + // Empty boundary: no lowered commands, so no EvalDone + // will match. Freeze it now with a zero-second timer + // and cascade to the next. + self.pending_input_boundaries[i].completed = true; + if let Some(entry) = self.scrollback.get_mut(idx) { + entry.completed_at = Some(now); + } + i += 1; + } + } + + /// Stamp `completed_at` on every still-running Input entry + /// from the most recent batch. Called from the `EvalDone` + /// handler on `last_in_batch` so the per-input timers freeze + /// at their final duration once the batch has finished + /// evaluating. For `--load` echoed batches with multiple + /// Input entries (one per top-level statement) all entries + /// freeze at the same wall time — finer-grained per-statement + /// timing would require carrying the eval-to-input mapping + /// through the worker round-trip, which is more plumbing + /// than the v1 timer needs. + fn mark_inputs_completed(&mut self) { + let now = std::time::Instant::now(); + for entry in self.scrollback.iter_mut().rev() { + if matches!(entry.kind, ScrollbackKind::Input) + && entry.completed_at.is_none() + { + entry.completed_at = Some(now); + } else if entry.completed_at.is_some() + && matches!(entry.kind, ScrollbackKind::Input) + { + // Already-completed Input from a prior batch — + // we've walked past the current batch's inputs. + break; + } + } + } + + /// Apply a signed scroll delta (positive = down toward newer + /// content, negative = up toward older content). Disengages + /// tail-follow when the user scrolls up; re-engages when + /// they scroll down past the bottom — same semantics as a + /// scroll-wheel in a terminal emulator. + /// + /// Anchors the new offset against `last_rendered_scroll` (set + /// by the renderer each frame) rather than `scrollback_scroll`, + /// because while tail-follow is on `scrollback_scroll` is stale + /// — the renderer computes the effective bottom-aligned offset + /// without writing it back to that field. Starting the manual + /// delta from the rendered offset is what lets Ctrl-K from + /// tail-follow mode actually move up by 5 instead of jumping + /// to position 5. + fn scroll_by(&mut self, delta: i32) { + let base = self.last_rendered_scroll as i32; + let new = base.saturating_add(delta).max(0) as u16; + if delta < 0 { + self.scrollback_follow = false; + } else if delta > 0 && new >= self.last_max_scroll { + // Downward scroll that reaches (or passes) the bottom + // re-engages tail-follow — standard terminal-emulator + // behavior. Safe now that `last_max_scroll` is the + // renderer's exact wrapped-row math, not the old + // raw-line-count heuristic that misfired on wrapped + // multi-MB entries (spuriously snapping back to bottom + // on any scroll-up and making scroll appear dead). + self.scrollback_follow = true; + } + self.scrollback_scroll = new; + // Predictively mirror the new offset into + // `last_rendered_scroll`. Without this, drag-to-select + // auto-scrolls but the subsequent `cell_to_buffer` call + // in the same event still uses the previously-rendered + // value — so the selection cursor lags one drag event + // behind the scroll. The renderer will write the + // actually-rendered offset back next frame (which may + // clamp to max_scroll), so this is at worst a one-frame + // optimistic preview. + self.last_rendered_scroll = new; + } +} + +// --------------------------------------------------------------------- +// Worker task: owns the Vivado backend, serializes evals. +// --------------------------------------------------------------------- + +#[allow(clippy::too_many_arguments)] +async fn worker_task( + mut rx: mpsc::Receiver, + tx: mpsc::UnboundedSender, + verbose: bool, + verbose_log: Option, + info_with_stack: bool, + part: Option, + variant: Option, + rpc_workspace_root: Option, + preload: vw_vivado::SharedPreload, +) { + // Auto-project bootstrap: same rule as `vw run`. If the + // enclosing workspace declares `[[target-parts]]` or + // `[[workspace.variants]]`, create an in-memory project up + // front so `ip::check`, `get_ipdefs`, and every other + // project-scoped call have a project to read at the first + // user eval. `part` (part-mode workspaces) and `variant` + // (variant-mode workspaces) come from `--part` / `--variant` + // respectively; the workspace shape decides which applies. + let ws_utf8 = rpc_workspace_root + .as_deref() + .and_then(camino::Utf8Path::from_path) + .map(|p| p.to_path_buf()); + let (auto_project, active_variant) = match ws_utf8.as_deref() { + Some(ws) => match resolve_worker_selection( + ws, + part.as_deref(), + variant.as_deref(), + ) { + Ok(pair) => pair, + Err(e) => { + let _ = tx.send(WorkerEvent::StartFailed( + vw_eda::BackendError::Worker(e), + )); + return; + } + }, + None => (None, None), + }; + // RPC handler — mirrors `vw run`'s. `vw::workspace_root` + // answers with the entry / cwd's nearest `vw.toml` parent; + // unknown methods fail loudly so future htcl calls surface + // a clear "unknown method" instead of hanging. The + // session-scoped `active_variant` is the fallback the + // `vhdl_design_sources` filter uses when no per-call kwarg + // overrides it. + // Raw byte-log for the session: /target/logs/vivado-.log. + // Under the REPL's alternate screen we can't safely eprintln! the + // path (it'd race the TUI), so we swallow errors silently and note + // the path once the terminal is restored via `info!`. + let raw_log = rpc_workspace_root.as_deref().and_then(|ws| { + match vw_vivado::raw_log_path_for_workspace(ws) { + Ok(p) => { + tracing::info!(path = %p.display(), "raw vivado log"); + Some(p) + } + Err(e) => { + tracing::warn!(error = %e, "raw log unavailable"); + None + } + } + }); + // RPC handler with the shared preload map — App owns the + // Arc's other end and updates the map after every + // `session.commit()` from `session.loaded_paths()`. See the + // `SharedPreload` docs in vw-vivado. + // + // Also carries a session-scoped CW counter. The sink below + // bumps it on every `Severity::CriticalWarning` chunk, and + // `vw::synth` / `vw::place` read it via + // `vw::critical_warning_count` to gate their checkpoint + // writes on a CW-clean phase. Same wiring as `vw run` — the + // REPL must not diverge or the exact same htcl session would + // persist a checkpoint here that `vw run` would refuse. + let cw_count: vw_vivado::SharedCriticalWarningCount = + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let rpc_handler = vw_vivado::make_handler_full( + rpc_workspace_root, + active_variant, + preload, + cw_count.clone(), + ); + let backend = vw_vivado::VivadoBackend::spawn(vw_vivado::VivadoConfig { + verbose, + verbose_log, + info_with_stack, + rpc_handler: Some(rpc_handler), + auto_project, + raw_log, + ..Default::default() + }) + .await; + let mut backend = match backend { + Ok(b) => { + let _ = tx.send(WorkerEvent::Started { + child_pid: b.child_pid(), + }); + b + } + Err(e) => { + let _ = tx.send(WorkerEvent::StartFailed(e)); + return; + } + }; + + // Stream chunks to the UI as they arrive. The closure + // captures the unbounded sender so it can fire without + // awaiting. The kind tag (`StreamKind::Stdout` for user `puts` + // output, `Warning`/`Error`/`Info` for Vivado's own message + // lines harvested from the PTY) flows through unchanged so + // the UI can colour them appropriately. + let stdout_tx = tx.clone(); + let cw = cw_count.clone(); + backend.set_stdout_sink(move |kind, chunk: &str| { + // Bump the CW counter exposed via the + // `critical_warning_count` RPC. Only exact + // CriticalWarning — errors already halt eval before any + // htcl checkpoint-write branch runs, so counting them + // here would double-report. + if matches!(kind, vw_vivado::StreamKind::CriticalWarning) { + cw.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + let _ = stdout_tx.send(WorkerEvent::Stream { + kind, + data: chunk.to_string(), + }); + }); + + while let Some(cmd) = rx.recv().await { + match cmd { + WorkerCmd::EvalBatch(items) => { + let total = items.len(); + for (i, item) in items.into_iter().enumerate() { + // Wrap each command's Tcl body with a + // shim-level origin marker so any traceless + // warning it emits stays tagged with THIS + // command's origin even when its PTY bytes lag + // the protocol response into the next eval's + // window. Without this the origin fallback in + // `tag_streamed_message` uses whatever + // `pending_eval_index` points at — which for a + // batch's synthetic prelude commands means + // `line=0` / `line=1` no-op tags. + let wrapped = crate::wrap_tcl_with_origin_marker( + &item.tcl, + &item.origin, + ); + let result = backend.eval(&wrapped).await; + let failed = result.is_err(); + let last_in_batch = i + 1 == total || failed; + let _ = tx.send(WorkerEvent::EvalDone { + origin: item.origin, + result, + last_in_batch, + }); + // Stop the batch at the first failure — running + // the rest of a script after an error confuses + // the user and risks side effects nobody + // intended. + if failed { + break; + } + } + } + WorkerCmd::Shutdown => break, + } + } + let _ = backend.shutdown().await; +} + +// --------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------- + +/// Render a Vivado error as a clean Python-style stack trace — +/// one frame per `file:line` from the outermost `src` the user +/// typed, down through any nested `src` imports, the leaf +/// statement we shipped, and any `(procedure X line N)` frames the +/// Tcl interpreter reported. The error message itself comes last. +/// +/// ```text +/// ip/cips.htcl:1 +/// src @cips +/// ~/src/htcl/amd/cips/module.htcl:3 +/// ip::check -name "xilinx.com:ip:versal_cips:3.4" +/// ~/src/htcl/amd/vivado-cmd/ip.htcl:18 +/// set ip_obj [get_ipdefs -all "$name"] +/// ERROR: [Common 17-53] No open project. ... +/// ``` +fn render_eval_error( + app: &mut App, + origin: &crate::lower::Origin, + err: vw_eda::BackendError, +) { + let mut frames: Vec = Vec::new(); + + // Outermost first: walk the `via` chain in reverse so the + // entry `src` lands at the top. + for f in origin.via.iter().rev() { + frames.push(Frame { + file: f.file.clone(), + line: f.line, + snippet: f.snippet.clone(), + }); + } + // Leaf htcl statement — the actual call site that triggered + // the Tcl evaluation. + frames.push(Frame { + file: origin.file.clone(), + line: origin.line, + snippet: origin.snippet.clone(), + }); + + // If Vivado gave us a Tcl trace, drill into any + // `(procedure "X" line N)` frames whose proc we recognize, so + // the user sees the actual failing line inside the proc body + // — not just the call to it. + let (message, code, info, stdout) = match err { + vw_eda::BackendError::Tcl { + message, + code, + info, + stdout, + } => (message, code, info, stdout), + other => { + for frame in &frames { + push_frame(app, frame); + } + app.push(ScrollbackKind::Error, format!("{other}")); + return; + } + }; + if let Some(info) = info.as_deref() { + let session_guard = app.session.read().unwrap(); + for tcl_frame in parse_tcl_proc_frames(info) { + // Check the in-flight batch first (the lowering that + // just ran), then fall back to prior session batches. + // This is what gives wrappers declared in earlier + // inputs a real `.htcl` path in the drill-down trace + // instead of an `(input):N` line in a vanished scratch. + let loc = app + .pending_batch + .as_ref() + .and_then(|b| b.procs.get(&tcl_frame.proc)) + .or_else(|| session_guard.lookup_proc(&tcl_frame.proc)); + let Some(loc) = loc else { continue }; + let Some((abs_line, content)) = + loc.resolve_body_line(tcl_frame.line) + else { + continue; + }; + frames.push(Frame { + file: loc.file.clone(), + line: abs_line, + snippet: content.trim().to_string(), + }); + } + } + + if !stdout.is_empty() { + app.push( + ScrollbackKind::Stdout, + stdout.trim_end_matches('\n').to_string(), + ); + } + + for frame in &frames { + push_frame(app, frame); + } + // Split `ERROR: [X] Msg\n Resolution: …` so the tagged + // first line pushes as its own single-line entry (full + // brightness, no `▼` dimming) — see `split_leading_diagnostic`. + let (leading, trailing) = split_leading_diagnostic(message.trim()); + app.push(ScrollbackKind::Error, leading); + if let Some(trailing) = trailing { + app.push(ScrollbackKind::Chatter, trailing); + } + if let Some(code) = code.filter(|s| !s.is_empty() && s != "NONE") { + app.push(ScrollbackKind::Notice, format!("({code})")); + } +} + +struct Frame { + file: Option, + line: u32, + snippet: String, +} + +fn push_frame(app: &mut App, frame: &Frame) { + let where_ = render_origin_path(frame.file.as_deref(), frame.line); + app.push(ScrollbackKind::Notice, where_); + if frame.snippet.is_empty() { + return; + } + // Indent every line of the snippet — for a multi-line + // command (`set proj [\n create_project\n -name x\n]`) + // this preserves the user's relative indentation so the + // structure is readable, while the gutter prefix added by + // `entry_lines` distinguishes the first line from the + // continuations. + let body = frame + .snippet + .lines() + .map(|line| format!(" {line}")) + .collect::>() + .join("\n"); + app.push(ScrollbackKind::Notice, body); +} + +/// Parse `(procedure "NAME" line N)` annotations out of Tcl's +/// `$errorInfo`. Returned in the order they appear in `info`, +/// which is innermost-first per Tcl convention — but the renderer +/// wants OUTERMOST-first (we already have the outer leaf frame from +/// the htcl side), so we reverse here and yield the inner frames +/// in execution order. +fn parse_tcl_proc_frames(info: &str) -> Vec { + let mut out = Vec::new(); + for line in info.lines() { + let trimmed = line.trim(); + // Expected shape: `(procedure "NAME" line N)` + let Some(rest) = trimmed.strip_prefix("(procedure \"") else { + continue; + }; + let Some((name, rest)) = rest.split_once("\" line ") else { + continue; + }; + let Some(num) = rest.strip_suffix(')') else { + continue; + }; + let Ok(n) = num.parse::() else { continue }; + out.push(TclProcFrame { + proc: name.to_string(), + line: n, + }); + } + // errorInfo lists innermost first; we want execution order + // (outermost first) so reverse. + out.reverse(); + out +} + +struct TclProcFrame { + proc: String, + line: u32, +} + +/// Rewrite `:N in ::procname` frames in a Vivado message to +/// point at the actual htcl source file and line. Delegates the +/// per-line parsing + dedup to [`crate::trace`], which is shared +/// with the `vw run` CLI driver so both surfaces render the same. +/// This wrapper closes over the REPL's session+pending proc lookup. +fn resolve_stack_frames( + msg: &str, + session: &Session, + pending: Option<&SessionBatch>, + input_file: Option<&std::path::Path>, +) -> String { + crate::trace::resolve_stack_frames_with( + msg, + |name| { + pending + .and_then(|b| b.procs.get(name)) + .or_else(|| session.lookup_proc(name)) + .cloned() + }, + input_file, + ) +} + +fn render_origin_path(file: Option<&std::path::Path>, line: u32) -> String { + match file { + Some(p) => format!("{}:{line}", display_path(p)), + None => format!("(input):{line}"), + } +} + +/// Shorten a path for display: drop the cwd prefix when it lines +/// up, leave it absolute otherwise. Saves screen real estate when +/// reporting errors from a dep cached deep under `~/.vw/deps/...`. +fn display_path(path: &std::path::Path) -> String { + if let Ok(cwd) = std::env::current_dir() { + if let Ok(rel) = path.strip_prefix(&cwd) { + return rel.display().to_string(); + } + if let Some(home) = dirs::home_dir() { + if let Ok(rel) = path.strip_prefix(&home) { + return format!("~/{}", rel.display()); + } + } + } + path.display().to_string() +} + +/// Decide whether the input buffer parses cleanly enough to ship to +/// Write an OSC 52 set-clipboard escape to stdout, base64-encoding +/// `text` per the protocol. The terminal puts the decoded text on +/// the system clipboard — no DISPLAY/Wayland-socket/pbcopy +/// dependency, and the same code path works over SSH. +/// +/// Some terminals cap the payload size at ~74KB (the original xterm +/// limit) or somewhere similar; selections larger than that may be +/// truncated by the terminal. Encoding/IO errors are swallowed — +/// the caller has nowhere useful to surface them, since OSC 52 is +/// fire-and-forget (the terminal doesn't ack). +fn send_osc52(text: &str) { + use base64::engine::general_purpose::STANDARD; + use base64::Engine; + use std::io::Write; + let encoded = STANDARD.encode(text.as_bytes()); + let payload = format!("\x1b]52;c;{encoded}\x07"); + let mut stdout = std::io::stdout(); + let _ = stdout.write_all(payload.as_bytes()); + let _ = stdout.flush(); +} + +/// Vivado, or whether the user is still in the middle of typing +/// (unterminated brace, etc.). We re-use the htcl parser since +/// it already understands every multi-line construct (procs, +/// `[ … ]` substitutions, braced groups). +/// Walk a Document looking for `proc ` (recursing into +/// `namespace eval` blocks). Returns the `Command.doc_comments` +/// slice when found. Used by the signature-help lookup path to +/// surface the proc's `##` docs alongside its argument list. +/// +/// Matches the recursion shape that `vw_htcl::signature_table` +/// uses — qualified names like `util::props` resolve to a proc +/// declared inside `namespace eval util { … }`. +/// Same rule as `vw-cli::resolve_workspace_selection`, mirrored +/// here so the REPL worker doesn't take a cross-crate dep on the +/// CLI. Returns `(auto_project, active_variant)` when the +/// workspace is happy with the flags, `Err(String)` for +/// mode-mismatch or bad selectors. +fn resolve_worker_selection( + ws: &camino::Utf8Path, + part: Option<&str>, + variant: Option<&str>, +) -> Result<(Option, Option), String> { + let Ok(cfg) = vw_lib::load_workspace_config(ws) else { + return Ok((None, None)); + }; + let ws_info = &cfg.workspace; + if variant.is_some() && ws_info.variants.is_empty() { + return Err(format!( + "workspace at {ws} has no `[[workspace.variants]]`; \ + remove `--variant` or add variants to vw.toml", + )); + } + if part.is_some() && !ws_info.variants.is_empty() { + return Err(format!( + "workspace at {ws} is variant-mode; use `--variant ` \ + instead of `--part`", + )); + } + if !ws_info.variants.is_empty() { + let Some(v) = + ws_info.select_variant(variant).map_err(|e| e.to_string())? + else { + return Ok((None, None)); + }; + let persist_dir = prepare_repl_persist_dir(ws, &ws_info.name); + return Ok(( + Some(vw_vivado::AutoProject { + name: ws_info.name.clone(), + part: v.part.clone(), + persist_dir, + }), + Some(v.name.clone()), + )); + } + let selected = ws_info + .select_target_part(part) + .map_err(|e| e.to_string())?; + let persist_dir = prepare_repl_persist_dir(ws, &ws_info.name); + Ok(( + selected.map(|p| vw_vivado::AutoProject { + name: ws_info.name.clone(), + part: p.to_string(), + persist_dir: persist_dir.clone(), + }), + None, + )) +} + +/// REPL sibling of `vw-cli::prepare_persist_dir`. Same behavior: +/// runs Phase-6 legacy IP-cache cleanup + Phase-3 staleness wipe, +/// returns `Some(/target/vw-project)` for the worker to +/// `open_project`/`create_project -dir` into, or `None` if the +/// bootstrap fails (falling back to in-memory). +/// +/// Kept mirrored (not shared through a common crate) for the same +/// reason `resolve_worker_selection` is duplicated: `vw-repl` +/// doesn't take a cross-crate dep on `vw-cli`. +fn prepare_repl_persist_dir( + ws: &camino::Utf8Path, + name: &str, +) -> Option { + match vw_lib::prepare_vw_project_dir(ws, name) { + Ok(prep) => { + if prep.legacy_cache_removed > 0 { + tracing::info!( + "removed {} legacy IP cache entr{y} under \ + {ws}/target/ip — replaced by on-disk Vivado project", + prep.legacy_cache_removed, + y = if prep.legacy_cache_removed == 1 { + "y" + } else { + "ies" + }, + ); + } + if let Some(wiped) = &prep.wiped_project { + tracing::info!( + "wiped stale Vivado project at {wiped} \ + (source fingerprint changed or manifest missing)" + ); + } + Some(prep.project_dir.into_std_path_buf()) + } + Err(e) => { + tracing::warn!( + "failed to prepare on-disk Vivado project dir under \ + {ws}/target/vw-project ({e}); falling back to in-memory \ + project (state won't persist across sessions)" + ); + None + } + } +} + +fn lookup_proc_doc_comments<'a>( + doc: &'a vw_htcl::Document, + qualified_name: &str, +) -> Option<&'a [String]> { + lookup_in_stmts(&doc.stmts, "", qualified_name) +} + +fn lookup_in_stmts<'a>( + stmts: &'a [vw_htcl::Stmt], + prefix: &str, + qualified_name: &str, +) -> Option<&'a [String]> { + use vw_htcl::CommandKind; + for stmt in stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(name) = proc.name.as_deref() { + let qualified = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + if qualified == qualified_name { + return Some(&cmd.doc_comments); + } + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(name) = ns.name.as_deref() { + let nested = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + if let Some(d) = + lookup_in_stmts(&ns.body, &nested, qualified_name) + { + return Some(d); + } + } + } + _ => {} + } + } + None +} + +/// Index of the parameter the cursor is on within `sig.args`. +/// Mirrors the logic in `vw_htcl::signature_help::active_parameter`: +/// the active arg is whichever `-flag` was most recently completed +/// on the line, or — when the partial is still being typed — the +/// first arg whose name has the partial as a prefix. +fn compute_active_parameter( + sig: &vw_htcl::ProcSignature, + line: &vw_htcl::cmdline::CmdLine<'_>, +) -> Option { + let mut active = None; + for word in line.words.iter().skip(1) { + if let Some(flag) = word.strip_prefix('-') { + if let Some(i) = sig.args.iter().position(|a| a.name == flag) { + active = Some(i as u32); + } + } + } + if let Some(flag) = line.partial.strip_prefix('-') { + if !flag.is_empty() { + if let Some(i) = + sig.args.iter().position(|a| a.name.starts_with(flag)) + { + return Some(i as u32); + } + } + } + active +} + +/// Identifier under the cursor — bare-word-shape including the `::` +/// namespace separator so `util::props` resolves as one symbol. +/// Used by the hover lookup path to find what the user is pointing +/// at when [`vw_htcl::hover_at`] can't see the proc (because it's +/// defined in a session batch, not the in-flight input). +fn ident_under_cursor(text: &str, offset: u32) -> Option<&str> { + let bytes = text.as_bytes(); + let o = (offset as usize).min(bytes.len()); + let is_word_byte = + |b: u8| -> bool { b.is_ascii_alphanumeric() || b == b'_' || b == b':' }; + let mut start = o; + while start > 0 && is_word_byte(bytes[start - 1]) { + start -= 1; + } + let mut end = o; + while end < bytes.len() && is_word_byte(bytes[end]) { + end += 1; + } + if start < end { + Some(&text[start..end]) + } else { + None + } +} + +/// Build the title line for a hover popup that points at a proc — +/// shows the proc's name plus its signature in compact one-line form +/// (`name -arg1: type -arg2: type → ret`). The body of the popup is +/// the reflowed doc-comment block. +fn render_proc_title(name: &str, sig: &vw_htcl::ProcSignature) -> String { + let mut out = name.to_string(); + for arg in &sig.args { + out.push_str(" -"); + out.push_str(&arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + out.push_str(": "); + out.push_str(&render_type(ty)); + } + if let Some(default) = format_default_value(arg) { + out.push_str(" = "); + out.push_str(&default); + } + } + if let Some(ret) = sig.return_type.as_ref() { + out.push_str(" → "); + out.push_str(&render_type(ret)); + } + out +} + +/// Convert a [`vw_htcl::hover::HoverTarget`] into our owned +/// [`crate::popup::HoverPopup`]. Returns `None` when the target +/// can't be rendered usefully (anonymous procs, missing signatures). +fn hover_target_to_popup( + target: vw_htcl::HoverTarget<'_>, + anchor: (u16, u16), +) -> Option { + use vw_htcl::HoverTarget; + let (title, body) = match target { + HoverTarget::ProcDef { proc, .. } => { + let name = proc.name.clone()?; + let sig = proc.signature.as_ref()?; + // Doc comments live on the enclosing Command. The hover + // module doesn't surface them here — accept that we + // show only the signature for in-buffer proc decls; + // the user can re-hover the call site for full docs. + (render_proc_title(&name, sig), String::new()) + } + HoverTarget::ProcArgDef { arg, .. } => { + let mut title = format!("-{}", arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + title.push_str(": "); + title.push_str(&render_type(ty)); + } + let body = vw_htcl::doc::reflow_doc_comments(&arg.doc_comments); + (title, body) + } + HoverTarget::CallSite { + proc_name, + signature, + .. + } => (render_proc_title(&proc_name, signature), String::new()), + HoverTarget::CallArg { arg, .. } => { + let mut title = format!("-{}", arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + title.push_str(": "); + title.push_str(&render_type(ty)); + } + let body = vw_htcl::doc::reflow_doc_comments(&arg.doc_comments); + (title, body) + } + HoverTarget::LocalVar { name, .. } => { + (format!("${name}"), String::from("local variable")) + } + HoverTarget::EnumDef { decl, .. } => { + let name = decl.name.clone()?; + let variants: Vec = decl + .variants + .iter() + .map(|v| { + if let Some(ty) = v.payload.as_ref() { + format!("{}: {}", v.name, render_type(ty)) + } else { + v.name.clone() + } + }) + .collect(); + let title = format!("enum {name}"); + let body = format!("{{ {} }}", variants.join("; ")); + (title, body) + } + HoverTarget::TypeDef { decl, .. } => { + let name = decl.name.clone()?; + let title = format!("type {name}"); + let body = match decl.underlying.as_ref() { + Some(ty) => format!("= {}", render_type(ty)), + None => String::new(), + }; + (title, body) + } + }; + Some(crate::popup::HoverPopup { + title, + body, + anchor, + }) +} + +/// Return the indices of `sig.args` in completion-popup / +/// signature-help **display order**: required arguments (no +/// `@default(...)` attribute) first, then optional ones, both +/// groups sorted alphabetically within. Source declaration order +/// often follows IP-XACT or generator conventions that don't match +/// what users want to scan visually — surfacing required args at +/// the top makes "what MUST I supply?" answerable at a glance. +/// +/// The display order is just a permutation of `sig.args` indices; +/// callers map the `active_parameter` (which is computed in +/// declaration-order space) into display space via `.position()`. +fn sorted_arg_indices(sig: &vw_htcl::ProcSignature) -> Vec { + let mut indices: Vec = (0..sig.args.len()).collect(); + indices.sort_by(|&a, &b| { + let a_arg = &sig.args[a]; + let b_arg = &sig.args[b]; + let a_has_default = a_arg.attribute("default").is_some(); + let b_has_default = b_arg.attribute("default").is_some(); + // false < true → no-default (required) sorts before has-default. + a_has_default + .cmp(&b_has_default) + .then_with(|| a_arg.name.cmp(&b_arg.name)) + }); + indices +} + +/// Detail string shown next to a `-flag` row in the completion popup. +/// Combines the arg's type annotation (when present) with its +/// `@default(...)` value (when present). Returns `None` when the +/// arg has neither so the popup row stays compact for untyped +/// undefaulted args. +fn build_flag_detail(arg: &vw_htcl::ast::ProcArg) -> Option { + let ty = arg.type_annotation.as_ref().map(render_type); + let default = format_default_value(arg); + match (ty, default) { + (None, None) => None, + (Some(t), None) => Some(t), + (None, Some(d)) => Some(format!("= {d}")), + (Some(t), Some(d)) => Some(format!("{t} = {d}")), + } +} + +/// Extract a proc arg's `@default(...)` value, formatted as a short +/// display string. Returns `None` when the arg has no default. Long +/// values (multi-KB paired-dict literals from IP-XACT generators) +/// are truncated to the first ~32 chars with an ellipsis so the +/// signature-help / hover / completion popups don't blow wide. +pub fn format_default_value(arg: &vw_htcl::ast::ProcArg) -> Option { + let attr = arg.attribute("default")?; + let first = attr.values.first()?; + let raw = first.to_tcl_literal(); + const MAX: usize = 32; + if raw.chars().count() > MAX { + let truncated: String = raw.chars().take(MAX - 1).collect(); + Some(format!("{truncated}…")) + } else { + Some(raw) + } +} + +/// One-line rendering of a `TypeExpr` — `string`, `dict`, etc. +/// Used by the signature-help popup to show arg + return types +/// alongside the names. +fn render_type(ty: &vw_htcl::TypeExpr) -> String { + use vw_htcl::TypeExpr; + match ty { + TypeExpr::Named { name, .. } => name.clone(), + TypeExpr::Generic { name, args, .. } => { + let inner: Vec = args.iter().map(render_type).collect(); + format!("{name}<{}>", inner.join(", ")) + } + TypeExpr::Qualified { + namespace, variant, .. + } => format!("{namespace}::{variant}"), + } +} + +/// Decide whether the current input buffer is ready to submit. +/// +/// Two hard "no": unterminated brace/bracket errors from the parser, +/// or a "missing required argument" diagnostic from the validator. +/// The latter is the key ergonomic hook: when the user types +/// `vivado_cmd::assign_bd_address` alone and hits Enter, the +/// validator sees a call to a proc with required args that haven't +/// been supplied. Treat as incomplete → the REPL appends a newline +/// so the user can continue with `-offset …` on the next line. +/// Once every required arg is supplied, the diagnostic clears and +/// Enter submits. +/// +/// The signature table comes from the app's session so procs +/// defined earlier in the same REPL run are visible; passing an +/// empty map degrades gracefully to "unterminated-only" behavior +/// (Slice 5's compositional constructors have required args but +/// aren't in the session table for unit tests). +fn is_buffer_complete( + text: &str, + sig_table: &std::collections::HashMap, +) -> bool { + let parsed = vw_htcl::parse(text); + if parsed + .errors + .iter() + .any(|e| e.message.contains("unterminated")) + { + return false; + } + // Ask the validator whether required-arg gaps remain. Any + // other kind of diagnostic (unknown proc, type error, etc.) is + // a real error the user should see; incomplete is reserved + // for the specific "waiting for more flags" state. + let diags = + vw_htcl::validate_with_signatures(&parsed.document, text, sig_table); + let waiting = diags.iter().any(|d| { + matches!(d.severity, vw_htcl::Severity::Error) + && d.message.starts_with("missing required argument") + }); + !waiting +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_sigs( + ) -> std::collections::HashMap + { + std::collections::HashMap::new() + } + + #[test] + fn buffer_complete_for_simple_statement() { + assert!(is_buffer_complete("set x 1", &empty_sigs())); + assert!(is_buffer_complete("puts \"hi\"", &empty_sigs())); + } + + #[test] + fn buffer_incomplete_with_unterminated_brace() { + assert!(!is_buffer_complete( + "set x [\n create_cpm5\n -name cpm5", + &empty_sigs() + )); + assert!(!is_buffer_complete("proc foo {", &empty_sigs())); + } + + #[test] + fn buffer_complete_for_multiline_well_formed_proc() { + assert!(is_buffer_complete( + "proc foo {\n @default(1) x\n} {\n puts $x\n}", + &empty_sigs() + )); + } + + /// The core UX fix: when a proc call is missing required args, + /// the buffer is considered incomplete so Enter appends a + /// newline instead of submitting — the user can continue with + /// `-flag value` continuations. + #[test] + fn buffer_incomplete_when_required_args_missing() { + use vw_htcl::{parse, signature_table}; + let src = "\ +proc greet { + name + msg +} unit { + puts \"$msg $name\" +} +"; + let parsed = parse(src); + let sigs = signature_table(&parsed.document); + // Bare call — both required args missing. + assert!(!is_buffer_complete("greet", &sigs)); + // One required arg missing — still incomplete. + assert!(!is_buffer_complete("greet -name there", &sigs)); + // Both provided — complete. + assert!(is_buffer_complete("greet -name there -msg hi", &sigs)); + } + + // --- stack-frame resolution --------------------------------- + + use crate::lower::ProcLocation; + use crate::session::SessionBatch; + use std::collections::HashMap; + use std::path::PathBuf; + use vw_htcl::{parse, LoadedProgram}; + + fn session_with_proc( + proc: &str, + file: PathBuf, + body_start_line: u32, + body_lines: Vec, + ) -> Session { + // Session stores proc names without the leading `::` — + // see `lower::qualify`. + let key = proc.strip_prefix("::").unwrap_or(proc); + let src = format!("proc {key} {{}} {{}}\n"); + let parsed = parse(&src); + let mut procs = HashMap::new(); + procs.insert( + key.to_string(), + ProcLocation { + file: Some(file), + body_start_line, + body_lines, + }, + ); + let batch = SessionBatch { + program: LoadedProgram { + source: src, + files: Vec::new(), + regions: Vec::new(), + }, + document: parsed.document, + procs, + }; + let mut s = Session::new(); + s.commit(batch); + s + } + + #[test] + fn rewrite_resolves_input_line_to_absolute_file_line() { + let session = session_with_proc( + "::configure_cips", + "ip/cips.htcl".into(), + 95, + (0..30).map(|i| format!("body line {i}")).collect(), + ); + let frame = crate::trace::rewrite_stack_line( + " at :14 in ::configure_cips", + |name| session.lookup_proc(name).cloned(), + None, + ) + .expect("should resolve"); + // body line 14 = body_start_line (95) + (14 - 1) = 108 + assert!( + frame.formatted.contains("ip/cips.htcl:108"), + "got {:?}", + frame.formatted + ); + assert_eq!(frame.line, 108); + } + + #[test] + fn rewrite_resolves_namespaced_proc() { + // Tcl reports `::port::plumb_if_pin` (with leading `::`) + // but the session indexes it as `port::plumb_if_pin`. + let session = session_with_proc( + "::port::plumb_if_pin", + "vivado-cmd/port.htcl".into(), + 70, + (0..10).map(|i| format!("line {i}")).collect(), + ); + let frame = crate::trace::rewrite_stack_line( + " at :5 in ::port::plumb_if_pin", + |name| session.lookup_proc(name).cloned(), + None, + ) + .expect("should resolve namespaced proc"); + assert!( + frame.formatted.contains("vivado-cmd/port.htcl:74"), + "got {:?}", + frame.formatted + ); + } + + #[test] + fn rewrite_passes_unknown_proc_through() { + let session = Session::new(); + assert!(crate::trace::rewrite_stack_line( + " at :14 in ::vivado_builtin_thing", + |name| session.lookup_proc(name).cloned(), + None, + ) + .is_none()); + } + + #[test] + fn rewrite_skips_non_frame_lines() { + let session = Session::new(); + assert!(crate::trace::rewrite_stack_line( + "WARNING: [Common 17-1] something", + |name| session.lookup_proc(name).cloned(), + None, + ) + .is_none()); + assert!(crate::trace::rewrite_stack_line( + "", + |name| session.lookup_proc(name).cloned(), + None, + ) + .is_none()); + } + + #[test] + fn resolve_dedupes_adjacent_same_proc_frames() { + // Two consecutive `:N in ::port::plumb_if_pin` frames + // resolving to the same absolute line should collapse to one. + let session = session_with_proc( + "::port::plumb_if_pin", + "vivado-cmd/port.htcl".into(), + 70, + (0..10).map(|i| format!("line {i}")).collect(), + ); + let msg = "\ +WARNING: [port::plumb_if_pin-1] skipping foo + at :5 in ::port::plumb_if_pin + at :5 in ::port::plumb_if_pin"; + let out = resolve_stack_frames(msg, &session, None, None); + // Only one resolved frame line should remain. + let count = out + .lines() + .filter(|l| l.contains("port::plumb_if_pin")) + .count(); + assert_eq!(count, 2, "got:\n{out}"); // header + 1 frame + } + + // --- request/response ordering --------------------------------- + + /// Regression: when a batch's Nth command fails, the (N+1)th + /// boundary's echo must NOT be pushed to scrollback ahead of the + /// error trace. The auto-load repro looked like this: `set cips + /// [configure_cips]` failed, then `› set clk [configure_clock]` + /// appeared, then the trace + error for `set cips`. The error + /// belongs to `set cips` and has to land BEFORE the next + /// statement's echo (or preferably: the next echo shouldn't + /// appear at all, since the batch aborts on failure). + #[tokio::test] + async fn failed_eval_does_not_activate_next_boundary_before_error() { + let (worker_tx, _worker_rx) = + tokio::sync::mpsc::channel::(8); + let (event_tx, event_rx) = + tokio::sync::mpsc::unbounded_channel::(); + let mut app = App::new( + ReplOptions::default(), + worker_tx, + event_rx, + event_tx, + CollapseMode::Normal, + std::sync::Arc::new(std::sync::RwLock::new( + std::collections::HashMap::new(), + )), + ); + + // Two-boundary batch. Only boundary 0's echo lives in + // scrollback (deferred push means boundary 1 stays lazy + // until its predecessor closes). + app.push( + ScrollbackKind::Input, + "set cips [configure_cips]".to_string(), + ); + app.pending_input_boundaries = vec![ + InputBoundary { + scrollback_idx: Some(0), + snippet: "set cips [configure_cips]".to_string(), + last_command_idx: Some(0), + completed: false, + }, + InputBoundary { + scrollback_idx: None, + snippet: "set clk [configure_clock]".to_string(), + last_command_idx: Some(1), + completed: false, + }, + ]; + let origin0 = crate::lower::Origin { + file: None, + line: 14, + snippet: "set cips [configure_cips]".to_string(), + via: Vec::new(), + }; + app.pending_origins = vec![ + origin0.clone(), + crate::lower::Origin { + file: None, + line: 15, + snippet: "set clk [configure_clock]".to_string(), + via: Vec::new(), + }, + ]; + app.pending_return_types = vec![None, None]; + + // Failed EvalDone for the first command, last_in_batch=true + // (mirrors what worker_task sends when it breaks on error). + let err = vw_eda::BackendError::Tcl { + message: "[Common 17-163] Missing value for option 'objects'" + .into(), + code: None, + info: None, + stdout: String::new(), + }; + app.handle_worker_event(WorkerEvent::EvalDone { + origin: origin0, + result: Err(err), + last_in_batch: true, + }) + .await; + + // Walk scrollback and find the two positions we care about: + // the `set clk` Input entry (if any) and the Error entry + // carrying the failure message. + let clk_pos = app + .scrollback() + .iter() + .position(|e| e.text.contains("set clk")); + let err_pos = app.scrollback().iter().position(|e| { + matches!(e.kind, ScrollbackKind::Error) + && e.text.contains("Missing value") + }); + assert!( + err_pos.is_some(), + "expected an Error entry, got scrollback: {:#?}", + app.scrollback() + .iter() + .map(|e| (e.kind, e.text.clone())) + .collect::>() + ); + if let Some(clk) = clk_pos { + let err = err_pos.unwrap(); + assert!( + err < clk, + "error at {err} must land before `set clk` echo at \ + {clk} — the trace belongs to `set cips` which came \ + first. scrollback: {:#?}", + app.scrollback() + .iter() + .map(|e| (e.kind, e.text.clone())) + .collect::>() + ); + } + } + + /// Regression: a pure `puts` output (which classifies as + /// `StreamKind::Stdout` → `Severity::None`) has to reach + /// scrollback even when NO diagnostic follows it. The block + /// segmenter only flushes pending NONE content when a + /// classified chunk arrives; without an explicit flush on + /// `EvalDone`, `puts "muffins"` on a quiet REPL sits invisible + /// in `pending_none` until the next diagnostic (which for a + /// small interactive session may never come). The fix: flush + /// the accumulator at eval-done boundaries so every eval's own + /// output surfaces before the next input echo. + #[tokio::test] + async fn plain_puts_output_flushes_at_eval_done() { + let (worker_tx, _worker_rx) = + tokio::sync::mpsc::channel::(8); + let (event_tx, event_rx) = + tokio::sync::mpsc::unbounded_channel::(); + let mut app = App::new( + ReplOptions::default(), + worker_tx, + event_rx, + event_tx, + CollapseMode::Normal, + std::sync::Arc::new(std::sync::RwLock::new( + std::collections::HashMap::new(), + )), + ); + // Emulate the sequence a `puts "muffins"` eval produces: + // one Stream chunk carrying the output, then EvalDone. + // Both events land through `handle_worker_event`, same as + // the real worker task drives them. + app.handle_worker_event(WorkerEvent::Stream { + kind: vw_vivado::StreamKind::Stdout, + data: "muffins\n".to_string(), + }) + .await; + // Before EvalDone the accumulator is still holding the + // chunk — nothing in scrollback yet. + assert!( + !app.scrollback().iter().any(|e| e.text.contains("muffins")), + "muffins should still be in pending_none before EvalDone" + ); + app.handle_worker_event(WorkerEvent::EvalDone { + origin: crate::lower::Origin { + file: None, + line: 1, + snippet: "puts \"muffins\"".to_string(), + via: Vec::new(), + }, + result: Ok(vw_eda::EvalOutput::default()), + last_in_batch: true, + }) + .await; + // After EvalDone the pending NONE content flushes into + // scrollback — the user actually sees their output. + assert!( + app.scrollback().iter().any(|e| e.text.contains("muffins")), + "muffins should have flushed into scrollback on EvalDone. \ + scrollback: {:#?}", + app.scrollback() + .iter() + .map(|e| (e.kind, e.text.clone())) + .collect::>() + ); + } +} diff --git a/vw-repl/src/config.rs b/vw-repl/src/config.rs new file mode 100644 index 0000000..f43dfb5 --- /dev/null +++ b/vw-repl/src/config.rs @@ -0,0 +1,157 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Workspace-local REPL configuration loaded from `/.vw/repl.toml`. +//! +//! Optional file — the REPL runs fine without it. Present so a +//! project can pin per-workspace UI preferences (currently just the +//! auto-collapse policy for scrollback entries) without touching +//! `vw.toml`, which is the *build* manifest and shouldn't be +//! littered with editor-UX knobs. + +use std::path::Path; + +use serde::Deserialize; + +/// How aggressively the REPL auto-collapses multi-line scrollback +/// entries when they land. Single-line entries are never +/// collapsible regardless of mode — a `▶` around one row of text +/// is worse UX than the row itself. +#[derive(Copy, Clone, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum CollapseMode { + /// Auto-collapse only past [`crate::app::COLLAPSE_AUTO_THRESHOLD`] + /// lines. Smaller multi-line entries land expanded but stay + /// toggleable via Shift+click. The default — matches the + /// out-of-the-box behavior users see without a config file. + #[default] + Normal, + /// Every collapsible entry (>=2 lines) starts collapsed. + /// Turns the scrollback into a compact index of `▶`-marked + /// placeholders that expand on demand — useful when running + /// long batches where most output is chatter you scroll past. + Aggressive, +} + +/// Deserialized `/.vw/repl.toml`. All fields optional so a +/// stub `[ui]` section is legal — missing keys fall back to +/// [`Default`]. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct ReplConfig { + pub ui: UiConfig, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct UiConfig { + pub collapse: CollapseMode, +} + +/// Load `/.vw/repl.toml` if it exists. Absent file → default +/// config. Malformed file → default config + a `tracing::warn!` so +/// the user notices in the verbose log but the REPL still starts; +/// a config error shouldn't be a fatal boot condition for an +/// interactive tool. +pub fn load(workspace_root: Option<&Path>) -> ReplConfig { + let Some(ws) = workspace_root else { + return ReplConfig::default(); + }; + let path = ws.join(".vw").join("repl.toml"); + let raw = match std::fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return ReplConfig::default(); + } + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "failed to read repl.toml — using defaults", + ); + return ReplConfig::default(); + } + }; + match toml::from_str::(&raw) { + Ok(cfg) => cfg, + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "malformed repl.toml — using defaults", + ); + ReplConfig::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_file_is_default() { + let tmp = tempfile::tempdir().unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } + + #[test] + fn no_workspace_root_is_default() { + let cfg = load(None); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } + + #[test] + fn parses_aggressive() { + let tmp = tempfile::tempdir().unwrap(); + let vw_dir = tmp.path().join(".vw"); + std::fs::create_dir_all(&vw_dir).unwrap(); + std::fs::write( + vw_dir.join("repl.toml"), + "[ui]\ncollapse = \"aggressive\"\n", + ) + .unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Aggressive); + } + + #[test] + fn parses_normal_explicit() { + let tmp = tempfile::tempdir().unwrap(); + let vw_dir = tmp.path().join(".vw"); + std::fs::create_dir_all(&vw_dir).unwrap(); + std::fs::write( + vw_dir.join("repl.toml"), + "[ui]\ncollapse = \"normal\"\n", + ) + .unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } + + #[test] + fn empty_file_is_default() { + let tmp = tempfile::tempdir().unwrap(); + let vw_dir = tmp.path().join(".vw"); + std::fs::create_dir_all(&vw_dir).unwrap(); + std::fs::write(vw_dir.join("repl.toml"), "").unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } + + #[test] + fn malformed_file_falls_back() { + let tmp = tempfile::tempdir().unwrap(); + let vw_dir = tmp.path().join(".vw"); + std::fs::create_dir_all(&vw_dir).unwrap(); + std::fs::write( + vw_dir.join("repl.toml"), + "[ui]\ncollapse = \"chaotic\"\n", + ) + .unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } +} diff --git a/vw-repl/src/diag_search.rs b/vw-repl/src/diag_search.rs new file mode 100644 index 0000000..ebe2945 --- /dev/null +++ b/vw-repl/src/diag_search.rs @@ -0,0 +1,707 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Diagnostics fuzzy-finder — Ctrl-F opens a centered modal listing +//! every diagnostic entry (Error / Warning / Notice) currently in the +//! scrollback. Typing filters via [`nucleo_matcher`]; ↑/↓ navigates; +//! Enter closes the modal and jumps the scrollback viewport to the +//! chosen entry, dropping a persistent left-gutter marker so the user +//! can spot it in a busy log. Alt-C clears the marker. +//! +//! Kind-filter checkboxes at the top let the user toggle inclusion of +//! each severity independently — Ctrl-E, Ctrl-W, Ctrl-N. Defaults +//! are Error+Warning on, Notice off (chatty INFO messages usually +//! aren't what someone reaches for `find diagnostic` to see). +//! +//! Modeled on [`crate::symbol_search`] — same nucleo-matcher engine, +//! same modal shape (70% × 75% of frame). + +use nucleo_matcher::pattern::{CaseMatching, Normalization, Pattern}; +use nucleo_matcher::{Config, Matcher, Utf32String}; +use ratatui::layout::Rect; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{ + Block, Borders, Clear, List, ListItem, ListState, Paragraph, Scrollbar, + ScrollbarOrientation, ScrollbarState, +}; +use ratatui::Frame; + +use crate::app::{ScrollbackEntry, ScrollbackKind}; + +/// One scrollback entry surfaced in the picker. `scrollback_idx` is +/// stable relative to the App's `scrollback` Vec at open time — the +/// picker holds a snapshot, and while it's open new appends land at +/// higher indices without perturbing existing ones (scrollback is +/// append-only), so accepting an item still jumps to the right row. +#[derive(Clone, Debug)] +pub struct DiagItem { + pub scrollback_idx: usize, + pub kind: ScrollbackKind, + /// True when the source entry was a Vivado CRITICAL WARNING + /// (kind is still `Error` — CW and plain Error share the same + /// scrollback visual bucket). The picker uses this to answer + /// the `Critical` filter checkbox; a critical entry also + /// passes the `Error` filter, so turning on both is fine and + /// won't duplicate rows. + pub is_critical_warning: bool, + /// First non-empty line of the entry — what shows in the result + /// list. Truncated to a sensible width by the renderer. + pub preview: String, + /// Full text used as the fuzzy-match haystack. Multi-line + /// diagnostics (WARNING body + attached stack) match on any + /// content, so a query for a function name in the trace still + /// finds the top-level warning. + pub full: String, +} + +/// Scored entry after applying the current query + kind filters. +/// Sorted by descending `score` at the end of `recompute`. +#[derive(Clone, Copy, Debug)] +pub struct Scored { + pub item_idx: usize, + pub score: u32, +} + +/// Diagnostic-picker overlay state. +#[derive(Debug)] +pub struct DiagnosticPicker { + /// Snapshot of Error/Warning/Notice entries from the scrollback, + /// taken at open time. Fixed for the picker's lifetime — new + /// scrollback appends aren't reflected until the user reopens. + pub items: Vec, + pub query: String, + pub results: Vec, + pub selected: usize, + /// Which kinds pass the filter row's checkboxes. + pub filter_error: bool, + pub filter_warning: bool, + pub filter_notice: bool, + /// Critical-warning subset filter. CW entries carry + /// `kind == Error` (shared visual bucket) — this checkbox + /// lets the user surface JUST the criticals within that + /// bucket. An item passes when either its kind matches an + /// enabled kind filter OR its CW flag matches `filter_critical`. + pub filter_critical: bool, +} + +impl DiagnosticPicker { + /// Build the picker from an App scrollback slice. Filters the + /// slice down to diagnostic-kind entries; entries with empty + /// text are dropped (nothing to preview or match against). + pub fn from_scrollback(scrollback: &[ScrollbackEntry]) -> Self { + let items: Vec = scrollback + .iter() + .enumerate() + .filter(|(_, e)| is_diagnostic(e.kind)) + .filter_map(|(idx, e)| { + let preview = e.text.lines().next().unwrap_or("").to_string(); + if preview.is_empty() && e.text.is_empty() { + return None; + } + Some(DiagItem { + scrollback_idx: idx, + kind: e.kind, + is_critical_warning: e.is_critical_warning, + preview, + full: e.text.clone(), + }) + }) + .collect(); + let mut p = Self { + items, + query: String::new(), + results: Vec::new(), + selected: 0, + // Defaults: Error+Warning+Critical on, Notice off. + // Notice buckets INFO-severity Vivado messages, which + // are chatty and usually not what someone reaches for + // a diagnostics finder to see — but it's one keystroke + // (Ctrl-N) away. Critical is on by default because + // it's a subset of Error, so having it enabled changes + // nothing until the user turns Error off to isolate + // criticals. + filter_error: true, + filter_warning: true, + filter_notice: false, + filter_critical: true, + }; + p.recompute(); + p + } + + pub fn move_up(&mut self) { + if self.selected > 0 { + self.selected -= 1; + } + } + + pub fn move_down(&mut self) { + if self.selected + 1 < self.results.len() { + self.selected += 1; + } + } + + pub fn push_char(&mut self, c: char) { + self.query.push(c); + self.recompute(); + } + + pub fn pop_char(&mut self) { + self.query.pop(); + self.recompute(); + } + + /// Toggle inclusion of a specific kind. Ignores kinds outside + /// the picker's diagnostic set — passing `Chatter` is a no-op. + pub fn toggle_kind(&mut self, kind: ScrollbackKind) { + match kind { + ScrollbackKind::Error => self.filter_error = !self.filter_error, + ScrollbackKind::Warning => { + self.filter_warning = !self.filter_warning; + } + ScrollbackKind::Notice => self.filter_notice = !self.filter_notice, + _ => return, + } + self.recompute(); + } + + /// Toggle the Critical-warning subset filter. Independent of + /// the kind filters — with `filter_critical` on and every + /// kind off, the picker shows just critical warnings. + pub fn toggle_critical(&mut self) { + self.filter_critical = !self.filter_critical; + self.recompute(); + } + + /// Currently-selected item, if any. Returns the item struct + /// (contains `scrollback_idx` and `kind`) rather than just the + /// index so callers can pass kind into the marker-styling code + /// without a second lookup. + pub fn current(&self) -> Option<&DiagItem> { + let scored = self.results.get(self.selected)?; + self.items.get(scored.item_idx) + } + + /// Whether `item` passes the current filter row. An entry + /// passes when EITHER its kind's checkbox is on OR (for + /// critical warnings) the Critical checkbox is on — so a CW + /// entry surfaces under Error, under Critical, or both, + /// without appearing twice. + fn item_allowed(&self, item: &DiagItem) -> bool { + let kind_pass = match item.kind { + ScrollbackKind::Error => self.filter_error, + ScrollbackKind::Warning => self.filter_warning, + ScrollbackKind::Notice => self.filter_notice, + _ => false, + }; + let critical_pass = item.is_critical_warning && self.filter_critical; + kind_pass || critical_pass + } + + /// Rebuild `results` from current query + filter state. O(N) + /// over the snapshot. Called from `new`, `push_char`, + /// `pop_char`, `toggle_kind`, `toggle_critical`. + fn recompute(&mut self) { + let allowed: Vec = self + .items + .iter() + .enumerate() + .filter(|(_, it)| self.item_allowed(it)) + .map(|(i, _)| i) + .collect(); + + if self.query.is_empty() { + // No query: show all kind-allowed items in scrollback + // order (== descending scrollback_idx would be reverse- + // chronological; ascending == chronological, which is + // what appears in the log itself — pick the latter so + // the picker order matches what the user's eye scanned). + self.results = allowed + .into_iter() + .map(|item_idx| Scored { item_idx, score: 0 }) + .collect(); + self.clamp_selected(); + return; + } + + let mut matcher = Matcher::new(Config::DEFAULT); + let pattern = Pattern::parse( + &self.query, + CaseMatching::Smart, + Normalization::Smart, + ); + let mut scored: Vec = allowed + .iter() + .filter_map(|&item_idx| { + let it = &self.items[item_idx]; + let hay = Utf32String::from(it.full.as_str()); + let score = pattern.score(hay.slice(..), &mut matcher)?; + if score == 0 { + None + } else { + Some(Scored { item_idx, score }) + } + }) + .collect(); + scored.sort_by_key(|s| std::cmp::Reverse(s.score)); + scored.truncate(500); + self.results = scored; + self.clamp_selected(); + } + + fn clamp_selected(&mut self) { + if self.selected >= self.results.len() { + self.selected = self.results.len().saturating_sub(1); + } + } +} + +/// True when `kind` is one of the diagnostic kinds the picker +/// surfaces. Broken out so `from_scrollback` and `kind_allowed` +/// stay in sync — adding a new diagnostic kind means updating +/// this predicate alone. +fn is_diagnostic(kind: ScrollbackKind) -> bool { + matches!( + kind, + ScrollbackKind::Error + | ScrollbackKind::Warning + | ScrollbackKind::Notice + ) +} + +/// Render the diagnostic picker as a centered modal. Sized to +/// match [`crate::symbol_search::draw_symbol_picker`] so the two +/// finders feel like siblings. +pub fn draw_diagnostic_picker(f: &mut Frame, picker: &DiagnosticPicker) { + let frame = f.area(); + let width = (frame.width as f32 * 0.7) as u16; + let height = (frame.height as f32 * 0.75) as u16; + let x = frame.x + (frame.width.saturating_sub(width)) / 2; + let y = frame.y + (frame.height.saturating_sub(height)) / 2; + let area = Rect { + x, + y, + width: width.min(frame.width), + height: height.min(frame.height), + }; + + f.render_widget(Clear, area); + // Title shows a live match / snapshot-size count so the user + // knows how selective their filters + query are. `snapshot` + // is the total diagnostics captured when the picker opened; + // `matches` is the count after filter+query. When the two + // are equal the count reads as e.g. `(12/12)` = "everything + // in scrollback passes". + let title = format!( + " find diagnostic ({}/{}) — Esc to close ", + picker.results.len(), + picker.items.len(), + ); + let block = Block::default().borders(Borders::ALL).title(title); + let inner = block.inner(area); + f.render_widget(block, area); + if inner.width == 0 || inner.height < 4 { + return; + } + + // Vertical layout: filter row (1) + query row (1) + separator + // (1) + result list (remaining). + let filter_row = Rect { + x: inner.x, + y: inner.y, + width: inner.width, + height: 1, + }; + let query_row = Rect { + x: inner.x, + y: inner.y + 1, + width: inner.width, + height: 1, + }; + let sep_row = Rect { + x: inner.x, + y: inner.y + 2, + width: inner.width, + height: 1, + }; + let list_area = Rect { + x: inner.x, + y: inner.y + 3, + width: inner.width, + height: inner.height.saturating_sub(3), + }; + + draw_filter_row(f, picker, filter_row); + draw_query_row(f, picker, query_row); + f.render_widget( + Paragraph::new(Line::from(Span::styled( + "─".repeat(inner.width as usize), + Style::default().add_modifier(Modifier::DIM), + ))), + sep_row, + ); + draw_result_list(f, picker, list_area); +} + +fn draw_filter_row(f: &mut Frame, picker: &DiagnosticPicker, area: Rect) { + let cell = |on: bool, glyph: &str, label: &str, key: &str, color: Color| { + let box_style = if on { + Style::default().fg(color).add_modifier(Modifier::BOLD) + } else { + Style::default().add_modifier(Modifier::DIM) + }; + vec![ + Span::styled(glyph.to_string(), box_style), + Span::raw(" "), + Span::styled(label.to_string(), box_style), + Span::raw(" "), + Span::styled( + format!("({key})"), + Style::default().add_modifier(Modifier::DIM), + ), + Span::raw(" "), + ] + }; + let mut spans: Vec> = Vec::new(); + spans.push(Span::styled( + " filter: ".to_string(), + Style::default().add_modifier(Modifier::DIM), + )); + spans.extend(cell( + picker.filter_error, + if picker.filter_error { "[x]" } else { "[ ]" }, + "Error", + "^E", + Color::Red, + )); + spans.extend(cell( + picker.filter_critical, + if picker.filter_critical { "[x]" } else { "[ ]" }, + "Critical", + "^K", + Color::Rgb(255, 90, 90), + )); + spans.extend(cell( + picker.filter_warning, + if picker.filter_warning { "[x]" } else { "[ ]" }, + "Warning", + "^W", + Color::Rgb(255, 140, 0), + )); + spans.extend(cell( + picker.filter_notice, + if picker.filter_notice { "[x]" } else { "[ ]" }, + "Info", + "^N", + Color::Gray, + )); + f.render_widget(Paragraph::new(Line::from(spans)), area); +} + +fn draw_query_row(f: &mut Frame, picker: &DiagnosticPicker, area: Rect) { + let spans = vec![ + Span::styled( + " › ".to_string(), + Style::default() + .fg(Color::Rgb(180, 130, 220)) + .add_modifier(Modifier::BOLD), + ), + Span::styled(picker.query.clone(), Style::default().fg(Color::White)), + // Cursor position — a solid block on the end of the query + // makes the input row read as an editable field even + // without a real terminal cursor there (ratatui's cursor + // is bound to the input editor below the popup). + Span::styled( + "▏".to_string(), + Style::default().fg(Color::Rgb(180, 130, 220)), + ), + ]; + f.render_widget(Paragraph::new(Line::from(spans)), area); +} + +fn draw_result_list(f: &mut Frame, picker: &DiagnosticPicker, area: Rect) { + if picker.results.is_empty() { + let msg = if picker.items.is_empty() { + "no diagnostics in scrollback" + } else if picker.query.is_empty() { + "no diagnostics match current filters" + } else { + "no diagnostics match query" + }; + f.render_widget( + Paragraph::new(Line::from(Span::styled( + format!(" {msg}"), + Style::default().add_modifier(Modifier::DIM), + ))), + area, + ); + return; + } + + // Reserve the rightmost column for a scrollbar when the + // result list overflows the visible area. Matches the main + // scrollback pane's auto-hide behavior — no wasted column + // when everything fits. + let needs_scrollbar = picker.results.len() > area.height as usize; + let (list_area, scrollbar_area) = if needs_scrollbar && area.width >= 2 { + let narrower = Rect { + width: area.width - 1, + ..area + }; + (narrower, Some(area)) + } else { + (area, None) + }; + + let max_preview_width = (list_area.width as usize) + .saturating_sub(6 /* kind badge + space */); + + let items: Vec = picker + .results + .iter() + .filter_map(|s| picker.items.get(s.item_idx)) + .enumerate() + .map(|(row_idx, it)| { + // Badge: distinguish critical warnings from plain + // errors even though they share ScrollbackKind::Error + // — the whole point of the Critical filter is that + // the two are semantically distinct. + let (badge_str, badge_style) = match it.kind { + ScrollbackKind::Error if it.is_critical_warning => ( + "C ", + Style::default() + .fg(Color::Rgb(255, 90, 90)) + .add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Error => ( + "E ", + Style::default() + .fg(Color::Red) + .add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Warning => ( + "W ", + Style::default() + .fg(Color::Rgb(255, 140, 0)) + .add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Notice => ( + "i ", + Style::default() + .fg(Color::Gray) + .add_modifier(Modifier::BOLD), + ), + _ => ("? ", Style::default()), + }; + let preview = truncate_chars(&it.preview, max_preview_width); + let is_selected = row_idx == picker.selected; + let preview_style = if is_selected { + Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(Color::Gray) + }; + let row_style = if is_selected { + Style::default().bg(Color::Rgb(40, 40, 60)) + } else { + Style::default() + }; + let spans = vec![ + Span::styled(badge_str.to_string(), badge_style), + Span::styled(preview, preview_style), + ]; + ListItem::new(Line::from(spans)).style(row_style) + }) + .collect(); + + let mut list_state = ListState::default(); + list_state.select(Some(picker.selected)); + let list = List::new(items) + .highlight_style(Style::default().bg(Color::Rgb(40, 40, 60))); + f.render_stateful_widget(list, list_area, &mut list_state); + + // Scrollbar reflects the selection's position within the + // full result set (not just the visible viewport) — that's + // what the user cares about when they've paged Down deep + // into hits. + if let Some(sb_area) = scrollbar_area { + let mut sb_state = ScrollbarState::new(picker.results.len()) + .position(picker.selected) + .viewport_content_length(list_area.height as usize); + let scrollbar = Scrollbar::default() + .orientation(ScrollbarOrientation::VerticalRight); + f.render_stateful_widget(scrollbar, sb_area, &mut sb_state); + } +} + +/// Truncate `s` to `max_chars` display cells, appending `…` when +/// clipped. Character-count based (not byte-based) so multi-byte +/// content — non-ASCII proc names, stack-frame arrows — clip +/// cleanly. +fn truncate_chars(s: &str, max_chars: usize) -> String { + if max_chars < 2 { + return String::new(); + } + let count = s.chars().count(); + if count <= max_chars { + return s.to_string(); + } + let take = max_chars.saturating_sub(1); + let mut out: String = s.chars().take(take).collect(); + out.push('…'); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(kind: ScrollbackKind, text: &str) -> ScrollbackEntry { + ScrollbackEntry { + kind, + text: text.to_string(), + started_at: None, + completed_at: None, + collapse_state: None, + is_critical_warning: false, + } + } + + fn critical_entry(text: &str) -> ScrollbackEntry { + ScrollbackEntry { + kind: ScrollbackKind::Error, + text: text.to_string(), + started_at: None, + completed_at: None, + collapse_state: None, + is_critical_warning: true, + } + } + + #[test] + fn snapshot_filters_to_diagnostics() { + let sb = vec![ + entry(ScrollbackKind::Input, "some input"), + entry(ScrollbackKind::Error, "unresolved symbol foo"), + entry(ScrollbackKind::Stdout, "hello"), + entry(ScrollbackKind::Warning, "deprecated call"), + entry(ScrollbackKind::Chatter, "banner"), + entry(ScrollbackKind::Notice, "vivado: ready"), + ]; + let picker = DiagnosticPicker::from_scrollback(&sb); + assert_eq!(picker.items.len(), 3); + // scrollback indices preserved + assert_eq!(picker.items[0].scrollback_idx, 1); + assert_eq!(picker.items[1].scrollback_idx, 3); + assert_eq!(picker.items[2].scrollback_idx, 5); + } + + #[test] + fn default_filter_hides_notice() { + let sb = vec![ + entry(ScrollbackKind::Error, "e1"), + entry(ScrollbackKind::Warning, "w1"), + entry(ScrollbackKind::Notice, "n1"), + ]; + let picker = DiagnosticPicker::from_scrollback(&sb); + // 3 items in the snapshot but only 2 pass default filters. + assert_eq!(picker.items.len(), 3); + assert_eq!(picker.results.len(), 2); + } + + #[test] + fn toggle_notice_shows_it() { + let sb = vec![ + entry(ScrollbackKind::Error, "e1"), + entry(ScrollbackKind::Notice, "n1"), + ]; + let mut picker = DiagnosticPicker::from_scrollback(&sb); + assert_eq!(picker.results.len(), 1); + picker.toggle_kind(ScrollbackKind::Notice); + assert_eq!(picker.results.len(), 2); + } + + #[test] + fn critical_shows_under_error_and_critical() { + let sb = vec![ + entry(ScrollbackKind::Error, "plain error"), + critical_entry("critical warning body"), + ]; + let mut picker = DiagnosticPicker::from_scrollback(&sb); + // Both Error+Critical filters on by default → both entries. + assert_eq!(picker.results.len(), 2); + // Error off, Critical on → just the CW (it still passes + // via the critical predicate even though its kind is + // Error). + picker.toggle_kind(ScrollbackKind::Error); + assert_eq!(picker.results.len(), 1); + assert!(picker.current().unwrap().is_critical_warning); + // Critical off too → nothing (both filter paths off). + picker.toggle_critical(); + assert_eq!(picker.results.len(), 0); + // Error back on, Critical still off → BOTH entries show: + // CW passes via its kind (Error) even without the + // critical checkbox. This is the "if they show up when + // error is selected that's fine" behavior — the Critical + // filter is additive, not exclusive. + picker.toggle_kind(ScrollbackKind::Error); + assert_eq!(picker.results.len(), 2); + } + + #[test] + fn fuzzy_query_narrows() { + let sb = vec![ + entry(ScrollbackKind::Error, "unresolved symbol foo"), + entry(ScrollbackKind::Error, "cannot open file bar"), + entry(ScrollbackKind::Warning, "deprecated call foo"), + ]; + let mut picker = DiagnosticPicker::from_scrollback(&sb); + assert_eq!(picker.results.len(), 3); + picker.push_char('f'); + picker.push_char('o'); + picker.push_char('o'); + // Both `foo`-mentioning entries survive; the `bar` one doesn't. + assert_eq!(picker.results.len(), 2); + for r in &picker.results { + assert!(picker.items[r.item_idx].full.contains("foo")); + } + } + + #[test] + fn move_updown_clamps() { + let sb = vec![ + entry(ScrollbackKind::Error, "a"), + entry(ScrollbackKind::Error, "b"), + ]; + let mut picker = DiagnosticPicker::from_scrollback(&sb); + assert_eq!(picker.selected, 0); + picker.move_up(); + assert_eq!(picker.selected, 0); + picker.move_down(); + assert_eq!(picker.selected, 1); + picker.move_down(); + assert_eq!(picker.selected, 1); + } + + #[test] + fn current_maps_to_scrollback_idx() { + let sb = vec![ + entry(ScrollbackKind::Input, "ignored"), + entry(ScrollbackKind::Warning, "hit"), + ]; + let picker = DiagnosticPicker::from_scrollback(&sb); + let current = picker.current().unwrap(); + assert_eq!(current.scrollback_idx, 1); + assert_eq!(current.kind, ScrollbackKind::Warning); + } + + #[test] + fn empty_scrollback_no_panic() { + let picker = DiagnosticPicker::from_scrollback(&[]); + assert_eq!(picker.results.len(), 0); + assert!(picker.current().is_none()); + } +} diff --git a/vw-repl/src/highlight.rs b/vw-repl/src/highlight.rs new file mode 100644 index 0000000..01a5c00 --- /dev/null +++ b/vw-repl/src/highlight.rs @@ -0,0 +1,525 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Syntax highlighter for compiler-emitted enum reprs. +//! +//! Reprs follow a uniform shape regardless of which enum produced +//! them (the compiler emits `Variant`, `Variant(payload)`, or +//! `Variant(\n inner\n)` for any user-declared enum, and +//! dict/list reprs join entries with `\n`). This module recognizes +//! that shape line-by-line and emits styled +//! [`ratatui::text::Span`]s — keys in blue, variant names in teal, +//! punctuation in dim, scalar payloads in green. +//! +//! Shape-based, not name-based: the highlighter has no knowledge +//! of `Property` / `Properties` / any specific enum. It recognizes +//! the structural pattern (`IDENT '(' … ')'` for variant calls, +//! `KEY SP VARIANT …` for dict entries, bare `)` for multi-line +//! close), so adding a new enum to the htcl source automatically +//! gets the same highlighting on its repr output. +//! +//! Falls back to plain text when a line doesn't parse — non-repr +//! content (raw `puts` output, error messages, etc.) renders +//! normally. +//! +//! Color palette is exported so [`crate::render::entry_lines`] +//! can apply the same fallback `body_style` for unparsed runs. + +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::Span; +use winnow::ascii::space0; +use winnow::combinator::repeat; +use winnow::error::ContextError; +use winnow::token::take_while; +use winnow::{ModalResult, Parser}; + +/// Semantic role of a highlighted piece of text. Backend-agnostic: +/// callers (the REPL's ratatui renderer, `vw run`'s ANSI stdout +/// renderer, or anything else consuming +/// [`highlight_line_pieces`]) map each variant to whatever styling +/// primitives their target supports. The RGB values in +/// [`key_style`] / [`variant_style`] / [`scalar_style`] and the +/// dim modifier in [`punct_style`] are the CANONICAL palette; every +/// backend should reproduce these colors so the two rendering +/// surfaces look identical. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StyleKind { + /// Uncolored spacing (indent, inter-token whitespace, + /// trailing runs). Backends emit the text verbatim. + Plain, + /// Dict key (`CONFIG`, `CPM_PCIE0_MODES`, …). + Key, + /// Enum variant name (`Scalar`, `Nested`, …). + Variant, + /// Structural punctuation (`(` / `)`). + Punct, + /// Scalar payload (the string inside `Scalar(…)`). + Scalar, +} + +/// One styled fragment of a highlighted repr line. +#[derive(Debug, Clone)] +pub struct Piece { + pub text: String, + pub kind: StyleKind, +} + +impl Piece { + fn plain(text: impl Into) -> Self { + Self { + text: text.into(), + kind: StyleKind::Plain, + } + } + fn styled(text: impl Into, kind: StyleKind) -> Self { + Self { + text: text.into(), + kind, + } + } +} + +/// Style for dict keys (`CONFIG`, `CPM_PCIE0_MODES`, …) — the +/// identifier immediately preceding a value. +pub fn key_style() -> Style { + Style::default().fg(Color::Rgb(80, 150, 255)) +} + +/// Style for enum variant names (`Scalar`, `Nested`, …) — the +/// identifier immediately preceding `(`. +pub fn variant_style() -> Style { + Style::default().fg(Color::Rgb(100, 200, 200)) +} + +/// Style for structural punctuation (`(` and `)`) — dimmed so the +/// nesting structure recedes visually next to keys and values. +pub fn punct_style() -> Style { + Style::default().add_modifier(Modifier::DIM) +} + +/// Style for scalar payloads — the string inside `Scalar(…)`. +pub fn scalar_style() -> Style { + Style::default().fg(Color::Rgb(120, 200, 120)) +} + +/// Convert a [`StyleKind`] to the ratatui [`Style`] the REPL's +/// scrollback uses. Kept as a single map so the REPL and any other +/// ratatui-based consumer stay in sync with the canonical palette. +pub fn ratatui_style(kind: StyleKind) -> Style { + match kind { + StyleKind::Plain => Style::default(), + StyleKind::Key => key_style(), + StyleKind::Variant => variant_style(), + StyleKind::Punct => punct_style(), + StyleKind::Scalar => scalar_style(), + } +} + +/// Try to recognize `line` as a compiler-emitted enum-repr line +/// and return a backend-agnostic piece sequence. Returns `None` +/// when the line doesn't match the repr grammar — callers fall +/// back to rendering the raw text with their default body style. +/// +/// Callers targeting a ratatui surface (the REPL scrollback) can +/// use [`highlight_line`] for a Span-typed shortcut; callers +/// targeting a plain terminal (`vw run`'s stdout stream) consume +/// pieces directly and produce ANSI escapes. +pub fn highlight_line_pieces(line: &str) -> Option> { + let mut input = line; + parse_line.parse_next(&mut input).ok().filter(|pieces| { + // Reject parses that didn't consume the whole line — a + // partial match means we'd silently style some tokens + // and drop the rest. Better to fall through to plain. + input.is_empty() && !pieces.is_empty() + }) +} + +/// Ratatui-flavored wrapper around [`highlight_line_pieces`]. The +/// REPL's scrollback renderer consumes ratatui `Span`s directly. +pub fn highlight_line(line: &str) -> Option>> { + highlight_line_pieces(line).map(|pieces| { + pieces + .into_iter() + .map(|p| match p.kind { + StyleKind::Plain => Span::raw(p.text), + _ => Span::styled(p.text, ratatui_style(p.kind)), + }) + .collect() + }) +} + +// Top-level line shapes: +// `INDENT? ')' [SP] EOL` — multi-line close +// `INDENT? KEY SP VALUE [SP]? EOL` — dict entry +fn parse_line(input: &mut &str) -> ModalResult> { + let mut pieces: Vec = Vec::new(); + let indent = space0::<_, ContextError>.parse_next(input)?; + if !indent.is_empty() { + pieces.push(Piece::plain(indent)); + } + // Multi-line-close line: bare `)`, optionally followed by trailing whitespace. + if input.starts_with(')') { + let close = ")"; + *input = &input[1..]; + pieces.push(Piece::styled(close, StyleKind::Punct)); + let trailing = space0::<_, ContextError>.parse_next(input)?; + if !trailing.is_empty() { + pieces.push(Piece::plain(trailing)); + } + return Ok(pieces); + } + // Dict-entry line: KEY SP VALUE + let key = parse_dict_key(input)?; + pieces.push(Piece::styled(key, StyleKind::Key)); + let sp = take_while(1.., |c: char| c == ' ').parse_next(input)?; + pieces.push(Piece::plain(sp)); + // Top-level: require the value to use parens to avoid false- + // positives on plain prose `puts "Word Other"` Stdout lines. + let value_pieces = parse_value(input, true)?; + pieces.extend(value_pieces); + Ok(pieces) +} + +// VALUE is one of: +// IDENT '(' INNER ')' — single-line variant call with payload +// IDENT '(' — multi-line open (line ends after `(`) +// IDENT — empty-payload variant +// +// `require_parens = true` rejects the bare-IDENT form. We use that +// at the TOP LEVEL of a Stdout/Result line, where "KEY VARIANT" +// without parens is far more often plain prose (e.g. +// `puts "Configuring CIPS"` → `Configuring CIPS`) than an actual +// repr — styling that prose as if it were a typed value produces +// distracting false-positives. Inside an inline payload (sub-entries +// like `Nested(K1 Empty K2 Other(x))`) we accept bare IDENT because +// we've already seen the surrounding `Nested(`, so the context is +// unambiguous. +fn parse_value( + input: &mut &str, + require_parens: bool, +) -> ModalResult> { + let variant = parse_ident(input)?; + let mut out = vec![Piece::styled(variant, StyleKind::Variant)]; + if !input.starts_with('(') { + if require_parens { + return Err(winnow::error::ErrMode::Backtrack(ContextError::new())); + } + return Ok(out); + } + *input = &input[1..]; + out.push(Piece::styled("(", StyleKind::Punct)); + if input.is_empty() { + // `Variant(` at end of line — multi-line open. The + // closing `)` will appear on a later line and be matched + // by the close-only branch in `parse_line`. + return Ok(out); + } + // Inline payload. The payload is everything up to the + // matching close paren, with `(`/`)` balanced. Could be: + // - a scalar string (no inner parens): color green + // - a sub-entry KEY VARIANT(...) [SP KEY VARIANT(...)]*: recurse + let payload_pieces = parse_inline_payload(input)?; + out.extend(payload_pieces); + if input.starts_with(')') { + *input = &input[1..]; + out.push(Piece::styled(")", StyleKind::Punct)); + } + Ok(out) +} + +// Inline payload between `(` and its matching `)`. Recognizes +// either a single scalar (text with no parens) or a sequence of +// inline dict-entry-shaped sub-values (`KEY VARIANT(...) ...`). +// Stops at the closing `)` of the surrounding call. +fn parse_inline_payload(input: &mut &str) -> ModalResult> { + // Look ahead: does the payload look like `IDENT SP IDENT (`? + // If so it's a sub-entry — recurse. Otherwise treat it as a + // scalar value. + if looks_like_sub_entry(input) { + let mut out = Vec::new(); + // First sub-entry. + let entry = parse_sub_entry(input)?; + out.extend(entry); + // Optional further sub-entries separated by space (for + // dicts with multiple inline children). + let more: Vec> = repeat( + 0.., + ( + take_while(1.., |c: char| c == ' ') + .map(|s: &str| Piece::plain(s)), + parse_sub_entry, + ) + .map(|(sp, mut e)| { + e.insert(0, sp); + e + }), + ) + .parse_next(input)?; + for chunk in more { + out.extend(chunk); + } + Ok(out) + } else { + // Scalar payload: take everything up to the matching close + // paren of the surrounding call. Track paren depth so + // scalar values containing their own `(…)` (e.g. Vivado's + // `RS(544) CL119` FEC-config string) don't get cut short at + // the FIRST `)` — that used to abort the parse, drop the + // trailing text onto the caller's leftover input, and + // fall the whole line back to gray. + let start = *input; + let mut depth: usize = 0; + let mut end = 0; + for (i, c) in start.char_indices() { + match c { + '(' => depth += 1, + ')' if depth == 0 => { + end = i; + break; + } + ')' => depth -= 1, + _ => {} + } + end = i + c.len_utf8(); + } + let scalar = &start[..end]; + *input = &start[end..]; + Ok(vec![Piece::styled(scalar, StyleKind::Scalar)]) + } +} + +// A sub-entry inside an inline payload: KEY SP VARIANT [( ... )]. +fn parse_sub_entry(input: &mut &str) -> ModalResult> { + let mut out = Vec::new(); + let key = parse_ident(input)?; + out.push(Piece::styled(key, StyleKind::Key)); + let sp = take_while(1.., |c: char| c == ' ').parse_next(input)?; + out.push(Piece::plain(sp)); + // Sub-entry: bare-ident variant allowed (we're already inside + // an established `Variant(...)` so the context is unambiguous). + let value_pieces = parse_value(input, false)?; + out.extend(value_pieces); + Ok(out) +} + +// Best-effort lookahead: does the input start with `IDENT SP IDENT` +// (which would indicate a KEY SP VARIANT sub-entry rather than a +// bare scalar payload)? Doesn't consume input. +fn looks_like_sub_entry(input: &&str) -> bool { + let s = input; + let mut it = s.chars(); + // First ident + let first_ok = matches!( + it.next(), + Some(c) if c.is_ascii_alphabetic() || c == '_', + ); + if !first_ok { + return false; + } + let mut saw_first = 1; + for c in it.by_ref() { + if c.is_ascii_alphanumeric() || c == '_' { + saw_first += 1; + } else if c == ' ' { + break; + } else { + return false; + } + } + if saw_first == 0 { + return false; + } + // Next must be IDENT (after the space we just consumed). + let mut second_count = 0; + for c in it { + if c.is_ascii_alphanumeric() || c == '_' { + second_count += 1; + } else { + // A sub-entry's second ident is the variant name — must + // be followed by `(` to count. + return second_count > 0 && c == '('; + } + } + false +} + +// `[A-Za-z_][A-Za-z0-9_]*` — take identifier-shaped chars then +// verify the leading character is letter/underscore (we can't +// match the leading-letter constraint and the run cleanly with a +// single `take_while`, but it's fine to take everything plausible +// then reject if the leading char would have made it digit-led). +fn parse_ident<'a>(input: &mut &'a str) -> ModalResult<&'a str> { + let ident = + take_while(1.., |c: char| c.is_ascii_alphanumeric() || c == '_') + .parse_next(input)?; + match ident.chars().next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' => Ok(ident), + _ => Err(winnow::error::ErrMode::Backtrack(ContextError::new())), + } +} + +/// Same as [`parse_ident`] but also accepts `.` in the middle — +/// used for dict KEYS, not variant names. Vivado's property keys +/// are `CONFIG.` style (a dot-composed namespace), so a +/// `::Config`-style repr line like `CONFIG.CPM_PCIE0_MODES +/// Scalar(None)` needs to accept the `.` as part of the key or +/// the whole line fails to parse and falls back to plain rendering +/// — which is what "the highlighter isn't working" looked like in +/// practice. Variant names (`Scalar`, `Nested`, `Empty`, …) still +/// use [`parse_ident`] so this stays confined to KEYS only. +fn parse_dict_key<'a>(input: &mut &'a str) -> ModalResult<&'a str> { + let ident = take_while(1.., |c: char| { + c.is_ascii_alphanumeric() || c == '_' || c == '.' + }) + .parse_next(input)?; + match ident.chars().next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' => { + // Reject leading dot / trailing dot / consecutive dots + // shape — those are structurally malformed keys, and + // accepting them would silently paint bogus prose. + if ident.starts_with('.') + || ident.ends_with('.') + || ident.contains("..") + { + Err(winnow::error::ErrMode::Backtrack(ContextError::new())) + } else { + Ok(ident) + } + } + _ => Err(winnow::error::ErrMode::Backtrack(ContextError::new())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn close_only_line() { + let spans = highlight_line(")").expect("parses"); + assert!(!spans.is_empty()); + // Last span content is ")" + let last = &spans[spans.len() - 1]; + assert_eq!(last.content.as_ref(), ")"); + } + + #[test] + fn indented_close_line() { + let spans = highlight_line(" )").expect("parses"); + // First span = " " (indent), last span = ")" + assert_eq!(spans[0].content.as_ref(), " "); + assert_eq!(spans.last().unwrap().content.as_ref(), ")"); + } + + #[test] + fn simple_scalar_entry() { + let spans = highlight_line("CONFIG Scalar(foo)").expect("parses"); + // Should have spans for CONFIG, " ", Scalar, "(", foo, ")" + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!( + contents.contains(&"CONFIG"), + "missing CONFIG span: {contents:?}" + ); + assert!( + contents.contains(&"Scalar"), + "missing Scalar span: {contents:?}" + ); + assert!(contents.contains(&"foo"), "missing foo span: {contents:?}"); + } + + #[test] + fn variant_open_multiline() { + let spans = highlight_line("CONFIG Nested(").expect("parses"); + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!(contents.contains(&"CONFIG")); + assert!(contents.contains(&"Nested")); + assert_eq!(spans.last().unwrap().content.as_ref(), "("); + } + + /// The `::Config::repr` shape emits `CONFIG. + /// Scalar(value)` — a dot-composed Vivado property key. The + /// key parser has to accept dots or the whole line falls + /// through to unstyled plain text. + #[test] + fn dotted_config_key_parses() { + let spans = highlight_line("CONFIG.CPM_PCIE0_MODES Scalar(None)") + .expect("parses"); + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!( + contents.contains(&"CONFIG.CPM_PCIE0_MODES"), + "dotted key not captured whole: {contents:?}" + ); + assert!(contents.contains(&"Scalar"), "{contents:?}"); + assert!(contents.contains(&"None"), "{contents:?}"); + } + + /// Consecutive-dot / leading-dot / trailing-dot keys are + /// rejected so genuine prose (`. Alignment ...`) doesn't + /// silently get repainted as a repr. + #[test] + fn malformed_dotted_key_rejected() { + assert!(highlight_line(".leading Scalar(x)").is_none()); + assert!(highlight_line("trailing. Scalar(x)").is_none()); + assert!(highlight_line("dou..ble Scalar(x)").is_none()); + } + + #[test] + fn nested_inline_entry() { + let spans = + highlight_line("CONFIG Nested(CPM_PCIE0_MODES Scalar(None))") + .expect("parses"); + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!(contents.contains(&"CONFIG")); + assert!(contents.contains(&"Nested")); + assert!(contents.contains(&"CPM_PCIE0_MODES")); + assert!(contents.contains(&"Scalar")); + assert!(contents.contains(&"None")); + } + + #[test] + fn non_repr_line_returns_none() { + assert!(highlight_line("INFO: vivado started").is_none()); + assert!(highlight_line("just some random text").is_none()); + assert!(highlight_line("").is_none()); + } + + /// Regression: `Scalar(RS(544) CL119)` — Vivado's FEC-config + /// property value contains its own `(…)` pair. The scalar + /// payload parser must balance parens and stop only at the + /// matching outer `)`, otherwise the whole line falls back to + /// the gray no-repr styling. + #[test] + fn scalar_payload_with_inner_parens_still_highlights() { + let spans = highlight_line("FEC_SLICE0_CFG_C0 Scalar(RS(544) CL119)") + .expect("parses"); + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!( + contents.contains(&"FEC_SLICE0_CFG_C0"), + "missing key: {contents:?}" + ); + assert!( + contents.contains(&"Scalar"), + "missing variant: {contents:?}" + ); + assert!( + contents.contains(&"RS(544) CL119"), + "missing scalar payload: {contents:?}" + ); + } + + #[test] + fn plain_two_word_prose_not_styled_as_repr() { + // Regression: stdout text like `puts "Configuring CIPS"` used + // to match the `KEY EmptyVariant` shape and get colored. The + // top-level parser must require parens to avoid this. + assert!(highlight_line("Configuring CIPS").is_none()); + assert!(highlight_line("CPM 5 USER PROPS").is_none()); + assert!(highlight_line("Hello World").is_none()); + } +} diff --git a/vw-repl/src/highlight_htcl.rs b/vw-repl/src/highlight_htcl.rs new file mode 100644 index 0000000..1bdf6f9 --- /dev/null +++ b/vw-repl/src/highlight_htcl.rs @@ -0,0 +1,1049 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Syntax highlighter for htcl source text. +//! +//! Single-pass character scanner with a small context-state machine. +//! Walks the source one byte at a time, recognizing comments, strings, +//! `$variable` refs, command substitutions, attributes, numeric +//! literals, and bare-word identifiers. The state machine tracks +//! "what was the previous token" so an identifier after `proc` is +//! styled as a declaration name, an identifier after `set` is styled +//! as a variable, etc. +//! +//! Why not the AST? The user-facing requirement is **stable** +//! highlighting — tokens keep the same color whether or not the +//! source as a whole parses. An AST-based highlighter has to +//! choose between (a) emitting nothing for unparseable regions +//! (flickers off mid-edit) or (b) merging with a separate lexical +//! pass that produces different output than the AST pass +//! (flickers between two styles when the parse status changes). +//! Neither is acceptable. +//! +//! A scanner-with-state produces the same output for `proc foo` +//! whether followed by `{x} unit { ... }` (complete) or by `{x` +//! (incomplete) — every recognizable byte run gets its consistent +//! token kind in one pass. Tree-sitter's grammar-with-error-recovery +//! gives the same property; a stateful scanner is the lighter-weight +//! analog for our small language. +//! +//! Token kinds: +//! +//! - **Keyword** — `proc`, `set`, `type`, `enum`, `src`, `namespace`, +//! plus the control-flow / Tcl-builtin set. +//! - **Function (builtin)** — `puts`, `lappend`, `dict`, etc. +//! - **Declaration** — the identifier immediately following a +//! declaration keyword (`proc foo`, `type T`, `enum E`). +//! - **Variable** — `$name`, `${name}`, and the identifier after +//! `set`. +//! - **Type** — the identifier following `:` (proc arg type +//! annotation), and the bare word after a proc decl's args +//! brace (the return-type slot). +//! - **Parameter** — bare identifiers inside the first `{...}` of +//! a proc decl (the args block). +//! - **Attribute** — `@name` on proc args. +//! - **String** — `"..."` quoted, and `{...}` braced when not a +//! script body. +//! - **Comment / DocComment** — `#` / `##` to end of line, only +//! when in command-start position. +//! - **Numeric** — integer literals. +//! - **Punctuation** — `[`, `]`, `(`, `)`, `:`. Brackets recurse: +//! the scanner re-enters at `[` with fresh state for the inner +//! command, then resumes the outer state after the matching `]`. + +use std::ops::Range; + +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::Span as RatatuiSpan; + +/// Built-in command / control-flow words. +const BUILTIN_KEYWORDS: &[&str] = &[ + "if", "elseif", "else", "while", "for", "foreach", "switch", "catch", + "return", "break", "continue", "expr", "eval", "uplevel", "upvar", + "global", "variable", "try", "throw", "finally", +]; + +/// Built-in commands; styled distinct from user-proc calls. +/// `putr` is compile-time-rewritten to `puts [T::repr -v $x]` by +/// `vw_htcl::putr::rewrite` before any code reaches Tcl, but the +/// user writes it exactly like they write `puts` — same color for +/// visual consistency. +const BUILTIN_FUNCS: &[&str] = &[ + "puts", "putr", "lappend", "lindex", "llength", "lrange", "lsearch", + "lset", "lsort", "dict", "list", "string", "incr", "format", "scan", + "regexp", "regsub", "join", "split", "concat", "subst", "info", +]; + +/// Declaration keywords that name the next identifier as a decl. +const DECL_KEYWORDS: &[&str] = &["proc", "type", "enum", "namespace"]; + +/// One styled byte range. Used by both the scrollback renderer +/// (slice 2) and the input editor (slice 3) to apply per-token +/// styles on top of the entry's default body color. +#[derive(Clone, Debug, PartialEq)] +pub struct TokenSpan { + pub range: Range, + pub style: Style, +} + +// --- color palette -------------------------------------------------- + +pub fn keyword_style() -> Style { + Style::default() + .fg(Color::Rgb(180, 130, 220)) + .add_modifier(Modifier::BOLD) +} +pub fn builtin_func_style() -> Style { + Style::default().fg(Color::Rgb(180, 130, 220)) +} +pub fn function_style() -> Style { + Style::default().fg(Color::Rgb(230, 200, 120)) +} +pub fn declaration_style() -> Style { + Style::default() + .fg(Color::Rgb(230, 200, 120)) + .add_modifier(Modifier::BOLD) +} +pub fn parameter_style() -> Style { + Style::default().fg(Color::Rgb(220, 220, 220)) +} +pub fn variable_style() -> Style { + Style::default().fg(Color::Rgb(130, 200, 230)) +} +pub fn string_style() -> Style { + Style::default().fg(Color::Rgb(140, 200, 130)) +} +pub fn comment_style() -> Style { + Style::default() + .fg(Color::Rgb(110, 110, 110)) + .add_modifier(Modifier::DIM) +} +pub fn doc_comment_style() -> Style { + Style::default() + .fg(Color::Rgb(130, 160, 170)) + .add_modifier(Modifier::ITALIC) +} +pub fn type_style() -> Style { + Style::default().fg(Color::Rgb(100, 200, 200)) +} +pub fn attribute_style() -> Style { + Style::default().fg(Color::Rgb(200, 130, 180)) +} +pub fn numeric_style() -> Style { + Style::default().fg(Color::Rgb(180, 200, 130)) +} +pub fn punct_style() -> Style { + Style::default().add_modifier(Modifier::DIM) +} + +// --- public entry points -------------------------------------------- + +/// Scan `source` and return a sorted, non-overlapping list of +/// [`TokenSpan`]s. Output is stable across edits — the same byte +/// sequence produces the same tokens regardless of whether the +/// surrounding source parses successfully. +pub fn highlight_source(source: &str) -> Vec { + let mut s = Scanner::new(source); + s.scan_script(usize::MAX); + s.out +} + +/// Slice `text` per-line using the same token classifier as +/// [`highlight_source`], returning one styled-`Span` Vec per line. +/// Lines are split on `\n`. Bytes outside any token use `body_style`. +pub fn highlight_per_line( + text: &str, + body_style: Style, +) -> Vec>> { + let tokens = highlight_source(text); + let mut out = Vec::new(); + let mut line_start: usize = 0; + for line in text.split('\n') { + let line_end = line_start + line.len(); + let mut spans: Vec> = Vec::new(); + let mut cursor = line_start; + for ts in tokens + .iter() + .filter(|t| t.range.start < line_end && t.range.end > line_start) + { + let start = ts.range.start.max(line_start); + let end = ts.range.end.min(line_end); + if cursor < start { + spans.push(RatatuiSpan::styled( + text[cursor..start].to_string(), + body_style, + )); + } + if start < end { + spans.push(RatatuiSpan::styled( + text[start..end].to_string(), + ts.style, + )); + cursor = end; + } + } + if cursor < line_end { + spans.push(RatatuiSpan::styled( + text[cursor..line_end].to_string(), + body_style, + )); + } + out.push(spans); + line_start = line_end + 1; + } + out +} + +// --- scanner -------------------------------------------------------- + +struct Scanner<'a> { + source: &'a str, + bytes: &'a [u8], + pos: usize, + out: Vec, +} + +/// Per-command classification state. Reset at every command +/// boundary (`\n`, `;`, or the start of a `[...]` interior). +#[derive(Clone, Copy, Default)] +struct CmdState { + /// What the previous bare-word token was. Used to classify the + /// NEXT bare word in context (e.g. "after `proc`" → decl). + prev: PrevToken, + /// Word index within the current command (0 = command name, + /// 1 = first arg, ...). Used to detect the `proc NAME ARGS RET` + /// shape: at word 2 inside a proc decl, the args brace is + /// expected; at word 3, the return-type ident. + word_idx: usize, + /// True when this command's word-0 was `proc`. Drives the + /// "args-block braces hold parameters" + "word-3 is return + /// type" behaviour. + is_proc_decl: bool, + /// True when word-0 was `dict`. Drives the `dict for` / + /// `dict get` / etc. compound-head recognition — word-1 is a + /// sub-command keyword rather than a plain arg. + is_dict: bool, +} + +#[derive(Clone, Copy, Default, PartialEq, Eq)] +enum PrevToken { + #[default] + None, + Keyword, // proc / type / enum / namespace + SetKeyword, // `set` +} + +impl<'a> Scanner<'a> { + fn new(source: &'a str) -> Self { + Self { + source, + bytes: source.as_bytes(), + pos: 0, + out: Vec::new(), + } + } + + fn peek(&self) -> Option { + self.bytes.get(self.pos).copied() + } + + fn peek_at(&self, off: usize) -> Option { + self.bytes.get(self.pos + off).copied() + } + + fn push(&mut self, range: Range, style: Style) { + if range.start < range.end { + self.out.push(TokenSpan { range, style }); + } + } + + /// Scan a script: a sequence of commands until either end-of- + /// source or a closing-bracket terminator (when we're inside a + /// `[...]` substitution). `limit` is a byte ceiling on the scan + /// (usize::MAX for top-level). + fn scan_script(&mut self, limit: usize) { + while self.pos < self.bytes.len() && self.pos < limit { + self.skip_horizontal_ws(); + if self.pos >= limit { + break; + } + match self.peek() { + None => break, + Some(b']') => return, // end of `[...]` interior + Some(b'\n') | Some(b';') => { + self.pos += 1; + continue; + } + Some(b'#') => { + // Comment in command position. + self.scan_comment(); + continue; + } + _ => {} + } + self.scan_command(limit); + } + } + + /// Skip spaces and tabs. NOT newlines (which are command + /// terminators). + fn skip_horizontal_ws(&mut self) { + while let Some(c) = self.peek() { + if c == b' ' || c == b'\t' { + self.pos += 1; + } else if c == b'\\' && self.peek_at(1) == Some(b'\n') { + // Line continuation. + self.pos += 2; + } else { + break; + } + } + } + + fn scan_command(&mut self, limit: usize) { + let mut state = CmdState::default(); + loop { + self.skip_horizontal_ws(); + if self.pos >= limit { + return; + } + match self.peek() { + None => return, + Some(b'\n') | Some(b';') | Some(b']') => return, + _ => {} + } + self.scan_word(&mut state); + state.word_idx += 1; + // Guard against an inner scan (bare-word / braced / + // bracketed) overshooting the parent's byte limit. The + // narrow case that used to bite: `{lib srcs}` in + // command-arg position — scan_braced_as_script hands the + // interior to scan_script(close_pos), scan_command then + // consumes past the closing `}` because it only looked + // for `\n`/`;`/`]` as terminators. After the outer scan + // resumed, every span from `$deps { … }` was emitted a + // second time, and the per-line renderer duplicated the + // corresponding text on screen. + if self.pos >= limit { + return; + } + } + } + + fn scan_word(&mut self, state: &mut CmdState) { + match self.peek() { + Some(b'"') => { + self.scan_quoted(); + state.prev = PrevToken::None; + } + Some(b'{') => { + if state.is_proc_decl && state.word_idx == 2 { + // Proc args block: scan interior for params/types. + self.scan_proc_args(); + } else { + // All other braces — proc body (word 3 with no + // return type, or word 4 with one), control-flow + // condition / body, generic braced word — get + // scanned as a script. Tcl-convention braces are + // scripts; we always treat them that way so + // identifiers inside `if {…}`, `while {…}`, proc + // bodies, etc. get live highlighting. The narrow + // loss: braced return-type annotations + // (`proc foo {} {dict} { … }`) + // get their interior scanned as a script rather + // than as a type — acceptable since the bare-word + // form is far more common. + self.scan_braced_as_script(); + } + state.prev = PrevToken::None; + } + Some(b'[') => { + self.scan_bracket_subst(); + state.prev = PrevToken::None; + } + Some(b'$') => { + self.scan_var_ref(); + // After a $var, the next ident isn't a decl/var/type. + state.prev = PrevToken::None; + } + Some(b'@') if state.word_idx > 0 => { + self.scan_attribute(); + state.prev = PrevToken::None; + } + Some(c) if c.is_ascii_digit() => { + self.scan_numeric(); + state.prev = PrevToken::None; + } + Some(b'-') => { + // Could be a flag (`-name`) or a negative number. + let next = self.peek_at(1); + if matches!(next, Some(c) if c.is_ascii_digit()) { + self.scan_numeric(); + } else { + self.scan_bare_word(state); + } + state.prev = PrevToken::None; + } + Some(_) => { + self.scan_bare_word(state); + } + None => (), + } + } + + /// Scan a `#` or `##` comment to end of line. Only called when + /// `#` appears in command position. + fn scan_comment(&mut self) { + let start = self.pos; + let is_doc = self.peek_at(1) == Some(b'#'); + while self.pos < self.bytes.len() && self.bytes[self.pos] != b'\n' { + self.pos += 1; + } + let style = if is_doc { + doc_comment_style() + } else { + comment_style() + }; + self.push(start..self.pos, style); + } + + fn scan_quoted(&mut self) { + let start = self.pos; + self.pos += 1; // consume opening `"` + while let Some(c) = self.peek() { + if c == b'\\' && self.peek_at(1).is_some() { + self.pos += 2; + continue; + } + if c == b'"' { + self.pos += 1; + break; + } + self.pos += 1; + } + self.push(start..self.pos, string_style()); + } + + /// Scan a `{...}` proc-args block: emit braces as punct, walk + /// the interior recognizing parameter names, `@attributes`, + /// `:type` annotations, and `;` separators. + fn scan_proc_args(&mut self) { + let open = self.pos; + self.pos += 1; + self.push(open..open + 1, punct_style()); + let mut depth = 1usize; + // Walk the interior emitting param-style for bare idents, + // colon-then-type for annotations, attribute style for `@`. + let mut expect_type = false; + while let Some(c) = self.peek() { + if c == b'{' { + depth += 1; + self.pos += 1; + continue; + } + if c == b'}' { + if depth == 1 { + let close = self.pos; + self.pos += 1; + self.push(close..close + 1, punct_style()); + return; + } + depth -= 1; + self.pos += 1; + continue; + } + if c == b' ' || c == b'\t' || c == b'\n' || c == b';' { + self.pos += 1; + continue; + } + if c == b'#' { + self.scan_comment(); + continue; + } + if c == b':' { + let p = self.pos; + self.pos += 1; + self.push(p..p + 1, punct_style()); + expect_type = true; + continue; + } + if c == b'@' { + self.scan_attribute(); + continue; + } + if c.is_ascii_alphabetic() || c == b'_' { + let start = self.pos; + while let Some(d) = self.peek() { + if d.is_ascii_alphanumeric() + || d == b'_' + || d == b'<' + || d == b'>' + || d == b',' + { + self.pos += 1; + } else if d == b':' && self.peek_at(1) == Some(b':') { + self.pos += 2; + } else { + break; + } + } + let style = if expect_type { + type_style() + } else { + parameter_style() + }; + self.push(start..self.pos, style); + expect_type = false; + continue; + } + // Unknown byte — skip. + self.pos += 1; + } + } + + /// Scan `{...}` as a proc body: recurse with full script + /// classification on the interior. + fn scan_braced_as_script(&mut self) { + let open = self.pos; + self.pos += 1; + self.push(open..open + 1, punct_style()); + let mut depth = 1usize; + // Find the matching close brace position (best-effort). + let mut p = self.pos; + while p < self.bytes.len() { + let c = self.bytes[p]; + if c == b'\\' && p + 1 < self.bytes.len() { + p += 2; + continue; + } + if c == b'{' { + depth += 1; + } else if c == b'}' { + depth -= 1; + if depth == 0 { + break; + } + } + p += 1; + } + // Recurse on the interior [self.pos .. p). + let close_pos = p; + self.scan_script(close_pos); + // Skip past the matching close brace if we found one. + if close_pos < self.bytes.len() && self.bytes[close_pos] == b'}' { + self.pos = close_pos; + self.push(close_pos..close_pos + 1, punct_style()); + self.pos += 1; + } else { + // EOF without close — leave the cursor at end. + self.pos = close_pos.max(self.pos); + } + } + + /// `[command-substitution]` — recurse on the interior as a + /// fresh script. + fn scan_bracket_subst(&mut self) { + let open = self.pos; + self.push(open..open + 1, punct_style()); + self.pos += 1; + self.scan_script(usize::MAX); + if let Some(b']') = self.peek() { + let close = self.pos; + self.push(close..close + 1, punct_style()); + self.pos += 1; + } + } + + fn scan_var_ref(&mut self) { + let start = self.pos; + self.pos += 1; // $ + if self.peek() == Some(b'{') { + self.pos += 1; + while let Some(c) = self.peek() { + self.pos += 1; + if c == b'}' { + break; + } + } + } else { + while let Some(c) = self.peek() { + if c.is_ascii_alphanumeric() + || c == b'_' + || c == b':' && self.peek_at(1) == Some(b':') + { + self.pos += 1; + } else { + break; + } + } + } + if self.pos > start + 1 { + self.push(start..self.pos, variable_style()); + } + } + + fn scan_attribute(&mut self) { + let start = self.pos; + self.pos += 1; // `@` + while let Some(c) = self.peek() { + if c.is_ascii_alphanumeric() || c == b'_' { + self.pos += 1; + } else { + break; + } + } + if self.pos > start + 1 { + self.push(start..self.pos, attribute_style()); + } + // Optional `(...)` payload — consume as a balanced group. + if self.peek() == Some(b'(') { + let p = self.pos; + self.pos += 1; + self.push(p..p + 1, punct_style()); + let mut depth = 1usize; + while let Some(c) = self.peek() { + if c == b'(' { + depth += 1; + self.pos += 1; + } else if c == b')' { + depth -= 1; + self.pos += 1; + if depth == 0 { + self.push(self.pos - 1..self.pos, punct_style()); + return; + } + } else if c == b'"' { + self.scan_quoted(); + } else if c.is_ascii_digit() { + self.scan_numeric(); + } else if c.is_ascii_alphabetic() || c == b'_' { + let s = self.pos; + while let Some(d) = self.peek() { + if d.is_ascii_alphanumeric() || d == b'_' { + self.pos += 1; + } else { + break; + } + } + self.push(s..self.pos, string_style()); + } else { + self.pos += 1; + } + } + } + } + + fn scan_numeric(&mut self) { + let start = self.pos; + if self.peek() == Some(b'-') { + self.pos += 1; + } + while let Some(c) = self.peek() { + if c.is_ascii_digit() { + self.pos += 1; + } else { + break; + } + } + if self.pos > start { + self.push(start..self.pos, numeric_style()); + } + } + + fn scan_bare_word(&mut self, state: &mut CmdState) { + let start = self.pos; + while let Some(c) = self.peek() { + if c.is_ascii_alphanumeric() + || c == b'_' + || c == b'-' + || c == b'.' + || c == b'/' + { + self.pos += 1; + } else if c == b':' && self.peek_at(1) == Some(b':') { + self.pos += 2; + } else { + break; + } + } + if self.pos == start { + // No bare-word chars (probably punctuation we don't know). + self.pos += 1; + return; + } + let word = &self.source[start..self.pos]; + let style = match state.prev { + PrevToken::Keyword => { + // Declaration name follows a decl keyword. + state.prev = PrevToken::None; + Some(declaration_style()) + } + PrevToken::SetKeyword => { + state.prev = PrevToken::None; + Some(variable_style()) + } + PrevToken::None => { + if state.word_idx == 0 { + // Command position: keyword / builtin / function. + if DECL_KEYWORDS.contains(&word) { + state.prev = PrevToken::Keyword; + if word == "proc" { + state.is_proc_decl = true; + } + Some(keyword_style()) + } else if word == "set" { + state.prev = PrevToken::SetKeyword; + Some(keyword_style()) + } else if BUILTIN_KEYWORDS.contains(&word) { + Some(keyword_style()) + } else if BUILTIN_FUNCS.contains(&word) { + if word == "dict" { + state.is_dict = true; + } + Some(builtin_func_style()) + } else { + Some(function_style()) + } + } else if state.is_proc_decl && state.word_idx == 3 { + // Return-type slot (bare-ident form). + Some(type_style()) + } else if state.is_dict && state.word_idx == 1 { + // `dict for`, `dict get`, `dict set`, … + // Compound-head sub-command; color like the + // top-level keyword it stands in for. + Some(keyword_style()) + } else if word == "true" || word == "false" { + // Boolean literals in arg position — same + // treatment as top-level bool sniffing so + // `-flag true` reads consistently. + Some(keyword_style()) + } else if word.starts_with('-') && word.len() > 1 { + // Flag-style word. + Some(attribute_style()) + } else { + // Argument to a call — leave to body color, with + // the exception that if the next char is `:` we + // still treat this as a parameter name (rare in + // call args but harmless to leave unstyled). + None + } + } + }; + if let Some(s) = style { + self.push(start..self.pos, s); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn highlights(source: &str) -> Vec<(String, &'static str)> { + highlight_source(source) + .into_iter() + .map(|s| { + let text = source[s.range.clone()].to_string(); + let tag = style_tag(&s.style); + (text, tag) + }) + .collect() + } + + fn style_tag(style: &Style) -> &'static str { + if style == &keyword_style() { + "keyword" + } else if style == &builtin_func_style() { + "builtin_func" + } else if style == &function_style() { + "function" + } else if style == &declaration_style() { + "decl" + } else if style == ¶meter_style() { + "param" + } else if style == &variable_style() { + "var" + } else if style == &string_style() { + "string" + } else if style == &comment_style() { + "comment" + } else if style == &doc_comment_style() { + "doc" + } else if style == &type_style() { + "type" + } else if style == &attribute_style() { + "attr" + } else if style == &numeric_style() { + "num" + } else if style == &punct_style() { + "punct" + } else { + "?" + } + } + + fn has(h: &[(String, &str)], text: &str, tag: &str) -> bool { + h.iter().any(|(t, g)| t == text && *g == tag) + } + + #[test] + fn empty_input() { + assert!(highlight_source("").is_empty()); + } + + #[test] + fn set_keyword_var_value() { + let h = highlights("set foo 42"); + assert!(has(&h, "set", "keyword"), "{h:?}"); + assert!(has(&h, "foo", "var"), "{h:?}"); + assert!(has(&h, "42", "num"), "{h:?}"); + } + + #[test] + fn generic_call_first_word_function() { + let h = highlights("foo bar baz"); + assert!(has(&h, "foo", "function"), "{h:?}"); + } + + #[test] + fn builtin_control_flow_keyword() { + let h = highlights("if {$x > 0} { puts hi }"); + assert!(has(&h, "if", "keyword"), "{h:?}"); + // `puts` inside the body braces should still highlight as a + // builtin func because the body is scanned as a script. + assert!(has(&h, "puts", "builtin_func"), "{h:?}"); + // $x inside the condition braces gets var styling via the + // braced-as-string scanner... but the condition braces are + // treated as a string today. v1 accepts that — the cost of + // braced-string consistency is occasional loss of $var + // styling inside conditions. Acceptable trade. + let _ = h; + } + + #[test] + fn proc_decl_keywords_and_names() { + let h = highlights("proc foo {x: int} bool { return $x }"); + assert!(has(&h, "proc", "keyword")); + assert!(has(&h, "foo", "decl")); + assert!(has(&h, "x", "param")); + assert!(has(&h, "int", "type")); + assert!(has(&h, "bool", "type")); + // Body scanned as script. + assert!(has(&h, "return", "keyword")); + assert!(has(&h, "$x", "var")); + } + + #[test] + fn proc_with_attribute() { + let h = highlights("proc foo {@default(0) x: int} unit {}"); + assert!(has(&h, "@default", "attr")); + assert!(has(&h, "0", "num")); + assert!(has(&h, "x", "param")); + assert!(has(&h, "int", "type")); + assert!(has(&h, "unit", "type")); + } + + #[test] + fn type_decl() { + let h = highlights("type Properties = {dict}"); + assert!(has(&h, "type", "keyword")); + assert!(has(&h, "Properties", "decl")); + } + + #[test] + fn enum_decl() { + let h = highlights("enum Direction = {North South East West}"); + assert!(has(&h, "enum", "keyword")); + assert!(has(&h, "Direction", "decl")); + } + + #[test] + fn doc_comment_vs_regular() { + let h = highlights("# regular\n## doc\nset x 1"); + assert!(h + .iter() + .any(|(t, tag)| t.contains("regular") && *tag == "comment")); + assert!(h.iter().any(|(t, tag)| t.contains("doc") && *tag == "doc")); + } + + #[test] + fn cmd_substitution_recurses() { + let h = highlights("set y [foo $x]"); + assert!(has(&h, "set", "keyword")); + assert!(has(&h, "foo", "function")); + assert!(has(&h, "$x", "var")); + assert!(has(&h, "[", "punct")); + assert!(has(&h, "]", "punct")); + } + + #[test] + fn quoted_string() { + let h = highlights("puts \"hello world\""); + assert!(has(&h, "puts", "builtin_func")); + assert!(has(&h, "\"hello world\"", "string")); + } + + // The crucial stability tests: incomplete input must produce + // the SAME classifications for the tokens that ARE present. + #[test] + fn stable_on_unclosed_proc_brace() { + let complete = highlights("proc foo {x: int} bool { return 0 }"); + let incomplete = highlights("proc foo {x: int} bool { return 0"); + // The tokens shared between the two must classify the same. + // Specifically: proc/foo/x/int/bool/return all match. + for (text, tag) in &[ + ("proc", "keyword"), + ("foo", "decl"), + ("x", "param"), + ("int", "type"), + ("bool", "type"), + ("return", "keyword"), + ] { + assert!( + has(&complete, text, tag), + "complete missing ({text}, {tag}): {complete:?}" + ); + assert!( + has(&incomplete, text, tag), + "incomplete missing ({text}, {tag}): {incomplete:?}" + ); + } + } + + #[test] + fn stable_on_unclosed_outer_brace() { + let complete = highlights("proc foo {}"); + let incomplete = highlights("proc foo {"); + for (text, tag) in &[("proc", "keyword"), ("foo", "decl")] { + assert!(has(&complete, text, tag), "complete missing"); + assert!(has(&incomplete, text, tag), "incomplete missing"); + } + } + + #[test] + fn stable_on_unclosed_bracket() { + let complete = highlights("set y [foo $x]"); + let incomplete = highlights("set y [foo $x"); + for (text, tag) in + &[("set", "keyword"), ("foo", "function"), ("$x", "var")] + { + assert!( + has(&complete, text, tag), + "complete missing ({text}, {tag})" + ); + assert!( + has(&incomplete, text, tag), + "incomplete missing ({text}, {tag}): {incomplete:?}" + ); + } + } + + #[test] + fn no_panic_on_garbage() { + let _ = highlight_source("[[["); + let _ = highlight_source("{{{"); + let _ = highlight_source("$$$"); + let _ = highlight_source(""); + let _ = highlight_source("# unterminated\nproc \"weird name\""); + } + + #[test] + fn plain_prose_not_styled_as_decl() { + // Top-level plain words shouldn't get function styling for + // their non-first words (i.e. just the first word styles + // as function-call; the rest are left to body color). + let h = highlights("just some words"); + assert!(has(&h, "just", "function")); + // "some" and "words" are call args — no special styling. + assert!(!has(&h, "some", "decl")); + assert!(!has(&h, "words", "decl")); + } + + #[test] + fn proc_body_contents_scan_as_script() { + // Regression: the body of a proc decl (without a return-type + // annotation) used to be styled as a single type span, + // painting the whole interior teal. Body must scan as a + // normal script so `set`, `[...]`, `$vars`, etc. all + // classify normally. + let h = + highlights("proc foo { x: int } { set foo [reticulate -value x] }"); + // Inside the body: + assert!(has(&h, "set", "keyword"), "{h:?}"); + // `foo` after `set` is the variable being assigned. + assert!( + h.iter().any(|(t, tag)| t == "foo" && *tag == "var"), + "expected `foo` styled as var inside body: {h:?}" + ); + assert!(has(&h, "reticulate", "function"), "{h:?}"); + assert!(has(&h, "[", "punct")); + assert!(has(&h, "]", "punct")); + } + + #[test] + fn body_with_and_without_return_type_scan_same() { + // Identical body content should classify the same whether + // the proc has a return-type annotation or not. + let with = highlights("proc f {} unit { set x 1 }"); + let without = highlights("proc f {} { set x 1 }"); + for (text, tag) in &[("set", "keyword"), ("x", "var")] { + assert!(has(&with, text, tag), "with: missing ({text}, {tag})"); + assert!( + has(&without, text, tag), + "without: missing ({text}, {tag}): {without:?}" + ); + } + } + + #[test] + fn per_line_preserves_content() { + let src = "set a 1\nproc f {} unit {}\nputs hi"; + let lines = highlight_per_line(src, Style::default()); + let mut reconstructed = String::new(); + for (i, line) in lines.iter().enumerate() { + if i > 0 { + reconstructed.push('\n'); + } + for span in line { + reconstructed.push_str(span.content.as_ref()); + } + } + assert_eq!(reconstructed, src); + } + + #[test] + fn braced_word_scan_does_not_overshoot_close() { + // Regression: `{lib srcs}` as an arg to `dict for` was + // recursed into as a script, but `scan_command` inside + // that recursion consumed past the closing `}` because + // it only checked for `\n`/`;`/`]`. The outer scan then + // re-tokenized everything from `$deps` onward, and the + // renderer duplicated the corresponding text on screen. + // + // Assert token spans are non-overlapping and monotonic. + let src = "dict for {lib srcs} $deps {\n puts $lib\n}"; + let tokens = highlight_source(src); + let mut prev_end = 0usize; + for t in &tokens { + assert!( + t.range.start >= prev_end, + "overlap at {:?} after prev end {prev_end} — full: {tokens:?}", + t.range, + ); + prev_end = t.range.end; + } + // And reconstructing per-line output round-trips. + let lines = highlight_per_line(src, Style::default()); + let mut reconstructed = String::new(); + for (i, line) in lines.iter().enumerate() { + if i > 0 { + reconstructed.push('\n'); + } + for span in line { + reconstructed.push_str(span.content.as_ref()); + } + } + assert_eq!(reconstructed, src); + } +} diff --git a/vw-repl/src/history.rs b/vw-repl/src/history.rs new file mode 100644 index 0000000..ed6ad72 --- /dev/null +++ b/vw-repl/src/history.rs @@ -0,0 +1,236 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Persistent input history with incremental search. +//! +//! Entries are appended to a newline-delimited file (one entry per +//! line, with embedded newlines escaped) under the platform's state +//! dir — typically `~/.local/state/vw/repl-history` on Linux. The +//! file is loaded once at startup; new entries are appended both to +//! memory and to the file as soon as they're recorded so a crashed +//! session doesn't lose history. +//! +//! Ctrl-R triggers an *incremental* search: as the user types, we +//! find the most recent entry whose text contains the query as a +//! substring (case-insensitive). Repeated Ctrl-R steps to the next- +//! older match. Esc cancels; Enter accepts the match into the input +//! buffer. + +use std::fs::{create_dir_all, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::PathBuf; + +const ESCAPED_NEWLINE: &str = "\\n"; +const ESCAPED_BACKSLASH: &str = "\\\\"; + +/// In-memory history, backed by an on-disk file. Indexed +/// most-recent-last; `entries[entries.len() - 1]` is the freshest +/// record, matching how Readline / Reedline order things. +#[derive(Debug)] +pub struct History { + file_path: PathBuf, + entries: Vec, +} + +impl History { + /// Load history from the default location. Returns an empty + /// store (and skips disk writes) when no state dir is available + /// — the REPL still runs, just without persistence. + pub fn load_default() -> Self { + let path = default_history_path(); + match path { + Some(p) => Self::load_from(p), + None => Self { + file_path: PathBuf::new(), + entries: Vec::new(), + }, + } + } + + /// Load history from a specific file. Missing file → empty + /// history (the file gets created on first append). + pub fn load_from(file_path: PathBuf) -> Self { + let entries = read_entries(&file_path).unwrap_or_default(); + Self { file_path, entries } + } + + #[allow(dead_code)] // public API for the in-progress completion slice + pub fn entries(&self) -> &[String] { + &self.entries + } + + /// Append `entry` to the in-memory log and persist it. Empty or + /// whitespace-only entries are ignored. An entry identical to + /// the most recent one is also ignored (the common case of + /// re-running the same command shouldn't bloat the file). + pub fn append(&mut self, entry: &str) { + let trimmed = entry.trim(); + if trimmed.is_empty() { + return; + } + if self.entries.last().map(String::as_str) == Some(entry) { + return; + } + self.entries.push(entry.to_string()); + if self.file_path.as_os_str().is_empty() { + return; + } + if let Some(parent) = self.file_path.parent() { + let _ = create_dir_all(parent); + } + if let Ok(mut f) = OpenOptions::new() + .create(true) + .append(true) + .open(&self.file_path) + { + let _ = writeln!(f, "{}", encode_line(entry)); + } + } + + /// Find the most recent entry whose text contains `query` as a + /// substring (case-insensitive). Returns the index in + /// [`Self::entries`] plus the entry itself. `start_before` is an + /// exclusive upper bound — passing `Some(prev_idx)` resumes the + /// search at the next-older entry, which is how repeated + /// `Ctrl-R` steps backward. + pub fn search_back( + &self, + query: &str, + start_before: Option, + ) -> Option<(usize, &str)> { + if query.is_empty() { + return None; + } + let upper = start_before.unwrap_or(self.entries.len()); + let needle = query.to_lowercase(); + for i in (0..upper).rev() { + if self.entries[i].to_lowercase().contains(&needle) { + return Some((i, self.entries[i].as_str())); + } + } + None + } +} + +fn default_history_path() -> Option { + let state = dirs::state_dir().or_else(dirs::data_local_dir)?; + Some(state.join("vw").join("repl-history")) +} + +fn read_entries(path: &PathBuf) -> Option> { + let f = std::fs::File::open(path).ok()?; + let mut entries = Vec::new(); + for line in BufReader::new(f).lines().map_while(Result::ok) { + entries.push(decode_line(&line)); + } + Some(entries) +} + +fn encode_line(s: &str) -> String { + // Single-line newline-delimited file format: backslashes and + // embedded newlines get a literal escape so a multi-line htcl + // buffer round-trips cleanly. + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '\\' => out.push_str(ESCAPED_BACKSLASH), + '\n' => out.push_str(ESCAPED_NEWLINE), + other => out.push(other), + } + } + out +} + +fn decode_line(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.next() { + Some('n') => out.push('\n'), + Some('\\') => out.push('\\'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } else { + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + + #[test] + fn append_persists_and_dedupes_consecutive_duplicates() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("h"); + let mut h = History::load_from(p.clone()); + h.append("foo"); + h.append("foo"); + h.append("bar"); + h.append(""); + h.append(" "); + assert_eq!(h.entries(), &["foo".to_string(), "bar".to_string()]); + // File round-trips. + let mut buf = String::new(); + std::fs::File::open(&p) + .unwrap() + .read_to_string(&mut buf) + .unwrap(); + assert_eq!(buf, "foo\nbar\n"); + } + + #[test] + fn multiline_entries_round_trip_with_escapes() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("h"); + { + let mut h = History::load_from(p.clone()); + h.append("set x [\n create_cpm5 -name cpm5\n]"); + h.append("with \\ backslash"); + } + let h2 = History::load_from(p); + assert_eq!( + h2.entries(), + &[ + "set x [\n create_cpm5 -name cpm5\n]".to_string(), + "with \\ backslash".to_string(), + ] + ); + } + + #[test] + fn search_back_finds_most_recent_match() { + let dir = tempfile::tempdir().unwrap(); + let mut h = History::load_from(dir.path().join("h")); + h.append("set x 1"); + h.append("create_project foo"); + h.append("set y 2"); + let (idx, hit) = h.search_back("set", None).unwrap(); + assert_eq!(hit, "set y 2"); + assert_eq!(idx, 2); + // Step to the next older. + let (idx2, hit2) = h.search_back("set", Some(idx)).unwrap(); + assert_eq!(hit2, "set x 1"); + assert_eq!(idx2, 0); + // Nothing older. + assert!(h.search_back("set", Some(idx2)).is_none()); + } + + #[test] + fn search_is_case_insensitive() { + let dir = tempfile::tempdir().unwrap(); + let mut h = History::load_from(dir.path().join("h")); + h.append("Create_Project foo"); + let (_, hit) = h.search_back("create", None).unwrap(); + assert_eq!(hit, "Create_Project foo"); + } +} diff --git a/vw-repl/src/lib.rs b/vw-repl/src/lib.rs new file mode 100644 index 0000000..286494d --- /dev/null +++ b/vw-repl/src/lib.rs @@ -0,0 +1,131 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Interactive REPL for htcl scripts. +//! +//! A ratatui-driven shell that talks to a long-lived Vivado worker +//! via [`vw_vivado::VivadoBackend`]. The session document model +//! (every successful eval appended to an in-memory script + the +//! current input as its tail) lets the analyzer power the same +//! features the LSP gives editors — completion, hover, signature +//! help — without any REPL-specific machinery. +//! +//! v1 (this slice) ships the foundation: screen layout, multi-line +//! input with Readline-quality editing, persistent history with +//! Ctrl-R search, a long-lived Vivado worker, and `:load `. +//! Tab completion, signature help, hover overlay, command palette, +//! and structured-result rendering layer on top in subsequent +//! slices. + +mod app; +pub mod config; +pub mod diag_search; +pub mod highlight; +pub mod highlight_htcl; +mod history; +pub mod lower; +mod popup; +mod render; +mod session; +mod symbol_index; +mod symbol_search; +pub mod trace; +mod ui; + +use camino::Utf8PathBuf; +use thiserror::Error; + +pub use app::App; +pub use lower::{build_proc_locations, Origin, OriginFrame, ProcLocation}; +pub use session::Session; +pub use trace::{ + display_path, resolve_stack_frames_with, rewrite_stack_line, RewrittenFrame, +}; + +/// Wrap a Tcl body with shim-side origin markers so any traceless +/// warning/error emitted during the eval is tagged with `origin` +/// via the marker stack — not with whatever the REPL / CLI happens +/// to have as `pending_eval_index` when the message eventually +/// arrives. +/// +/// The race this fixes: Vivado's C++ writes warning bytes to the +/// PTY, then Tcl sends the eval response over the protocol socket. +/// The pump thread's latency can put the response ahead of the +/// warning at the receiver, so the warning lands during the *next* +/// eval and inherits its origin — often a synthetic prelude +/// command with `line=0`. Wrapping the body means the origin frame +/// sits on the shim's marker stack from `emit_pty_ctx_begin` (just +/// before the body runs) until `emit_pty_ctx_end` (just after), +/// and every straggler in that window tags off the top of the +/// stack. +/// +/// The wrapped body preserves rc/result/errorcode/-errorinfo via +/// `return -options`, so callers see identical behavior to the +/// unwrapped `tcl`. +pub fn wrap_tcl_with_origin_marker(tcl: &str, origin: &Origin) -> String { + // Build a single frame string in the same ":" + // shape `capture_stack` emits, so the downstream renderer's + // stack-frame regex handles both uniformly. No proc part — + // this frame is the *statement's* file/line, not a Tcl + // proc-body line. + let file_repr = origin + .file + .as_deref() + .map(display_path) + .unwrap_or_else(|| "".to_string()); + let frame = format!("{file_repr}:{}", origin.line); + // Tcl list-quote via braces. The frame content is a file path + // + integer, so braces alone are sufficient — no metachars to + // escape. + format!( + "::vw::emit_pty_ctx_begin [list {{{frame}}}]\n\ + set _vw_wrap_rc [catch {{\n{tcl}\n}} _vw_wrap_r _vw_wrap_o]\n\ + ::vw::emit_pty_ctx_end\n\ + return -options $_vw_wrap_o $_vw_wrap_r" + ) +} + +#[derive(Debug, Error)] +pub enum ReplError { + #[error("terminal I/O: {0}")] + Io(#[from] std::io::Error), + #[error("backend: {0}")] + Backend(#[from] vw_eda::BackendError), +} + +/// Tunable knobs supplied by the CLI invocation. +#[derive(Clone, Debug, Default)] +pub struct ReplOptions { + /// Minimum severity that renders in scrollback. `Debug` shows + /// every block raw (including Vivado's banners, tables, and + /// other non-diagnostic noise); higher levels collapse + /// non-diagnostic content into a toggleable placeholder and + /// hide diagnostics below the threshold. + pub log_level: vw_vivado::LogLevel, + /// If set, source this file into the session immediately after + /// the Vivado worker comes up. Equivalent to typing `:load + /// ` as the first input. + pub initial_load: Option, + /// If true, INFO-severity Vivado messages carry their full Tcl + /// stack frames into the scrollback. Off by default — INFO is + /// noisy enough without stack traces — but useful when diagnosing + /// where a particular INFO is emitted from. + pub info_with_stack: bool, + /// Optional `--part ` selector — picks a non-default + /// `[[target-parts]]` entry to drive the auto-project. `None` + /// uses the workspace default. Mutually exclusive with + /// `variant`; the CLI enforces this via clap. + pub part: Option, + /// Optional `--variant ` selector — picks a + /// `[[workspace.variants]]` entry to drive the auto-project + /// AND to filter design sources via the session-scoped + /// active-variant fallback in the RPC handler. + pub variant: Option, +} + +/// Run the REPL until the user exits. Owns the terminal alternate +/// screen for the duration; restores it on every exit path. +pub async fn run(opts: ReplOptions) -> Result<(), ReplError> { + app::run(opts).await +} diff --git a/vw-repl/src/lower.rs b/vw-repl/src/lower.rs new file mode 100644 index 0000000..891bf91 --- /dev/null +++ b/vw-repl/src/lower.rs @@ -0,0 +1,1643 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Lower a REPL input buffer to a sequence of `(htcl-origin, Tcl)` +//! commands the Vivado worker can evaluate one at a time. +//! +//! Shipping one statement per `eval` (rather than a single +//! concatenated script) is what lets us render Vivado errors against +//! htcl source. The loader's [`vw_htcl::LoadedProgram::locate_span`] +//! tells us which `.htcl` file each top-level statement came from; +//! we keep that mapping alongside the lowered Tcl so the REPL can +//! report `× : ` instead of a Tcl stack +//! trace pointing into our shim. + +use std::io::Write; +use std::path::{Path, PathBuf}; + +use camino::Utf8Path; +use vw_htcl::{LineIndex, Resolver}; + +use crate::session::{Session, SessionBatch}; + +struct NoopObserver; +impl vw_htcl::LoadObserver for NoopObserver {} + +#[derive(Debug, thiserror::Error)] +pub enum LowerError { + #[error("writing scratch input file: {0}")] + Io(#[from] std::io::Error), + #[error("loading htcl: {0}")] + Load(#[from] vw_htcl::LoadError), + #[error("{0}")] + Parse(String), +} + +/// Where in the loaded htcl tree a particular command came from. +/// Drives the error renderer in the App. +#[derive(Clone, Debug)] +pub struct Origin { + /// `.htcl` file the command was declared in, when known. `None` + /// only when the input itself wasn't backed by a real file + /// (e.g. interactive REPL input lowered before any imports). + pub file: Option, + /// 1-based line number in `file` (or in the input buffer when + /// `file` is `None`). + pub line: u32, + /// First line of the command as written by the user — used as + /// the "what was running" line in the error renderer. + pub snippet: String, + /// The chain of `src` imports that brought this command's file + /// into scope, ordered nearest-first (so the last frame is the + /// entry file / user input). Empty when the command lives + /// directly in the entry, since there's nothing to chain. + pub via: Vec, +} + +/// One frame in the `via` chain: a `src` statement in some +/// importing file, captured as that importer's path, the line the +/// `src` lives on, and the snippet of that line (so the user sees +/// `src ip/cips` and not just `src`). +#[derive(Clone, Debug)] +pub struct OriginFrame { + pub file: Option, + pub line: u32, + pub snippet: String, +} + +#[derive(Clone, Debug)] +pub struct PreparedCommand { + pub tcl: String, + pub origin: Origin, + /// Declared return type of the expression this command + /// evaluates, when knowable from static analysis. `None` for + /// expressions whose head we couldn't resolve to a known proc + /// (untyped calls, control flow, raw Tcl, etc.). The App uses + /// this to suppress the Result push entirely on `unit` and to + /// skip the heuristic fallback formatter on every other typed + /// case (since the wrapped `tcl` already returns a formatted + /// string from the type's `repr` proc). + pub expected_return_type: Option, + /// True when the top-level command is `set VAR ` — a + /// binding operation. The App suppresses the Result echo for + /// these: the user picked a name to hold the value; they + /// didn't ask to see it. `puts $VAR` is the explicit + /// "show me" form when they want the value displayed. + /// + /// Applies to the LITERAL `set` command only. `[set x y]` + /// inside a bracketed expression doesn't count — that's a + /// nested Tcl call, not a top-level binding. + pub is_set_binding: bool, +} + +#[derive(Debug)] +pub struct Prepared { + /// Each top-level statement in the loaded program, in source + /// order. The worker fires `eval` once per item and stops at + /// the first failure. + pub commands: Vec, + /// The parsed program + proc map for this batch. Stays out of + /// the session document until every command in [`commands`] + /// succeeds — at which point the App calls + /// [`Session::commit`](crate::session::Session::commit) to + /// fold it into the running session. On failure the batch is + /// dropped, which is what keeps a half-applied state from + /// polluting the analyzer. + pub batch: SessionBatch, + /// Pre-flight findings worth surfacing to the user *before* we + /// ship anything to Vivado. The most common one is "this call + /// uses `-flag` keyword args but the proc isn't a loaded htcl + /// wrapper" — Vivado's underlying builtin almost always parses + /// the arguments differently, and the resulting error message + /// makes no sense without that context. + pub warnings: Vec, + /// Top-level statements that lived directly in the entry file + /// (the user's `--load` target, or the typed REPL input), + /// regardless of whether they lowered to any Tcl. Captured so + /// the `--load` echo path can show `src` directives next to + /// the calls that produce Tcl — without this, `src @vivado-cmd` + /// would never get its `›` echo because its lowering is empty + /// (consumed at load time by the loader). + pub entry_top_level: Vec, +} + +#[derive(Clone, Debug)] +pub struct PrepareWarning { + pub origin: Origin, + pub message: String, +} + +/// Where a proc's body lives in htcl source. `body_start_line` is +/// the 1-based absolute line of the first body line in `file`; line +/// N of the proc's body is `body_start_line + N - 1` in `file` and +/// `body_lines[N - 1]` carries that line's text. +#[derive(Clone, Debug)] +pub struct ProcLocation { + pub file: Option, + pub body_start_line: u32, + pub body_lines: Vec, +} + +impl ProcLocation { + /// Resolve a 1-based body line into a renderable + /// (absolute_line, content) pair. Returns `None` when the + /// reported line is past the end of the body — happens when + /// Tcl points at a line we can't account for (synthesized + /// content, off-by-one in some wrapper, etc.); the caller + /// gracefully skips the frame. + pub fn resolve_body_line(&self, n: u32) -> Option<(u32, String)> { + let idx = n.checked_sub(1)? as usize; + let content = self.body_lines.get(idx).cloned()?; + Some((self.body_start_line + idx as u32, content)) + } +} + +pub fn prepare( + input: &str, + cwd: &Path, + session: &Session, +) -> Result { + let mut noop = NoopObserver; + prepare_with_observer(input, cwd, session, &mut noop) +} + +/// Same as [`prepare`], with an extra hook the loader fires per +/// parsed file. Used by the perf regression test to assert that a +/// new batch only parses its own content (plus any transitive +/// `src` imports), never the entire prior-session prelude. +pub fn prepare_with_observer( + input: &str, + cwd: &Path, + session: &Session, + observer: &mut dyn vw_htcl::LoadObserver, +) -> Result { + let workspace_dir = vw_lib::find_workspace_dir(cwd); + let resolver = build_resolver(workspace_dir.as_deref()); + + let scratch_dir = workspace_dir + .as_deref() + .map(Utf8Path::as_std_path) + .unwrap_or(cwd); + + // The scratch contains ONLY the new input — never a prepended + // prelude. Prior batches contribute parsed signatures and proc + // locations directly via `session`, so we never re-parse the + // entire session on each keystroke. This is what keeps the + // REPL responsive after several `src @lib` imports have built + // up hundreds of thousands of lines of wrapper declarations. + let scratch = ScratchFile::new(scratch_dir, input)?; + + // Seed the loader's already-loaded set from prior batches so + // an incoming `src ip/gtm` (already sourced by an earlier + // batch) short-circuits at each preloaded file instead of + // re-parsing thousands of transitive imports. Every REPL + // submit prior to this change re-parsed the full transitive + // tree from scratch — 879 vivado-cmd files can dominate + // wall-clock for tens of seconds even though the analyzer + // already has all of it in prior session batches. + let preloaded = session.loaded_paths(); + let program = vw_htcl::load_program_with_preloaded( + &scratch.path, + &resolver, + observer, + &preloaded, + )?; + let parsed = vw_htcl::parse(&program.source); + + if let Some(err) = parsed.errors.first() { + let idx = LineIndex::new(&program.source); + let (start, _) = idx.range(err.span); + let where_ = + render_location(&program, err.span, start.line + 1, &scratch.path); + return Err(LowerError::Parse(format!("{where_}: {}", err.message))); + } + + // Validator runs first so unknown-keyword-call errors land + // before we ship anything. Prior-batch signatures + types + + // enums + top-level var names are merged in so calls, type + // refs, and `$var` references to prior-batch state all + // resolve. These are hard errors (not pre-flight warnings); + // routing them back as `LowerError` keeps the App's existing + // error-rendering path unchanged. + let prior_sigs = session.signature_table(); + let prior_types = session.type_decl_table(); + let prior_vars = session.top_level_var_names(); + // Prior-batch variable TYPES for the putr rewrite. Without + // this seed, `putr $prior_var` at a fresh prompt would fall + // through to plain `puts` (the var came from a previous + // batch's `set`, invisible to the current parse alone) and + // dump the raw tagged Tcl list instead of dispatching + // through the type's `repr`. + let prior_var_types = session.top_level_var_types(); + + // Build the `putr` rewrite map: for every `putr ` + // command in the document, the value's replacement Tcl. The + // lowering consults this map per-command via + // `vw_htcl::lower_command_with_putr`. Empty when the source + // contained no `putr` calls; safe (and cheap) to build + // unconditionally. + let putr_map = vw_htcl::putr::rewrite_with_extras( + &program.source, + &parsed.document, + &prior_sigs, + &prior_var_types, + ); + // Names of every dep the workspace resolver knows about. + // Passed to the validator so `src @` where `` + // isn't in vw.toml fires a spanned Error diagnostic. + let dep_names: std::collections::HashSet = + resolver.deps().map(|(name, _)| name.to_string()).collect(); + let validator_diags = vw_htcl::validate_with_all_extras_and_vars( + &parsed.document, + &program.source, + &prior_sigs, + &prior_types, + &std::collections::HashMap::new(), + &prior_vars, + &dep_names, + ); + if let Some(first_err) = validator_diags + .iter() + .find(|d| matches!(d.severity, vw_htcl::Severity::Error)) + { + let idx = LineIndex::new(&program.source); + let (start, _) = idx.range(first_err.span); + let where_ = render_location( + &program, + first_err.span, + start.line + 1, + &scratch.path, + ); + return Err(LowerError::Parse(format!( + "{where_}: {}", + first_err.message + ))); + } + + // Build the lowering table by merging prior-batch signatures + // with the new doc's own. The new doc's entries shadow prior + // ones (Tcl's "second `proc` redefines" semantics) — done by + // starting from the prior table and `extend`-ing with the new + // doc's table, since `extend` overwrites on key collision. + let mut table = prior_sigs; + table.extend(vw_htcl::signature_table(&parsed.document)); + let line_index = LineIndex::new(&program.source); + // Parse the *raw* input (not `program.source`) to capture every + // top-level statement as the user wrote it, including `src` + // directives. The loader rewrites `src` into the imported file's + // content before parsing `program.source`, so the loader-expanded + // document no longer contains a Stmt::Command for `src @foo`. + // We need that statement to drive the `--load` echo path. + let entry_top_level: Vec = { + let entry_parsed = vw_htcl::parse(input); + let entry_idx = LineIndex::new(input); + let mut out = Vec::new(); + for stmt in &entry_parsed.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + let (line, _) = entry_idx.range(cmd.span); + let snippet = input[cmd.span.start as usize..cmd.span.end as usize] + .trim_end() + .to_string(); + out.push(Origin { + file: None, + line: line.line + 1, + snippet, + via: Vec::new(), + }); + } + out + }; + + let mut commands = Vec::new(); + let mut extern_names: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + + // Auto-emit machinery for enums + overload dispatchers. Both + // ship as synthetic PreparedCommand entries up front so the + // user's statements (which may construct enum values or call + // overloaded procs) find the supporting Tcl already in scope. + // The classification + overload-table build also re-runs the + // multi-decl signature collection — diagnostics from THAT pass + // already fired through the validator above, so we discard + // them here. + let mut _ignored_diags = Vec::new(); + let enum_decl_table = + vw_htcl::build_enum_decl_table(&parsed.document, &mut _ignored_diags); + // Merge prior-batch type declarations so wrap_with_repr can + // see newtypes declared in earlier `src @lib` batches (e.g. + // `type Properties = dict` from + // @vivado-cmd, when the user types + // `util::props -object $cips` at a later REPL prompt). + // Without this merge, the wrap can't recurse into + // Properties's underlying to ship + // `dict_string_Property::repr`, and the user's + // `Properties::repr` body fails with `invalid command + // name`. + let mut type_decl_table = session.type_decl_table(); + let batch_type_decls = + vw_htcl::build_type_decl_table(&parsed.document, &mut _ignored_diags); + for (name, td) in batch_type_decls { + type_decl_table.insert(name, td); + } + let newtype_names: std::collections::HashSet = + type_decl_table.keys().cloned().collect(); + let (_full_sig_table, overload_table) = + vw_htcl::build_signature_table_with_overloads( + &parsed.document, + &newtype_names, + &mut _ignored_diags, + ); + for ed in enum_decl_table.values() { + let prelude = vw_htcl::emit_enum_prelude(ed); + if prelude.is_empty() { + continue; + } + commands.push(PreparedCommand { + tcl: prelude, + origin: Origin { + file: None, + line: 0, + snippet: format!( + "", + ed.name.as_deref().unwrap_or("?") + ), + via: Vec::new(), + }, + expected_return_type: None, + is_set_binding: false, + }); + } + for info in overload_table.values() { + let dispatcher = vw_htcl::emit_dispatcher(info); + commands.push(PreparedCommand { + tcl: dispatcher, + origin: Origin { + file: None, + line: 0, + snippet: format!("", info.public_name), + via: Vec::new(), + }, + expected_return_type: None, + is_set_binding: false, + }); + } + + // Eagerly emit the primitive repr prelude (string/int/bool/ + // unit) so user procs that call e.g. `extern::string::repr` + // from inside their bodies see those procs in scope. Without + // this, the primitives are only emitted by `wrap_with_repr` + // at top-level REPL eval sites, leaving inner uses dead. + for proc in vw_htcl::repr::emit_primitive_prelude() { + commands.push(PreparedCommand { + tcl: proc, + origin: Origin { + file: None, + line: 0, + snippet: "".into(), + via: Vec::new(), + }, + expected_return_type: None, + is_set_binding: false, + }); + } + + // Eagerly emit monomorphized generic reprs for every declared + // type alias whose underlying is a generic + // (`dict<…>` / `list<…>`). Without this, a user-written + // `T::repr` body that delegates to the compiler-synthesized + // monomorphized name (e.g. `Properties::repr` calling + // `extern::dict_string_Property::repr`) errors at runtime + // when invoked from inside a proc body — `wrap_with_repr` + // only emits the monomorphization chain at top-level REPL + // eval sites, not for inner uses. By emitting here, the + // procs are in scope everywhere within the session. + // + // Dedup-by-text within the batch prevents shipping the same + // monomorphization more than once when two type aliases + // resolve to the same underlying generic. + let mut emitted_mono_reprs: std::collections::HashSet = + std::collections::HashSet::new(); + for td in type_decl_table.values() { + let Some(underlying) = td.underlying.as_ref() else { + continue; + }; + if !matches!(underlying, vw_htcl::TypeExpr::Generic { .. }) { + continue; + } + let emission = + vw_htcl::repr::emit_repr_with_types(underlying, &type_decl_table); + for proc in emission.procs { + if !emitted_mono_reprs.insert(proc.clone()) { + continue; + } + commands.push(PreparedCommand { + tcl: proc, + origin: Origin { + file: None, + line: 0, + snippet: format!( + "", + td.name.as_deref().unwrap_or("?") + ), + via: Vec::new(), + }, + expected_return_type: None, + is_set_binding: false, + }); + } + } + + for stmt in &parsed.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + let (line_one_based, _) = line_index.range(cmd.span); + let origin = build_origin( + &program, + cmd.span, + line_one_based.line + 1, + &scratch.path, + ); + // If this command is a proc that's been classified as an + // overload specialization, lower it under its mangled name + // so the dispatcher (shipped above) can find it. Otherwise + // take the normal path. + let lowered_raw = + match overload_specialization_mangle(cmd, &overload_table) { + Some(mangled) => { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + unreachable!() + }; + vw_htcl::lower_proc_decl_with_name_and_index( + proc, + &program.source, + &table, + Some(&mangled), + &putr_map, + &line_index, + ) + } + None => vw_htcl::lower_command_with_putr_and_index( + cmd, + &program.source, + &table, + &putr_map, + &line_index, + ), + }; + let rewritten = vw_htcl::rewrite_externs(&lowered_raw); + for name in rewritten.names { + extern_names.insert(name); + } + if rewritten.text.trim().is_empty() { + continue; + } + // Resolve the command's expected return type and, if any, + // wrap the lowered Tcl so it dispatches through the type's + // `repr` proc. The wrapped form runs the user's expression + // into a sentinel local then formats via the repr; the + // sentinel-binding step preserves `set var [...]`-style + // bindings (the user's `$var` still gets the raw value). + let expected_return_type = resolve_return_type(cmd, &table); + let final_tcl = match expected_return_type.as_ref() { + Some(ty) => wrap_with_repr(&rewritten.text, ty, &type_decl_table), + None => rewritten.text, + }; + let is_set_binding = matches!(cmd.kind, vw_htcl::CommandKind::Set); + commands.push(PreparedCommand { + tcl: final_tcl, + origin, + expected_return_type, + is_set_binding, + }); + } + + // No prelude needed in the current architecture: wrappers + // live in the `vivado::` namespace and `extern::name` rewrites + // to `::name`, which Tcl resolves at the global root regardless + // of the calling namespace. We still drain `extern_names` so + // the analyzer can grow future per-extern bookkeeping without + // rewiring this path. + let _ = extern_names; + + let procs = build_proc_locations(&parsed.document, &program, &scratch.path); + // The dedicated pre-flight `collect_warnings` is gone — the + // validator now treats "unknown call with `-flag` args" as a + // hard error and the REPL has already returned via `LowerError` + // above when one fires. + let warnings: Vec = Vec::new(); + + Ok(Prepared { + commands, + batch: SessionBatch { + program, + document: parsed.document, + procs, + }, + warnings, + entry_top_level, + }) +} + +/// Walk every proc declaration (top-level + nested inside +/// `namespace eval` blocks) and record its body's source location +/// keyed by the proc's qualified name. Same recursion shape as +/// `vw_htcl::validate::collect_signatures` — kept in sync by +/// convention rather than refactor so this crate stays a leaf +/// consumer of vw-htcl. +pub fn build_proc_locations( + doc: &vw_htcl::Document, + program: &vw_htcl::LoadedProgram, + scratch_path: &Path, +) -> std::collections::HashMap { + use std::collections::HashMap; + let mut out: HashMap = HashMap::new(); + collect_procs(&doc.stmts, "", program, scratch_path, &mut out); + out +} + +fn collect_procs( + stmts: &[vw_htcl::Stmt], + prefix: &str, + program: &vw_htcl::LoadedProgram, + scratch_path: &Path, + out: &mut std::collections::HashMap, +) { + use vw_htcl::CommandKind; + for stmt in stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + if let Some(loc) = + proc_body_location(program, proc.body_span, scratch_path) + { + out.insert(qualified, loc); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + let nested = qualify(prefix, name); + collect_procs(&ns.body, &nested, program, scratch_path, out); + } + _ => {} + } + } +} + +fn qualify(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + } +} + +fn proc_body_location( + program: &vw_htcl::LoadedProgram, + body_span: vw_htcl::Span, + scratch_path: &Path, +) -> Option { + let (file_index, file_span) = program.locate_span(body_span)?; + let file = &program.files[file_index]; + let file_path = if file.path == scratch_path { + None + } else { + Some(file.path.clone()) + }; + // Tcl's `(procedure "X" line N)` counts the line **containing + // the opening `{`** as line 1, the next line as line 2, etc. + // `body_span.start` is the byte right after the `{`, so the + // `{` itself sits at `file_span.start - 1`. The line at that + // byte is what Tcl calls "line 1." When the proc body is on a + // single line (`proc f {x} {puts $x}`) that line is also the + // content line. + let brace_pos = file_span.start.saturating_sub(1); + let body_start_line = file_line_at(&file.source, brace_pos); + // For the body_lines vector we want every file line from the + // one with the `{` up to (and including) the one with the + // matching `}` — so `resolve_body_line(N)` returns the + // corresponding source. Anything past the body is irrelevant. + let body_end_line = + file_line_at(&file.source, file_span.end.saturating_sub(1)); + let body_lines: Vec = file + .source + .lines() + .skip(body_start_line.saturating_sub(1) as usize) + .take((body_end_line - body_start_line + 1) as usize) + .map(str::to_string) + .collect(); + Some(ProcLocation { + file: file_path, + body_start_line, + body_lines, + }) +} + +/// Return the declared return type of `cmd`'s head call, when we +/// can resolve it from the signature table. Currently handles two +/// shapes: +/// +/// - Direct call: `proc-name arg arg …` → look up `proc-name`'s +/// return type in the table. +/// - Bracket-bound assignment: `set var [proc-name …]` → look up +/// the inner bracketed call's return type (since `set` returns +/// the value being set, which is the type of the inner call). +/// +/// Anything else (control flow, variable substitution, raw Tcl, +/// unknown commands) returns `None`. The App falls back to the +/// untyped-display path for those. +fn resolve_return_type( + cmd: &vw_htcl::ast::Command, + table: &std::collections::HashMap, +) -> Option { + let head = cmd.words.first()?.as_text()?; + if head == "set" { + // `set var [EXPR]` → recurse into the bracketed + // expression on the third word (words[2]). Other `set` + // shapes (set var literal, set var $other) leave the + // type unknown — we'd need real expression type-inference + // to do better, and that's out of scope for v1. + let val_word = cmd.words.get(2)?; + // Look for a CmdSubst part — `[…]` — at the top of the + // value word. If found, recurse into the bracketed + // command's first statement. + for part in &val_word.parts { + if let vw_htcl::WordPart::CmdSubst { body, .. } = part { + let vw_htcl::Stmt::Command(inner) = body.first()? else { + continue; + }; + return resolve_return_type(inner, table); + } + } + return None; + } + let sig = table.get(head)?; + sig.return_type.clone() +} + +/// Wrap the lowered Tcl `inner` so that, after evaluating it, the +/// result is fed through `::repr` (or the appropriate +/// monomorphized generic repr) to produce a display string. +/// +/// Prepends: +/// 1. The primitive prelude (`string` / `int` / `bool` / `unit` +/// triplets) — cheap to redefine per-eval; Tcl `proc` +/// redefinition is idempotent. +/// 2. Any per-instantiation generic reprs needed for `ty`, in +/// topological order so each proc is defined before its +/// dependents call it. +/// 3. `set __vw_result []` — captures the user expression's +/// raw value into a sentinel local. This preserves any +/// `set var [...]` bindings the user wrote, since `set`'s +/// side effect runs before our sentinel-capture wraps it. +/// 4. ` $__vw_result` — calls the type's repr proc on +/// the captured value. The eval returns this formatted string. +fn wrap_with_repr( + inner: &str, + ty: &vw_htcl::TypeExpr, + types: &std::collections::HashMap, +) -> String { + use std::fmt::Write; + let mut out = String::new(); + for p in vw_htcl::repr::emit_primitive_prelude() { + out.push_str(&p); + } + // Walks the dispatch type's underlying when `ty` is a newtype + // — necessary for `Properties` (newtype wrapping + // `dict`) so the body of `Properties::repr` + // can call the monomorphized `dict_string_Property::repr`. + let emission = vw_htcl::repr::emit_repr_with_types(ty, types); + for p in &emission.procs { + out.push_str(p); + } + writeln!(out, "set __vw_result [{}]", inner.trim_end()) + .expect("writeln to String never fails"); + // All reprs (compiler-emitted primitives + generics + user- + // written newtype reprs + auto-generated enum reprs) share a + // single `{args}` envelope that uses `::vw::kwargs` to bind + // `$v`. The dispatch site always calls them as + // ` -v ` so the kwargs envelope binds + // uniformly regardless of which class of repr is being + // invoked. + write!(out, "{} -v $__vw_result", emission.dispatch) + .expect("write to String never fails"); + out +} + +/// If `cmd` is a top-level `proc` whose name appears in the +/// overload table AND whose first arg is a qualified-variant +/// annotation, return the mangled internal name that this +/// specialization should lower under. Otherwise `None`. +/// +/// This is what reroutes user-written `proc handle_prop {v: +/// Property::Scalar} { … }` from emitting under the literal +/// `handle_prop` name (which would collide with the synthesized +/// dispatcher) to emitting under `__handle_prop__Scalar` (which +/// the dispatcher's switch arm calls). +fn overload_specialization_mangle( + cmd: &vw_htcl::Command, + overloads: &vw_htcl::OverloadTable, +) -> Option { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + return None; + }; + let name = proc.name.as_deref()?; + if !overloads.contains_key(name) { + return None; + } + let sig = proc.signature.as_ref()?; + let first = sig.args.first()?; + let vw_htcl::TypeExpr::Qualified { variant, .. } = + first.type_annotation.as_ref()? + else { + return None; + }; + Some(vw_htcl::mangle_specialization(name, variant)) +} + +fn build_origin( + program: &vw_htcl::LoadedProgram, + span: vw_htcl::Span, + flat_line: u32, + scratch_path: &Path, +) -> Origin { + // Full span — for a multi-line `set proj [ … ]` the snippet + // includes every line of the command so the trace shows what + // the user actually wrote, not just `set proj [`. The renderer + // is responsible for indenting continuation lines. + let snippet = program.source[span.start as usize..span.end as usize] + .trim_end() + .to_string(); + + if let Some((file_index, file_span)) = program.locate_span(span) { + let file = &program.files[file_index]; + let file_path = if file.path == scratch_path { + None + } else { + Some(file.path.clone()) + }; + let file_line = file_line_at(&file.source, file_span.start); + let via = build_via_chain(program, file_index, scratch_path); + return Origin { + file: file_path, + line: file_line, + snippet, + via, + }; + } + Origin { + file: None, + line: flat_line, + snippet, + via: Vec::new(), + } +} + +/// Walk the loader's import chain from the leaf file back toward +/// the entry, turning each [`vw_htcl::ImportEdge`] into a renderable +/// frame. Nearest first. +fn build_via_chain( + program: &vw_htcl::LoadedProgram, + leaf_file: usize, + scratch_path: &Path, +) -> Vec { + program + .ancestry(leaf_file) + .map(|edge| { + let importer = &program.files[edge.importer_file]; + let line = file_line_at(&importer.source, edge.src_span.start); + let snippet = first_line( + &importer.source, + edge.src_span.start as usize, + edge.src_span.end as usize, + ); + OriginFrame { + file: if importer.path == scratch_path { + None + } else { + Some(importer.path.clone()) + }, + line, + snippet, + } + }) + .collect() +} + +fn first_line(source: &str, start: usize, end: usize) -> String { + let line_end = source[start..].find('\n').map(|n| start + n).unwrap_or(end); + source[start..line_end].trim().to_string() +} + +fn file_line_at(source: &str, offset: u32) -> u32 { + let upto = offset.min(source.len() as u32) as usize; + 1 + source[..upto].bytes().filter(|b| *b == b'\n').count() as u32 +} + +fn render_location( + program: &vw_htcl::LoadedProgram, + span: vw_htcl::Span, + flat_line: u32, + scratch_path: &Path, +) -> String { + if let Some((file_index, file_span)) = program.locate_span(span) { + let file = &program.files[file_index]; + if file.path != scratch_path { + let line = file_line_at(&file.source, file_span.start); + return format!("{}:{line}", file.path.display()); + } + } + format!("(input):{flat_line}") +} + +fn build_resolver(workspace_dir: Option<&Utf8Path>) -> Resolver { + let mut resolver = Resolver::new(); + let Some(ws) = workspace_dir else { + return resolver; + }; + if let Ok(paths) = vw_lib::transitive_dep_cache_paths(ws) { + for (name, path) in paths { + resolver = resolver.with_dep(name, path); + } + } + resolver +} + +struct ScratchFile { + path: PathBuf, +} + +impl ScratchFile { + fn new(dir: &Path, contents: &str) -> std::io::Result { + let name = format!(".vw-repl-input-{}.htcl", std::process::id()); + let path = dir.join(name); + let mut f = std::fs::File::create(&path)?; + f.write_all(contents.as_bytes())?; + Ok(Self { path }) + } +} + +impl Drop for ScratchFile { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_session() -> Session { + Session::new() + } + + /// User-statement commands only — strips the synthetic + /// prelude entries (enum reprs, overload dispatchers, + /// primitive reprs, monomorphized generic reprs) the + /// preparer ships before each batch. Tests that assert + /// command count / shape only care about what the user + /// wrote, not the prelude scaffolding. + fn user_commands(prep: &Prepared) -> Vec<&PreparedCommand> { + prep.commands + .iter() + .filter(|c| !c.origin.snippet.starts_with('<')) + .collect() + } + + #[test] + fn unknown_keyword_call_inside_bracket_errors() { + // Mirrors the metroid project.htcl shape: a call to an + // unknown proc with keyword args, nested inside a `[ … ]` + // substitution. The validator now treats this as a hard + // error so the lowering returns `Err` and nothing ships + // to Vivado — the user is forced to either `src` a + // wrapper module or write `extern::create_project`. + let dir = tempfile::tempdir().unwrap(); + let err = prepare( + "set proj [\n create_project\n -in_memory 1\n -name foo\n]\n", + dir.path(), + &empty_session(), + ) + .unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("create_project"), "{msg}"); + assert!(msg.contains("extern::"), "{msg}"); + } + + #[test] + fn extern_prefixed_call_is_accepted() { + // The opt-out: `extern::create_project` is explicitly a + // raw Tcl call, no wrapper required. Lowering strips the + // prefix so the bare native resolves through Tcl's global + // namespace at runtime — no rename plumbing, no prelude. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "extern::create_project -name foo\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 1, "{:?}", cmds); + assert!( + cmds[0].tcl.contains("create_project -name foo"), + "{}", + cmds[0].tcl + ); + assert!(!cmds[0].tcl.contains("extern::"), "{}", cmds[0].tcl); + } + + #[test] + fn prior_batch_procs_resolve_in_next_batch() { + // Reproduces the REPL "src @lib then call" pattern: a + // wrapper declared in a previous batch should be visible + // to the analyzer/lowering when we lower a bare call in + // the next batch — and the new batch should ship only + // its own statement (not re-emit the wrapper). + let dir = tempfile::tempdir().unwrap(); + let mut session = Session::new(); + // Batch 1: declare the wrapper. Commit so it joins the + // session — same flow the App follows on successful eval. + let first = prepare( + "namespace eval vivado {\n \ + proc current_project {\n \ + @enum(0, 1) @default(0) quiet\n \ + @enum(0, 1) @default(0) verbose\n \ + @default(\"\") project\n \ + } string {\n \ + set cmd [list ::current_project]\n \ + return [{*}$cmd]\n \ + }\n\ + }\n", + dir.path(), + &session, + ) + .unwrap(); + // The first batch ships its own declaration to the worker + // exactly once — that's what makes the wrapper exist in + // Tcl. Subsequent batches must NOT re-emit it. + assert!( + first + .commands + .iter() + .any(|c| c.tcl.contains("namespace eval")), + "first batch must ship the namespace decl: {:?}", + first.commands + ); + session.commit(first.batch); + + // Batch 2: bare call to the wrapper. Should ship as-is + // (htcl is keyword-only at the call site; the wrapper + // parses its own kwargs at runtime via the ::vw::kwargs + // prelude), with no rewriting and no re-emission of the + // prior batch's declaration. + let prep = + prepare("vivado::current_project\n", dir.path(), &session).unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 1, "{:?}", cmds); + assert!( + cmds[0].tcl.contains("vivado::current_project"), + "{}", + cmds[0].tcl + ); + // And nothing in the new batch's source mentions the + // wrapper body — we never re-parsed the prior batch. + assert!( + !prep.batch.program.source.contains("namespace eval vivado"), + "{}", + prep.batch.program.source + ); + } + + #[test] + fn known_keyword_call_is_not_errored() { + // When the called proc IS in scope, no error fires. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "proc create_project { @default(\"\") name } { }\n\ + set proj [ create_project -name foo ]\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + assert!(prep.warnings.is_empty(), "{:?}", prep.warnings); + } + + #[test] + fn lowers_plain_proc_call_to_tcl() { + let dir = tempfile::tempdir().unwrap(); + let prep = prepare("puts hello", dir.path(), &empty_session()).unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 1); + assert!(cmds[0].tcl.contains("puts hello")); + // Input is at line 1 of the buffer. + assert_eq!(cmds[0].origin.line, 1); + assert!(cmds[0].origin.file.is_none()); + assert_eq!(cmds[0].origin.snippet, "puts hello"); + } + + #[test] + fn each_statement_gets_its_own_origin() { + let dir = tempfile::tempdir().unwrap(); + let prep = + prepare("set x 1\nset y 2\nset z 3", dir.path(), &empty_session()) + .unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 3); + assert_eq!(cmds[0].origin.line, 1); + assert_eq!(cmds[1].origin.line, 2); + assert_eq!(cmds[2].origin.line, 3); + } + + #[test] + fn proc_body_line_resolution_matches_tcl_line_counting() { + // Tcl counts the proc-body line **containing the opening + // `{`** as line 1 — so a `(procedure "ip::check" line 2)` + // frame should point at the first content line of the body, + // not the line after it. + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("dep"); + std::fs::create_dir_all(&dep).unwrap(); + // Lines 1-2: blank + the namespace header; line 3 has `{` + // (the proc body opener); content lives on lines 4+. + std::fs::write( + dep.join("module.htcl"), + "namespace eval foo {\n proc bar {} {\n puts hi\n error oh-no\n }\n}\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.dep]\npath = \"{}\"\n", + dep.display() + ), + ) + .unwrap(); + let prep = prepare("src @dep", dir.path(), &empty_session()).unwrap(); + let loc = prep + .batch + .procs + .get("foo::bar") + .expect("expected foo::bar in proc map"); + // The proc body opens on file line 2 (the `} {`-style line + // here is just `proc bar {} {`), so Tcl line 1 → line 2 of + // the file. + assert_eq!(loc.body_start_line, 2); + // Tcl line 2 → file line 3 → `puts hi`. + let (line, content) = loc.resolve_body_line(2).unwrap(); + assert_eq!(line, 3); + assert_eq!(content.trim(), "puts hi"); + // Tcl line 3 → file line 4 → `error oh-no`. + let (line, content) = loc.resolve_body_line(3).unwrap(); + assert_eq!(line, 4); + assert_eq!(content.trim(), "error oh-no"); + } + + #[test] + fn origin_via_chain_walks_back_through_src_imports() { + // entry → mid → leaf, all via `src`. A command in `leaf` + // should carry a 2-frame via chain (mid → entry/input). + let dir = tempfile::tempdir().unwrap(); + let mid_dep = dir.path().join("mid_dep"); + let leaf_dep = dir.path().join("leaf_dep"); + std::fs::create_dir_all(&mid_dep).unwrap(); + std::fs::create_dir_all(&leaf_dep).unwrap(); + std::fs::write(leaf_dep.join("module.htcl"), "set leaf_var 1\n") + .unwrap(); + std::fs::write(mid_dep.join("module.htcl"), "src @leaf\n").unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.mid]\npath = \"{}\"\n\ + [dependencies.leaf]\npath = \"{}\"\n", + mid_dep.display(), + leaf_dep.display() + ), + ) + .unwrap(); + + let prep = prepare("src @mid", dir.path(), &empty_session()).unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 1); + let origin = &cmds[0].origin; + // Leaf-most command lives in leaf_dep/module.htcl. + assert!( + origin + .file + .as_ref() + .unwrap() + .ends_with("leaf_dep/module.htcl"), + "{:?}", + origin.file + ); + // The via chain: leaf was imported by mid (line 1), and mid + // was imported by the entry input (line 1). + assert_eq!(origin.via.len(), 2, "{:?}", origin.via); + assert!(origin.via[0] + .file + .as_ref() + .unwrap() + .ends_with("mid_dep/module.htcl")); + assert_eq!(origin.via[0].snippet, "src @leaf"); + // The outermost frame is the user's input (file = None). + assert!(origin.via[1].file.is_none(), "{:?}", origin.via[1].file); + assert_eq!(origin.via[1].snippet, "src @mid"); + } + + #[test] + fn src_imported_statements_resolve_to_imported_file() { + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("dep"); + std::fs::create_dir_all(&dep).unwrap(); + std::fs::write( + dep.join("module.htcl"), + "proc hello {} { puts world }\nhello\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.dep]\npath = \"{}\"\n", + dep.display() + ), + ) + .unwrap(); + + let prep = prepare("src @dep", dir.path(), &empty_session()).unwrap(); + // Two commands from the imported file: `proc hello` and the + // bare `hello` call. Both must carry the imported file's + // path as origin. + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 2); + for cmd in &cmds { + let file = cmd.origin.file.as_ref().expect("import has file"); + assert!(file.ends_with("dep/module.htcl"), "{:?}", file); + } + // Line numbers point into the imported file. + assert_eq!(cmds[0].origin.line, 1); + assert_eq!(cmds[1].origin.line, 2); + } + + #[test] + fn second_batch_parses_only_its_own_files() { + // Regression guard against the lag bug: after `src @dep` + // commits, a subsequent bare call must NOT cause the + // loader to re-parse the dep's files. We assert by hooking + // the loader's per-file observer and counting parses on + // each batch. + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("dep"); + std::fs::create_dir_all(&dep).unwrap(); + std::fs::write( + dep.join("module.htcl"), + "namespace eval lib {\n \ + proc f { @default(0) x } int { return $x }\n\ + }\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.dep]\npath = \"{}\"\n", + dep.display() + ), + ) + .unwrap(); + + #[derive(Default)] + struct Counter { + parsed: Vec, + } + impl vw_htcl::LoadObserver for Counter { + fn on_parsed(&mut self, file: &Path, _raw: Option<&str>) { + self.parsed.push(file.to_path_buf()); + } + } + + let mut session = Session::new(); + + // First batch: imports the dep. Two files parse — the + // entry scratch and the dep's module.htcl. + let mut counter = Counter::default(); + let first = prepare_with_observer( + "src @dep\n", + dir.path(), + &session, + &mut counter, + ) + .unwrap(); + assert_eq!( + counter.parsed.len(), + 2, + "first batch should parse entry + dep, got {:?}", + counter.parsed + ); + session.commit(first.batch); + + // Second batch: bare call to the wrapper. The prior + // batch's signatures are merged in via `session`, so the + // loader must NOT re-read the dep's file — only the new + // scratch parses. + let mut counter = Counter::default(); + let _second = prepare_with_observer( + "lib::f -x 1\n", + dir.path(), + &session, + &mut counter, + ) + .unwrap(); + assert_eq!( + counter.parsed.len(), + 1, + "second batch should parse only the new input, got {:?}", + counter.parsed + ); + // And the one file parsed is the scratch, not the dep. + let only = &counter.parsed[0]; + assert!( + !only.starts_with(&dep), + "the dep's files must not be re-parsed on a fresh \ + batch: {:?}", + only + ); + } + + #[test] + fn prior_batch_proc_location_survives_for_drilldown() { + // The user-reported bug: `src @vivado-cmd` declares + // `vivado::create_bd_design` in batch A, then a later + // `vivado::create_bd_design -name metroid` fires in batch + // B and the Tcl error frame names that proc. The + // proc-location lookup must resolve to the REAL .htcl + // file the wrapper came from — not the disposable scratch + // path of either batch. + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("vivado_cmd"); + std::fs::create_dir_all(&dep).unwrap(); + std::fs::write( + dep.join("module.htcl"), + "namespace eval vivado {\n \ + proc create_bd_design {\n \ + @default(\"\") name\n \ + } string {\n \ + set cmd [list ::create_bd_design]\n \ + return [{*}$cmd]\n \ + }\n\ + }\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.vivado-cmd]\npath = \"{}\"\n", + dep.display() + ), + ) + .unwrap(); + + let mut session = Session::new(); + // Batch A: pull the wrapper in. + let first = prepare("src @vivado-cmd\n", dir.path(), &session).unwrap(); + session.commit(first.batch); + + // Batch B: call the wrapper. Look up its location through + // the session — which is exactly the path the App's error + // renderer takes when resolving a Tcl drill-down frame. + let _second = prepare( + "vivado::create_bd_design -name metroid\n", + dir.path(), + &session, + ) + .unwrap(); + let loc = session.lookup_proc("vivado::create_bd_design").expect( + "wrapper from a prior `src @vivado-cmd` batch must be \ + reachable through session.lookup_proc", + ); + // The crucial assertion: the file pointer is the REAL + // imported .htcl, not `None` (the scratch) and not some + // huge synthetic offset. + let file = loc.file.as_ref().expect( + "wrapper from imported module must carry its real \ + .htcl path, not the disposable scratch", + ); + assert!( + file.ends_with("vivado_cmd/module.htcl"), + "expected the imported module path, got {:?}", + file + ); + // And `body_start_line` is the file-local line of the + // proc body opener — small, not a combined-scratch offset. + assert!( + loc.body_start_line < 100, + "body_start_line should be a small file-local number, \ + got {}", + loc.body_start_line + ); + } + + // --- typed-expression wrap (step 3) ---------------------------- + + #[test] + fn typed_proc_call_wraps_with_repr_dispatch() { + let dir = tempfile::tempdir().unwrap(); + // A proc annotated dict, called bare. The + // wrap should: + // - capture the call's result into __vw_result + // - invoke the monomorphized dict repr proc on it + // PreparedCommand.expected_return_type carries the type. + let prep = prepare( + "proc props {} dict { return {} }\n\ + props\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + // Two commands: proc decl (drops to empty Tcl) + call. + // proc decl ships as a regular command; the call should + // be wrapped. + let call = prep + .commands + .iter() + .find(|c| c.tcl.contains("__vw_result")) + .expect("expected the `props` call to be repr-wrapped"); + assert!( + call.tcl.contains("set __vw_result [props]"), + "tcl: {}", + call.tcl + ); + assert!( + call.tcl + .contains("dict_string_string::repr -v $__vw_result"), + "tcl: {}", + call.tcl + ); + // Primitive prelude is included so the dict repr's + // element calls (string::repr) resolve. Both the + // primitive procs and the monomorphized generic procs + // are wrapped in explicit `namespace eval` blocks so + // Tcl's namespace-conflict heuristic doesn't reject the + // declaration (the bare `proc string::repr` form trips + // over Tcl's built-in `string` command). + assert!( + call.tcl.contains("namespace eval string"), + "expected primitive prelude in wrapped tcl: {}", + call.tcl + ); + // Plus the dict repr itself. + assert!( + call.tcl.contains("namespace eval dict_string_string"), + "expected monomorphized dict repr: {}", + call.tcl + ); + // The expected_return_type rides along for App-side use. + let ty = call + .expected_return_type + .as_ref() + .expect("expected_return_type set"); + match ty { + vw_htcl::TypeExpr::Generic { name, args, .. } => { + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + } + _ => panic!("expected Generic, got {:?}", ty), + } + } + + #[test] + fn set_var_call_inherits_inner_return_type() { + // `set cips [props]` — `set` returns the value being set, + // so its type is whatever `props` returns. The wrap should + // bind `$cips` correctly AND dispatch on the inner call's + // declared type. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "proc props {} dict { return {} }\n\ + set x [props]\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + let set_cmd = prep + .commands + .iter() + .find(|c| c.tcl.contains("__vw_result")) + .expect("expected the `set x [...]` to be repr-wrapped"); + assert!( + set_cmd.tcl.contains("set __vw_result [set x [props]]"), + "expected the original set to be inner-wrapped: {}", + set_cmd.tcl + ); + assert!( + set_cmd + .tcl + .contains("dict_string_string::repr -v $__vw_result"), + "tcl: {}", + set_cmd.tcl + ); + } + + #[test] + fn unannotated_call_is_not_wrapped() { + // No return type → no wrap, no `__vw_result` capture, + // and `expected_return_type` is None. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "proc plain {} { puts whatever }\n\ + plain\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + let plain_call = prep + .commands + .iter() + .find(|c| c.tcl.trim() == "plain") + .expect("expected raw `plain` call without wrap"); + assert!(plain_call.expected_return_type.is_none()); + assert!( + !plain_call.tcl.contains("__vw_result"), + "unannotated calls shouldn't get the repr wrap: {}", + plain_call.tcl + ); + } + + #[test] + fn unit_typed_call_is_wrapped_with_unit_dispatch() { + // `unit`-typed expressions still get wrapped — the wrap + // returns the empty string from `unit::repr`. The App's + // EvalDone handler is what suppresses the Result push; + // the lowerer is uniform. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "proc do_thing {} unit { puts hi }\n\ + do_thing\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + let call = prep + .commands + .iter() + .find(|c| c.tcl.contains("__vw_result")) + .expect("expected repr-wrap on unit-typed call"); + assert!(call.tcl.contains("unit::repr -v $__vw_result")); + let ty = call.expected_return_type.as_ref().unwrap(); + match ty { + vw_htcl::TypeExpr::Named { name, .. } => { + assert_eq!(name, "unit"); + } + _ => panic!(), + } + } + + // --- enum / overload pipeline (step 5) ------------------------- + + #[test] + fn enum_decl_ships_namespace_eval_prelude() { + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "enum Direction = {\n North\n South\n}\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + // The prelude is shipped as a synthetic PreparedCommand. + let prelude = prep + .commands + .iter() + .find(|c| c.tcl.contains("namespace eval Direction")) + .expect("expected enum prelude in prepared commands"); + assert!(prelude.tcl.contains("proc North {}")); + assert!(prelude.tcl.contains("proc South {}")); + assert!(prelude.tcl.contains("proc tag {v}")); + assert!(prelude.tcl.contains("proc payload {v}")); + assert!(prelude.tcl.contains("proc repr {args}")); + } + + #[test] + fn overload_set_ships_dispatcher_and_mangled_specializations() { + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "enum E = {\n A: string\n B: int\n}\n\ + proc f {v: E::A} string { return $v }\n\ + proc f {v: E::B} string { return $v }\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + // Dispatcher emitted with the switch body. The dispatcher + // takes the standard kwargs envelope (`{args}`), walks + // kwargs for `-v `, then switches on the + // tag. + let dispatcher = prep + .commands + .iter() + .find(|c| { + c.tcl.contains("proc f {args}") + && c.tcl.contains("switch") + && c.tcl.contains("__f__") + }) + .expect("expected dispatcher for `f`"); + assert!(dispatcher.tcl.contains("__f__A")); + assert!(dispatcher.tcl.contains("__f__B")); + // Specializations emitted under mangled names — look for + // the `proc __f__A` declaration (rather than `proc f`). + assert!( + prep.commands + .iter() + .any(|c| c.tcl.contains("proc __f__A {args}")), + "expected specialization under __f__A: tcls={:?}", + prep.commands.iter().map(|c| &c.tcl).collect::>() + ); + assert!( + prep.commands + .iter() + .any(|c| c.tcl.contains("proc __f__B {args}")), + "expected specialization under __f__B" + ); + // The user-visible name `f` should NOT appear as a + // user-procedure declaration — only as the dispatcher. + // (The dispatcher's body has `proc f {v args}` which we + // already accounted for above; what we're guarding + // against is a leaked `proc f {args} { ::vw::kwargs ... }` + // specialization.) + let leaked_f = prep + .commands + .iter() + .filter(|c| c.tcl.contains("proc f {args} { ::vw::kwargs")) + .count(); + assert_eq!( + leaked_f, 0, + "specialization should NOT have leaked under public name `f`" + ); + } + + #[test] + fn cross_batch_newtype_recursion_emits_generic_repr() { + // Reproduces the user-reported regression: batch 1 + // declares `type Properties = dict` and a + // proc returning Properties; batch 2 calls that proc. + // The wrap_with_repr in batch 2 should walk Properties's + // underlying (the dict generic) and emit + // `dict_string_string::repr` so the user's + // `Properties::repr` body can find it. + // + // Pre-fix: type_decl_table was per-batch, so batch 2 + // couldn't see Properties; the recursion didn't fire; + // dict_string_string::repr was never emitted; the + // user's body errored with `invalid command name`. + let dir = tempfile::tempdir().unwrap(); + let mut session = Session::new(); + let first = prepare( + "type Properties = {dict}\n\ + proc Properties::repr {v} { return $v }\n\ + proc Properties::from {v} { return $v }\n\ + proc Properties::to {v} { return $v }\n\ + proc get_props {} Properties { return {a 1 b 2} }\n", + dir.path(), + &session, + ) + .unwrap(); + session.commit(first.batch); + + // Batch 2: just the call. parsed.document doesn't have + // the type decl — it must come from `session`. + let second = prepare("get_props\n", dir.path(), &session).unwrap(); + let call = second + .commands + .iter() + .find(|c| c.tcl.contains("__vw_result")) + .expect("expected wrapped call to get_props"); + // The wrap must include the monomorphized dict repr + // (reached by recursing through Properties's underlying). + assert!( + call.tcl.contains("namespace eval dict_string_string"), + "expected dict_string_string::repr in wrap (newtype \ + recursion across batches): {}", + call.tcl + ); + // And the top-level dispatch goes through Properties::repr + // with the `-v` form, not positional. + assert!( + call.tcl.contains("Properties::repr -v $__vw_result"), + "expected Properties::repr dispatch via -v form: {}", + call.tcl + ); + } +} diff --git a/vw-repl/src/popup.rs b/vw-repl/src/popup.rs new file mode 100644 index 0000000..476553c --- /dev/null +++ b/vw-repl/src/popup.rs @@ -0,0 +1,758 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Overlay popups attached to the input editor — completion (slice 4), +//! signature help (slice 5), hover (slice 6). +//! +//! Each popup is a [`PopupState`] variant held on the App as +//! `Option`. The key handler in +//! [`crate::app::App::handle_terminal_event`] intercepts navigation +//! and dismissal keys when a popup is active, BEFORE the catch-all +//! editor handoff — so Up/Down/Enter/Esc go to the popup rather than +//! moving the text cursor. +//! +//! Popups render in [`crate::ui::draw`] as an additional pass on top +//! of the input editor; anchor coordinates are derived from the +//! cursor's screen position so they appear next to where the user is +//! typing. +//! +//! Coexistence: only one popup is active at a time. Completion +//! takes precedence over signature help; both dismiss when hover +//! opens; any popup dismisses on Esc. + +use ratatui::layout::Rect; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{ + Block, Borders, Clear, List, ListItem, ListState, Paragraph, +}; +use ratatui::Frame; +use vw_htcl::complete::{Completion, CompletionKind}; + +/// One of the popup kinds the input editor can show. +#[derive(Debug)] +pub enum PopupState { + Completion(CompletionPopup), + SignatureHelp(SignatureHelpPopup), + Hover(HoverPopup), + Help(HelpPopup), + /// Fuzzy symbol picker (Ctrl-T). The picker owns its own + /// matcher state and rendering; this enum just multiplexes + /// into [`crate::symbol_search`]. + SymbolSearch(crate::symbol_search::SymbolPicker), + /// Fuzzy diagnostic finder (Ctrl-F). Snapshots scrollback + /// Error/Warning/Notice entries; Enter jumps the viewport to + /// the chosen entry and drops a persistent left-gutter + /// marker so the user can find it in a busy log. + DiagnosticSearch(crate::diag_search::DiagnosticPicker), +} + +/// Keybinding cheat-sheet shown by Ctrl-H. Lists every key chord the +/// REPL responds to, with a short description, so users don't have +/// to read the source to discover features. Any key dismisses. +#[derive(Debug)] +pub struct HelpPopup; + +/// Per-keystroke signature help — shows the proc's args + active +/// parameter while the user is typing arg values. Owns its strings +/// so we can store it on the App across frames without borrowing +/// from any short-lived `vw_htcl` document. Recomputed on every +/// buffer-mutating keystroke. +#[derive(Clone, Debug)] +pub struct SignatureHelpPopup { + pub proc_name: String, + pub args: Vec, + pub return_type: Option, + pub doc_brief: Option, + /// Index into `args` of the parameter under the cursor. + pub active: Option, + /// Cursor cell where the popup should anchor (above the cursor + /// — the renderer flips to below if there's no room above). + pub anchor: (u16, u16), + /// First arg index to show — wide-signature procs + /// (`create_cpm5_*` has 50+ args) overflow the popup's vertical + /// budget and get truncated. Ctrl-↑ / Ctrl-↓ adjusts this so + /// the user can scroll through the full list. Preserved across + /// `refresh_signature_help` rebuilds (App reads the old popup's + /// value before overwriting) so manual scrolling sticks while + /// the user is typing. + pub scroll_offset: usize, +} + +#[derive(Clone, Debug)] +pub struct SigHelpArg { + pub name: String, + pub type_str: Option, + /// Pre-formatted `@default(...)` value when the arg declares + /// one. Already truncated by the caller (see + /// `crate::app::format_default_value`) to a sensible width so + /// the popup doesn't blow wide on multi-KB paired-dict + /// defaults from generated IP wrappers. + pub default_str: Option, +} + +/// Hover popup — Ctrl-K opens it, any keystroke dismisses. Shows the +/// proc/var the cursor sits on, with full doc-comment body when +/// available. Distinct from signature help: hover is explicit + shows +/// the FULL docs; sig help is auto + shows a one-line brief. +#[derive(Clone, Debug)] +pub struct HoverPopup { + /// First line — usually the proc signature or `$variable: type`. + pub title: String, + /// Full doc body, reflowed. May be empty. + pub body: String, + pub anchor: (u16, u16), +} + +/// Render the hover popup at the cursor anchor. Sized to fit the +/// content, capped to ~70% of the frame so a long doc doesn't +/// swallow the whole screen. +pub fn draw_hover_popup(f: &mut Frame, popup: &HoverPopup, frame_area: Rect) { + let max_width = ((frame_area.width as usize) * 7 / 10).max(40); + let max_height = ((frame_area.height as usize) * 6 / 10).max(8); + + // Wrap body lines to fit max_width-2 (border padding). + let inner_w = max_width.saturating_sub(2); + let mut body_lines: Vec = Vec::new(); + for paragraph in popup.body.split('\n') { + if paragraph.is_empty() { + body_lines.push(String::new()); + continue; + } + let mut line = String::new(); + for word in paragraph.split_whitespace() { + if line.chars().count() + word.chars().count() + 1 > inner_w { + body_lines.push(std::mem::take(&mut line)); + } + if !line.is_empty() { + line.push(' '); + } + line.push_str(word); + } + if !line.is_empty() { + body_lines.push(line); + } + } + let total_lines = 1 + body_lines.len(); // title + body + let height = (total_lines + 2).min(max_height) as u16; + let width = max_width as u16; + + let (anchor_x, anchor_y) = popup.anchor; + let y = if anchor_y + height < frame_area.y + frame_area.height { + (anchor_y + 1) + .min(frame_area.y + frame_area.height.saturating_sub(height)) + } else if anchor_y >= frame_area.y + height { + anchor_y.saturating_sub(height) + } else { + frame_area.y + }; + let x = anchor_x.min(frame_area.x + frame_area.width.saturating_sub(width)); + let area = Rect { + x, + y, + width: width.min(frame_area.width), + height, + }; + f.render_widget(Clear, area); + + let mut lines: Vec> = Vec::new(); + lines.push(Line::from(Span::styled( + popup.title.clone(), + Style::default() + .fg(Color::Rgb(230, 200, 120)) + .add_modifier(Modifier::BOLD), + ))); + for body in body_lines { + lines.push(Line::from(Span::styled( + body, + Style::default().fg(Color::Gray), + ))); + } + + let para = Paragraph::new(lines).block( + Block::default() + .borders(Borders::ALL) + .title(" hover — any key to dismiss "), + ); + f.render_widget(para, area); +} + +/// Render the signature help popup as an overlay anchored just above +/// the cursor cell. If there's not enough vertical room above, the +/// renderer flips it below. +pub fn draw_signature_help_popup( + f: &mut Frame, + popup: &SignatureHelpPopup, + frame_area: Rect, +) { + // Multi-line layout: proc name on line 1, then one indented + // arg per line, then `→ return_type` (when annotated), then an + // optional doc-brief line. + // + // Single-line layouts blew past the popup width on procs with + // many args — `create_versal_cips` has ~20 — and ratatui + // truncates beyond the box. One-arg-per-line keeps each arg + // readable and scales naturally. The active-arg highlighting + // (bold + underline on the arg's name) still works per-line. + let name_style_keyword = Style::default() + .fg(Color::Rgb(230, 200, 120)) + .add_modifier(Modifier::BOLD); + let arg_name_style_default = Style::default().fg(Color::Gray); + let arg_name_style_active = Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD | Modifier::UNDERLINED); + let type_style = Style::default().fg(Color::Rgb(100, 200, 200)); + let dim = Style::default().add_modifier(Modifier::DIM); + let default_style = Style::default() + .fg(Color::Rgb(180, 200, 130)) + .add_modifier(Modifier::DIM); + + // Build the proc name line (always visible — never scrolled + // off) and the body lines (arg lines + return + doc) as + // separate vectors so we can apply scroll_offset to the body + // independently of the title. + let name_line = + Line::from(Span::styled(popup.proc_name.clone(), name_style_keyword)); + let mut body_lines: Vec> = Vec::new(); + for (i, arg) in popup.args.iter().enumerate() { + let active = popup.active == Some(i); + let mut spans: Vec> = vec![Span::raw(" ")]; + let name_style = if active { + arg_name_style_active + } else { + arg_name_style_default + }; + spans.push(Span::styled(format!("-{}", arg.name), name_style)); + if let Some(ty) = arg.type_str.as_deref() { + spans.push(Span::styled(": ".to_string(), dim)); + spans.push(Span::styled(ty.to_string(), type_style)); + } + if let Some(default) = arg.default_str.as_deref() { + spans.push(Span::styled(" = ".to_string(), dim)); + spans.push(Span::styled(default.to_string(), default_style)); + } + body_lines.push(Line::from(spans)); + } + if let Some(ret) = popup.return_type.as_deref() { + body_lines.push(Line::from(vec![ + Span::styled(" → ".to_string(), dim), + Span::styled(ret.to_string(), type_style), + ])); + } + if let Some(brief) = popup.doc_brief.as_deref() { + if !brief.is_empty() { + body_lines.push(Line::from("")); + body_lines.push(Line::from(Span::styled( + brief.to_string(), + Style::default().fg(Color::DarkGray), + ))); + } + } + + // Body budget: total height budget minus borders, the proc + // name row, and (when scrolled / overflowing) one row for the + // "(N more · ctrl-↑/↓ to scroll)" footer hint. + let max_height_rows = ((frame_area.height as usize) * 7 / 10).max(4); + let chrome_rows = 2 /* borders */ + 1 /* proc name */; + let max_body_rows = max_height_rows.saturating_sub(chrome_rows); + + // Apply scroll_offset, clamping so we never scroll past a point + // where the visible window would underfill: the user can scroll + // until the LAST body line is at the bottom of the window. + let total_body = body_lines.len(); + let needs_scroll_hint = total_body > max_body_rows; + let visible_body_rows = if needs_scroll_hint { + max_body_rows.saturating_sub(1) // reserve one row for the hint + } else { + max_body_rows + }; + let max_offset = total_body.saturating_sub(visible_body_rows); + let scroll_offset = popup.scroll_offset.min(max_offset); + let body_end = (scroll_offset + visible_body_rows).min(total_body); + let mut lines: Vec> = Vec::with_capacity( + 1 + (body_end - scroll_offset) + needs_scroll_hint as usize, + ); + lines.push(name_line); + lines.extend(body_lines[scroll_offset..body_end].iter().cloned()); + if needs_scroll_hint { + let visible_lo = scroll_offset + 1; + let visible_hi = body_end; + let hint = format!( + " … showing {visible_lo}-{visible_hi} of {total_body} \ + · shift-↑/shift-↓ to scroll", + ); + lines.push(Line::from(Span::styled( + hint, + Style::default().add_modifier(Modifier::DIM), + ))); + } + + // Width: longest line + borders, capped at 70% of frame so + // wide types don't push the popup off-screen. + let widest: usize = lines + .iter() + .map(|l| { + l.spans + .iter() + .map(|s| s.content.chars().count()) + .sum::() + }) + .max() + .unwrap_or(20); + let max_width = ((frame_area.width as usize) * 7 / 10).max(40); + let width = ((widest + 2).clamp(20, max_width)) as u16; + // Recompute height now that the line list is bounded. + let height = ((lines.len() + 2) as u16) + .min(frame_area.height.saturating_sub(1).max(3)); + + let (anchor_x, anchor_y) = popup.anchor; + // Anchor ABOVE cursor when room exists (don't fight the completion + // popup which anchors below). Falls back to below, then clamps to + // the frame bottom — never lets the rect extend past the buffer + // (would panic inside ratatui's `render_widget`). + let y_above = anchor_y.saturating_sub(height); + let y_below = anchor_y.saturating_add(1); + let y = if anchor_y >= frame_area.y + height { + y_above + } else { + y_below + }; + let max_y = frame_area + .y + .saturating_add(frame_area.height) + .saturating_sub(height); + let y = y.min(max_y).max(frame_area.y); + let x = anchor_x.min(frame_area.x + frame_area.width.saturating_sub(width)); + let area = Rect { + x, + y, + width, + height, + }; + + f.render_widget(Clear, area); + let para = Paragraph::new(lines) + .block(Block::default().borders(Borders::ALL).title(" signature ")); + f.render_widget(para, area); +} + +/// One row in the cheat-sheet. `keys` is shown left-aligned in its +/// own column; `description` fills the rest of the row. +struct HelpRow { + keys: &'static str, + description: &'static str, +} + +/// The full keybinding + meta-command catalog. Keep in sync with +/// `crate::app::App::handle_terminal_event` (key chords) and +/// `crate::app::META_COMMANDS` (the `:foo` REPL commands) — every +/// recognized chord and meta-command should appear here so the +/// help is authoritative. +/// +/// Rows are grouped with blank-spacer rows between sections; the +/// renderer treats `("", "")` as a section separator. +const HELP_ROWS: &[HelpRow] = &[ + // --- popups + completion --- + HelpRow { + keys: "Ctrl-H", + description: "show this help", + }, + HelpRow { + keys: "Tab", + description: "open completion popup (procs, flags, :commands)", + }, + HelpRow { + keys: "↑ / ↓ / Enter", + description: "navigate / accept inside any popup", + }, + HelpRow { + keys: "Esc", + description: "dismiss popup or reverse search", + }, + HelpRow { + keys: "", + description: "", + }, + // --- input / submit --- + HelpRow { + keys: "Enter", + description: "submit input (newline if parse incomplete)", + }, + HelpRow { + keys: "Shift+Enter, Ctrl+Enter, Alt+Enter", + description: "insert literal newline (Shift+Enter reaches the app as Ctrl+J on legacy-encoding terminals; we bind both)", + }, + HelpRow { + keys: "Ctrl-P / Ctrl-N", + description: "prev / next in input history", + }, + HelpRow { + keys: "Ctrl-R", + description: "reverse-search history", + }, + HelpRow { + keys: "Ctrl-C", + description: "clear current input", + }, + HelpRow { + keys: "", + description: "", + }, + // --- scrollback --- + HelpRow { + keys: "Alt-K, PageUp", + description: "scroll scrollback up", + }, + HelpRow { + keys: "Alt-J, PageDown", + description: "scroll scrollback down", + }, + HelpRow { + keys: "End, Ctrl-G", + description: "jump to bottom (resume tail-follow)", + }, + HelpRow { + keys: "Mouse wheel", + description: "scroll scrollback (mouse capture must be on)", + }, + HelpRow { + keys: "Mouse drag", + description: "select for clipboard copy (auto-scrolls past edges)", + }, + HelpRow { + keys: "Shift + click", + description: "expand / collapse a ▶ / ▼ non-diagnostic block", + }, + HelpRow { + keys: "Ctrl-C (during eval)", + description: "cancel the running Vivado eval (session survives)", + }, + HelpRow { + keys: "F2", + description: "toggle mouse capture (mouse-app vs terminal-native)", + }, + HelpRow { + keys: "", + description: "", + }, + // --- session / exit --- + HelpRow { + keys: "Ctrl-D", + description: "exit REPL", + }, + HelpRow { + keys: ":quit / :q / :exit", + description: "exit REPL via meta-command", + }, + HelpRow { + keys: ":load ", + description: "evaluate the contents of a file in this session", + }, + HelpRow { + keys: ":libs", + description: "list loaded libraries and their symbol counts", + }, + HelpRow { + keys: ":restart", + description: + "restart the Vivado worker (stubbed — not yet implemented)", + }, + HelpRow { + keys: "", + description: "", + }, + HelpRow { + keys: "(auto)", + description: "signature help shows while you type a call's args", + }, + HelpRow { + keys: "Shift-↑ / Shift-↓", + description: "scroll the signature-help popup (long arg lists)", + }, + HelpRow { + keys: "Ctrl-Y", + description: "hover docs under cursor", + }, + HelpRow { + keys: "Ctrl-S", + description: "fuzzy symbol search (Tab toggles libraries view)", + }, + HelpRow { + keys: "Ctrl-F", + description: "find diagnostic in scrollback (Ctrl-E/K/W/N toggle Error/Critical/Warning/Info filters; Enter jumps + marks)", + }, + HelpRow { + keys: "Alt-C", + description: "clear the diagnostic-finder jump marker", + }, +]; + +/// Render the help modal centered in the frame. Width fits the +/// longest row + small padding; height fits all rows + borders. +pub fn draw_help_popup(f: &mut Frame, frame_area: Rect) { + let key_col_w: usize = HELP_ROWS + .iter() + .map(|r| r.keys.chars().count()) + .max() + .unwrap_or(8); + let desc_col_w: usize = HELP_ROWS + .iter() + .map(|r| r.description.chars().count()) + .max() + .unwrap_or(20); + // +4 = " │ " separator + outer 1-cell padding on each side. + let width = ((key_col_w + desc_col_w + 6).clamp(40, 80)) as u16; + let height = (HELP_ROWS.len() + 2) as u16; // +2 borders + let x = frame_area.x + frame_area.width.saturating_sub(width) / 2; + let y = frame_area.y + frame_area.height.saturating_sub(height) / 2; + let area = Rect { + x, + y, + width: width.min(frame_area.width), + height: height.min(frame_area.height), + }; + f.render_widget(Clear, area); + + let key_style = Style::default() + .fg(Color::Rgb(180, 130, 220)) + .add_modifier(Modifier::BOLD); + let desc_style = Style::default().fg(Color::Gray); + let sep_style = Style::default().add_modifier(Modifier::DIM); + + let lines: Vec> = HELP_ROWS + .iter() + .map(|row| { + if row.keys.is_empty() && row.description.is_empty() { + return Line::from(""); + } + let pad = key_col_w.saturating_sub(row.keys.chars().count()); + Line::from(vec![ + Span::styled(format!(" {}", row.keys), key_style), + Span::raw(" ".repeat(pad)), + Span::styled(" │ ", sep_style), + Span::styled(row.description.to_string(), desc_style), + ]) + }) + .collect(); + + let para = Paragraph::new(lines).block( + Block::default() + .borders(Borders::ALL) + .title(" help — press any key to dismiss "), + ); + f.render_widget(para, area); +} + +/// Completion popup state. Owns the list of items the call to +/// `vw_htcl::complete_at` produced, plus the currently-selected +/// index and the cursor offset at which the popup was anchored +/// (used by Enter to know what range to replace). +#[derive(Debug)] +pub struct CompletionPopup { + pub items: Vec, + pub selected: usize, + /// Screen-anchor cell — where the popup's top-left should appear + /// relative to the frame. Set by the caller (which knows the + /// terminal cursor position) and used by the renderer. + pub anchor: (u16, u16), +} + +impl CompletionPopup { + pub fn new(items: Vec, anchor: (u16, u16)) -> Option { + if items.is_empty() { + None + } else { + Some(Self { + items, + selected: 0, + anchor, + }) + } + } + + pub fn move_up(&mut self) { + if self.selected > 0 { + self.selected -= 1; + } + } + + pub fn move_down(&mut self) { + if self.selected + 1 < self.items.len() { + self.selected += 1; + } + } + + /// The item the user would accept by pressing Enter. + pub fn current(&self) -> Option<&Completion> { + self.items.get(self.selected) + } +} + +/// Render a completion popup as an overlay. Sized to fit a sensible +/// row count from the available frame height (10 rows default, less +/// when near the bottom edge). Anchored just below-and-right of the +/// cursor position the popup was created at; clamped to stay inside +/// the frame. +pub fn draw_completion_popup( + f: &mut Frame, + popup: &CompletionPopup, + frame_area: Rect, +) { + if popup.items.is_empty() { + return; + } + let max_rows = popup.items.len().min(10) as u16; + // Width: longest label + space + longest detail + padding, capped + // at 60 cells. + let max_label = popup + .items + .iter() + .map(|c| c.label.chars().count()) + .max() + .unwrap_or(8); + let max_detail = popup + .items + .iter() + .map(|c| c.detail.as_deref().unwrap_or("").chars().count()) + .max() + .unwrap_or(0); + let width = (max_label + max_detail + 4).clamp(20, 60) as u16; + let height = max_rows + 2; // +2 for borders + + let (anchor_x, anchor_y) = popup.anchor; + let x = anchor_x.min(frame_area.x + frame_area.width.saturating_sub(width)); + let y = (anchor_y + 1) + .min(frame_area.y + frame_area.height.saturating_sub(height)); + let area = Rect { + x, + y, + width, + height, + }; + + f.render_widget(Clear, area); + + let items: Vec = popup + .items + .iter() + .enumerate() + .map(|(i, item)| { + let kind_glyph = match item.kind { + CompletionKind::Proc => "·", + CompletionKind::Flag => "-", + CompletionKind::EnumValue => "=", + CompletionKind::Constructor => "+", + }; + let kind_style = match item.kind { + CompletionKind::Proc => { + Style::default().fg(Color::Rgb(230, 200, 120)) + } + CompletionKind::Flag => { + Style::default().fg(Color::Rgb(180, 130, 220)) + } + CompletionKind::EnumValue => { + Style::default().fg(Color::Rgb(100, 200, 200)) + } + CompletionKind::Constructor => { + Style::default().fg(Color::Rgb(120, 200, 140)) + } + }; + let detail = item + .detail + .as_deref() + .map(|d| format!(" {d}")) + .unwrap_or_default(); + let mut spans = vec![ + Span::styled(format!("{kind_glyph} "), kind_style), + Span::styled( + item.label.clone(), + if i == popup.selected { + Style::default() + .add_modifier(Modifier::BOLD) + .fg(Color::White) + } else { + Style::default().fg(Color::Gray) + }, + ), + ]; + if !detail.is_empty() { + spans.push(Span::styled( + detail, + Style::default().add_modifier(Modifier::DIM), + )); + } + let style = if i == popup.selected { + Style::default().bg(Color::Rgb(40, 40, 60)) + } else { + Style::default() + }; + ListItem::new(Line::from(spans)).style(style) + }) + .collect(); + + let mut list_state = ListState::default(); + list_state.select(Some(popup.selected)); + + let list = List::new(items) + .block( + Block::default() + .borders(Borders::ALL) + .title(" completions "), + ) + .highlight_style(Style::default().bg(Color::Rgb(40, 40, 60))); + f.render_stateful_widget(list, area, &mut list_state); +} + +#[cfg(test)] +mod tests { + use super::*; + use vw_htcl::span::Span as HtclSpan; + + fn item(label: &str, kind: CompletionKind) -> Completion { + Completion { + label: label.to_string(), + kind, + detail: None, + documentation: None, + replace: HtclSpan { start: 0, end: 0 }, + insert_text: None, + snippet: false, + } + } + + #[test] + fn new_returns_none_on_empty() { + assert!(CompletionPopup::new(Vec::new(), (0, 0)).is_none()); + } + + #[test] + fn move_up_down_clamps() { + let items = vec![ + item("a", CompletionKind::Proc), + item("b", CompletionKind::Proc), + ]; + let mut p = CompletionPopup::new(items, (0, 0)).unwrap(); + assert_eq!(p.selected, 0); + p.move_up(); + assert_eq!(p.selected, 0); // can't go below 0 + p.move_down(); + assert_eq!(p.selected, 1); + p.move_down(); + assert_eq!(p.selected, 1); // can't go past last + p.move_up(); + assert_eq!(p.selected, 0); + } + + #[test] + fn current_returns_selected_item() { + let items = vec![ + item("a", CompletionKind::Proc), + item("b", CompletionKind::Flag), + ]; + let mut p = CompletionPopup::new(items, (0, 0)).unwrap(); + assert_eq!(p.current().unwrap().label, "a"); + p.move_down(); + assert_eq!(p.current().unwrap().label, "b"); + } +} diff --git a/vw-repl/src/render.rs b/vw-repl/src/render.rs new file mode 100644 index 0000000..8acdb54 --- /dev/null +++ b/vw-repl/src/render.rs @@ -0,0 +1,676 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Scrollback rendering helpers shared between `ui::draw_scrollback` +//! and `App` mouse-selection. Both need the same view of "how does the +//! scrollback look on screen, row by row" — the UI to render it, the +//! App to map mouse clicks to text positions and extract the selected +//! substring on copy. +//! +//! The flow is: [`entry_lines`] turns each `ScrollbackEntry` into one +//! styled [`Line`] per source line; [`wrap_lines`] then breaks each of +//! those at the rendered column width into screen-row–sized chunks. +//! After wrapping, screen-row N is `wrapped[scroll + N]` — that 1:1 +//! mapping is what makes mouse-cell → text-cell trivial. With +//! ratatui's built-in `Wrap { trim: false }` we'd have to replay +//! ratatui's word-wrap to find the same mapping, which we don't want +//! to maintain in lockstep. + +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; + +use crate::app::{ScrollbackEntry, ScrollbackKind}; + +/// One styled [`Line`] per source line in `entry`. The leading +/// 2-cell column is the kind-prefix (`› `, `· `, `⚠ `, etc.) on the +/// first source line and two spaces on continuation lines, so a +/// multi-line entry visually hangs together. +/// +/// `area_width` is the terminal column count — used to +/// right-justify the per-input timer marker on the first line of +/// an `Input` entry. Pass the same width the renderer will wrap +/// to so the timer ends up flush at the right margin. +/// Backwards-compatible full-entry render. Same output as +/// [`entry_lines_windowed`] called with `0..u32::MAX`. Used by tests +/// and by the clipboard-copy path (which needs every source line +/// regardless of what's on screen). +pub fn entry_lines( + entry: &ScrollbackEntry, + area_width: u16, +) -> Vec> { + entry_lines_windowed(entry, area_width, 0..u32::MAX).0 +} + +/// Produce styled [`Line`] objects for only the source lines whose +/// **wrapped** rows overlap `window` (0-indexed within this entry's +/// wrapped row range). +/// +/// Returns `(lines, offset_into_first)` where `offset_into_first` is +/// how many wrapped rows of the first emitted line's own wrapped +/// output precede `window.start`. Callers pass this back through the +/// ratatui `Paragraph::scroll` residual so the on-screen scroll +/// alignment stays byte-for-byte identical to what the full-entry +/// render would have produced. +/// +/// This is the primary path — a huge scrollback entry (200K lines of +/// a `list` repr, say) that intersects the viewport was +/// previously O(entry.text.lines().count()) work per redraw because +/// `entry_lines` walked every source line, called `highlight_line` on +/// each, allocated a `Line` per line, and then handed the whole Vec +/// to `wrap_lines` which processed it end-to-end. Only the visible +/// window (typically ≤ area.height rows) ever mattered. This function +/// still walks the source-line iterator to *count* rows for skipped +/// lines (that's cheap — just `chars().count()` + arithmetic per +/// line), but does no allocation or highlighting until we hit the +/// window. Same output visually; O(entry) → O(visible + prefix_scan). +pub fn entry_lines_windowed( + entry: &ScrollbackEntry, + area_width: u16, + window: std::ops::Range, +) -> (Vec>, u32) { + // Collapsed entries: exactly one placeholder row regardless of + // window — the placeholder itself is the whole render. + if entry.collapse_state == Some(true) { + return (collapsible_lines(entry, true), 0); + } + // Expanded (Some(false)) and non-collapsible (None) both walk + // the windowed source-line path with the entry-kind's normal + // prefix/style. Expanded collapsible entries additionally get + // a `▼` marker in the leftmost 2-cell column so the toggleable + // affordance is visible — users can see this cell is collapsible + // without having to try Shift+click on every entry. None entries + // (single-line, no meaningful collapse) skip the marker column. + let orange = Color::Rgb(255, 140, 0); + let (kind_prefix_str, prefix_style) = kind_prefix(entry.kind, orange); + let marker_style = Style::default().fg(Color::DarkGray); + let has_marker = entry.collapse_state == Some(false); + // Continuation rows get an indent matching row 0's total prefix + // width so text stays visually aligned across a wrapped entry. + let cont_indent = if has_marker { " " } else { " " }; + let body_style = match entry.kind { + ScrollbackKind::Input => Style::default().fg(Color::White), + ScrollbackKind::Result => Style::default().fg(Color::Gray), + ScrollbackKind::Stdout => Style::default().fg(Color::White), + ScrollbackKind::Error => Style::default().fg(Color::Red), + ScrollbackKind::Warning => Style::default().fg(orange), + ScrollbackKind::Notice => Style::default().fg(Color::DarkGray), + ScrollbackKind::Chatter => Style::default() + .fg(Color::DarkGray) + .add_modifier(Modifier::DIM), + }; + // For Input entries with a timer, render `` flush + // right on the first line. Color follows whether it's still + // running (dim while live) vs. completed (subtle gray). + let timer = timer_for(entry); + // Highlighting strategy per kind: + // - Result / Stdout: repr highlighter (per-line shape recognition + // for compiler-emitted enum reprs). + // - Input: htcl-source highlighter (whole-entry parse, per-line + // span slicing) — colors keywords, calls, $vars, types, + // comments etc. the same as the input editor. + // - Error / Warning / Notice: single body color (not repr-formatted). + let repr_highlight = + matches!(entry.kind, ScrollbackKind::Result | ScrollbackKind::Stdout); + let input_highlight = matches!(entry.kind, ScrollbackKind::Input); + // Input entries: parse the whole entry text once and slice per-line. + // The body_style on Input is the bright cyan we'd otherwise apply + // flatly; the htcl highlighter overrides it for recognized tokens + // and leaves it for the gaps. + let input_per_line: Option>>> = if input_highlight { + Some(crate::highlight_htcl::highlight_per_line( + &entry.text, + body_style, + )) + } else { + None + }; + let mut out = Vec::new(); + // Per-source-line wrapped-row accounting — matches the formula + // in `count_wrapped_rows` exactly. Any drift here would misalign + // the caller's global row math with the actual rendered rows. + let w = area_width.max(1) as usize; + let prefix_width = 2; + let mut cumulative: u32 = 0; + let mut first_emitted_row_start: Option = None; + let mut had_lines = false; + for (i, line) in entry.text.lines().enumerate() { + had_lines = true; + let body_chars = line.chars().count(); + let total = body_chars.saturating_add(prefix_width).max(1); + let line_rows = (total.div_ceil(w).max(1)) as u32; + let line_end = cumulative.saturating_add(line_rows); + // Skip source lines whose wrapped-row range ends before the + // window even starts. Cheap — just chars().count() + arith, + // no allocations, no highlight calls. + if line_end <= window.start { + cumulative = line_end; + continue; + } + // Stop once we've moved past the window's end. `cumulative` + // here is the FIRST row this line contributes; if that's + // already past the window, everything remaining is offscreen. + if cumulative >= window.end { + break; + } + if first_emitted_row_start.is_none() { + first_emitted_row_start = Some(cumulative); + } + // Row-0 gutter for a collapsible expanded entry: `▼ ` marker + // in dim gray, then the entry-kind's normal prefix. Row 0+ of + // the same entry uses `cont_indent` (4 spaces to reserve room + // for both the marker column and the kind prefix) so wrapped + // text hangs cleanly under row 0's body. + let mut spans: Vec> = Vec::new(); + if i == 0 { + if has_marker { + spans.push(Span::styled("▼ ".to_string(), marker_style)); + } + spans.push(Span::styled(kind_prefix_str.to_string(), prefix_style)); + } else { + spans.push(Span::styled(cont_indent.to_string(), prefix_style)); + } + if let Some(per_line) = input_per_line.as_ref() { + if let Some(line_spans) = per_line.get(i) { + spans.extend(line_spans.iter().cloned()); + } else { + spans.push(Span::styled(line.to_string(), body_style)); + } + } else if repr_highlight { + if let Some(highlighted) = crate::highlight::highlight_line(line) { + spans.extend(highlighted); + } else { + spans.push(Span::styled(line.to_string(), body_style)); + } + } else { + spans.push(Span::styled(line.to_string(), body_style)); + } + if i == 0 { + if let Some((label, label_style)) = timer.as_ref() { + let used: usize = + spans.iter().map(|s| display_cells(&s.content)).sum(); + let label_w = display_cells(label); + if (used + label_w + 1) as u16 <= area_width { + let pad = area_width as usize - used - label_w; + spans.push(Span::raw(" ".repeat(pad))); + spans.push(Span::styled(label.clone(), *label_style)); + } + } + } + out.push(Line::from(spans)); + cumulative = line_end; + } + if !had_lines && out.is_empty() { + // Empty text: match the pre-windowed behavior (one blank + // prefix-only line). Only render it if window includes row 0. + if window.start == 0 && window.end > 0 { + let mut blank_spans: Vec> = Vec::new(); + if has_marker { + blank_spans.push(Span::styled("▼ ".to_string(), marker_style)); + } + blank_spans + .push(Span::styled(kind_prefix_str.to_string(), prefix_style)); + out.push(Line::from(blank_spans)); + first_emitted_row_start = Some(0); + } + } + let offset = first_emitted_row_start + .map(|s| window.start.saturating_sub(s)) + .unwrap_or(0); + (out, offset) +} + +/// Prefix + style for an entry-kind's row-0 gutter. Extracted so +/// the collapsed-placeholder renderer can use the same colors as +/// the expanded path — a collapsed `Result` reads with the same +/// gray body as an expanded `Result`, a collapsed `Warning` reads +/// with the same orange bold, etc. Only the entry's Chatter kind +/// specifically stays dim (that's the "background noise" bucket). +fn kind_prefix(kind: ScrollbackKind, orange: Color) -> (&'static str, Style) { + match kind { + ScrollbackKind::Input => ( + "› ", + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Result => (" ", Style::default().fg(Color::Gray)), + ScrollbackKind::Stdout => (" ", Style::default().fg(Color::White)), + ScrollbackKind::Error => ( + "✗ ", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Warning => ( + "⚠ ", + Style::default().fg(orange).add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Notice => ("· ", Style::default().fg(Color::DarkGray)), + // Chatter: 2-space prefix (no diagnostic glyph) + DIM + // DarkGray — background-noise bucket for classifier-produced + // NONE blocks. Multi-line Chatter still auto-collapses per + // COLLAPSE_AUTO_THRESHOLD; the dim style just signals "this + // is elidable" even before the user thinks about collapsing. + ScrollbackKind::Chatter => ( + " ", + Style::default() + .fg(Color::DarkGray) + .add_modifier(Modifier::DIM), + ), + } +} + +/// Render the single-row placeholder for a **collapsed** entry: +/// dimmed preview of the first non-empty content line plus a +/// `(N lines hidden)` tail. Uses a uniform dim dark-gray body +/// style regardless of entry kind — the whole point of collapse +/// is "this is elided content; expand to read." Rendering it in +/// the kind's normal color (bright red for Error, orange for +/// Warning, etc.) makes the placeholder compete visually with +/// entries that aren't collapsed, defeating the "elided noise" +/// signal. When the user Shift-clicks to expand, the entry +/// reverts to its normal kind coloring — that's when you actually +/// want the Warning to look like a Warning. +fn collapsible_lines( + entry: &ScrollbackEntry, + collapsed: bool, +) -> Vec> { + debug_assert!( + collapsed, + "expanded entries go through entry_lines_windowed's \ + regular source-line path — collapsible_lines is the \ + collapsed-only placeholder helper" + ); + let marker_style = Style::default().fg(Color::DarkGray); + let dim = Style::default() + .fg(Color::DarkGray) + .add_modifier(Modifier::DIM); + let source_lines: Vec<&str> = entry.text.lines().collect(); + // Preview: first non-empty line (else first line, else ""). + let preview = source_lines + .iter() + .find(|l| !l.trim().is_empty()) + .copied() + .unwrap_or_default(); + let count = source_lines.len(); + let suffix = if count > 1 { + format!(" ({count} lines hidden)") + } else { + String::new() + }; + let mut spans = vec![Span::styled("▶ ".to_string(), marker_style)]; + spans.push(Span::styled(preview.to_string(), dim)); + if !suffix.is_empty() { + spans.push(Span::styled(suffix, dim)); + } + vec![Line::from(spans)] +} + +/// `(label, style)` for an entry's elapsed-time marker, or `None` +/// when the entry isn't timed. Color hints whether the timer is +/// still live (running) or frozen (completed). +fn timer_for(entry: &ScrollbackEntry) -> Option<(String, Style)> { + let start = entry.started_at?; + let end = entry.completed_at.unwrap_or_else(std::time::Instant::now); + let elapsed = end.saturating_duration_since(start); + let label = format_duration(elapsed); + let style = if entry.completed_at.is_some() { + // Frozen at final value — quiet, post-fact. + Style::default().fg(Color::DarkGray) + } else { + // Live — slightly more present so the user sees it's + // still moving. + Style::default().fg(Color::Yellow) + }; + Some((label, style)) +} + +/// Format a duration as `Ns`, `M:SS`, or `H:MM:SS` depending on +/// magnitude. Always second-granularity; never fractional. Matches +/// what users expect for "how long did this take" markers. +pub fn format_duration(d: std::time::Duration) -> String { + let total = d.as_secs(); + if total < 60 { + format!("{total}s") + } else if total < 3600 { + format!("{}:{:02}", total / 60, total % 60) + } else { + let h = total / 3600; + let m = (total % 3600) / 60; + let s = total % 60; + format!("{h}:{m:02}:{s:02}") + } +} + +/// Crude width estimator — counts chars, treating each as one +/// terminal cell. Good enough for our prefix glyphs (`› ` etc., +/// each rendered as one cell in monospace terminals) and ASCII +/// timer labels. A full unicode-width crate would be more +/// correct but isn't worth the dep for the small set of +/// characters this code emits. +fn display_cells(s: &str) -> usize { + s.chars().count() +} + +/// Split each input line into screen-row-sized chunks of `width` +/// columns, preserving span styles across the split. The output +/// renders 1:1 against screen rows when fed to a `Paragraph` with no +/// further wrapping, so screen-row N is `out[scroll + N]`. +/// +/// Splitting is character-based (no word-boundary respect) — this is +/// REPL output, not prose; long Vivado property dicts and Tcl errors +/// don't have natural break points. +/// Cheap pre-computation of how many wrapped terminal rows an +/// entry will occupy at the given width — WITHOUT actually +/// allocating wrapped lines. O(text length) per entry, no heap +/// allocations beyond the iterator. +/// +/// Used by the viewport-slicing render path to find which +/// entries intersect the visible window in linear time, so the +/// expensive [`entry_lines`] + [`wrap_lines`] only runs on the +/// handful of entries actually in view. Without this, a huge +/// entry (e.g. the formatted `util::props` output) gets +/// fully re-wrapped on every draw — turning every wheel event +/// into multi-MB of per-char allocation. +/// +/// The count must match what [`entry_lines`] + [`wrap_lines`] +/// actually produce: each natural text line contributes +/// `ceil((prefix + body_chars) / width)` wrapped rows (min 1). +/// The Input-entry timer suffix is ignored — when it fits it +/// pads the first line to exactly `width` (still 1 row); when +/// it doesn't fit it isn't added (so the body wraps normally +/// without it). Either way the row count matches. +pub fn count_wrapped_rows(entry: &ScrollbackEntry, width: u16) -> u32 { + if width == 0 { + return 1; + } + let w = width as usize; + // Row-0 gutter width matches what `entry_lines_windowed` emits: + // - collapsible + expanded (Some(false)): 4 cells ("▼ " + + // kind prefix) on row 0, 4-space `cont_indent` on + // continuation rows. + // - collapsed (Some(true)): handled by the `▶` placeholder + // branch below. + // - non-collapsible (None): 2 cells ("kind_prefix"). + let prefix_width: usize = if entry.collapse_state == Some(false) { + 4 + } else { + 2 + }; + // Collapsed entry: exactly one placeholder row of content, + // wrapped like any other line if the preview + suffix exceed + // the terminal width. The formula has to MATCH what + // `collapsible_lines` actually emits, character for character + // — any drift shifts every downstream entry's row index by the + // rounding error, and a click on the visible content maps to + // an adjacent buffer row (visible: "A total of 4711…", copies: + // "· INFO: [Common 17-83] Releasing license…"). + if let Some(true) = entry.collapse_state { + let preview = entry + .text + .lines() + .find(|l| !l.trim().is_empty()) + .unwrap_or(""); + let count = entry.text.lines().count(); + // Suffix exactly matches the `format!(" ({count} lines + // hidden)")` in `collapsible_lines`: 3 (`" ("`) + digits + + // 14 (`" lines hidden)"`). + let suffix_width = if count > 1 { + 3 + count.to_string().chars().count() + 14 + } else { + 0 + }; + let body_chars = preview.chars().count(); + let total = body_chars + .saturating_add(prefix_width) + .saturating_add(suffix_width) + .max(1); + return total.div_ceil(w).max(1) as u32; + } + let mut rows: u32 = 0; + let mut had_lines = false; + for line in entry.text.lines() { + had_lines = true; + let body_chars = line.chars().count(); + let total = body_chars.saturating_add(prefix_width).max(1); + let line_rows = total.div_ceil(w).max(1); + rows = rows.saturating_add(line_rows as u32); + } + if !had_lines { + // Empty text → entry_lines emits one blank line. + rows = 1; + } + rows +} + +pub fn wrap_lines(input: Vec>, width: u16) -> Vec> { + if width == 0 { + return input; + } + let w = width as usize; + let mut out = Vec::with_capacity(input.len()); + for line in input { + // Flatten spans → (char, style) so chunking can ignore the + // span boundaries and only care about per-cell style. + let mut chars: Vec<(char, Style)> = Vec::new(); + for span in &line.spans { + for c in span.content.chars() { + chars.push((c, span.style)); + } + } + if chars.is_empty() { + out.push(Line::from("")); + continue; + } + for chunk in chars.chunks(w) { + out.push(merge_to_line(chunk)); + } + } + out +} + +fn merge_to_line(chunk: &[(char, Style)]) -> Line<'static> { + let mut spans = Vec::new(); + let mut buf = String::new(); + let mut cur_style = chunk[0].1; + for (c, st) in chunk { + if *st != cur_style { + spans.push(Span::styled(std::mem::take(&mut buf), cur_style)); + cur_style = *st; + } + buf.push(*c); + } + if !buf.is_empty() { + spans.push(Span::styled(buf, cur_style)); + } + Line::from(spans) +} + +/// Plain-text content of a [`Line`] — span styles dropped, content +/// concatenated. Used to extract the selected substring for clipboard +/// copy. +pub fn line_plain_text(line: &Line<'_>) -> String { + let mut out = String::new(); + for span in &line.spans { + out.push_str(span.content.as_ref()); + } + out +} + +/// Re-style cells in `lines` that fall inside the selection range, +/// `[start, end)`. Both endpoints are `(row, col)` indices into +/// `lines` (the post-wrap, post-scroll Vec). The range may be +/// inverted (cursor before anchor); callers should normalize first. +pub fn apply_selection_highlight( + lines: &mut [Line<'static>], + start: (usize, usize), + end: (usize, usize), +) { + let (sr, sc) = start; + let (er, ec) = end; + for (row_idx, line) in lines.iter_mut().enumerate() { + if row_idx < sr || row_idx > er { + continue; + } + let row_start = if row_idx == sr { sc } else { 0 }; + let row_end = if row_idx == er { ec } else { usize::MAX }; + highlight_cols(line, row_start, row_end); + } +} + +fn highlight_cols(line: &mut Line<'static>, start: usize, end: usize) { + // Rebuild spans, splitting any that straddle the selection + // boundary so the REVERSED modifier applies to exactly the cells + // in [start, end). + let mut new_spans: Vec> = Vec::new(); + let mut col = 0usize; + for span in line.spans.drain(..) { + let span_chars: Vec = span.content.chars().collect(); + let len = span_chars.len(); + let span_start = col; + let span_end = col + len; + col = span_end; + + if span_end <= start || span_start >= end { + // Wholly outside selection — push unchanged. + new_spans.push(span); + continue; + } + + // Compute the three potential sub-pieces [..lo, lo..hi, hi..] + // where lo, hi are local offsets within span_chars. + let lo = start.saturating_sub(span_start).min(len); + let hi = end.saturating_sub(span_start).min(len); + + if lo > 0 { + let s: String = span_chars[..lo].iter().collect(); + new_spans.push(Span::styled(s, span.style)); + } + if hi > lo { + let s: String = span_chars[lo..hi].iter().collect(); + new_spans.push(Span::styled( + s, + span.style.add_modifier(Modifier::REVERSED), + )); + } + if hi < len { + let s: String = span_chars[hi..].iter().collect(); + new_spans.push(Span::styled(s, span.style)); + } + } + line.spans = new_spans; +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn duration_seconds_under_minute() { + assert_eq!(format_duration(Duration::from_secs(0)), "0s"); + assert_eq!(format_duration(Duration::from_secs(1)), "1s"); + assert_eq!(format_duration(Duration::from_secs(59)), "59s"); + } + + #[test] + fn duration_mss_minute_to_hour() { + assert_eq!(format_duration(Duration::from_secs(60)), "1:00"); + assert_eq!(format_duration(Duration::from_secs(75)), "1:15"); + assert_eq!(format_duration(Duration::from_secs(3599)), "59:59"); + } + + #[test] + fn duration_hmmss_hour_plus() { + assert_eq!(format_duration(Duration::from_secs(3600)), "1:00:00"); + assert_eq!(format_duration(Duration::from_secs(3661)), "1:01:01"); + assert_eq!(format_duration(Duration::from_secs(36_000)), "10:00:00"); + } + + #[test] + fn duration_truncates_subsecond() { + // 5.9s should render as "5s" — second granularity only, + // never fractional. + assert_eq!(format_duration(Duration::from_millis(5_900)), "5s"); + } + + // ------------------------------------------------------------------ + // Windowed slicing regression + correctness + // ------------------------------------------------------------------ + + fn result_entry(text: &str) -> crate::app::ScrollbackEntry { + crate::app::ScrollbackEntry { + kind: crate::app::ScrollbackKind::Result, + text: text.to_string(), + started_at: None, + completed_at: None, + collapse_state: None, + is_critical_warning: false, + } + } + + /// A huge Result entry is what triggered the REPL lock-up: 200K + /// source lines, each cheap on its own but O(entry) work per + /// redraw when we allocated `Line`s for every one. `entry_lines` + /// (which delegates to the windowed impl with a full range) + /// still produces every row on demand — for the clipboard-copy + /// path — but `entry_lines_windowed` with a tight range should + /// produce only what fits. + #[test] + fn windowed_slicer_emits_only_visible_source_lines() { + // Build an entry where each source line fits on ONE wrapped + // row (short body + 2-cell prefix < area_width). Then the + // number of wrapped rows the entry contributes is exactly + // its source-line count, making the assertions easy to + // reason about. + let source: String = + (0..200_000).map(|i| format!("pin_{i}\n")).collect(); + let entry = result_entry(&source); + let (lines, offset) = + entry_lines_windowed(&entry, 80, 100_000..100_030); + // 30 lines requested, 30 lines emitted. + assert_eq!(lines.len(), 30); + // Window starts exactly on a source-line boundary — no + // sub-line offset. + assert_eq!(offset, 0); + } + + /// The FIRST emitted source line may land mid-window when the + /// window's start lands inside a line's wrapped output. The + /// slicer emits the whole line (row 0 of that line) and + /// reports the offset so ratatui's Paragraph::scroll can trim + /// the top. + #[test] + fn windowed_slicer_returns_sub_line_offset_when_window_starts_mid_line() { + // Build lines that wrap TO exactly 2 wrapped rows each on + // a width-10 area (2 prefix + 12 body chars → 14 total → + // ceil(14/10) = 2 rows). + let source: String = + (0..10).map(|i| format!("abcdefghjkl_{i:02}\n")).collect(); + let entry = result_entry(&source); + // Window that starts on the SECOND row of source line 3. + // Line 3's wrapped rows are [6, 8) globally; window [7, 12) + // should emit lines 3..6 with an offset of 1 wrapped row. + let (lines, offset) = entry_lines_windowed(&entry, 10, 7..12); + assert!(!lines.is_empty(), "expected some lines emitted"); + assert_eq!( + offset, 1, + "first line's row 0 is one row before window.start" + ); + } + + /// Full-range windowing must be identical to `entry_lines` — + /// otherwise the `entry_lines` shim would silently render + /// something different from what tests / clipboard-copy expect. + #[test] + fn windowed_slicer_full_range_matches_entry_lines() { + let source = "one\ntwo\nthree\nfour\nfive\nsix\nseven\neight\nnine\n"; + let entry = result_entry(source); + let unwindowed = entry_lines(&entry, 80); + let (windowed, offset) = entry_lines_windowed(&entry, 80, 0..u32::MAX); + assert_eq!(unwindowed.len(), windowed.len()); + for (a, b) in unwindowed.iter().zip(windowed.iter()) { + assert_eq!(a.spans.len(), b.spans.len()); + } + assert_eq!(offset, 0); + } +} diff --git a/vw-repl/src/session.rs b/vw-repl/src/session.rs new file mode 100644 index 0000000..468cb3a --- /dev/null +++ b/vw-repl/src/session.rs @@ -0,0 +1,341 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! In-memory REPL session, held as parsed batches rather than a +//! re-concatenated text blob. +//! +//! Every successful input contributes one [`SessionBatch`] — the +//! loaded program (own source + import edges), its parsed +//! [`vw_htcl::Document`], and the map from proc-name to +//! [`ProcLocation`] for every proc the batch declared. Prior +//! batches are read by [`crate::lower::prepare`] when lowering the +//! next input: their signatures resolve unknown calls; their proc +//! locations let the error renderer translate Tcl's `(procedure +//! "X" line N)` frames back to the real `.htcl` file the wrapper +//! body was declared in. +//! +//! Why this shape (vs. the original text-blob prelude): +//! +//! 1. **Performance.** After a few `src @lib` imports the prelude +//! is hundreds of thousands of lines. Re-parsing and re-walking +//! it on every input is what made the REPL feel laggy. Storing +//! parsed state means each new input parses only its own +//! content + transitive imports — O(new), not O(total). +//! 2. **Error rendering.** A drill-down frame for a wrapper proc +//! declared in an earlier batch knows the real `.htcl` path it +//! came from, so `(procedure "vivado::create_bd_design" line +//! 2)` resolves to `vivado-cmd/bd.htcl:42` instead of +//! `(input):199185` of the giant combined scratch. + +use std::collections::HashMap; + +use vw_htcl::{Document, LoadedProgram, ProcSignature, TypeDecl}; + +use crate::lower::ProcLocation; + +/// One committed input: the parsed program it produced, plus the +/// proc-location map the lowerer derived from it. Stored on a +/// per-batch basis so signatures and proc lookups can fold across +/// the whole session without ever re-parsing a prior batch. +#[derive(Debug)] +pub struct SessionBatch { + /// Loader output for this batch — file paths, import edges, + /// and the flattened source. Held alongside the parsed + /// document so future analyzer features (completion, goto- + /// def, hover) can walk back to per-file context without + /// re-running the loader. Spans inside [`document`] are + /// offsets into [`program.source`](LoadedProgram::source); + /// keeping the program alive keeps those spans meaningful. + #[allow(dead_code)] + pub program: LoadedProgram, + pub document: Document, + pub procs: HashMap, +} + +/// A live REPL session: every committed batch in order. +#[derive(Debug, Default)] +pub struct Session { + batches: Vec, +} + +impl Session { + pub fn new() -> Self { + Self::default() + } + + /// Append a batch — called from the App after every successful + /// eval (including the pure-`src` no-Tcl-to-eval case, which + /// commits immediately because no eval can fail). + pub fn commit(&mut self, batch: SessionBatch) { + self.batches.push(batch); + } + + /// Build a merged signature table covering every proc declared + /// in the session so far. Later batches shadow earlier ones, + /// matching Tcl's "second `proc` redefines" semantics. The + /// returned map borrows from `self`; held only for the duration + /// of the next batch's prepare() call. + pub fn signature_table(&self) -> HashMap { + let mut table: HashMap = HashMap::new(); + for batch in &self.batches { + // Per-batch table merges into the running table; later + // batches' entries overwrite earlier ones via `insert`. + let batch_table = vw_htcl::signature_table(&batch.document); + for (name, sig) in batch_table { + table.insert(name, sig); + } + } + table + } + + /// Same as [`signature_table`] but for `type NAME = …` + /// declarations. Needed when wrapping a typed expression's + /// result through its `repr` proc — the dispatch type may be + /// a newtype declared in a prior batch (e.g. `Properties` + /// from a sourced `@vivado-cmd` library), and the repr + /// codegen walks the underlying to emit the dependent generic + /// repr (`dict_string_Property::repr` in that case). + pub fn type_decl_table(&self) -> HashMap { + let mut table: HashMap = HashMap::new(); + for batch in &self.batches { + let mut diags = Vec::new(); + let batch_table = + vw_htcl::build_type_decl_table(&batch.document, &mut diags); + for (name, td) in batch_table { + table.insert(name, td); + } + } + table + } + + /// Union of every top-level variable name defined across every + /// batch. Passed to the validator so the undef-var pass doesn't + /// false-positive `set p …` in an earlier REPL input followed + /// by `$p` in a later one. Only top-level names — proc-body + /// locals don't leak across evals (Tcl semantics). + pub fn top_level_var_names(&self) -> std::collections::HashSet { + let mut names = std::collections::HashSet::new(); + for batch in &self.batches { + names.extend(vw_htcl::top_level_var_names( + &batch.document, + &batch.program.source, + )); + } + names + } + + /// Companion to [`top_level_var_names`] returning inferred + /// types for the top-level `set` bindings across every + /// committed batch. Later batches shadow earlier ones so + /// re-binding `set foo […]` overrides the previous entry. + /// + /// Signature lookup is CUMULATIVE: batch N's `set foo [bar]` + /// resolves against every proc defined in batches ≤ N. This + /// matters at the REPL when a user runs `src @vw` in one + /// batch (populating `vw::vhdl_dependency_sources`) and then + /// `set deps [vw::vhdl_dependency_sources]` in the next — + /// without the cumulative table, `deps` would type-infer as + /// `None` and the next batch's `putr $deps` would fall + /// through to plain `puts` and dump the flat Tcl list. + pub fn top_level_var_types( + &self, + ) -> std::collections::HashMap { + let mut types = std::collections::HashMap::new(); + // Accumulated signatures across every committed batch — + // owned entries because each batch's sig_table borrows + // from the batch's own document; keeping references + // across batches would require self-referential lifetimes. + let mut cumulative_sigs: std::collections::HashMap< + String, + vw_htcl::ProcSignature, + > = std::collections::HashMap::new(); + for batch in &self.batches { + let mut sig_diags = Vec::new(); + let batch_sigs = vw_htcl::validate::build_signature_table( + &batch.document, + &mut sig_diags, + ); + // Merge into the cumulative store — later batches win + // on name collisions (Tcl re-definition semantics). + for (name, sig) in &batch_sigs { + cumulative_sigs.insert(name.clone(), (*sig).clone()); + } + // Re-project as `&ProcSignature` for + // `top_level_var_types`, which takes borrows. + let sig_view: std::collections::HashMap< + String, + &vw_htcl::ProcSignature, + > = cumulative_sigs + .iter() + .map(|(n, s)| (n.clone(), s)) + .collect(); + let batch_types = + vw_htcl::top_level_var_types(&batch.document, &sig_view); + for (name, ty) in batch_types { + types.insert(name, ty); + } + } + types + } + + /// Per-file `(path, mtime-at-load-time)` map covering every + /// committed batch's loaded files. Passed to the next + /// batch's loader as the `preloaded` set: the loader + /// short-circuits `src ` only when the current on-disk + /// mtime matches the stored one, so a user editing a + /// `.htcl` file and re-running `src` at the REPL actually + /// picks up the change. Later batches shadow earlier ones + /// on overlapping paths (matches "most-recent read wins"). + /// + /// Cross-batch this is what avoids re-parsing every + /// transitive dep when the target hasn't changed — a + /// `src ip/gtm` after `--load prime.htcl` used to re-parse + /// hundreds of vivado-cmd files unconditionally; now it + /// stats each preloaded file once, matches mtimes, and + /// skips. + pub fn loaded_paths( + &self, + ) -> std::collections::HashMap + { + let mut out = std::collections::HashMap::new(); + for batch in &self.batches { + for f in &batch.program.files { + // Files whose mtime we couldn't capture at + // load time stay OUT of the map — the loader + // treats "not preloaded" as "always reload", + // which is the safe default. + if let Some(t) = f.mtime { + out.insert(f.path.clone(), t); + } + } + } + out + } + + /// Look up the most-recent proc location across every batch. + /// Returns `None` when no batch has declared that proc — the + /// error renderer's drill-down path silently skips such frames + /// (Tcl proc frames for builtins, dynamically-defined procs, + /// etc.). + /// Iterate committed batches newest-first. Used by the + /// signature-help / hover paths to walk back through documents + /// looking for a proc's doc comments — Tcl's "later proc + /// shadows earlier" semantics mean the most-recent definition + /// is the one the user expects to see described. + pub fn batches_for_doc_search( + &self, + ) -> impl Iterator { + self.batches.iter().rev() + } + + pub fn lookup_proc(&self, name: &str) -> Option<&ProcLocation> { + for batch in self.batches.iter().rev() { + if let Some(loc) = batch.procs.get(name) { + return Some(loc); + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use vw_htcl::parse; + + fn batch_from(source: &str) -> SessionBatch { + // Build a minimal in-memory LoadedProgram from a string for + // tests that don't care about the loader pipeline. + let parsed = parse(source); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + SessionBatch { + program: LoadedProgram { + source: source.to_string(), + files: Vec::new(), + regions: Vec::new(), + }, + document: parsed.document, + procs: HashMap::new(), + } + } + + #[test] + fn signature_table_folds_across_batches() { + let mut s = Session::new(); + s.commit(batch_from("proc foo { x } { }\n")); + s.commit(batch_from("proc bar { y } { }\n")); + let table = s.signature_table(); + assert!(table.contains_key("foo")); + assert!(table.contains_key("bar")); + } + + #[test] + fn later_batch_shadows_earlier_signature() { + // Second `proc foo` redefines the first — the merged table + // returns the newer signature. + let mut s = Session::new(); + s.commit(batch_from("proc foo { x } { }\n")); + s.commit(batch_from("proc foo { y z } { }\n")); + let table = s.signature_table(); + let sig = table.get("foo").unwrap(); + let arg_names: Vec<&str> = + sig.args.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(arg_names, vec!["y", "z"]); + } + + #[test] + fn top_level_var_types_resolves_across_batches() { + // Regression: batch 1 defines `proc lookup {} dict { ... }`; + // batch 2 does `set d [lookup]`. `top_level_var_types` must + // report `d: dict` — before the fix, each batch's inference + // saw only its own procs, so `d` came back untyped and the + // downstream `putr $d` fell through to plain `puts`. + let mut s = Session::new(); + s.commit(batch_from("proc lookup {} dict { return {a 1} }\n")); + s.commit(batch_from("set d [lookup]\n")); + let types = s.top_level_var_types(); + let ty = types + .get("d") + .expect("`d` should have an inferred type across batches"); + match ty { + vw_htcl::TypeExpr::Named { name, .. } => { + assert_eq!(name, "dict") + } + other => panic!("expected `dict`, got {other:?}"), + } + } + + #[test] + fn lookup_proc_returns_latest_batch() { + // Two batches both register `foo` in their `procs` map (the + // lowerer normally does this, but here we set it manually). + // `lookup_proc` returns the entry from the most recent + // batch. + let mut a = batch_from("proc foo { x } { }\n"); + let mut b = batch_from("proc foo { y } { }\n"); + a.procs.insert( + "foo".into(), + ProcLocation { + file: None, + body_start_line: 10, + body_lines: vec!["from-a".into()], + }, + ); + b.procs.insert( + "foo".into(), + ProcLocation { + file: None, + body_start_line: 20, + body_lines: vec!["from-b".into()], + }, + ); + let mut s = Session::new(); + s.commit(a); + s.commit(b); + let got = s.lookup_proc("foo").unwrap(); + assert_eq!(got.body_start_line, 20); + assert_eq!(got.body_lines[0], "from-b"); + assert!(s.lookup_proc("missing").is_none()); + } +} diff --git a/vw-repl/src/symbol_index.rs b/vw-repl/src/symbol_index.rs new file mode 100644 index 0000000..414fd5b --- /dev/null +++ b/vw-repl/src/symbol_index.rs @@ -0,0 +1,497 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Catalog of every named thing the session knows about — procs +//! (including overload dispatchers), type aliases, enum +//! declarations and their variants. Each entry carries the +//! **library** it came from (the `src @` import that brought +//! it into scope, or `` for the user's directly-typed +//! batch) plus its doc comments and a one-line signature brief. +//! +//! Consumed by the fuzzy symbol-search popup (slice 8) and the +//! libraries view (slice 9). The picker doesn't need to know +//! about parse state; it just needs a flat list of `Symbol`s with +//! enough metadata to display, filter, and rank. +//! +//! The index is purely structural — no fuzzy-matching here. That's +//! the picker's job; the index just hands it the candidate list. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use vw_htcl::ast::{CommandKind, Document, EnumVariant, ProcSignature, Stmt}; +use vw_htcl::loader::LoadedProgram; +use vw_htcl::span::Span; + +use crate::session::{Session, SessionBatch}; + +#[derive(Clone, Debug)] +pub struct Symbol { + /// Fully-qualified name (`util::props`, `Property::Scalar`, etc.). + pub name: String, + pub kind: SymbolKind, + pub library: LibraryRef, + /// First paragraph of the doc comments, reflowed for a compact + /// summary in the picker's result row. + pub doc_summary: String, + /// Full reflowed doc body, shown in the picker's preview pane + /// (when added) and in the hover popup (already wired). + pub doc_full: String, + /// One-line "signature brief" — `name -arg: type -arg: type → ret` + /// for procs, `enum Name = { V1; V2 }` for enums, etc. + pub signature_brief: String, + /// Origin span in the source the symbol was declared in. `None` + /// for variables (whose `set` site isn't a "declaration" in the + /// AST sense we want to jump to). Used by a future goto-def + /// keybinding; the picker doesn't need it. + pub def_span: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum SymbolKind { + Proc, + Type, + EnumDecl, + EnumVariant, + Variable, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LibraryRef { + /// Symbol was declared in the user's directly-typed batch (no + /// `src @` import). Shown as `` in the picker. + Entry, + /// Symbol was declared in a file pulled in via a `src @` + /// import (transitively — nested imports still attribute to the + /// top-level `@`). + Import { name: String, path: PathBuf }, +} + +impl LibraryRef { + /// Short display name — `` or the library's import name. + /// Used as the prefix in picker rows. + pub fn display(&self) -> String { + match self { + LibraryRef::Entry => "".to_string(), + LibraryRef::Import { name, .. } => name.clone(), + } + } +} + +#[derive(Clone, Debug)] +pub struct LibraryInfo { + pub library: LibraryRef, + /// Number of symbols this library contributes to the index. + pub symbol_count: usize, +} + +/// Flat catalog of every symbol the session + in-flight input +/// knows about. Build via [`SymbolIndex::build`]; iterate via +/// [`SymbolIndex::all`] or [`SymbolIndex::libraries`]. +#[derive(Clone, Debug, Default)] +pub struct SymbolIndex { + all: Vec, +} + +impl SymbolIndex { + pub fn build( + session: &Session, + pending: Option<&SessionBatch>, + in_flight: Option<&Document>, + ) -> Self { + let mut symbols = Vec::new(); + for batch in session.batches_for_doc_search() { + collect_from_batch(batch, &mut symbols); + } + // Pending batch — the in-flight eval. During a long + // `src @vivado-cmd` load, every proc the user just sourced + // lives here, NOT yet in session.signature_table. Including + // it makes Ctrl-S / :libs work mid-eval, same way the + // Tab-completion fix did. + if let Some(batch) = pending { + collect_from_batch(batch, &mut symbols); + } + if let Some(doc) = in_flight { + collect_from_doc(doc, None, &mut symbols); + } + // Dedupe by (name, kind) — later-batch decls shadow earlier + // (Tcl's "later proc redefines" semantics). Since we walked + // session newest-first via batches_for_doc_search, the FIRST + // occurrence in our list is the one to keep. + let mut seen: BTreeMap<(String, SymbolKind), ()> = BTreeMap::new(); + symbols.retain(|s| seen.insert((s.name.clone(), s.kind), ()).is_none()); + Self { all: symbols } + } + + pub fn all(&self) -> &[Symbol] { + &self.all + } + + /// Distinct libraries with their symbol counts, sorted by + /// descending count (heavyweights like `vivado-cmd` first). + pub fn libraries(&self) -> Vec { + let mut counts: BTreeMap = BTreeMap::new(); + for sym in &self.all { + let entry = counts + .entry(sym.library.display()) + .or_insert_with(|| (sym.library.clone(), 0)); + entry.1 += 1; + } + let mut out: Vec = counts + .into_values() + .map(|(library, symbol_count)| LibraryInfo { + library, + symbol_count, + }) + .collect(); + out.sort_by_key(|b| std::cmp::Reverse(b.symbol_count)); + out + } +} + +fn collect_from_batch(batch: &SessionBatch, out: &mut Vec) { + collect_from_doc(&batch.document, Some(&batch.program), out); +} + +fn collect_from_doc( + doc: &Document, + program: Option<&LoadedProgram>, + out: &mut Vec, +) { + walk_stmts(&doc.stmts, "", program, out); +} + +fn walk_stmts( + stmts: &[Stmt], + prefix: &str, + program: Option<&LoadedProgram>, + out: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { + continue; + }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let Some(sig) = proc.signature.as_ref() else { + continue; + }; + let qualified = qualify(prefix, name); + let library = library_for_span(program, cmd.span); + let signature_brief = render_signature(&qualified, sig); + let doc_summary = + vw_htcl::doc::brief(&cmd.doc_comments).unwrap_or_default(); + let doc_full = + vw_htcl::doc::reflow_doc_comments(&cmd.doc_comments); + out.push(Symbol { + name: qualified, + kind: SymbolKind::Proc, + library, + doc_summary, + doc_full, + signature_brief, + def_span: Some(proc.name_span), + }); + // Recurse into proc body so nested procs / type decls + // also enter the index. Rare but supported. + walk_stmts(&proc.body, prefix, program, out); + } + CommandKind::TypeDecl(td) => { + let Some(name) = td.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + let library = library_for_span(program, cmd.span); + let underlying = td + .underlying + .as_ref() + .map(render_type) + .unwrap_or_else(|| "?".into()); + let signature_brief = + format!("type {qualified} = {underlying}"); + let doc_summary = + vw_htcl::doc::brief(&cmd.doc_comments).unwrap_or_default(); + let doc_full = + vw_htcl::doc::reflow_doc_comments(&cmd.doc_comments); + out.push(Symbol { + name: qualified, + kind: SymbolKind::Type, + library, + doc_summary, + doc_full, + signature_brief, + def_span: Some(td.name_span), + }); + } + CommandKind::EnumDecl(ed) => { + let Some(name) = ed.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + let library = library_for_span(program, cmd.span); + let variants_brief: Vec = ed + .variants + .iter() + .map(|v| { + if let Some(ty) = v.payload.as_ref() { + format!("{}: {}", v.name, render_type(ty)) + } else { + v.name.clone() + } + }) + .collect(); + let signature_brief = format!( + "enum {qualified} = {{ {} }}", + variants_brief.join("; ") + ); + let doc_summary = + vw_htcl::doc::brief(&cmd.doc_comments).unwrap_or_default(); + let doc_full = + vw_htcl::doc::reflow_doc_comments(&cmd.doc_comments); + out.push(Symbol { + name: qualified.clone(), + kind: SymbolKind::EnumDecl, + library: library.clone(), + doc_summary: doc_summary.clone(), + doc_full: doc_full.clone(), + signature_brief, + def_span: Some(ed.name_span), + }); + // One Symbol per variant so users can search for + // variant names directly (`Scalar`, `Nested`, …). + for v in &ed.variants { + push_variant(&qualified, v, library.clone(), out); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + let nested = qualify(prefix, name); + walk_stmts(&ns.body, &nested, program, out); + } + CommandKind::Set => { + // Top-level / proc-body variable. Take the first + // word as the variable name. Variables are tagged + // with the library of the enclosing batch but get + // no def_span (set isn't a declaration in the + // jump-to-def sense we want). + if let Some(name_word) = cmd.words.get(1) { + if let Some(name) = name_word.as_text() { + let library = library_for_span(program, cmd.span); + out.push(Symbol { + name: qualify(prefix, name), + kind: SymbolKind::Variable, + library, + doc_summary: String::new(), + doc_full: String::new(), + signature_brief: format!("set {name}"), + def_span: None, + }); + } + } + } + _ => {} + } + } +} + +fn push_variant( + enum_qualified: &str, + v: &EnumVariant, + library: LibraryRef, + out: &mut Vec, +) { + let qualified = format!("{enum_qualified}::{}", v.name); + let signature_brief = if let Some(ty) = v.payload.as_ref() { + format!("{qualified}({})", render_type(ty)) + } else { + qualified.clone() + }; + out.push(Symbol { + name: qualified, + kind: SymbolKind::EnumVariant, + library, + doc_summary: String::new(), + doc_full: String::new(), + signature_brief, + def_span: Some(v.name_span), + }); +} + +fn qualify(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + } +} + +/// Attribute a source span to its originating library. Walks the +/// loader's `regions` + `files` tables to find which file the span +/// came from, then climbs the `imported_via` chain to the +/// top-level import (the `src @` directly under the entry +/// file). Returns `LibraryRef::Entry` when the span lies in the +/// entry file itself or when no `LoadedProgram` is provided +/// (in-flight input that hasn't been loaded through the import +/// machinery). +fn library_for_span(program: Option<&LoadedProgram>, span: Span) -> LibraryRef { + let Some(program) = program else { + return LibraryRef::Entry; + }; + let Some((file_idx, _)) = program.locate(span.start) else { + return LibraryRef::Entry; + }; + // Build the chain: [origin_file, ..., entry_file]. + let mut chain = vec![file_idx]; + let mut cur = file_idx; + while let Some(edge) = program.files[cur].imported_via { + chain.push(edge.importer_file); + cur = edge.importer_file; + } + if chain.len() <= 1 { + // The origin is the entry file. + return LibraryRef::Entry; + } + // The element just before the entry is the top-level imported + // file — that's our library. + let top_imported = chain[chain.len() - 2]; + let file = &program.files[top_imported]; + let name = library_name_for_path(&file.path); + LibraryRef::Import { + name, + path: file.path.clone(), + } +} + +/// Short, user-facing library name. We use the parent directory's +/// name when it's available (so +/// `/home/ry/src/htcl/amd/vivado-cmd/module.htcl` becomes +/// `vivado-cmd`); otherwise fall back to the file's stem. +fn library_name_for_path(path: &std::path::Path) -> String { + if let Some(parent) = path.parent() { + if let Some(name) = parent.file_name() { + return name.to_string_lossy().into_owned(); + } + } + path.file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.display().to_string()) +} + +fn render_type(ty: &vw_htcl::TypeExpr) -> String { + use vw_htcl::TypeExpr; + match ty { + TypeExpr::Named { name, .. } => name.clone(), + TypeExpr::Generic { name, args, .. } => { + let inner: Vec = args.iter().map(render_type).collect(); + format!("{name}<{}>", inner.join(", ")) + } + TypeExpr::Qualified { + namespace, variant, .. + } => format!("{namespace}::{variant}"), + } +} + +fn render_signature(name: &str, sig: &ProcSignature) -> String { + let mut out = name.to_string(); + for arg in &sig.args { + out.push_str(" -"); + out.push_str(&arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + out.push_str(": "); + out.push_str(&render_type(ty)); + } + } + if let Some(ret) = sig.return_type.as_ref() { + out.push_str(" → "); + out.push_str(&render_type(ret)); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use vw_htcl::parser::parse; + + #[test] + fn empty_session_yields_empty_index() { + let session = Session::default(); + let idx = SymbolIndex::build(&session, None, None); + assert!(idx.all().is_empty()); + assert!(idx.libraries().is_empty()); + } + + #[test] + fn in_flight_proc_decl_indexed() { + let session = Session::default(); + let parsed = parse("proc foo {x: int} bool { return $x }"); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + assert_eq!(idx.all().len(), 1); + let sym = &idx.all()[0]; + assert_eq!(sym.name, "foo"); + assert_eq!(sym.kind, SymbolKind::Proc); + assert_eq!(sym.library, LibraryRef::Entry); + assert!(sym.signature_brief.contains("foo")); + assert!(sym.signature_brief.contains("int")); + assert!(sym.signature_brief.contains("bool")); + } + + #[test] + fn namespaced_procs_qualified() { + let session = Session::default(); + let src = "namespace eval util {\n proc props {x: int} string { return $x }\n}"; + let parsed = parse(src); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + assert!( + idx.all().iter().any(|s| s.name == "util::props"), + "expected util::props in index: {:?}", + idx.all().iter().map(|s| &s.name).collect::>() + ); + } + + #[test] + fn enum_emits_decl_and_variants() { + let session = Session::default(); + let src = + "enum Property = {\n Scalar: string\n Nested: Properties\n}"; + let parsed = parse(src); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + let names: Vec<&str> = + idx.all().iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"Property"), "{names:?}"); + assert!(names.contains(&"Property::Scalar"), "{names:?}"); + assert!(names.contains(&"Property::Nested"), "{names:?}"); + } + + #[test] + fn type_decl_indexed() { + let session = Session::default(); + let parsed = parse("type Properties = {dict}"); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + let ty = idx + .all() + .iter() + .find(|s| s.name == "Properties" && s.kind == SymbolKind::Type) + .expect("Properties not found"); + assert!(ty.signature_brief.contains("type")); + assert!(ty.signature_brief.contains("dict")); + } + + #[test] + fn libraries_count_symbols() { + let session = Session::default(); + let parsed = parse( + "proc foo {} unit {}\nproc bar {} unit {}\nproc baz {} unit {}", + ); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + let libs = idx.libraries(); + assert_eq!(libs.len(), 1); + assert_eq!(libs[0].symbol_count, 3); + } +} diff --git a/vw-repl/src/symbol_search.rs b/vw-repl/src/symbol_search.rs new file mode 100644 index 0000000..2e532e4 --- /dev/null +++ b/vw-repl/src/symbol_search.rs @@ -0,0 +1,488 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Fuzzy symbol picker — Ctrl-T opens a centered modal listing every +//! known procedure / type / enum / variant across the session and +//! the in-flight input. Typing filters; ↑/↓ navigates; Enter inserts +//! the symbol's qualified name at the cursor in the input editor. +//! +//! The matcher is [`nucleo_matcher`] — the same engine Helix and +//! several other Rust TUIs use. Each candidate is scored twice: +//! once against its `name` (high weight) and once against its +//! `doc_summary` (low weight). The final score takes the max of +//! the two, so a query that hits ONLY a doc-comment phrase still +//! surfaces the symbol but ranks below any name match. +//! +//! Tab inside the picker switches to the **library view** — +//! `(library, symbol-count)` rows. Enter on a library row filters +//! the symbol list to that library. + +use std::sync::Arc; + +use nucleo_matcher::pattern::{CaseMatching, Normalization, Pattern}; +use nucleo_matcher::{Config, Matcher, Utf32String}; +use ratatui::layout::Rect; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{ + Block, Borders, Clear, List, ListItem, ListState, Paragraph, +}; +use ratatui::Frame; + +use crate::symbol_index::{LibraryInfo, LibraryRef, SymbolIndex, SymbolKind}; + +/// Boost factor applied to name-match scores so name hits always +/// outrank doc-only hits. Picked by trial — a value of 3 keeps a +/// short doc match (e.g. one word) below any name fuzzy hit. +const NAME_WEIGHT: u32 = 3; + +/// Symbol-picker overlay state. +#[derive(Debug)] +pub struct SymbolPicker { + /// Snapshot of the symbol index when the picker was opened. + /// We don't update mid-search — the index may not change in + /// practice (no commits land while a popup is open), and a + /// stable index avoids selection-index churn while typing. + pub index: Arc, + /// Live query string the user is typing. + pub query: String, + /// Scored result indices into `index.all()` in display order. + pub results: Vec, + pub selected: usize, + /// When `Some(name)`, the result list is filtered to that + /// library. Set by accepting a row in the libraries sub-view. + pub library_filter: Option, + /// Toggle between symbol list and library list. + pub view: PickerView, + /// Cached library list (computed once on open). + pub libraries: Vec, + /// Selected library row (only used in `Libraries` view). + pub selected_library: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PickerView { + Symbols, + Libraries, +} + +#[derive(Clone, Copy, Debug)] +pub struct Scored { + pub sym_idx: usize, + pub score: u32, +} + +impl SymbolPicker { + pub fn new(index: Arc) -> Self { + let libraries = index.libraries(); + let mut p = Self { + index, + query: String::new(), + results: Vec::new(), + selected: 0, + library_filter: None, + view: PickerView::Symbols, + libraries, + selected_library: 0, + }; + p.recompute(); + p + } + + pub fn move_up(&mut self) { + match self.view { + PickerView::Symbols => { + if self.selected > 0 { + self.selected -= 1; + } + } + PickerView::Libraries => { + if self.selected_library > 0 { + self.selected_library -= 1; + } + } + } + } + + pub fn move_down(&mut self) { + match self.view { + PickerView::Symbols => { + if self.selected + 1 < self.results.len() { + self.selected += 1; + } + } + PickerView::Libraries => { + if self.selected_library + 1 < self.libraries.len() { + self.selected_library += 1; + } + } + } + } + + pub fn push_char(&mut self, c: char) { + self.query.push(c); + self.recompute(); + } + + pub fn pop_char(&mut self) { + self.query.pop(); + self.recompute(); + } + + pub fn toggle_view(&mut self) { + self.view = match self.view { + PickerView::Symbols => PickerView::Libraries, + PickerView::Libraries => PickerView::Symbols, + }; + } + + /// Apply a library filter — sets `library_filter` and switches + /// back to the symbols view. Used when Enter is pressed on a + /// library row. + pub fn apply_library_filter(&mut self) { + if let Some(lib) = self.libraries.get(self.selected_library) { + self.library_filter = Some(lib.library.display()); + self.view = PickerView::Symbols; + self.selected = 0; + self.recompute(); + } + } + + /// Currently-selected symbol, if any (Symbols view only). + pub fn current_symbol(&self) -> Option<&crate::symbol_index::Symbol> { + let idx = self.results.get(self.selected)?.sym_idx; + self.index.all().get(idx) + } + + /// Re-run the matcher with the current query and library + /// filter. Called from `new` / `push_char` / `pop_char` / + /// `apply_library_filter`. O(N) over the index. + fn recompute(&mut self) { + let mut matcher = Matcher::new(Config::DEFAULT); + let all = self.index.all(); + // Indices that survive the library filter (if any). + let candidate_indices: Vec = all + .iter() + .enumerate() + .filter(|(_, s)| match &self.library_filter { + Some(lib) => s.library.display() == *lib, + None => true, + }) + .map(|(i, _)| i) + .collect(); + + if self.query.is_empty() { + // No query: show all candidates sorted alphabetically, + // capped at 200 so the popup is bounded. + let mut sorted = candidate_indices.clone(); + sorted.sort_by(|a, b| all[*a].name.cmp(&all[*b].name)); + sorted.truncate(200); + self.results = sorted + .into_iter() + .map(|sym_idx| Scored { sym_idx, score: 0 }) + .collect(); + self.selected = + self.selected.min(self.results.len().saturating_sub(1)); + return; + } + + let pattern = Pattern::parse( + &self.query, + CaseMatching::Smart, + Normalization::Smart, + ); + let mut scored: Vec = candidate_indices + .iter() + .filter_map(|&sym_idx| { + let sym = &all[sym_idx]; + let name_haystack = Utf32String::from(sym.name.as_str()); + let doc_haystack = Utf32String::from(sym.doc_summary.as_str()); + let name_score = + pattern.score(name_haystack.slice(..), &mut matcher); + let doc_score = + pattern.score(doc_haystack.slice(..), &mut matcher); + let combined = match (name_score, doc_score) { + (None, None) => 0, + (Some(n), None) => n * NAME_WEIGHT, + (None, Some(d)) => d, + (Some(n), Some(d)) => (n * NAME_WEIGHT).max(d), + }; + if combined == 0 { + None + } else { + Some(Scored { + sym_idx, + score: combined, + }) + } + }) + .collect(); + scored.sort_by_key(|b| std::cmp::Reverse(b.score)); + scored.truncate(200); + self.results = scored; + if self.selected >= self.results.len() { + self.selected = self.results.len().saturating_sub(1); + } + } +} + +/// Render the picker as a centered modal. Sized at 70% width × 75% +/// height of the frame. +pub fn draw_symbol_picker(f: &mut Frame, picker: &SymbolPicker) { + let frame = f.area(); + let width = (frame.width as f32 * 0.7) as u16; + let height = (frame.height as f32 * 0.75) as u16; + let x = frame.x + (frame.width.saturating_sub(width)) / 2; + let y = frame.y + (frame.height.saturating_sub(height)) / 2; + let area = Rect { + x, + y, + width, + height, + }; + f.render_widget(Clear, area); + + // Top: query line (1 row). Middle: list (fills). Bottom: hint + // (1 row). Use a manual vertical split since this is a small + // fixed layout. + let inner_top = Rect { + x: area.x + 1, + y: area.y + 1, + width: area.width.saturating_sub(2), + height: 1, + }; + let inner_hint = Rect { + x: area.x + 1, + y: area.y + area.height.saturating_sub(2), + width: area.width.saturating_sub(2), + height: 1, + }; + let inner_list = Rect { + x: area.x + 1, + y: area.y + 2, + width: area.width.saturating_sub(2), + height: area.height.saturating_sub(4), + }; + + let title = match picker.view { + PickerView::Symbols => " symbol search ", + PickerView::Libraries => " libraries — Enter to filter ", + }; + f.render_widget(Block::default().borders(Borders::ALL).title(title), area); + + let prompt_style = Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD); + let dim = Style::default().add_modifier(Modifier::DIM); + let prompt_prefix = match picker.view { + PickerView::Symbols => "› ", + PickerView::Libraries => "» ", + }; + let prompt_text = match (picker.view, picker.library_filter.as_deref()) { + (PickerView::Symbols, Some(lib)) => { + format!("{prompt_prefix}[{lib}] {}", picker.query) + } + (PickerView::Symbols, None) => { + format!("{prompt_prefix}{}", picker.query) + } + (PickerView::Libraries, _) => "(use ↑/↓; Enter to filter)".to_string(), + }; + f.render_widget( + Paragraph::new(Span::styled(prompt_text, prompt_style)), + inner_top, + ); + + match picker.view { + PickerView::Symbols => render_symbols(f, picker, inner_list), + PickerView::Libraries => render_libraries(f, picker, inner_list), + } + + let hint = match picker.view { + PickerView::Symbols => "Tab: libraries · Enter: insert · Esc: dismiss", + PickerView::Libraries => "Tab: symbols · Enter: filter · Esc: dismiss", + }; + f.render_widget(Paragraph::new(Span::styled(hint, dim)), inner_hint); +} + +fn render_symbols(f: &mut Frame, picker: &SymbolPicker, area: Rect) { + let all = picker.index.all(); + let items: Vec = picker + .results + .iter() + .filter_map(|scored| all.get(scored.sym_idx)) + .map(|sym| { + let icon = match sym.kind { + SymbolKind::Proc => "·", + SymbolKind::Type => "≡", + SymbolKind::EnumDecl => "◆", + SymbolKind::EnumVariant => "◇", + SymbolKind::Variable => "$", + }; + let icon_style = match sym.kind { + SymbolKind::Proc => { + Style::default().fg(Color::Rgb(230, 200, 120)) + } + SymbolKind::Type + | SymbolKind::EnumDecl + | SymbolKind::EnumVariant => { + Style::default().fg(Color::Rgb(100, 200, 200)) + } + SymbolKind::Variable => { + Style::default().fg(Color::Rgb(130, 200, 230)) + } + }; + let lib_style = Style::default().add_modifier(Modifier::DIM); + let name_style = Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD); + let doc_style = Style::default().fg(Color::Gray); + + let mut spans = vec![ + Span::styled(format!("{icon} "), icon_style), + Span::styled(format!("{} ", sym.library.display()), lib_style), + Span::styled(":: ", lib_style), + Span::styled(sym.name.clone(), name_style), + ]; + if !sym.doc_summary.is_empty() { + spans.push(Span::raw(" ")); + spans.push(Span::styled(sym.doc_summary.clone(), doc_style)); + } + ListItem::new(Line::from(spans)) + }) + .collect(); + + let mut state = ListState::default(); + if !picker.results.is_empty() { + state.select(Some(picker.selected)); + } + let list = List::new(items).highlight_style( + Style::default() + .bg(Color::Rgb(40, 40, 60)) + .add_modifier(Modifier::BOLD), + ); + f.render_stateful_widget(list, area, &mut state); +} + +fn render_libraries(f: &mut Frame, picker: &SymbolPicker, area: Rect) { + let items: Vec = picker + .libraries + .iter() + .map(|info| { + let name = info.library.display(); + let path_str = match &info.library { + LibraryRef::Entry => "".to_string(), + LibraryRef::Import { path, .. } => path.display().to_string(), + }; + ListItem::new(Line::from(vec![ + Span::styled( + format!(" {:5} ", info.symbol_count), + Style::default() + .fg(Color::Rgb(230, 200, 120)) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + name, + Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD), + ), + Span::raw(" "), + Span::styled( + path_str, + Style::default().add_modifier(Modifier::DIM), + ), + ])) + }) + .collect(); + let mut state = ListState::default(); + if !picker.libraries.is_empty() { + state.select(Some(picker.selected_library)); + } + let list = List::new(items).highlight_style( + Style::default() + .bg(Color::Rgb(40, 40, 60)) + .add_modifier(Modifier::BOLD), + ); + f.render_stateful_widget(list, area, &mut state); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use crate::session::Session; + use crate::symbol_index::SymbolIndex; + + fn build_index_from_src(src: &str) -> Arc { + let session = Session::default(); + let parsed = vw_htcl::parse(src); + Arc::new(SymbolIndex::build(&session, None, Some(&parsed.document))) + } + + #[test] + fn empty_query_shows_all_alphabetic() { + let idx = build_index_from_src( + "proc foo {} unit {}\nproc bar {} unit {}\nproc baz {} unit {}", + ); + let picker = SymbolPicker::new(idx); + let names: Vec<&str> = picker + .results + .iter() + .map(|s| picker.index.all()[s.sym_idx].name.as_str()) + .collect(); + assert_eq!(names, vec!["bar", "baz", "foo"]); + } + + #[test] + fn query_filters_results() { + let idx = build_index_from_src( + "proc foo {} unit {}\nproc bar {} unit {}\nproc baz {} unit {}", + ); + let mut picker = SymbolPicker::new(idx); + picker.push_char('b'); + picker.push_char('a'); + let names: Vec<&str> = picker + .results + .iter() + .map(|s| picker.index.all()[s.sym_idx].name.as_str()) + .collect(); + assert!(names.contains(&"bar")); + assert!(names.contains(&"baz")); + assert!(!names.contains(&"foo")); + } + + #[test] + fn name_match_outranks_doc_only() { + // A proc whose name doesn't match but whose docs mention the + // query word should rank BELOW a proc whose name matches. + let src = "\ +## a wonderful procedure that does foobar things +proc unrelated {} unit {} +proc foobar_proc {} unit {}"; + let idx = build_index_from_src(src); + let mut picker = SymbolPicker::new(idx); + for c in "foobar".chars() { + picker.push_char(c); + } + let names: Vec<&str> = picker + .results + .iter() + .map(|s| picker.index.all()[s.sym_idx].name.as_str()) + .collect(); + // foobar_proc must come first + assert_eq!(names.first().copied(), Some("foobar_proc"), "{names:?}"); + } + + #[test] + fn library_view_toggles() { + let idx = build_index_from_src("proc foo {} unit {}"); + let mut picker = SymbolPicker::new(idx); + assert_eq!(picker.view, PickerView::Symbols); + picker.toggle_view(); + assert_eq!(picker.view, PickerView::Libraries); + picker.toggle_view(); + assert_eq!(picker.view, PickerView::Symbols); + } +} diff --git a/vw-repl/src/trace.rs b/vw-repl/src/trace.rs new file mode 100644 index 0000000..0340c42 --- /dev/null +++ b/vw-repl/src/trace.rs @@ -0,0 +1,163 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Stack-frame rewriting for Vivado error / warning messages. +//! +//! Vivado reports errors with frames like +//! ` at :14 in ::configure_cips` +//! +//! where `` is the scratch path the lowerer ships and the +//! line number is body-relative inside the proc. This module maps +//! those back to the original htcl source: +//! ` at ip/cips.htcl:69 in ::configure_cips` +//! +//! Both the REPL (driven by a multi-batch [`crate::session::Session`]) +//! and the `vw run` CLI driver (single batch) feed messages through +//! the same `resolve_stack_frames_with` machinery — they differ only +//! in how they answer the "where does proc P live?" question, supplied +//! as a closure. + +use std::path::Path; + +use crate::lower::ProcLocation; + +/// One stack-frame line after rewriting. Callers dedupe adjacent +/// frames that resolve to the same `(proc, line)` because Vivado +/// often emits two frames per logical site (one for the `proc`'s +/// `kwargs` wrapper, one for the real body). +pub struct RewrittenFrame { + pub proc: String, + pub line: u32, + pub formatted: String, +} + +/// Walk a message line-by-line, rewriting any `at :N in +/// ::proc` frames using `lookup`. Lines that don't match the +/// stack-frame grammar (regular message prose) pass through +/// unchanged. Adjacent frames that resolve to the same +/// `(proc, line)` are collapsed — the Vivado kwargs-wrapper + +/// body-call doubling becomes a single rendered frame. +pub fn resolve_stack_frames_with( + msg: &str, + lookup: F, + input_file: Option<&Path>, +) -> String +where + F: Fn(&str) -> Option, +{ + let mut out = String::with_capacity(msg.len()); + let mut last_resolved_key: Option<(String, u32)> = None; + for (i, line) in msg.lines().enumerate() { + if i > 0 { + out.push('\n'); + } + let Some(rewritten) = rewrite_stack_line(line, &lookup, input_file) + else { + out.push_str(line); + last_resolved_key = None; + continue; + }; + let key = (rewritten.proc.clone(), rewritten.line); + if last_resolved_key.as_ref() == Some(&key) { + if out.ends_with('\n') { + out.pop(); + } + continue; + } + last_resolved_key = Some(key); + out.push_str(&rewritten.formatted); + } + out +} + +/// Parse a single line like ` at :14 in ::configure_cips` +/// and rewrite it to point at the user's actual htcl source. +/// Returns `None` when the line isn't a stack frame (regular +/// message text) or when the proc isn't one we know about (Vivado +/// builtins, dynamic procs, etc.) — caller passes such lines +/// through unchanged. +pub fn rewrite_stack_line( + line: &str, + lookup: F, + input_file: Option<&Path>, +) -> Option +where + F: Fn(&str) -> Option, +{ + // Grammar emitted by `vw::format_frame`: + // " at :N in ::procname" ← lookup ProcLocation by name + // " at :N in ::procname" ← already absolute + // " at :N" ← anonymous eval / top-level + // " at " ← location-less + let rest = line.strip_prefix(" at ")?; + let (loc_str, proc_part) = match rest.split_once(" in ") { + Some((l, p)) => (l, Some(p.trim().to_string())), + None => (rest, None), + }; + let (file_part, line_part) = loc_str.rsplit_once(':')?; + let body_line: u32 = line_part.parse().ok()?; + + // Top-level `:N` frame (no proc). + let Some(proc) = proc_part else { + if file_part != "" { + return None; + } + let path = input_file?; + return Some(RewrittenFrame { + proc: String::new(), + line: body_line, + formatted: format!(" at {}:{body_line}", display_path(path)), + }); + }; + + // Already-absolute frames don't need rewriting; pass through + // (dedup downstream still benefits from parsed proc+line). + if file_part != "" { + return Some(RewrittenFrame { + proc, + line: body_line, + formatted: line.to_string(), + }); + } + // `:N in ::proc` — Tcl reports "line N of the proc + // body." Resolve through the lookup. Tcl always reports + // fully-qualified names (leading `::`); the proc table + // indexes them without (see `lower::qualify`), so strip + // before lookup. + let lookup_name = proc.strip_prefix("::").unwrap_or(&proc); + let loc = lookup(lookup_name)?; + let (abs_line, _content) = loc.resolve_body_line(body_line)?; + let path_str = match loc.file.as_deref() { + Some(p) => display_path(p), + None => match input_file { + Some(p) => display_path(p), + None => "".to_string(), + }, + }; + Some(RewrittenFrame { + proc: proc.clone(), + line: abs_line, + formatted: format!(" at {path_str}:{abs_line} in {proc}"), + }) +} + +/// Pretty-print a file path for diagnostics: prefer the cwd- +/// relative form (`ip/cips.htcl`) when the path is under the +/// current working directory, then home-relative (`~/src/…`), +/// then the absolute form. Matches the REPL's scrollback so +/// vw run + vw repl render the same way. +pub fn display_path(path: &Path) -> String { + if let Ok(cwd) = std::env::current_dir() { + if let Ok(rel) = path.strip_prefix(&cwd) { + return rel.display().to_string(); + } + } + if let Ok(home) = std::env::var("HOME") { + let home_path = Path::new(&home); + if let Ok(rel) = path.strip_prefix(home_path) { + return format!("~/{}", rel.display()); + } + } + path.display().to_string() +} diff --git a/vw-repl/src/ui.rs b/vw-repl/src/ui.rs new file mode 100644 index 0000000..5dd082e --- /dev/null +++ b/vw-repl/src/ui.rs @@ -0,0 +1,561 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! ratatui rendering for the REPL. +//! +//! Layout (top-to-bottom): +//! +//! ```text +//! ┌──────────────────────────────────────────┐ +//! │ scrollback ▲│ +//! │ (eval log, oldest top, newest bottom) ║│ +//! │ ▼│ +//! ├──────────────────────────────────────────┤ +//! │ input (multi-line, tui-textarea owned) │ +//! ├──────────────────────────────────────────┤ +//! │ status: vivado state | hints │ +//! └──────────────────────────────────────────┘ +//! ``` +//! +//! When Ctrl-R is active, a centered overlay replaces the input area +//! with the search query and the matching history entry. + +use crate::app::{App, ReverseSearch}; +use ratatui::layout::{Constraint, Direction, Layout, Rect}; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{ + Block, Borders, Clear, Paragraph, Scrollbar, ScrollbarOrientation, + ScrollbarState, Wrap, +}; +use ratatui::Frame; + +pub fn draw(f: &mut Frame, app: &mut App) { + let layout = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Min(1), // scrollback (fills) + Constraint::Length(input_height(app)), // input + Constraint::Length(1), // status bar + ]) + .split(f.area()); + + draw_scrollback(f, layout[0], app); + draw_input(f, layout[1], app); + draw_status(f, layout[2], app); + + if let Some(rs) = app.reverse_search() { + draw_reverse_search(f, layout[1], rs); + } + if let Some(popup) = app.popup_state() { + match popup { + crate::popup::PopupState::Completion(c) => { + crate::popup::draw_completion_popup(f, c, f.area()); + } + crate::popup::PopupState::SignatureHelp(s) => { + crate::popup::draw_signature_help_popup(f, s, f.area()); + } + crate::popup::PopupState::Hover(h) => { + crate::popup::draw_hover_popup(f, h, f.area()); + } + crate::popup::PopupState::Help(_) => { + crate::popup::draw_help_popup(f, f.area()); + } + crate::popup::PopupState::SymbolSearch(p) => { + crate::symbol_search::draw_symbol_picker(f, p); + } + crate::popup::PopupState::DiagnosticSearch(p) => { + crate::diag_search::draw_diagnostic_picker(f, p); + } + } + } +} + +fn input_height(app: &App) -> u16 { + // Start at a 5-line minimum so the user has room to draft a + // multi-statement entry without the input box flickering taller + // mid-typing. Grows past 5 with the buffer, capped at 12 so + // very long entries don't squeeze the scrollback out. + let lines = app.input_line_count().clamp(5, 12) as u16; + lines + 2 // +2 for the top/bottom block border +} + +fn draw_scrollback(f: &mut Frame, area: Rect, app: &mut App) { + if area.width == 0 || area.height == 0 { + app.set_scrollback_area(area); + return; + } + + // Pass 1: cheap per-entry wrapped-row count, no allocations. + // O(text length) for each, vs. the old approach which built + // and wrapped every entry per draw — turning a single huge + // entry into multi-MB of per-char allocation on every wheel + // tick. With this pass, total work per draw is O(scrollback) + // for counting + O(viewport) for actually wrapping the + // visible window. + // + // Auto-hide scrollbar policy: count once at full width to + // decide "does content fit?"; if it doesn't, narrow the + // paragraph by one column to make room for the scrollbar and + // recount. This costs a second pass ONLY on frames where the + // scrollbar is visible — the common overflow-free case pays + // just one pass. + let counts_full: Vec = app + .scrollback() + .iter() + .map(|e| crate::render::count_wrapped_rows(e, area.width)) + .collect(); + let total_full: u32 = + counts_full.iter().fold(0u32, |a, b| a.saturating_add(*b)); + let needs_scrollbar = total_full > area.height as u32; + + // `paragraph_area` is where wrapped text actually renders. + // When the scrollbar is on, it takes the rightmost column of + // `area`, leaving `area.width - 1` for text. Also the + // scrollback_area we hand back to App (for mouse coord + // mapping) so a click on the scrollbar column doesn't get + // interpreted as a text-selection click. + let (paragraph_area, counts, total) = if needs_scrollbar && area.width >= 2 + { + let narrower = Rect { + width: area.width - 1, + ..area + }; + let cs: Vec = app + .scrollback() + .iter() + .map(|e| crate::render::count_wrapped_rows(e, narrower.width)) + .collect(); + let t: u32 = cs.iter().fold(0u32, |a, b| a.saturating_add(*b)); + (narrower, cs, t) + } else { + (area, counts_full, total_full) + }; + // Hand paragraph_area (not `area`) back to App: mouse-coord + // translation now excludes the scrollbar column. + app.set_scrollback_area(paragraph_area); + + let max_scroll = total.saturating_sub(paragraph_area.height as u32); + // Consume any pending "jump this entry into view" request from + // the diagnostics-finder popup. Translation happens here (not + // in App) because it needs the per-entry wrapped-row counts we + // just computed — the popup key handler doesn't know + // area.width. Center-aligned when possible so the marked + // entry sits in the middle of the viewport; otherwise + // clamped by max_scroll. + if let Some(jump_idx) = app.take_pending_jump() { + let mut acc: u32 = 0; + for (i, c) in counts.iter().enumerate() { + if i == jump_idx { + break; + } + acc = acc.saturating_add(*c); + } + let center_offset = acc + .saturating_sub(paragraph_area.height as u32 / 3) + .min(max_scroll) + .min(u32::from(u16::MAX)) as u16; + // Writing scrollback_scroll (via App's setter) requires a + // path through the crate; simpler is to write the offset + // through set_last_rendered_scroll AND scrollback_scroll — + // we do both here so tail-follow disengage (done by the + // popup handler) sticks and the effective offset the + // renderer uses below is the jumped one. + app.set_scrollback_scroll(center_offset); + } + let scroll_offset = if app.scrollback_follow() { + max_scroll + } else { + u32::from(app.scrollback_scroll()).min(max_scroll) + }; + app.set_last_rendered_scroll(scroll_offset.min(u32::from(u16::MAX)) as u16); + app.set_last_max_scroll(max_scroll.min(u32::from(u16::MAX)) as u16); + + // Pass 2: walk entries; build wrapped lines only for those + // intersecting the viewport. Entries entirely above viewport + // are skipped (their row count contributes to the offset we + // pass to ratatui's `Paragraph::scroll`). Entries entirely + // below viewport stop the walk. + let viewport_start = scroll_offset; + let viewport_end = + viewport_start.saturating_add(paragraph_area.height as u32); + + let mut visible: Vec> = + Vec::with_capacity(paragraph_area.height as usize + 16); + let mut accumulated: u32 = 0; + // `skipped_rows` counts wrapped rows preceding the first row we + // actually emit into `visible`. For entries fully above the + // viewport we add their whole `count`; for the FIRST partially- + // visible entry we add whatever wrapped rows come between the + // entry's start and the first source line the windowed slicer + // emits. Combined, this keeps `local_scroll = viewport_start - + // skipped_rows` correct even when we slice inside an entry. + let mut skipped_rows: u32 = 0; + { + let scrollback = app.scrollback(); + for (entry, &count) in scrollback.iter().zip(counts.iter()) { + let entry_end = accumulated.saturating_add(count); + if entry_end <= viewport_start { + // Entirely above viewport — count its rows toward + // the local scroll offset and move on without + // wrapping. + skipped_rows = entry_end; + accumulated = entry_end; + continue; + } + if accumulated >= viewport_end { + break; + } + // Windowed slice: emit only source lines whose wrapped- + // row range overlaps the viewport. Local window is in + // this entry's own row space (0-indexed), so subtract + // `accumulated` first. + let local_start = viewport_start.saturating_sub(accumulated); + let local_end = viewport_end.saturating_sub(accumulated); + let (lines, offset) = crate::render::entry_lines_windowed( + entry, + paragraph_area.width, + local_start..local_end, + ); + // Only the FIRST windowed entry contributes an intra- + // entry offset; subsequent entries fall wholly within + // the viewport and start rendering at row 0 locally. + // For the first sliced entry, add the wrapped rows the + // slicer skipped (source lines above the visible + // window) to `skipped_rows` so downstream math stays + // symmetric with the whole-entry-skipped case. + if visible.is_empty() { + skipped_rows = + skipped_rows.saturating_add(local_start - offset); + } + let wrapped = + crate::render::wrap_lines(lines, paragraph_area.width); + visible.extend(wrapped); + accumulated = entry_end; + } + } + + // Selection highlight: coords are global wrapped-row indices. + // Subtract `skipped_rows` so they index into the local + // `visible` Vec instead. + if let Some(sel) = app.selection() { + let (start, end) = sel.ordered(); + let skipped = skipped_rows as usize; + let local_start = (start.0.saturating_sub(skipped), start.1); + let local_end = (end.0.saturating_sub(skipped), end.1); + crate::render::apply_selection_highlight( + &mut visible, + local_start, + local_end, + ); + } + + // We've already skipped entries above viewport; ratatui only + // needs to skip the remaining rows within the first visible + // entry (i.e. the offset from where that entry started to + // where the viewport actually begins). + let local_scroll = viewport_start + .saturating_sub(skipped_rows) + .min(u32::from(u16::MAX)) as u16; + + // Blank the scrollback area first. ratatui's Paragraph doesn't + // guarantee overwriting cells past its own content, so a frame + // that emits shorter/fewer wrapped lines than the previous one + // (very common under live streaming — a new INFO chunk shifts + // the viewport and the tail cells of the prior frame stay + // dirty) shows up as leftover fragments in the wrong color, + // usually looking like `INFO:` / `.v:` / hex-digit tails + // grafted onto the front of the current line. `Clear` writes + // spaces with the default style over `area`, so subsequent + // paragraph render lands on a clean slate. Clear covers the + // full `area` (including any scrollbar column) — Scrollbar + // will paint over that column below. + f.render_widget(Clear, area); + // No surrounding block: the scrollback's main job is to be + // copy-pastable. A box-drawing border around each visible row + // means any selection that spans full lines pulls in `│` chars + // at the start and end of every line. The input box below the + // scrollback still has its own border, which provides enough + // visual separation between the two regions. + let paragraph = Paragraph::new(visible).scroll((local_scroll, 0)); + f.render_widget(paragraph, paragraph_area); + + // Marker overlay for the entry the user jumped to via + // the diagnostics finder (Ctrl-F → Enter). Persistent until + // Alt-C clears it. Paints a bright colored bar in the + // leftmost column of every visible wrapped row of the marked + // entry. Color follows the entry's kind (red for Error, + // orange for Warning, gray for Notice) so the marker also + // reinforces the severity at a glance. + if let Some(marker_idx) = app.marker_entry() { + // Compute absolute row range of the marked entry. + let mut acc: u32 = 0; + let mut marker_range: Option<(u32, u32)> = None; + for (i, c) in counts.iter().enumerate() { + if i == marker_idx { + marker_range = Some((acc, acc.saturating_add(*c))); + break; + } + acc = acc.saturating_add(*c); + } + if let Some((start, end)) = marker_range { + // Intersect with viewport row range. + let iso_start = start.max(scroll_offset); + let iso_end = end.min( + scroll_offset.saturating_add(paragraph_area.height as u32), + ); + if iso_start < iso_end { + let kind = app.scrollback().get(marker_idx).map(|e| e.kind); + let marker_color = match kind { + Some(crate::app::ScrollbackKind::Error) => Color::Red, + Some(crate::app::ScrollbackKind::Warning) => { + Color::Rgb(255, 140, 0) + } + Some(crate::app::ScrollbackKind::Notice) => { + Color::Rgb(180, 130, 220) + } + _ => Color::Rgb(180, 130, 220), + }; + let marker_style = Style::default() + .fg(marker_color) + .add_modifier(Modifier::BOLD); + let buf = f.buffer_mut(); + for row in iso_start..iso_end { + let y = paragraph_area.y + (row - scroll_offset) as u16; + // `▎` = left one-eighth block — reads as a + // continuous vertical bar down the left edge + // when stacked across rows. + buf.set_string(paragraph_area.x, y, "▎", marker_style); + } + } + } + } + + // Scrollbar overlay — only when content overflows the viewport. + // ScrollbarState's `content_length` is the total scrollable + // range (max_scroll + 1 so the thumb can reach the very + // bottom); `position` is the current scroll offset. Rendered + // on the rightmost column of `area`; the paragraph was drawn + // in `paragraph_area` which excludes that column so there's + // no overpaint on wrapped text. + if needs_scrollbar && area.width >= 2 { + let content_len = max_scroll.saturating_add(1) as usize; + let mut sb_state = ScrollbarState::new(content_len) + .position(scroll_offset as usize) + .viewport_content_length(paragraph_area.height as usize); + let scrollbar = Scrollbar::default() + .orientation(ScrollbarOrientation::VerticalRight); + f.render_stateful_widget(scrollbar, area, &mut sb_state); + } +} + +fn draw_input(f: &mut Frame, area: Rect, app: &mut App) { + // TextArea remains the editing model (every keystroke still flows + // through `ta.input(key)` in `handle_terminal_event`'s catch-all + // arm). We replace ONLY the visual rendering layer here so we can + // paint per-token highlighter spans — tui-textarea's built-in + // renderer is monochrome and its `line_spans` is `pub(crate)`, so + // there's no hook for syntax styling without taking over the + // viewport ourselves. + let block = Block::default() + .borders(Borders::ALL) + .title(input_title(app)); + let inner = block.inner(area); + f.render_widget(block, area); + if inner.width == 0 || inner.height == 0 { + return; + } + + let ta = app.input_mut(); + let lines: Vec = ta.lines().to_vec(); + let (cursor_row, cursor_col) = ta.cursor(); + + // Run the htcl highlighter on the full buffer in one parse, then + // slice per-line for rendering. Per-frame cost is one + // `vw_htcl::parse` over the input text — fine for typical REPL + // inputs (tens of lines); revisit if it shows up in a profile. + let body_style = + ratatui::style::Style::default().fg(ratatui::style::Color::White); + let buffer = lines.join("\n"); + let highlighted = + crate::highlight_htcl::highlight_per_line(&buffer, body_style); + + // Vertical viewport: anchor so the cursor stays visible. When the + // buffer fits, anchor at top; when it overflows, scroll so cursor + // is on the last visible row. + let view_h = inner.height as usize; + let scroll_top = if lines.len() <= view_h { + 0 + } else { + cursor_row.saturating_sub(view_h.saturating_sub(1)) + }; + + // Render each visible line as a single-row Paragraph. + for (visible_idx, line_idx) in (scroll_top..lines.len()).enumerate() { + if visible_idx >= view_h { + break; + } + let row = inner.y + visible_idx as u16; + let line_spans = highlighted.get(line_idx).cloned().unwrap_or_default(); + let line_widget = Paragraph::new(ratatui::text::Line::from(line_spans)); + let line_area = Rect { + x: inner.x, + y: row, + width: inner.width, + height: 1, + }; + f.render_widget(line_widget, line_area); + } + + // Position the terminal-native text cursor so the user sees their + // edit point. Only set when the cursor row is visible — when + // scrolled away (shouldn't happen given the viewport math above, + // but defensive), we just leave the cursor hidden. + if cursor_row >= scroll_top && cursor_row < scroll_top + view_h { + let cursor_screen_row = inner.y + (cursor_row - scroll_top) as u16; + // Horizontal: clamp to area width. Cursor positions past the + // visible width get pinned to the last column rather than + // drifting off the block border. + let cursor_screen_col = + inner.x + (cursor_col as u16).min(inner.width.saturating_sub(1)); + f.set_cursor_position((cursor_screen_col, cursor_screen_row)); + } +} + +fn input_title(app: &App) -> String { + if app.eval_in_flight() { + " input — vivado: running ".to_string() + } else if app.input_is_complete() { + " input — Enter to run ".to_string() + } else { + " input — Enter for newline (parse incomplete) ".to_string() + } +} + +fn draw_status(f: &mut Frame, area: Rect, app: &App) { + let (label, bg) = match app.worker_state() { + // Indigo when Vivado is sitting idle, ready for input — + // the "you can interact" steady state. + WorkerStatusView::Ready => (" vivado: ready ", Color::Rgb(75, 0, 130)), + // Orange for transient / busy states — starting up or + // mid-eval. Catches the eye so the user notices they + // can't (yet, or currently) drive the session, but + // signals "wait" rather than "broken". + WorkerStatusView::Starting => { + (" vivado: starting ", Color::Rgb(255, 140, 0)) + } + WorkerStatusView::Running => { + (" vivado: running ", Color::Rgb(255, 140, 0)) + } + // Red for `down` — distinct from the orange busy states + // so the user can immediately tell the difference between + // "wait a moment" and "the worker died, `:restart` to + // recover". Sharing orange with `running` hid this. + WorkerStatusView::Down => (" vivado: down ", Color::Rgb(200, 30, 30)), + }; + let hint = if app.reverse_search().is_some() { + "Esc cancel · Enter accept · Ctrl-R older".to_string() + } else { + // Single key-chord hint — the full cheat-sheet lives in + // the Ctrl-H modal so we don't have to keep this row in + // sync with every binding we add or change. + "Ctrl-H for help".to_string() + }; + // Split the status bar into [hint (left, fills) | status + // indicator (right, fixed width)] so the status badge always + // anchors to the bottom-right corner. + let badge_width = label.chars().count() as u16; + let layout = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Min(1), Constraint::Length(badge_width)]) + .split(area); + f.render_widget( + Paragraph::new(Span::styled( + hint, + Style::default().fg(Color::DarkGray), + )), + layout[0], + ); + f.render_widget( + Paragraph::new(Span::styled( + label, + Style::default() + .bg(bg) + .fg(Color::White) + .add_modifier(Modifier::BOLD), + )), + layout[1], + ); +} + +fn draw_reverse_search(f: &mut Frame, anchor: Rect, rs: &ReverseSearch) { + let area = centered_rect(80, 5, f.area(), anchor); + f.render_widget(Clear, area); + let title = format!( + " reverse-i-search ({}) ", + if rs.match_index.is_some() { + "match" + } else if rs.query.is_empty() { + "type to search" + } else { + "no match" + } + ); + let body = vec![ + Line::from(vec![ + Span::styled("query: ", Style::default().fg(Color::DarkGray)), + Span::styled( + rs.query.clone(), + Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD), + ), + ]), + Line::from(Span::raw("")), + Line::from(vec![ + Span::styled("match: ", Style::default().fg(Color::DarkGray)), + Span::raw(rs.match_text.clone()), + ]), + ]; + let para = Paragraph::new(body) + .block( + Block::default() + .borders(Borders::ALL) + .title(title) + .style(Style::default().bg(Color::Black)), + ) + .wrap(Wrap { trim: false }); + f.render_widget(para, area); +} + +/// Compute a centered rectangle for a popup. `anchor` is the area the +/// popup is logically attached to (the input area); we expand around +/// the screen center but never overflow the parent. +fn centered_rect( + percent_x: u16, + height_lines: u16, + full: Rect, + _anchor: Rect, +) -> Rect { + let popup_w = full.width.saturating_mul(percent_x) / 100; + let popup_h = height_lines.min(full.height); + let x = (full.width.saturating_sub(popup_w)) / 2; + let y = (full.height.saturating_sub(popup_h)) / 2; + Rect { + x, + y, + width: popup_w, + height: popup_h, + } +} + +/// Worker status as the UI sees it. Lives here (not in `app`) so the +/// renderer doesn't have to know about the worker's internal state +/// machine. +pub enum WorkerStatusView { + Starting, + Ready, + Running, + Down, +} diff --git a/vw-vivado/Cargo.toml b/vw-vivado/Cargo.toml new file mode 100644 index 0000000..00e23eb --- /dev/null +++ b/vw-vivado/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "vw-vivado" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Vivado EDA backend: spawns a long-lived Vivado worker and drives it via the vw-eda wire protocol" + +[dependencies] +vw-eda = { path = "../vw-eda" } +vw-lib = { path = "../vw-lib" } +vw-htcl = { path = "../vw-htcl" } +camino.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +async-trait.workspace = true +tempfile.workspace = true +tracing.workspace = true +portable-pty.workspace = true +similar.workspace = true +colored = "2.0" + +# Unix-only. Used for `libc::kill(pid, SIGINT)` in +# `VivadoBackend::interrupt` — the eval-cancellation path documented on +# that method. Windows falls back to a no-op. +[target.'cfg(unix)'.dependencies] +libc = "0.2" diff --git a/vw-vivado/build.rs b/vw-vivado/build.rs new file mode 100644 index 0000000..f4c7f43 --- /dev/null +++ b/vw-vivado/build.rs @@ -0,0 +1,10 @@ +// Cargo doesn't track files included via `include_str!` for +// rebuild purposes — it only knows about `.rs` source files. +// The Vivado shim is `include_str!`'d into `worker.rs` and +// baked into the binary at compile time; without this build +// script edits to the shim go unnoticed until something else +// triggers a recompile of `vw-vivado`, leaving the deployed +// shim out of sync with the source. +fn main() { + println!("cargo:rerun-if-changed=shim/vivado-shim.tcl"); +} diff --git a/vw-vivado/shim/vivado-shim.tcl b/vw-vivado/shim/vivado-shim.tcl new file mode 100644 index 0000000..7e2e126 --- /dev/null +++ b/vw-vivado/shim/vivado-shim.tcl @@ -0,0 +1,1496 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# vw <-> Vivado wire protocol shim. +# +# Sourced by `vivado -mode tcl -source vivado-shim.tcl`. The shim +# connects back to vw over a loopback TCP socket on the port given in +# `$env(VW_PROTOCOL_ADDR)` and then reads newline-delimited JSON +# requests on that socket, writing responses on the same socket. +# +# Why a socket and not stdin/stdout: user TCL is free to call `puts`, +# Vivado prints its own banners and source-echo, and mixing all of +# that with the wire protocol on one stream forces either text +# markers (which a hostile or unlucky `puts` could spoof) or fragile +# OS-specific channels like FIFOs and chardevs. A loopback TCP socket +# works identically on Linux, macOS, and Windows, can't be polluted +# by anything the user writes to stdout, and frees vw to treat +# Vivado's stdout however it wants (forward for `vw run`, capture for +# the REPL/LSP). + +package require json + +namespace eval ::vw { + variable protocol_sock {} + # The eval id currently being processed. `puts` writes that would + # go to stdout while `capturing` is set are forwarded immediately + # as `{"id":N,"stream":"stdout","data":...}` notifications tagged + # with this id, so vw can stream output as it's produced rather + # than waiting for the final response. Required for any + # long-running command (synth_design, route_design, ...). + variable current_eval_id 0 + variable capturing 0 + # Reentrancy guard for the send_msg_id override. Without this, a + # message emitted *during* our own stack-walking or JSON encoding + # would recurse back into the override and either deadlock or + # double-emit. We just fall back to the original handler when + # already inside our wrapper. + variable in_send_msg_id 0 + # Cap on stack frames captured per message. Vivado's internal + # call chains can be 50+ frames deep through tclapp loaders; + # rendering all of them would drown the actual message. The + # cap is per-message, not per-session, so a future deeper trace + # still gets its first N frames. + variable stack_frame_cap 20 + # Set at startup from `VW_TRACE_STACK_CAPTURE`. When true, + # `capture_stack` emits a per-frame `[vw-stack]` log line with + # the info-frame dict, level-args probe, and keep/drop reason + # for every frame it examines. Useful when a warning tag + # renders with fewer frames than expected: the log shows + # exactly which frames existed and why each was kept or + # filtered. Zero cost when unset. + variable trace_stack_capture 0 + catch { + set trace_stack_capture \ + [expr {$::env(VW_TRACE_STACK_CAPTURE) eq "1"}] + } +} + +# ---------- puts capture ---------- +# +# Rename the real `puts` so we can install a wrapper that, while +# capturing, forwards each stdout write to vw as a streaming +# notification. Anything that targets a specific channel (stderr, +# the protocol socket, a file) passes through unchanged. Outside of +# eval (`capturing == 0`), stdout writes also pass through — Vivado's +# own messages between commands stay on the process's stdout where vw +# handles them per its `--verbose` setting. + +rename puts ::vw::real_puts + +proc puts {args} { + set len [llength $args] + set start 0 + set nonewline 0 + if {$len > 0 && [lindex $args 0] eq "-nonewline"} { + set nonewline 1 + set start 1 + } + set remaining [expr {$len - $start}] + if {$::vw::capturing} { + if {$remaining == 1} { + # `puts ?-nonewline? string` — implicit stdout + set str [lindex $args $start] + if {!$nonewline} { append str "\n" } + # Catch-wrap because attach_stack_if_message is + # defined later in this script. During shim sourcing + # there's a tiny window where puts exists (this proc) + # but the helper doesn't yet; we'd rather pass the + # raw string through than crash the puts itself. + catch {set str [::vw::attach_stack_if_message $str 2]} + ::vw::stream_stdout $::vw::current_eval_id $str + return + } elseif {$remaining == 2 \ + && [lindex $args $start] eq "stdout"} { + set str [lindex $args [expr {$start + 1}]] + if {!$nonewline} { append str "\n" } + catch {set str [::vw::attach_stack_if_message $str 2]} + ::vw::stream_stdout $::vw::current_eval_id $str + return + } + } + # Fall through to the real puts for: any non-stdout channel, or + # any stdout write when not capturing. + ::vw::real_puts {*}$args +} + +# ---------- JSON helpers ---------- + +# Hand-encode a string per RFC 8259. Vivado's bundled Tcllib doesn't +# include `json::write`, so we provide the minimum we need. +# +# Implementation: `string map` does the bulk-substitution in one +# native Tcl C call, vs. a per-char Tcl loop (which is what we +# used to do). The difference is dramatic at scale — a 1MB puts +# output (the kind `puts [util::props -object $cpm5]` produces) +# went from minutes of per-char `string index`/`scan`/`switch` +# iteration to ~100ms via `string map`. Rare control chars +# (codepoints < 0x20 other than the named whitespace escapes) +# trigger a slow per-char fallback; in practice Vivado property +# values don't contain them, so the fast path covers everything. +proc ::vw::json_string {value} { + # Order matters: backslash must be substituted FIRST so the + # backslashes we introduce for the other escapes aren't + # themselves re-escaped. + set escaped [string map [list \ + "\\" "\\\\" \ + "\"" "\\\"" \ + "\b" "\\b" \ + "\f" "\\f" \ + "\n" "\\n" \ + "\r" "\\r" \ + "\t" "\\t"] $value] + # Fast path: no remaining control chars → just wrap in quotes. + if {![regexp {[\x00-\x08\x0B\x0E-\x1F]} $escaped]} { + return "\"$escaped\"" + } + # Slow path: per-char loop for the remaining control chars. + # Only hit when the string contains rare control codepoints + # — Vivado property values shouldn't, but a user `puts` of + # binary-ish data might. + set out "\"" + set len [string length $escaped] + for {set i 0} {$i < $len} {incr i} { + set ch [string index $escaped $i] + scan $ch %c codepoint + if {$codepoint < 0x20} { + append out [format "\\u%04x" $codepoint] + } else { + append out $ch + } + } + append out "\"" + return $out +} + +# ---------- response helpers ---------- + +# Send a streaming stdout chunk during eval. These notifications are +# distinguishable from the final response by the presence of a +# `stream` field (and absence of `ok`). +proc ::vw::stream_stdout {id data} { + variable protocol_sock + set j [::vw::json_string $data] + ::vw::real_puts $protocol_sock \ + "{\"id\":$id,\"stream\":\"stdout\",\"data\":$j}" + flush $protocol_sock +} + +## `pending_response_id` / `pending_response_sent` — main-loop +## deadman for the "request in, no response out" case. The +## dispatch handler sets `pending_response_id` before invoking +## the per-op branch, and every `send_ok`/`send_err` clears it. +## If the main loop exits `::vw::dispatch` with an unclosed id +## (e.g. `interp cancel -unwind` from a Ctrl-C during `place_design` +## blew through the eval-branch's `catch` without letting the +## `send_err` branch run), the loop synthesises a fallback error +## response so the vw Rust worker's `read_response_for(id).await` +## unblocks instead of hanging the whole session. Without this +## guard, the REPL stays stuck at `vivado: running` forever. +variable ::vw::pending_response_id 0 +variable ::vw::pending_response_sent 0 + +proc ::vw::send_ok {id result} { + variable protocol_sock + set j_result [::vw::json_string $result] + # Use real_puts explicitly so the wrapper above can never + # accidentally divert protocol traffic into a stream notification. + ::vw::real_puts $protocol_sock \ + "{\"id\":$id,\"ok\":true,\"result\":$j_result}" + flush $protocol_sock + if {$::vw::pending_response_id == $id} { + set ::vw::pending_response_sent 1 + } +} + +proc ::vw::send_err {id message {code ""} {info ""}} { + variable protocol_sock + set j_msg [::vw::json_string $message] + set fields "\"message\":$j_msg" + if {$code ne ""} { + append fields ",\"code\":[::vw::json_string $code]" + } + if {$info ne ""} { + append fields ",\"info\":[::vw::json_string $info]" + } + ::vw::real_puts $protocol_sock \ + "{\"id\":$id,\"ok\":false,\"error\":{$fields}}" + flush $protocol_sock + if {$::vw::pending_response_id == $id} { + set ::vw::pending_response_sent 1 + } +} + +# ---------- shim-initiated RPC ---------- +# +# `vw::rpc_call METHOD [ARGS_JSON]` sends an RPC request to the +# `vw` Rust process and blocks until the response arrives. +# Returns the result value on success, throws with the vw-side +# error message on failure. +# +# Direction is the MIRROR of the eval loop: normally vw sends +# requests and this shim answers. Here the shim sends and vw +# answers. The response uses the same `{id, ok, result}` / +# `{id, ok:false, error}` shape as an eval response — no new +# response type — so it round-trips through the normal +# read/write machinery. +# +# ARGS_JSON is a JSON-encoded string (e.g. `[::json::dict2json +# {name value}]` or a bare literal like `null` / `{}`), NOT a +# Tcl dict. Callers that only need a bare call pass nothing; +# the arg defaults to `null`. +# +# Usage is meant to be called ONLY from inside an eval — during +# an eval we're the sole consumer of the protocol socket, so +# reading the response synchronously via `gets $sock` doesn't +# race the main dispatch loop. Calling from outside an eval +# would race that loop and swallow a real request; the guard +# below errors clearly rather than deadlocking. +variable ::vw::rpc_next_id 1 + +proc ::vw::rpc_call {method {args_json "null"}} { + variable protocol_sock + if {!$::vw::capturing} { + error "vw::rpc_call must be invoked from inside an eval" + } + set id $::vw::rpc_next_id + incr ::vw::rpc_next_id + set j_method [::vw::json_string $method] + ::vw::real_puts $protocol_sock \ + "{\"id\":$id,\"rpc\":true,\"method\":$j_method,\"args\":$args_json}" + flush $protocol_sock + # Read exactly one response line. Repeat if vw sends anything + # else in the meantime (e.g. an unexpected stream chunk); we + # only stop when we see a response tagged with our id. + while {1} { + if {[gets $protocol_sock line] < 0} { + if {[eof $protocol_sock]} { + error "vw::rpc_call: protocol socket closed before response" + } + continue + } + set line [string trim $line] + if {$line eq ""} { continue } + if {[catch {::json::json2dict $line} resp]} { + error "vw::rpc_call: unparseable response from vw: $resp" + } + if {![dict exists $resp id]} { continue } + if {[dict get $resp id] != $id} { continue } + if {[dict exists $resp ok] && [dict get $resp ok]} { + return [expr {[dict exists $resp result] \ + ? [dict get $resp result] : ""}] + } + set msg "" + if {[dict exists $resp error]} { + set err [dict get $resp error] + if {[dict exists $err message]} { + set msg [dict get $err message] + } + } + error "vw::rpc_call: $msg" + } +} + +proc ::vw::log {msg} { + puts stderr "\[vw-shim\] $msg" + flush stderr +} + +# Global-namespace call helper for wrappers. +# +# Each generated `vivado_cmd::` wrapper forwards to the +# underlying Vivado builtin via this proc so the forwarded call +# runs with the interp's current namespace set to `::`. That +# matters for builtins like `synth_ip` which source XDC files +# whose scripts use *unqualified* names (`create_clock`, +# `get_ports`, …). Without a global-namespace call, Tcl resolves +# those unqualified names against whatever the wrapper's own +# namespace happens to be (`::vivado_cmd::`), and picks the +# wrapper again — which then throws on the XDC's positional +# args because kwargs expects `-flag` form. +# +# We define the proc at `::` (via `namespace eval ::`) so its +# execution context is `::`. The body invokes its args via +# `{*}$cmd {*}$args` — a direct arg-expansion, NOT a string +# re-parse. That's the critical difference from +# `namespace eval :: [list …]` / `namespace inscope :: …` / +# `uplevel #0 [list …]`, all of which serialize their args to a +# script string and lose Tcl_Obj internal reps. bd_cell handles +# would round-trip to plain paths like `/cpm5`, which Vivado's +# `set_property -objects` then rejects as "Invalid option value". +namespace eval :: { + proc _vw_global_call {cmd args} { + {*}$cmd {*}$args + } +} + +# ---------- kwargs runtime ---------- +# +# Wrapper procs lowered from htcl declare themselves as +# `proc {args} { ::vw::kwargs $args {param default ...} ; }`. +# This helper parses `args` against `sig` (a dict of `param default +# param default ...`) and uses `upvar 1` to set each parameter as +# a local in the caller's frame. After this returns, the wrapper +# body sees `$dir`, `$cell`, `$name`, etc. just as if they were +# standard Tcl parameters with defaults. +# +# Why this exists: htcl is keyword-only at the call site, but Tcl +# proc dispatch is positional. Without this runtime parsing, the +# only way to make `wrap -name x` work would be to rewrite the +# call site to positional form at compile time — which our lowerer +# used to do, but only for top-level calls, not for calls inside +# proc bodies / namespace eval / [ ... ]. Moving the keyword parse +# to runtime makes every call site work uniformly. +# +# Arg shapes supported: +# `-flag value` — value-bearing flag, sets $flag = value +# `-flag` — bare boolean flag (end of args), sets $flag = 1 +# `-flag -other ...`— bare boolean flag (next token is another +# known flag), sets $flag = 1, continues +# +# The bare-flag heuristic matches Vivado's calling convention: +# their APIs and internal Tcl use `-quiet`/`-verbose`/etc. as bare +# booleans. The "is next token a known flag?" disambiguator avoids +# eating a legitimate value that happens to start with `-` (e.g. +# `-filter -name` where the user intended `-filter` to take `-name` +# as its value — but a leading `-` value is exotic enough that we +# accept the ambiguity). +proc ::vw::kwargs {argv sig} { + # Initialize each parameter to its declared default. Also + # initialize a `__vw_kw__set` flag to 0 — wrappers can + # check this to distinguish "user supplied this arg" from + # "we filled in the default", which matters for + # set_property -dict where setting unsupplied properties + # (with their defaults) re-validates the whole cell and + # rejects values Vivado considers out of range for the + # cell's current state. + foreach {name default} $sig { + upvar 1 $name var + set var $default + upvar 1 __vw_kw_${name}_set seen + set seen 0 + } + set n [llength $argv] + set i 0 + while {$i < $n} { + set flag [lindex $argv $i] + if {![string match -* $flag]} { + error "kwargs: expected -flag, got '$flag'" + } + set key [string range $flag 1 end] + if {![dict exists $sig $key]} { + set allowed [join [dict keys $sig] ", "] + error "kwargs: unknown flag '$flag'; allowed: $allowed" + } + # Decide whether the current flag is bare or takes a value. + # Bare iff: at end of args, OR next token is another known + # -flag. + set bare 1 + set next_i [expr {$i + 1}] + if {$next_i < $n} { + set peek [lindex $argv $next_i] + if {![string match -* $peek]} { + set bare 0 + } else { + set peek_key [string range $peek 1 end] + if {![dict exists $sig $peek_key]} { + # Peek looks like a flag but isn't ours — assume + # it's a value for the current flag (e.g. a CLI + # path or arg starting with `-`). + set bare 0 + } + } + } + upvar 1 $key var + upvar 1 __vw_kw_${key}_set seen + set seen 1 + if {$bare} { + set var 1 + incr i + } else { + set var [lindex $argv $next_i] + incr i 2 + } + } +} + +# ---------- bulk property fetch ---------- +# +# `::vw::props_dict ` returns a paired Tcl list (NAME VAL +# NAME VAL …) of every property on ``. The point is a +# single Vivado RPC instead of N: htcl wrappers that want the +# full property bag (e.g. `util::props`) would otherwise issue +# one `extern::get_property` per property × hundreds of +# properties on an IP cell. The PTY round-trip is the dominant +# cost; doing the iteration entirely Vivado-side cuts it to +# constant per call. +proc ::vw::props_dict {obj} { + set out [list] + foreach name [list_property $obj] { + lappend out $name [get_property $name $obj] + } + return $out +} + +# ---------- user-set property tracking ---------- +# +# Vivado offers no per-property "is this at the default?" API +# accessible from Tcl (`get_property`, `list_property`, +# `report_property -all`, `bd::get_properties` all return every +# property's current value with no user-vs-default distinction). +# The only system-of-record is `write_bd_tcl`'s output, which +# requires a full BD serialization round-trip per query. +# +# Instead we keep our own tally: every time +# `vivado_cmd::set_property` is invoked (the htcl-level chokepoint +# all wrappers and user code go through), it records the +# (object, name, value) triples here. `::vw::user_props_dict` / +# `::vw::user_props_nested` read back from this side-channel — +# returning ONLY the properties the user / wrapper explicitly +# pushed, never the ones Vivado cascaded as derived defaults. +# +# Cost: O(properties set) for record (dict insertion), O(props +# returned) for retrieval (dict walk). No file I/O, no full- +# design serialization. Persists across batches in the worker +# until `:restart`. +# +# Caveat: tracks only properties set via the +# `vivado_cmd::set_property` wrapper. Direct `extern::set_property` +# bypasses the recording. That's by design — the wrappers are +# the documented boundary, and bypassing them is an explicit +# opt-out of the tracking machinery. + +namespace eval ::vw { + variable user_set_props +} + +# Record one or more (name, value) pairs as user-set on `obj`. +# `args` is the paired list (name1 val1 name2 val2 …) — same +# shape the wrapper builds before calling `set_property -dict`. +# Last-set wins per property name within an object. +proc ::vw::record_user_props {obj args} { + variable user_set_props + if {![info exists user_set_props]} { + array set user_set_props {} + } + set key [::vw::_user_props_key $obj] + if {![info exists user_set_props($key)]} { + set user_set_props($key) [dict create] + } + set current $user_set_props($key) + foreach {n v} $args { + dict set current $n $v + } + set user_set_props($key) $current +} + +# Canonical key for the per-object side-channel storage. +# +# `PATH` uniquely identifies a BD cell (`/cips/cpm5` etc.), but +# it's not defined on every Vivado object type — a project-level +# IP handle from `create_ip` / `get_ips` doesn't have one, and +# querying it emits `[Vivado 12-1341] Failed to get property +# 'PATH' on IP 'foo'` via `send_msg_id`. Tcl's `catch` doesn't +# suppress that because Vivado routes the error through the +# message bus rather than returning a Tcl-level error. `-quiet` +# on the getter does suppress it — that's why we use it here. +# +# The lookup falls through PATH → NAME → the raw object string +# so any object type produces a stable key without noise. +proc ::vw::_user_props_key {obj} { + set p "" + catch { set p [get_property -quiet PATH $obj] } + if {$p ne ""} { return $p } + catch { set p [get_property -quiet NAME $obj] } + if {$p ne ""} { return $p } + return $obj +} + +# Parse a `::set_property` argv into (dict-pairs, objects) and +# funnel each object's pairs into [`record_user_props`]. Called by +# the [`install_set_property_recorder`] wrapper before the real +# set_property runs — so the tally reflects what the caller *tried* +# to set even if Vivado later rejects the write. +# +# Recognizes the two forms our IP wrappers actually emit: +# +# set_property -dict {NAME1 VAL1 NAME2 VAL2 …} -objects $cell +# set_property -name NAME -value VAL -objects $cell +# +# Positional-form `set_property NAME VAL $obj` also works. Unknown +# flags (`-quiet`, `-verbose`) are consumed silently. If we can't +# recover both a name/value dict and an object list, we return +# without recording — the write still happens; the tally just +# stays where it was. +proc ::vw::_record_set_property_call {args} { + set pairs [list] + set objects [list] + set positional [list] + set explicit_name "" + set explicit_value "" + set i 0 + while {$i < [llength $args]} { + set tok [lindex $args $i] + switch -- $tok { + -dict { + foreach {n v} [lindex $args [expr {$i + 1}]] { + lappend pairs $n $v + } + incr i 2 + } + -objects { + set objects [lindex $args [expr {$i + 1}]] + incr i 2 + } + -name { + set explicit_name [lindex $args [expr {$i + 1}]] + incr i 2 + } + -value { + set explicit_value [lindex $args [expr {$i + 1}]] + incr i 2 + } + -quiet - + -verbose { + incr i + } + default { + lappend positional $tok + incr i + } + } + } + # Positional shape: `set_property NAME VALUE OBJECTS`. + if {[llength $positional] == 3 && [llength $pairs] == 0 + && $explicit_name eq ""} { + lappend pairs [lindex $positional 0] [lindex $positional 1] + if {[llength $objects] == 0} { + set objects [lindex $positional 2] + } + } + # Trailing positional target: `set_property -dict {…} $cell`. + if {[llength $objects] == 0 && [llength $positional] == 1} { + set objects [lindex $positional 0] + } + if {$explicit_name ne "" && [llength $pairs] == 0} { + lappend pairs $explicit_name $explicit_value + } + if {[llength $pairs] == 0 || [llength $objects] == 0} { + return + } + foreach obj $objects { + catch { ::vw::record_user_props $obj {*}$pairs } + } +} + +# Install a `::set_property` wrapper that funnels every write +# through [`_record_set_property_call`] before delegating to +# whatever `::set_property` currently is (Vivado's C++ builtin, or +# the marker wrapper installed by +# [`install_all_context_wrappers`]). Regeneration-proof — the +# `vivado_cmd::set_property` htcl wrapper is generated from the +# Vivado command reference and re-emitted by `regenerate.sh`, so +# manual edits there don't survive. Keeping the recording here in +# the shim means the side-channel tally stays populated regardless +# of what the generated wrapper looks like. +# +# Called at startup AFTER [`install_all_context_wrappers`] so the +# recorder ends up OUTSIDE the marker wrapper: recording happens +# first, then BEGIN, then the real write, then END. +proc ::vw::install_set_property_recorder {} { + if {[info commands ::vw::orig_set_property_for_record] ne ""} { + return + } + if {[info commands ::set_property] eq ""} { return } + rename ::set_property ::vw::orig_set_property_for_record + proc ::set_property {args} { + catch { ::vw::_record_set_property_call {*}$args } + uplevel 1 [list ::vw::orig_set_property_for_record {*}$args] + } + ::vw::log "installed set_property recorder" +} + +# Return the recorded (name, value) paired list for `obj`. Empty +# list when nothing has been recorded. +proc ::vw::user_props_dict {obj} { + variable user_set_props + if {![info exists user_set_props]} { return [list] } + set key [::vw::_user_props_key $obj] + if {![info exists user_set_props($key)]} { return [list] } + set out [list] + dict for {k v} $user_set_props($key) { + lappend out $k $v + } + return $out +} + +# Same shape as `::vw::props_nested` but seeded from the recorded +# user-set property tally instead of from `list_property` + +# `get_property`. Each value is classified via the structural +# `_lift_value` helper and inserted by dot-split path so the +# result is a nested `Properties` with only the explicitly-set +# sub-keys present. +proc ::vw::user_props_nested {obj} { + set plain [dict create] + foreach {name raw} [::vw::user_props_dict $obj] { + set leaf [::vw::_lift_value $raw] + dict set plain {*}[split $name "."] $leaf + } + return [::vw::_wrap_nested $plain] +} + +# `::vw::props_nested ` returns the FULL output `util::props` +# wants — a nested `Properties` dict where dotted property names +# (CONFIG.X.Y) expand into hierarchy, and each leaf value is +# already a `[list Scalar ]` or `[list Nested ]` tuple. +# +# Lives in the shim (plain Tcl) rather than in user-side htcl +# because: +# - One Vivado RPC for the entire fetch + classification + +# nesting pipeline, vs. one RPC for the fetch + thousands +# of htcl-proc kwargs-envelope invocations per recursive +# sub-key for the post-processing. +# - CPM5 has ~200 top-level properties, each whose value is +# itself a paired-dict with dozens of sub-keys. The htcl- +# side post-processing was hitting tens of thousands of +# kwargs invocations × envelope overhead → minutes. Native +# Tcl inside Vivado does the same work in well under a +# second. +# +# The structural classifier (`::vw::_lift_value`) mirrors what +# `lift::lift_recursive` did in user-htcl: pure shape inference, +# no Vivado lookups. The wrap step (`::vw::_wrap_nested`) walks +# the plain nested dict once and tags intermediate levels as +# `Property::Nested(...)`. Leaves already carry their tag from +# `_lift_value`. +proc ::vw::props_nested {obj} { + set plain [dict create] + foreach name [list_property $obj] { + set raw [get_property $name $obj] + set leaf [::vw::_lift_value $raw] + dict set plain {*}[split $name "."] $leaf + } + return [::vw::_wrap_nested $plain] +} + +# Structural inference on a raw property value. Returns a +# `[list Scalar v]` or `[list Nested inner]` tuple. Mirror of +# lift::looks_like_paired_dict + lift::lift_recursive in plain +# Tcl with no kwargs envelope. +proc ::vw::_lift_value {raw} { + if {[catch {llength $raw} n]} { return [list Scalar $raw] } + if {$n == 0 || $n % 2 != 0} { return [list Scalar $raw] } + foreach {k _v} $raw { + if {![regexp {^[A-Za-z_][A-Za-z0-9_.]*$} $k]} { + return [list Scalar $raw] + } + } + set inner [dict create] + foreach {k v} $raw { + dict set inner $k [::vw::_lift_value $v] + } + return [list Nested $inner] +} + +# Walk a plain nested Tcl dict and wrap each intermediate +# level as `[list Nested ]`. A value is a leaf when +# it's a 2-element list whose head is "Scalar" or "Nested" +# (the existing Property tuple shape). Anything else is a +# sub-dict to descend into. +proc ::vw::_wrap_nested {plain} { + set out [dict create] + dict for {k v} $plain { + if {[llength $v] == 2 \ + && ([lindex $v 0] eq "Scalar" \ + || [lindex $v 0] eq "Nested")} { + dict set out $k $v + } else { + dict set out $k [list Nested [::vw::_wrap_nested $v]] + } + } + return $out +} + +# `::vw::config_from_dotted_pairs {pairs}` — lift a flat paired- +# list of `dotted.key raw-value` entries into a nested tagged +# `Properties` value. Same transform `::vw::props_nested` +# performs on `list_property` output — split each key on `.`, +# insert at the resulting path in a plain nested dict, `_wrap_nested` +# to tag intermediate levels, `_lift_value` to tag each leaf. +# +# The generated `::configure` procs call this in their bodies +# to convert the assembled `_vw_d` (built by `lappend _vw_d +# CONFIG. ` loops) into the proper `::Config` +# shape: a Properties value where CONFIG at the top wraps a +# `Property::Nested` containing every `` sub-key. Consumers +# then use `dict get [::Config::to -v $cfg] CONFIG` + +# `Property::as_nested -v ...` to extract the sub-tree, matching +# the pattern `props::get` documents. +proc ::vw::config_from_dotted_pairs {pairs} { + set plain [dict create] + foreach {name raw} $pairs { + set leaf [::vw::_lift_value $raw] + dict set plain {*}[split $name "."] $leaf + } + return [::vw::_wrap_nested $plain] +} + +# `::vw::config_to_dotted_flat {nested}` — inverse of the lift. +# Walks a nested tagged Properties tree and emits a flat paired +# list `TOP.LEAF value TOP.LEAF value ...` matching the shape +# Vivado's `set_property -dict` expects for an IP cell. +# +# **Depth invariant.** The generated `::configure` procs +# always assemble `_vw_d` with keys of the form `CONFIG.` +# (one dot at the top, two path segments). `Properties::from_ +# dotted_pairs` splits on `.` and inserts, producing a two-level +# tagged structure: root → Nested-wrapped CONFIG → tagged entries +# (one per Vivado property). +# +# So the flatten pairs off exactly two levels: iterate the root +# dict for TOP keys (`CONFIG`), unwrap that Nested to get the +# entries, then emit `TOP.LEAF = untag(value)` per entry. A Scalar +# entry unwraps to its bare string. A Nested entry — e.g. +# `CONFIG.CPM_CONFIG` where the caller passed a Properties value — +# unwraps to its raw paired-list dict (recursively stripping any +# further tags inside), which is what Vivado stores as the value +# of a nested-dict property. +# +# Naively recursing past the two-level structure would emit +# `CONFIG.CPM_CONFIG.CPM_PCIE0_MODES` which Vivado then rejects +# with `[BD 41-1276] Cannot set the parameter … Parameter does +# not exist`, since CPM_CONFIG is a single property (accepting a +# nested dict value), not a namespace. +proc ::vw::config_to_dotted_flat {nested} { + set out [list] + dict for {top_key top_val} $nested { + set top_tag [lindex $top_val 0] + set top_payload [lindex $top_val 1] + if {$top_tag ne "Nested"} { + # Unexpected shape at the root — configure-built Configs + # always wrap the top namespace as Nested via + # _wrap_nested. Emit under the raw key rather than + # silently drop. + lappend out $top_key [::vw::_untag_recursive $top_val] + continue + } + dict for {leaf_key leaf_val} $top_payload { + lappend out "$top_key.$leaf_key" \ + [::vw::_untag_recursive $leaf_val] + } + } + return $out +} + +# Recursively strip `Property::Scalar` / `Property::Nested` tags +# from a tagged Properties value. Scalar returns its bare string. +# Nested returns its inner dict with each value recursively +# untagged. Used inside `config_to_dotted_flat` to convert a +# Nested-typed property's tagged payload into the raw paired +# dict Vivado expects as that property's value. +proc ::vw::_untag_recursive {tagged} { + if {[llength $tagged] != 2} { return $tagged } + set tag [lindex $tagged 0] + set payload [lindex $tagged 1] + if {$tag eq "Scalar"} { + return $payload + } elseif {$tag eq "Nested"} { + set out [list] + dict for {k v} $payload { + lappend out $k [::vw::_untag_recursive $v] + } + return $out + } + return $tagged +} + +# ---------- send_msg_id override ---------- +# +# Why we override: when Vivado emits a WARNING/ERROR/INFO/CRITICAL +# WARNING via ::common::send_msg_id, the raw line goes to stdout +# with no call-context — the user sees `WARNING: [Common 17-1496] +# ...` and has no way to tell which Tcl proc triggered it. Hooking +# the Tcl entry point lets us capture the call stack at emit time +# and render it as `at file:line in proc` continuation lines. +# +# Tradeoffs to be aware of: +# - The original ::common::send_msg_id is NOT called. That means +# `set_msg_config -id X -suppress` won't suppress Tcl-emitted +# messages (it still works for messages Vivado's C code emits, +# which our PTY-level filter handles). Acceptable for v1; we +# can replicate suppression here if it becomes a real need. +# - Messages emitted from Vivado's C code (synth, route, etc.) +# bypass this override and are caught by the PTY-line filter +# in the worker, with no stack — that's a fundamental limit. + +# True when `str`'s first line looks like a Vivado-standard +# message: starts (after optional leading whitespace) with +# ERROR:/WARNING:/CRITICAL WARNING:/INFO:. Used by the puts +# wrapper to decide whether to attach a stack — we only want +# traces on message-formatted output, not on every `puts hi`. +proc ::vw::is_vivado_message {str} { + set first $str + set nl [string first "\n" $str] + if {$nl >= 0} { + set first [string range $str 0 [expr {$nl - 1}]] + } + set trimmed [string trimleft $first] + if {[string match "ERROR:*" $trimmed]} { return 1 } + if {[string match "CRITICAL WARNING:*" $trimmed]} { return 1 } + if {[string match "WARNING:*" $trimmed]} { return 1 } + if {[string match "INFO:*" $trimmed]} { return 1 } + return 0 +} + +# Severity of a Vivado-style message: one of `ERROR`, `CRITICAL`, +# `WARNING`, `INFO`. Returns empty for non-messages. Used by +# `attach_stack_if_message` to decide whether the stack is worth +# attaching (INFO is suppressed by default — see VW_INFO_WITH_STACK). +proc ::vw::message_severity {str} { + set first $str + set nl [string first "\n" $str] + if {$nl >= 0} { + set first [string range $str 0 [expr {$nl - 1}]] + } + set trimmed [string trimleft $first] + if {[string match "ERROR:*" $trimmed]} { return "ERROR" } + if {[string match "CRITICAL WARNING:*" $trimmed]} { return "CRITICAL" } + if {[string match "WARNING:*" $trimmed]} { return "WARNING" } + if {[string match "INFO:*" $trimmed]} { return "INFO" } + return "" +} + +# If `str` looks like a Vivado-style message, append the current +# Tcl call stack as `\n at ` continuation lines and +# return the augmented string. Otherwise return `str` unchanged. +# +# `skip_caller_frames` tells the stack walk how many wrapper +# layers to step past so the deepest reported frame is the user's +# code, not our shim's plumbing. For the puts wrapper that's 2 +# (this helper + the puts wrapper itself). +proc ::vw::attach_stack_if_message {str skip_caller_frames} { + if {![::vw::is_vivado_message $str]} { + return $str + } + # INFO messages are noisy under heavy Vivado activity (CIPS + # customization emits dozens per call). By default we suppress + # their stack so the scrollback stays scannable. The user opts + # in with `vw repl --info-with-stack` (or the `vw run` flag), + # which sets VW_INFO_WITH_STACK=1 on the spawned process. + # WARNING / ERROR / CRITICAL always keep their stacks. + if {[::vw::message_severity $str] eq "INFO"} { + set env_default 0 + catch { set env_default $::env(VW_INFO_WITH_STACK) } + if {$env_default ne "1"} { + return $str + } + } + set stack [::vw::capture_stack $skip_caller_frames] + if {[llength $stack] == 0} { + return $str + } + set has_trailing_nl 0 + set body $str + if {[string index $str end] eq "\n"} { + set has_trailing_nl 1 + set body [string range $str 0 end-1] + } + foreach frame $stack { + append body "\n at $frame" + } + if {$has_trailing_nl} { append body "\n" } + return $body +} + +# Walk the Tcl call stack starting at the caller of our override +# (`info frame 1` — skipping our wrapper itself) and build a list +# of "at file:line in proc" strings, deepest-first. Uses both +# `info frame` (gives script file/line) and `info level` (gives +# proc name + args) for each depth; merges whatever's available. +# Capped at `$::vw::stack_frame_cap` frames so a 50-deep tclapp +# loader chain doesn't drown the actual message. +# +# Returns at least one entry even when nothing is locatable — +# `(stack: depth=N, no locatable frames)` so the user can +# distinguish "override didn't fire" from "override fired but +# Tcl gave us nothing to render." +proc ::vw::capture_stack {skip_caller_frames} { + variable stack_frame_cap + variable trace_stack_capture + set out [list] + set depth [info frame] + set level_depth [info level] + # Skip our own frame plus whatever the caller asked us to skip. + set start [expr {1 + $skip_caller_frames}] + if {$trace_stack_capture} { + ::vw::log "\[vw-stack\] BEGIN capture skip=$skip_caller_frames\ + depth=$depth level_depth=$level_depth start=$start" + } + for {set i $start} {$i <= $depth} {incr i} { + if {[llength $out] >= $stack_frame_cap} { break } + set frame "" + catch {set frame [info frame -$i]} + # `info level -k` is indexed independently of `info frame` + # — k=0 is the current proc, k=-1 the caller, etc. We map + # frame index i to level index k by clamping; mismatches + # are common (frames can include non-proc evals) but worth + # trying as a fallback. + set level_args "" + set k [expr {$i - $skip_caller_frames - 1}] + if {$k > 0 && $k < $level_depth} { + catch {set level_args [info level -$k]} + } + set entry [::vw::format_frame $frame $level_args] + if {$trace_stack_capture} { + set kept "kept" + if {$entry eq ""} { set kept "dropped" } + ::vw::log "\[vw-stack\] i=$i k=$k $kept frame=\{$frame\}\ + level_args=\{$level_args\}\ + entry=\"$entry\"" + } + if {$entry ne ""} { lappend out $entry } + } + if {$trace_stack_capture} { + ::vw::log "\[vw-stack\] END capture out=\{[join $out { | }]\}" + } + if {[llength $out] == 0} { + lappend out "(stack: info-frame-depth=$depth\ + info-level-depth=$level_depth\ + — no locatable frames; message likely\ + emitted from byte-compiled or C-bridged Tcl)" + } + return $out +} + +# Turn one `info frame` dict (and an optional `info level` args +# list as a fallback proc-name source) into the human-readable +# string we render. Drops frames that have nothing locatable at +# all — they're just noise. +proc ::vw::format_frame {frame level_args} { + set proc "" + catch {set proc [dict get $frame proc]} + set file "" + catch {set file [dict get $frame file]} + set line "" + catch {set line [dict get $frame line]} + set cmd "" + catch {set cmd [dict get $frame cmd]} + # `info level -k` returns the proc invocation as `procname + # arg1 arg2 ...`; the first element is the proc name. + if {$proc eq "" && $level_args ne ""} { + set proc [lindex $level_args 0] + } + + # Drop frames that are part of our own plumbing — they're + # always noise to the user. The signal in a stack trace is + # "which line of MY code led to this message"; frames in + # the shim file, the ::vw:: namespace, our send_msg_id + # override, or the ::log:: helpers are all infrastructure. + if {[string match "*vivado-shim.tcl" $file]} { + return "" + } + if {[string match "::vw::*" $proc]} { + return "" + } + if {$proc eq "::common::send_msg_id"} { + return "" + } + if {[string match "::log::*" $proc]} { + return "" + } + + set location "" + if {$file ne "" && $line ne ""} { + set location "${file}:${line}" + } elseif {$line ne ""} { + # `eval` frames without a source file — common for our + # `uplevel #0 $tcl` shim entry — still tell the user + # "line N of the script you submitted." + set location ":${line}" + } + if {$location ne "" && $proc ne ""} { + return "${location} in ${proc}" + } elseif {$location ne ""} { + return $location + } elseif {$proc ne ""} { + return $proc + } elseif {$cmd ne ""} { + # Last-ditch: no proc and no location, but we know what + # command this frame was running. Truncate so a very long + # command doesn't blow out the trace. + set short [string range $cmd 0 80] + if {[string length $cmd] > 80} { append short "..." } + return "(cmd: $short)" + } + return "" +} + +# Severity normalizer. Vivado is inconsistent about case and +# uses underscores in CRITICAL_WARNING; we normalize to the same +# uppercase, space-separated form the PTY-line classifier expects +# so the worker can route warnings/errors to the right StreamKind. +proc ::vw::normalize_severity {sev} { + set s [string toupper [string trim $sev]] + switch -- $s { + "CRITICAL_WARNING" - + "CRITICAL WARNING" { return "CRITICAL WARNING" } + "ERROR" - + "FATAL" - + "FATAL_ERROR" { return "ERROR" } + "WARNING" { return "WARNING" } + "INFO" - + "STATUS" { return "INFO" } + default { return $s } + } +} + +# Install our wrapper *after* Vivado has had a chance to define +# ::common::send_msg_id. If the proc doesn't exist yet (very early +# init, headless mode without the common namespace), we silently +# skip — Vivado's PTY emission still works, just without our +# stack capture. +# +# Logs status once per successful install and once per skipped +# attempt (with the reason), so the user can see in the REPL +# whether the override is live without enabling --verbose. +proc ::vw::install_send_msg_override {} { + if {[info commands ::vw::orig_send_msg_id] ne ""} { + # Already installed — silent on the retry path so we don't + # spam the log on every eval. + return + } + set candidates [info commands ::common::send_msg*] + if {[info commands ::common::send_msg_id] eq ""} { + ::vw::log "::common::send_msg_id not present;\ + ::common::send_msg* = {$candidates};\ + stack-capture override NOT installed" + return + } + rename ::common::send_msg_id ::vw::orig_send_msg_id + + # The Vivado-Tcl signature is `send_msg_id id severity msg + # [optional args]`. We accept the same. + proc ::common::send_msg_id {id severity msg args} { + # Reentrancy guard — if our stack walk somehow triggers + # another send_msg_id, fall back to the original. + if {$::vw::in_send_msg_id} { + return [uplevel 1 [list ::vw::orig_send_msg_id \ + $id $severity $msg {*}$args]] + } + set ::vw::in_send_msg_id 1 + set ok [catch { + set sev_norm [::vw::normalize_severity $severity] + # INFO is noisy — suppress the stack by default, matching + # the puts-wrapper path. The user opts in with `vw repl + # --info-with-stack` (worker exports VW_INFO_WITH_STACK=1). + # WARNING / ERROR / CRITICAL always keep their stacks. + set attach_stack 1 + if {$sev_norm eq "INFO"} { + set env_default 0 + catch { set env_default $::env(VW_INFO_WITH_STACK) } + if {$env_default ne "1"} { set attach_stack 0 } + } + set out "${sev_norm}: \[${id}\] ${msg}" + if {$attach_stack} { + # Skip 1 caller frame so the deepest frame in the + # rendered stack is the one that called send_msg_id, + # not the user proc that called our wrapper. + set stack [::vw::capture_stack 1] + foreach frame $stack { + append out "\n at ${frame}" + } + } + if {$::vw::capturing} { + ::vw::stream_stdout $::vw::current_eval_id "$out\n" + } else { + # Outside an eval — fall back to the original so the + # message still appears wherever Vivado normally + # would have put it. + ::vw::orig_send_msg_id $id $severity $msg {*}$args + } + } err] + set ::vw::in_send_msg_id 0 + if {$ok != 0} { + # Our override threw — never let that prevent Vivado from + # at least seeing the message. Fall through to original. + ::vw::log "send_msg_id override failed: $err" + return [uplevel 1 [list ::vw::orig_send_msg_id \ + $id $severity $msg {*}$args]] + } + } + ::vw::log "installed send_msg_id override" +} + +# Commands wrapped with [`install_command_context`] so any +# traceless WARNINGs / ERRORs Vivado's C++ side emits during the +# call get the Tcl call stack attached. Each name MUST be a global +# Tcl command (no leading `::` — the wrapper installs as `::$name`). +# The list is open-ended: add a command here whenever a user hits a +# new noisy builtin and the worker filter shows the warning landing +# without a stack. There's no observable cost — the wrapper is a +# thin around-trace, only the message-tagging window is widened. +set ::vw::context_wrapped_commands { + set_property + generate_netlist_ip +} + +# Wrap a single Vivado command so we can attach the Tcl call stack +# to warnings/errors its C++ implementation emits. The C++ paths +# (notably `[IP_Flow 19-7090] Invalid parameter '…' provided, +# Ignoring` for `set_property`, `[Coretcl 2-176] No IPs found` for +# `generate_netlist_ip`) bypass `::common::send_msg_id` and write +# directly through Vivado's internal message bus to the PTY — +# there's no Tcl frame to grab by the time the bytes arrive at the +# Rust worker. So we capture the stack here, while the Tcl +# interpreter is *about* to enter the C++ command, emit it as a +# marker the worker recognizes and strips, then the worker tags any +# warnings that arrive while the marker is active. Markers go via +# `::vw::real_puts stdout` so they bypass our own `puts` override +# and land on the PTY directly. Idempotent — re-running this on +# every eval is harmless once the wrapper is in place. +proc ::vw::install_command_context {name} { + set orig "::vw::orig_${name}_for_ctx" + if {[info commands $orig] ne ""} { + return + } + if {[info commands ::$name] eq ""} { return } + rename ::$name $orig + # Build the wrapper body with `$orig` interpolated, NOT + # `$name` — the wrapper has to forward to the renamed original + # without a name-lookup detour. `set rc` is computed and + # forwarded so the wrapped command's return value, error code, + # and -errorinfo all flow back unchanged to the caller. + proc ::$name {args} [string map [list @ORIG@ $orig] { + # Skip 1 = this wrapper's own frame, so the deepest reported + # frame is the user proc that called the wrapped command. + set frames [::vw::capture_stack 1] + ::vw::emit_pty_ctx_begin $frames + set rc [catch { + uplevel 1 [list @ORIG@ {*}$args] + } result options] + ::vw::emit_pty_ctx_end + return -options $options $result + }] + ::vw::log "installed context wrap for ::$name" +} + +# Install context wrappers for every command in +# `::vw::context_wrapped_commands`. Called once after the protocol +# socket opens and again at the top of every eval — each +# `install_command_context` is idempotent, so re-attempts are cheap +# once installed and recover gracefully when a command first +# appears after a later library was sourced. +proc ::vw::install_all_context_wrappers {} { + foreach name $::vw::context_wrapped_commands { + catch {::vw::install_command_context $name} + } +} + +# Push a context marker onto the PTY. Format: a sentinel-prefixed +# line per frame plus begin/end bookends, so the Rust PTY filter +# can match line-by-line without needing a base64 decoder. +proc ::vw::emit_pty_ctx_begin {frames} { + ::vw::real_puts stdout "__VW_CTX_BEGIN__" + foreach f $frames { + ::vw::real_puts stdout "__VW_CTX_FRAME__:$f" + } + ::vw::real_puts stdout "__VW_CTX_READY__" + flush stdout +} + +proc ::vw::emit_pty_ctx_end {} { + ::vw::real_puts stdout "__VW_CTX_END__" + flush stdout +} + +# ---------- user-proc body wrap ---------- +# +# Traceless warnings from Vivado's C++ (e.g. `[Coretcl 2-176] No +# IPs found`) bypass `::common::send_msg_id`, so the per-command +# wrappers above only tag warnings emitted while THAT specific +# command is in flight. Real-world Vivado calls are deeper — a +# warning inside `generate_netlist_ip` may actually come from a +# nested `get_ips` call inside the C++ path — and enumerating +# every possible culprit isn't tractable. +# +# So we also instrument every USER-defined proc: rewrite each new +# proc's body to emit a marker BEGIN/READY at entry (with +# `capture_stack` from *inside* the body — the frame stack +# includes the proc itself) and an END on exit. The Rust +# marker-stack tracks nesting, so nested user procs each get their +# own frames and the innermost wins for tagging. Result: a warning +# fired anywhere under `configure_clock`'s call chain lands with +# `at ip/clock.htcl:N in ::configure_clock` attached, even when +# Vivado's C++ emits it silently. +# +# Only fires while `::vw::capturing == 1` — i.e. during a user +# eval — so Vivado's own Tcl-lib initialization defines procs +# unchanged. `::vw::*` procs are also skipped so our own plumbing +# doesn't recurse into itself. + +# Compute the fully-qualified name a `proc NAME ...` invocation +# would produce, given the caller's namespace. Used by the +# ::proc override's filter — we only wrap user procs that end up +# in the top-level `::` namespace, which is where htcl-lowered +# procs live. +proc ::vw::qualify_proc_name {name caller_ns} { + if {[string match ::* $name]} { return $name } + set caller_ns [string trimright $caller_ns ::] + if {$caller_ns eq ""} { return ::$name } + return ${caller_ns}::$name +} + +# Install the ::proc override. Idempotent — re-running is cheap +# once the wrapper is in place. The original `proc` is renamed to +# `::vw::orig_proc_for_body_wrap` and delegated to. +## Install a proc WITHOUT the body-wrap. Called by codegen-emitted +## infrastructure (enum variant constructors, `::to_raw` / +## `::from_raw` lifts, list/dict monomorphized repr walkers, +## primitive-type identity procs) where the wrap's per-call +## `capture_stack` + `emit_pty_ctx_begin/end` cost dwarfs the actual +## work — a proc that only does `return [list Scalar $v]` shouldn't +## take 100μs of instrumentation per call, and calling it 200K +## times during collection construction pushes total overhead into +## the multi-second range for no debug benefit. +## +## Named `codegen_proc` (not `proc_no_wrap` or similar) so the +## intent is clear at the emit site: this is for COMPILER-EMITTED +## infrastructure, not for user-authored procs the user might want +## to opt out of instrumentation on. Per-user opt-out — if we ever +## need it — is a separate mechanism. +## +## Behavior when the body-wrap isn't installed yet (very early +## shim startup, before `install_proc_body_wrap` runs): falls +## through to the raw `::proc` builtin. Codegen shipped before the +## wrap install still produces working procs; they don't gain the +## wrap retroactively once it installs, which is exactly the point +## — they opted out. +proc ::vw::codegen_proc {name spec body} { + if {[info commands ::vw::orig_proc_for_body_wrap] ne ""} { + uplevel 1 [list ::vw::orig_proc_for_body_wrap $name $spec $body] + } else { + uplevel 1 [list ::proc $name $spec $body] + } +} + +proc ::vw::install_proc_body_wrap {} { + if {[info commands ::vw::orig_proc_for_body_wrap] ne ""} { + return + } + rename ::proc ::vw::orig_proc_for_body_wrap + # Marker template: `@BODY@` gets literal-substituted with the + # user's body via `string map`, avoiding format/subst + # interpolation risks. `catch` preserves rc/result/errorcode/ + # errorinfo across the wrap so the wrapped proc behaves + # identically to the unwrapped original. + variable proc_body_template { + ::vw::emit_pty_ctx_begin [::vw::capture_stack 0] + set _vw_ctx_rc [catch {@BODY@} _vw_ctx_result _vw_ctx_opts] + ::vw::emit_pty_ctx_end + return -options $_vw_ctx_opts $_vw_ctx_result + } + ::vw::orig_proc_for_body_wrap ::proc {name spec body} { + # Delegate straight through when we're not inside a user + # eval — Vivado's own lib procs go untouched. + if {!$::vw::capturing} { + return [uplevel 1 [list ::vw::orig_proc_for_body_wrap \ + $name $spec $body]] + } + set caller_ns [uplevel 1 { namespace current }] + set qualified [::vw::qualify_proc_name $name $caller_ns] + # Skip our own helpers and any Vivado internal proc that a + # user eval might reach into. `::vw::*` covers our shim, + # `::tcl::*` guards Tcl's core, everything else in the + # top-level or user namespaces gets the wrap. + # + # Also skip codegen-emitted infrastructure procs — the + # compiler-generated per-type `repr` / `to` / `from` / + # `to_raw` / `from_raw` lifts, `tag` / `payload` / + # `empty` helpers, and the top-level `::putr`. Each of + # these is a tiny passthrough proc where the per-call + # `capture_stack` + `emit_pty_ctx_begin/end` cost + # dominates: a `list` return with a million + # elements calls `bd_pin::repr` a million times, and a + # `Properties`-typed proc constructing 200K entries calls + # `Property::Scalar` 200K times. The raw log for those + # cases shows the wait period is almost entirely + # `__VW_CTX_BEGIN__` / `__VW_CTX_END__` marker traffic, + # not the actual work. Stack context on a proc that just + # does `return [list Scalar $v]` adds nothing. + # + # Enum variant constructors (`Property::Scalar` etc.) + # have no distinctive name pattern, so we detect them + # structurally via the body-shape regex below. + if {[string match ::vw::* $qualified] + || [string match ::tcl::* $qualified] + || [string match *::repr $qualified] + || [string match *::to $qualified] + || [string match *::from $qualified] + || [string match *::to_raw $qualified] + || [string match *::from_raw $qualified] + || [string match *::tag $qualified] + || [string match *::payload $qualified] + || [string match *::empty $qualified] + || $qualified eq "::putr"} { + return [uplevel 1 [list ::vw::orig_proc_for_body_wrap \ + $name $spec $body]] + } + # Enum-variant-constructor detection: body is exactly + # `return [list ]` (empty payload) or + # `return [list $v]` (single-payload). This is the + # canonical shape `emit_constructor` in vw-htcl emits, and + # a user writing that body verbatim would get their proc + # silently un-instrumented — an acceptable false-positive + # since (a) no stack trace context is genuinely useful for + # a one-liner list-wrapper, and (b) the pattern is narrow + # enough to be near-impossible to hit by accident. + if {[regexp \ + {^\s*return\s+\[list\s+[A-Za-z_][A-Za-z0-9_]*(?:\s+\$v)?\]\s*$} \ + $body]} { + return [uplevel 1 [list ::vw::orig_proc_for_body_wrap \ + $name $spec $body]] + } + # Detect a re-wrap: if the body already contains our + # marker call, don't stack another layer around it. Redefs + # of a user proc during eval (e.g. re-`src`ing a library) + # would otherwise nest wrappers on every reload. + if {[string first "::vw::emit_pty_ctx_begin" $body] >= 0} { + return [uplevel 1 [list ::vw::orig_proc_for_body_wrap \ + $name $spec $body]] + } + set new_body [string map \ + [list @BODY@ $body] $::vw::proc_body_template] + uplevel 1 [list ::vw::orig_proc_for_body_wrap $name $spec $new_body] + } + ::vw::log "installed ::proc body wrap" +} + +# ---------- dispatch ---------- + +proc ::vw::dispatch {line} { + if {[catch {::json::json2dict $line} req]} { + ::vw::send_err 0 "protocol parse error: $req" + return + } + if {![dict exists $req id] || ![dict exists $req op]} { + ::vw::send_err 0 "missing id or op" + return + } + set id [dict get $req id] + set op [dict get $req op] + # Arm the main-loop deadman as soon as we've committed to a + # specific id — from here on, if this dispatch doesn't reach + # a `send_ok` / `send_err`, the main loop synthesises an + # error response for `id` so the client never blocks. + set ::vw::pending_response_id $id + set ::vw::pending_response_sent 0 + switch -- $op { + eval { + if {![dict exists $req tcl]} { + ::vw::send_err $id "eval request missing tcl field" + return + } + set tcl [dict get $req tcl] + set ::vw::current_eval_id $id + set ::vw::capturing 1 + set rc [catch {uplevel #0 $tcl} result opts] + set ::vw::capturing 0 + if {$rc != 0} { + set ecode "" + set einfo "" + catch {set ecode [dict get $opts -errorcode]} + catch {set einfo [dict get $opts -errorinfo]} + ::vw::send_err $id $result $ecode $einfo + } else { + ::vw::send_ok $id $result + } + } + shutdown { + ::vw::send_ok $id "" + ::vw::log "shim shutting down" + exit 0 + } + default { + ::vw::send_err $id "unknown op: $op" + } + } +} + +# ---------- main ---------- + +if {![info exists ::env(VW_PROTOCOL_ADDR)]} { + ::vw::log "VW_PROTOCOL_ADDR not set; exiting" + exit 1 +} + +if {![regexp {^(.*):(\d+)$} $::env(VW_PROTOCOL_ADDR) -> ::vw::host ::vw::port]} { + ::vw::log "invalid VW_PROTOCOL_ADDR: $::env(VW_PROTOCOL_ADDR)" + exit 1 +} + +if {[catch {socket $::vw::host $::vw::port} sock]} { + ::vw::log "failed to connect to $::vw::host:$::vw::port: $sock" + exit 1 +} + +set ::vw::protocol_sock $sock +fconfigure $sock -buffering line -translation lf + +::vw::log "connected to $::vw::host:$::vw::port" + +# Try installing the send_msg_id override now. If Vivado hasn't +# defined ::common::send_msg_id yet (unusual but possible in +# headless / minimal-mode configurations), the override will be +# re-attempted on the first eval — it's idempotent. +catch {::vw::install_send_msg_override} +catch {::vw::install_all_context_wrappers} +catch {::vw::install_set_property_recorder} +catch {::vw::install_proc_body_wrap} + +# Silence Vivado's per-command performance report — the +# `: Time (s): cpu = … Memory (MB): peak = …` chatter that +# appears after any command whose elapsed time exceeds +# `tcl.statsThreshold` seconds (default: quite low, so most +# `create_bd_cell` calls trip it). Users who WANT the stats can +# re-lower the threshold in their own script; setting it high +# by default keeps the REPL / `vw run` output focused on the +# user's actual results. +catch {set_param tcl.statsThreshold 9999999} + +while {1} { + if {[gets $sock line] < 0} { + if {[eof $sock]} { + ::vw::log "protocol socket closed; exiting" + break + } + continue + } + set line [string trim $line] + if {$line eq ""} { continue } + # Retry installs on each eval until they succeed — both procs + # bail out cheaply once installed. + catch {::vw::install_send_msg_override} + catch {::vw::install_all_context_wrappers} +catch {::vw::install_set_property_recorder} +catch {::vw::install_proc_body_wrap} + # Arm the response deadman. `::vw::dispatch` sets + # `pending_response_id` as soon as it parses the request id; + # `send_ok` / `send_err` clear `pending_response_sent`. Wrap + # the dispatch in `catch` so an `interp cancel -unwind` + # ripping through the eval-branch's own `catch` doesn't kill + # the main loop — we synthesise a fallback error response + # and keep serving requests. See the `pending_response_*` + # variable docs. + set ::vw::pending_response_id 0 + set ::vw::pending_response_sent 0 + catch {::vw::dispatch $line} dispatch_result dispatch_opts + set dispatch_rc [dict get $dispatch_opts -code] + # Restore capturing to a sane state — the eval-branch always + # clears it, but if `catch` unwound before the clear line ran + # it might still be 1 and would misroute the next eval's puts. + set ::vw::capturing 0 + if {$::vw::pending_response_id != 0 && !$::vw::pending_response_sent} { + set fallback_msg "shim dispatch aborted without sending a response" + if {$dispatch_rc != 0} { + append fallback_msg " (rc=$dispatch_rc): $dispatch_result" + } else { + append fallback_msg " (dispatch returned but did not send)" + } + ::vw::log "deadman firing for id=$::vw::pending_response_id: $fallback_msg" + ::vw::send_err $::vw::pending_response_id $fallback_msg + } +} + +close $sock +exit 0 diff --git a/vw-vivado/src/handlers.rs b/vw-vivado/src/handlers.rs new file mode 100644 index 0000000..bd6f404 --- /dev/null +++ b/vw-vivado/src/handlers.rs @@ -0,0 +1,1245 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Shared RPC handlers wired into every Vivado spawn (`vw run`, +//! `vw repl`, `vw test`). Each method here answers a `vw::` htcl +//! proc whose answer lives on the tool side rather than in Vivado. +//! +//! Methods: +//! - `workspace_root` — the discovered `vw.toml` parent dir. +//! - `critical_warning_count` — session-scoped monotonic count of +//! CRITICAL WARNING chunks the stream sink has classified so far. +//! `vw::synth` / `vw::place` snapshot this before + after each +//! phase to decide whether to persist a checkpoint (a CW delta +//! means the artifact would poison the next warm run). Vivado's +//! own `get_msg_config -count -severity` doesn't see CWs from +//! opt/place/phys_opt sub-processes, so we track them ourselves. +//! - `diff_files` — read two files and return a unified diff of +//! their contents. Used by `test::assert_file_eq` to render a +//! readable failure message instead of a two-line "files differ" +//! note. +//! - `vhdl_dependency_sources` — every VHDL file shipped by any +//! transitive dep (regular `[dependencies]` only), grouped by +//! target library. Consumed by `design.htcl` to feed +//! `read_vhdl -library …`. +//! - `vhdl_dependency_sources_with_test` — same, but the entry +//! workspace's `[test-dependencies]` are also included. +//! Consumed by `test/*.htcl` where test-deps are in scope. +//! - `vhdl_design_sources` — every VHDL file under +//! `/hdl/`. Consumed alongside dependency sources +//! to compile the workspace's own design. +//! - `vhdl_ip_sources` — every generated IP wrapper under +//! `/target/ip/**/*.vhd`. Populated by +//! `vw::make_wrapper`. Kept separate from design sources +//! because wrappers have their own regen lifecycle and +//! typically compile into a distinct library (`ip`). +//! - `design_constraints` — every Vivado constraint file under +//! `/constraints/**/*.{xdc,sdc}`. Fed to `read_xdc` +//! during synth prep. +//! - `design_synth_constraints` / `design_place_constraints` / +//! `design_route_constraints` — phase-scoped variants that +//! walk `constraints/synth/`, `constraints/place/`, +//! `constraints/route/` respectively. Used to attach USED_IN +//! flags to `read_xdc` so route-only constraints don't apply +//! during synthesis (and vice versa). +//! - `synth_needs_update` — content-hash comparison between the +//! checkpoint's sidecar manifest and the current tracked source +//! set (design VHDL, IP wrappers, synth XDC, workspace `.htcl`, +//! vw.toml, vw.lock). Returns `true` when the checkpoint OR +//! manifest is missing OR the fingerprints disagree. Backs +//! the `vw::synth` cache path. +//! - `synth_mark_checkpoint` — writes the sidecar manifest for a +//! freshly-produced checkpoint. Called after +//! `vivado_cmd::write_checkpoint` so the next invocation can +//! compare fingerprints and skip resynth on unchanged sources. +//! - `mark_project_configured` — records that the on-disk Vivado +//! project at `/target/vw-project//` has been +//! successfully configured against the current +//! `/ip/**/*.htcl` + `vw.toml` fingerprint. Called by +//! `vw::configure_ip` after `ip::configure` + `save_project` +//! complete. The freshness check itself runs Rust-side before +//! spawning Vivado (see `vw_lib::project_needs_wipe`), so there +//! is no matching `project_needs_update` RPC — invalidation +//! happens once per session, not per htcl call. +//! - `place_needs_update` / `place_mark_checkpoint` — same +//! shape again but scoped to the place stage. Fingerprint +//! covers `/constraints/place/**` plus the upstream synth +//! DCP file (which proxies for "everything synth depended +//! on" — if any synth-scope source changed, synth re-ran +//! and the DCP is fresh, invalidating place). Backs +//! `vw::place`. +//! - `compile_htcl_module` — parses + lowers an htcl module (any +//! `src`-shaped path resolved against the workspace root) and +//! returns the concatenated Tcl. `vw::configure_ip` uses this +//! to auto-load `/ip/module.htcl` when `::ip::configure` +//! isn't already defined, so a `src ip` in `design.htcl` isn't +//! a hidden requirement. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; +use std::time::SystemTime; + +use serde_json::Value; + +use crate::rpc::{FnHandler, RpcHandler}; + +/// Shared "already loaded in this Vivado session" map. Populated +/// by the CLI / REPL after each successful htcl load with the +/// paths + mtimes of every file that was shipped to Vivado. +/// Consulted by [`compile_htcl_module`] so on-demand +/// module compilation skips re-shipping files whose procs are +/// already installed — which is both a big perf win and, more +/// importantly, KEEPS the RPC path from re-loading dep files that +/// design.htcl never touched. Concretely: `design.htcl`'s +/// `src @vw` pulls @vw + @vivado-cmd; a workspace `ip/cips.htcl` +/// also does `src @cpm5` / `src @cips` / `src @clk-wizard`; those +/// three are NOT in the initial session because design.htcl +/// never reached them, and the auto-load path must actually +/// compile + ship them the first time. See the safety note on +/// [`SharedPreload`] for how the map's invariants are enforced. +pub type PreloadedPaths = HashMap; + +/// Shared handle callers hold to update the "already-loaded" +/// map after each session commit. The RPC handler holds the same +/// Arc so reads see the most recent update without any explicit +/// message passing. +/// +/// **Invariant**: only entries added to this map after Vivado +/// has finished evaling the corresponding Tcl are safe to skip. +/// Adding a path prematurely would cause `compile_htcl_module` +/// to omit content Vivado hasn't seen yet, leading to +/// `invalid command name` errors at runtime. The REPL updates +/// after every batch's `EvalDone`; the CLI updates once at the +/// end of the initial load. +pub type SharedPreload = Arc>; + +/// Session-scoped monotonic counter of CRITICAL WARNING (and +/// higher) chunks the stream classifier has seen. Vivado's own +/// `get_msg_config -count -severity "CRITICAL WARNING"` doesn't +/// see CWs emitted from sub-processes launched inside `opt_design` +/// / `place_design` / `phys_opt_design` (documented gotcha), so +/// htcl procs like `vw::synth` and `vw::place` need OUR counter +/// to decide whether to persist a checkpoint. Owned by whichever +/// call site sets up the stream sink (vw-cli / vw-repl); the RPC +/// handler holds the same Arc so `vw::critical_warning_count` +/// reads see the latest value with no message-passing. +pub type SharedCriticalWarningCount = Arc; + +/// Build the FnHandler used by every Vivado spawn. `workspace_root` +/// is the workspace-root path to serve for `vw::workspace_root`; +/// pass `None` when the caller couldn't discover one (the RPC +/// method then returns an error instead of a bogus path). +pub fn make_handler(workspace_root: Option) -> Arc { + make_handler_with_variant(workspace_root, None) +} + +/// Like [`make_handler`] but also carries a session-scoped active +/// variant name — the value the CLI's `--variant ` flag +/// picked. When present, RPC methods that filter by variant +/// (currently `vhdl_design_sources`) fall back to this instead +/// of the workspace default. Explicit per-call `variant` kwargs +/// still take precedence. +pub fn make_handler_with_variant( + workspace_root: Option, + active_variant: Option, +) -> Arc { + let preloaded: SharedPreload = Arc::new(RwLock::new(HashMap::new())); + make_handler_with_preloaded(workspace_root, active_variant, preloaded) +} + +/// Convenience over [`make_handler_full`] that allocates a fresh +/// (unshared) CW counter. Suitable for callers that don't need to +/// read the counter themselves (e.g. `vw test` — where each test +/// spawns its own worker and only cares about eval results, not +/// CW-gated checkpoint decisions). +pub fn make_handler_with_preloaded( + workspace_root: Option, + active_variant: Option, + preloaded: SharedPreload, +) -> Arc { + let cw = Arc::new(std::sync::atomic::AtomicU64::new(0)); + make_handler_full(workspace_root, active_variant, preloaded, cw) +} + +/// Full-detail constructor: carries every shared piece of state +/// the RPC surface needs. `preloaded` is the "already loaded in +/// this Vivado session" map [`compile_htcl_module`] consults — +/// callers who want on-demand htcl loading to skip files already +/// shipped to this Vivado session should clone the returned map's +/// Arc, hold onto it, and update it after every successful load. +/// `cw_count` is the session's monotonic CRITICAL WARNING counter +/// — callers that own the stream sink should clone this Arc and +/// bump it on every `Severity::CriticalWarning` chunk, and htcl +/// procs read it via the `critical_warning_count` RPC. +pub fn make_handler_full( + workspace_root: Option, + active_variant: Option, + preloaded: SharedPreload, + cw_count: SharedCriticalWarningCount, +) -> Arc { + let workspace_root = workspace_root.map(Arc::new); + let active_variant = active_variant.map(Arc::new); + FnHandler::new(move |method: String, args: Value| { + let ws = workspace_root.clone(); + let av = active_variant.clone(); + let pl = preloaded.clone(); + let cw = cw_count.clone(); + async move { + dispatch( + &method, + args, + ws.as_deref().map(|p| p.as_ref()), + av.as_deref().map(|s| s.as_str()), + &pl, + &cw, + ) + .await + } + }) +} + +async fn dispatch( + method: &str, + args: Value, + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, + preloaded: &SharedPreload, + cw_count: &SharedCriticalWarningCount, +) -> Result { + match method { + "workspace_root" => workspace_root + .map(|p| Value::String(p.to_string_lossy().into_owned())) + .ok_or_else(|| { + "no workspace root: entry file has no `vw.toml` in its \ + parent chain" + .to_string() + }), + "critical_warning_count" => Ok(Value::from( + cw_count.load(std::sync::atomic::Ordering::Relaxed), + )), + "active_variant" => { + Ok(active_variant_value(workspace_root, active_variant)) + } + "project_name" => project_name_value(workspace_root), + "diff_files" => diff_files(args), + "vhdl_dependency_sources" => { + vhdl_dependency_sources( + workspace_root, + /*include_test=*/ false, + extract_exclude_sim_only(&args), + ) + .await + } + "vhdl_dependency_sources_with_test" => { + vhdl_dependency_sources( + workspace_root, + /*include_test=*/ true, + extract_exclude_sim_only(&args), + ) + .await + } + "vhdl_design_sources" => vhdl_design_sources( + workspace_root, + extract_variant(&args).or_else(|| active_variant.map(String::from)), + ), + "vhdl_ip_sources" => vhdl_ip_sources(workspace_root), + "design_constraints" => design_constraints(workspace_root), + "design_synth_constraints" => { + design_phase_constraints(workspace_root, ConstraintPhase::Synth) + } + "design_place_constraints" => { + design_phase_constraints(workspace_root, ConstraintPhase::Place) + } + "design_route_constraints" => { + design_phase_constraints(workspace_root, ConstraintPhase::Route) + } + "synth_needs_update" => { + synth_needs_update(workspace_root, active_variant, args) + } + "synth_mark_checkpoint" => { + synth_mark_checkpoint(workspace_root, active_variant, args) + } + "mark_project_configured" => { + mark_project_configured(workspace_root, args) + } + "place_needs_update" => place_needs_update(workspace_root, args), + "place_mark_checkpoint" => place_mark_checkpoint(workspace_root, args), + "compile_htcl_module" => { + compile_htcl_module(workspace_root, args, preloaded).await + } + other => Err(format!("unknown RPC method: {other}")), + } +} + +/// `compile_htcl_module` — inputs `{path: ""}` where +/// `` follows the same resolution rules as an htcl `src` +/// statement (relative path, absolute path, `@dep/…`, or +/// directory-as-module). Loads the entry file + every transitive +/// import, lowers each command to Tcl, and returns the +/// concatenated Tcl string. +/// +/// Callers `eval` the returned string to install everything the +/// module exports into the current interpreter. Used by +/// `vw::configure_ip` to auto-load `/ip/module.htcl` when +/// `::ip::configure` isn't already defined — so the user doesn't +/// have to remember to `src ip` in their `design.htcl` before +/// calling the wrapper. +/// +/// The pipeline mirrors what `vw run` / `vw repl` do internally +/// (parse → sig-table → per-command lowering → extern rewrite) +/// but skips overload dispatchers, putr rewrites, and origin +/// markers. Those matter for user-facing tracebacks and +/// interactive `putr` — neither is needed for on-demand loading +/// of a well-formed module. +async fn compile_htcl_module( + workspace_root: Option<&std::path::Path>, + args: Value, + preloaded: &SharedPreload, +) -> Result { + // Extract owned inputs so the closure passed to + // `spawn_blocking` doesn't borrow anything from this async + // frame. `path` becomes a String, the preload map is + // snapshotted here (short critical section on the RwLock — + // NOT held across the compile). + let ws = workspace_root_or_error(workspace_root)?; + let obj = args.as_object().ok_or_else(|| { + "compile_htcl_module: args must be an object with a `path` \ + string field" + .to_string() + })?; + let path: String = obj + .get("path") + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| { + "compile_htcl_module: missing string `path`".to_string() + })?; + let preloaded_snapshot: std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + > = preloaded.read().map(|g| g.clone()).unwrap_or_default(); + let ws_owned = ws.clone(); + + // Offload the heavy work — file I/O, htcl parse, per-command + // lowering — to the blocking thread pool. Without this, the + // async frame runs to completion on the current runtime + // thread and starves every other task (in particular the + // REPL's 250ms redraw tick, which is why the input timer + // appeared to freeze during long compiles). Only mouse + // movement was un-freezing it because a mouse event would + // hit the `crossterm_events` branch of the select! and + // trigger a draw as a side effect of handling the event. + tokio::task::spawn_blocking(move || { + compile_htcl_module_blocking(ws_owned, path, preloaded_snapshot) + }) + .await + .map_err(|e| format!("compile_htcl_module: join error: {e}"))? +} + +/// Sync core of `compile_htcl_module` — runs on the tokio +/// blocking thread pool so the async runtime stays responsive. +/// All heavy work (loader recursion, parse, lower, extern +/// rewrite, disk writes) lives here. +/// +/// Disk cache: after a successful compile, writes a manifest at +/// `/target/.vw-compile-.tcl.manifest` +/// listing every loaded file's path + mtime. On subsequent +/// invocations, if the manifest exists AND every file's current +/// mtime matches, returns the cached `.tcl` verbatim in +/// milliseconds instead of re-running the loader. +/// +/// The cache is invalidated by any file's mtime changing OR the +/// preload set changing (folded into the manifest fingerprint). +/// Not invalidated by ADDING new files (a new `.htcl` under `ip/` +/// wouldn't be in the manifest); users adding sources should +/// `rm target/.vw-compile-*` to force recompile. +fn compile_htcl_module_blocking( + ws: camino::Utf8PathBuf, + path: String, + preloaded_snapshot: std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + >, +) -> Result { + // Compute cache paths early so we can short-circuit on hit. + let dbg_name = format!( + ".vw-compile-{}.tcl", + path.replace(['/', '\\'], "_").replace('@', "at_"), + ); + let target_dir = ws.join("target"); + let cache_tcl = target_dir.join(&dbg_name); + let cache_manifest_name = format!("{dbg_name}.manifest"); + let cache_manifest = target_dir.join(&cache_manifest_name); + + // Cache hit path: manifest exists AND every listed file's + // current mtime matches. The manifest also embeds a hash of + // the preload snapshot's paths (sorted) — if the caller now + // has a different set of files preloaded, the compile output + // would differ, so we invalidate on that too. + if let Some(cached) = try_load_cached_compile( + cache_manifest.as_std_path(), + cache_tcl.as_std_path(), + &preloaded_snapshot, + ) { + return Ok(Value::String(cached)); + } + // Build the resolver with the workspace's transitive deps plus + // the Cargo-parity self-injection. Mirrors vw-cli's + // `load_htcl_program_with_mode` at a minimum — we don't need + // test deps here (the auto-load path never fires from inside + // a test). + let mut resolver = vw_htcl::Resolver::new(); + if let Ok(paths) = vw_lib::transitive_dep_cache_paths(&ws) { + for (name, cache_path) in paths { + resolver = resolver.with_dep(name, cache_path); + } + } + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + resolver = resolver.with_dep_if_absent( + cfg.workspace.name.clone(), + ws.as_std_path().to_path_buf(), + ); + } + + // Resolve the import path against the workspace root (that's + // how `design.htcl` would resolve `src ip`). Directory-as- + // module + `.htcl` extension logic is inside `resolve()`. + let entry = resolver + .resolve(ws.as_std_path(), &path) + .map_err(|e| format!("compile_htcl_module: {e}"))?; + + // Preload: skip files the caller has already shipped to this + // Vivado session. + // + // Correctness rule: only files that are actually installed + // in the Vivado interp belong in this map. An earlier version + // of this code preloaded every `.htcl` under every dep root + // the resolver knew about — that was wrong because a + // workspace can declare deps (e.g. `@cpm5` / `@cips` / + // `@clk-wizard` in metroid) that `design.htcl`'s entry + // graph never actually pulls in. Those procs weren't + // installed at startup, and preloading their files caused + // `invalid command name "cpm5::cpm_pcie0"` at runtime. + // Trusting the caller-populated map avoids that class of + // bug entirely. + let mut noop = NoopLoadObserver; + let program = vw_htcl::loader::load_with_preloaded( + &entry, + &resolver, + &mut noop, + &preloaded_snapshot, + ) + .map_err(|e| format!("compile_htcl_module: loading {path}: {e}"))?; + + // Parse the flattened source once, then build the same set + // of auxiliary tables vw-cli's runner uses. Enum-prelude and + // overload-dispatcher emission both consult these — skipping + // them means user procs referencing an enum namespace (e.g. + // `proc Property::as_nested`) or a monomorphized generic + // repr (e.g. `dict_string_Property::repr`) reach Tcl before + // the namespace exists and error with `unknown namespace`. + let parsed = vw_htcl::parser::parse(&program.source); + let mut _ignored: Vec = Vec::new(); + let enum_decl_table = + vw_htcl::build_enum_decl_table(&parsed.document, &mut _ignored); + let type_decl_table = + vw_htcl::build_type_decl_table(&parsed.document, &mut _ignored); + let type_decl_names: std::collections::HashSet = + type_decl_table.keys().cloned().collect(); + let (_full_sigs, overload_table) = + vw_htcl::build_signature_table_with_overloads( + &parsed.document, + &type_decl_names, + &mut _ignored, + ); + let table = vw_htcl::signature_table(&parsed.document); + + let mut out = String::new(); + // Preludes first — namespace/proc definitions the lowered + // commands below depend on. Order matches vw-cli's runner: + // primitives create root type namespaces, enum preludes + // create per-enum namespaces + variant constructors, and + // overload dispatchers create the switch-arm procs. + // + // All three emissions are idempotent (Tcl `namespace eval X + // {}` on an existing X is a no-op, `proc` redefinition + // replaces). Safe to re-ship even when design.htcl already + // installed the same preludes at startup. + for p in vw_htcl::emit_primitive_prelude() { + out.push_str(&p); + out.push('\n'); + } + for ed in enum_decl_table.values() { + let prelude = vw_htcl::emit_enum_prelude(ed); + if !prelude.trim().is_empty() { + out.push_str(&prelude); + out.push('\n'); + } + } + for info in overload_table.values() { + let dispatcher = vw_htcl::emit_dispatcher(info); + if !dispatcher.trim().is_empty() { + out.push_str(&dispatcher); + out.push('\n'); + } + } + + // Now the lowered user commands. Same per-statement lowering + // vw-cli does (minus putr rewrites and origin markers — this + // path never fires from an interactive `putr` and the caller + // already has an origin frame from `vw::configure_ip`). + // + // Critically: proc declarations for overload specializations + // get lowered under a MANGLED internal name (e.g. + // `Property::as_nested::v_Property::Scalar`) so the + // dispatchers emitted above can route to the right variant. + // Without this, both overloads of `Property::as_nested` + // land at the same unmangled name and the second definition + // silently shadows the first via Tcl proc redefinition — + // which produces `called on Scalar value` errors when the + // dispatcher expects the mangled variants to exist. + // + // Perf: use the `_with_putr_and_index` / `_with_name_and_index` + // variants and pass a PRE-BUILT `LineIndex`. The non-`_index` + // variants rebuild a LineIndex per call — an O(source_size) + // newline scan. For a 16 MB / 13k-statement compile that was + // O(stmts × source_size) quadratic and dominated wall-clock + // at ~100s. With a shared index the loop is O(stmts × avg + // command body) which finishes in a couple of seconds. + let line_index = vw_htcl::LineIndex::new(&program.source); + let empty_putr: std::collections::HashMap = + std::collections::HashMap::new(); + for stmt in &parsed.document.stmts { + let vw_htcl::ast::Stmt::Command(cmd) = stmt else { + continue; + }; + let lowered = match overload_specialization_mangle(cmd, &overload_table) + { + Some(mangled) => { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + unreachable!( + "overload_specialization_mangle already \ + validated this is a Proc" + ) + }; + vw_htcl::lower_proc_decl_with_name_and_index( + proc, + &program.source, + &table, + Some(&mangled), + &empty_putr, + &line_index, + ) + } + None => vw_htcl::lower_command_with_putr_and_index( + cmd, + &program.source, + &table, + &empty_putr, + &line_index, + ), + }; + // `extern::name` → `::name` so wrapper bodies that forward + // via `extern::` reach Vivado as bare native names — same + // rewrite vw-cli applies before shipping to the backend. + let tcl = vw_htcl::rewrite_externs(&lowered).text; + if !tcl.trim().is_empty() { + out.push_str(&tcl); + out.push('\n'); + } + } + + // Persist the cache: compiled Tcl + a manifest listing every + // loaded file's path + mtime + a hash of the preload set. + // Next invocation short-circuits if all mtimes match AND the + // preload hash matches (same set of files already-loaded). + // Silent on write errors — cache misses on next run are + // annoying but not incorrect. + let _ = std::fs::create_dir_all(target_dir.as_std_path()); + let _ = std::fs::write(cache_tcl.as_std_path(), &out); + let _ = write_compile_manifest( + cache_manifest.as_std_path(), + &program, + &preloaded_snapshot, + ); + + Ok(Value::String(out)) +} + +/// Manifest file format (plain text): +/// ```text +/// preload-hash +/// +/// +/// ... +/// ``` +/// One line per loaded file. `preload-hash` is FNV-1a over the +/// sorted list of preload paths so a caller-side change to +/// what's already-loaded invalidates the cache too. +fn write_compile_manifest( + manifest_path: &std::path::Path, + program: &vw_htcl::LoadedProgram, + preloaded: &std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + >, +) -> std::io::Result<()> { + use std::fmt::Write; + let mut body = String::new(); + let ph = preload_fingerprint(preloaded); + writeln!(body, "preload-hash {ph}").ok(); + for f in &program.files { + let Some(mtime) = f.mtime else { continue }; + let Ok(dur) = mtime.duration_since(std::time::UNIX_EPOCH) else { + continue; + }; + writeln!(body, "{} {}", dur.as_nanos(), f.path.display()).ok(); + } + std::fs::write(manifest_path, body) +} + +/// Try to serve a compile from the cache. Returns `Some(tcl)` iff +/// - the manifest exists, +/// - every listed file's current mtime matches the recorded one, +/// - AND the recorded preload-hash matches the current one. +/// +/// Any mismatch (or missing file, or unreadable cache) → `None`, +/// and the caller falls through to a fresh compile. +fn try_load_cached_compile( + manifest_path: &std::path::Path, + cache_path: &std::path::Path, + preloaded: &std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + >, +) -> Option { + let manifest = std::fs::read_to_string(manifest_path).ok()?; + let mut lines = manifest.lines(); + let ph_line = lines.next()?; + let ph_str = ph_line.strip_prefix("preload-hash ")?; + let stored_ph: u64 = ph_str.parse().ok()?; + if stored_ph != preload_fingerprint(preloaded) { + return None; + } + for line in lines { + let mut parts = line.splitn(2, ' '); + let stored_ns: u128 = parts.next()?.parse().ok()?; + let path = std::path::Path::new(parts.next()?); + let meta = std::fs::metadata(path).ok()?; + let mtime = meta.modified().ok()?; + let current_ns = + mtime.duration_since(std::time::UNIX_EPOCH).ok()?.as_nanos(); + if current_ns != stored_ns { + return None; + } + } + // All checks pass — serve the cached Tcl. + std::fs::read_to_string(cache_path).ok() +} + +/// FNV-1a over the sorted preload path set. Deliberately ignores +/// mtimes — the file-mtime check above already covers content +/// changes for preloaded files; this fingerprint only detects +/// changes to WHICH files are preloaded (a different caller state). +fn preload_fingerprint( + preloaded: &std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + >, +) -> u64 { + let mut paths: Vec<&std::path::Path> = + preloaded.keys().map(|p| p.as_path()).collect(); + paths.sort(); + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for p in paths { + for b in p.to_string_lossy().as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + // Separator so `/a/b` + `/c` doesn't collide with `/a` + `/b/c`. + h ^= 0xff; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// `mark_project_configured` — inputs `{name: }`; +/// output is a JSON null. Writes the sidecar manifest recording +/// the current `/ip/**/*.htcl` + `vw.toml` fingerprint next +/// to the on-disk Vivado project's `.xpr` at +/// `/target/vw-project//.xpr.manifest`. Called +/// by `vw::configure_ip` after `ip::configure` and `save_project` +/// both succeed. +/// +/// The `name` arg is what Vivado's `[current_project]` returns — +/// canonically the workspace name that seeded [`AutoProject`]. +/// We take it as an arg rather than deriving it here so htcl +/// stays authoritative about the project name (and there's no +/// silent divergence if the workspace ever ships multiple named +/// projects). The freshness *check* runs Rust-side once per +/// session before Vivado spawns (see `vw_lib::project_needs_wipe` +/// consumed from `vw-cli` / `vw-repl`), so there is intentionally +/// no matching `project_needs_update` RPC. +fn mark_project_configured( + workspace_root: Option<&std::path::Path>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let obj = args.as_object().ok_or_else(|| { + "mark_project_configured: args must be an object with a `name` \ + string field" + .to_string() + })?; + let name = obj.get("name").and_then(Value::as_str).ok_or_else(|| { + "mark_project_configured: missing string `name` field".to_string() + })?; + let project_dir = vw_lib::vw_project_dir(&ws); + vw_lib::write_project_manifest(&ws, project_dir.as_std_path(), name) + .map_err(|e| format!("writing project manifest: {e}"))?; + Ok(Value::Null) +} + +/// `place_needs_update` — inputs +/// `{checkpoint: , synth_checkpoint: }`; +/// output is a JSON bool. Compares the place checkpoint's +/// sidecar manifest against the current fingerprint of every +/// place-scoped XDC under `/constraints/place/**` PLUS the +/// synth DCP file (which proxies for "everything synth +/// depended on"). `true` when the checkpoint / manifest is +/// missing or the fingerprints disagree. +fn place_needs_update( + workspace_root: Option<&std::path::Path>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let checkpoint = extract_checkpoint_arg(&args, "place_needs_update")?; + let synth_checkpoint = + extract_synth_checkpoint_arg(&args, "place_needs_update")?; + let needs = vw_lib::place_needs_update( + &ws, + std::path::Path::new(&checkpoint), + std::path::Path::new(&synth_checkpoint), + ) + .map_err(|e| format!("checking place checkpoint freshness: {e}"))?; + Ok(Value::Bool(needs)) +} + +/// `place_mark_checkpoint` — inputs +/// `{checkpoint: , synth_checkpoint: }`; +/// output is a JSON null. Writes the sidecar manifest recording +/// the current place-scope fingerprint. Called by `vw::place` +/// after `vivado_cmd::write_checkpoint`. +fn place_mark_checkpoint( + workspace_root: Option<&std::path::Path>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let checkpoint = extract_checkpoint_arg(&args, "place_mark_checkpoint")?; + let synth_checkpoint = + extract_synth_checkpoint_arg(&args, "place_mark_checkpoint")?; + vw_lib::write_place_checkpoint_manifest( + &ws, + std::path::Path::new(&checkpoint), + std::path::Path::new(&synth_checkpoint), + ) + .map_err(|e| format!("writing place checkpoint manifest: {e}"))?; + Ok(Value::Null) +} + +/// Sibling of [`extract_checkpoint_arg`] for the `synth_checkpoint` +/// slot on place-stage RPC methods. Kept as a distinct helper so +/// the error message names the exact missing key. +fn extract_synth_checkpoint_arg( + args: &Value, + method: &str, +) -> Result { + let obj = args.as_object().ok_or_else(|| { + format!( + "{method}: args must be an object with a `synth_checkpoint` \ + string field" + ) + })?; + obj.get("synth_checkpoint") + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| format!("{method}: missing string `synth_checkpoint`")) +} + +/// Silent [`vw_htcl::loader::LoadObserver`] — the RPC path has no +/// progress bar or CLI channel to talk to. Every callback stays +/// at the default no-op impl. +struct NoopLoadObserver; +impl vw_htcl::loader::LoadObserver for NoopLoadObserver {} + +/// Mirror of `vw-cli`'s `overload_specialization_mangle` and +/// `vw-repl/src/lower.rs`'s equivalent. If `cmd` is a top-level +/// `proc` whose name is an overload public name AND whose first +/// arg annotation is a qualified enum variant, return the +/// mangled internal name to emit it under. The dispatcher +/// (produced by `emit_dispatcher`) routes calls to these mangled +/// names by argument type. Skipping this in `compile_htcl_module` +/// caused both overloads of a proc to collapse onto the same +/// unmangled name — the second one silently shadowed the first +/// via Tcl proc redefinition, and the dispatcher's runtime +/// switch never found either specialization. +fn overload_specialization_mangle( + cmd: &vw_htcl::Command, + overloads: &vw_htcl::OverloadTable, +) -> Option { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + return None; + }; + let name = proc.name.as_deref()?; + if !overloads.contains_key(name) { + return None; + } + let sig = proc.signature.as_ref()?; + let first = sig.args.first()?; + let vw_htcl::TypeExpr::Qualified { variant, .. } = + first.type_annotation.as_ref()? + else { + return None; + }; + Some(vw_htcl::mangle_specialization(name, variant)) +} + +/// Shared arg extractor for the checkpoint-scoped RPC methods. +/// Every one takes `{checkpoint: }` and errors +/// identically on missing/wrong-typed input — factoring keeps +/// the message shape consistent across +/// `synth_*` / `ip_*` handlers. +fn extract_checkpoint_arg( + args: &Value, + method: &str, +) -> Result { + let obj = args.as_object().ok_or_else(|| { + format!( + "{method}: args must be an object with a `checkpoint` \ + string field" + ) + })?; + obj.get("checkpoint") + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| format!("{method}: missing string `checkpoint`")) +} + +/// `synth_mark_checkpoint` — inputs `{checkpoint: }`; +/// output is a JSON null. Writes the sidecar manifest recording +/// the tracked source set's current fingerprint next to the +/// checkpoint. Called by `vw::synth` immediately after +/// `vivado_cmd::write_checkpoint` succeeds so the next invocation +/// can compare fingerprints and skip resynthesis when the sources +/// are unchanged. +fn synth_mark_checkpoint( + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let obj = args.as_object().ok_or_else(|| { + "synth_mark_checkpoint: args must be an object with a \ + `checkpoint` string field" + .to_string() + })?; + let checkpoint = + obj.get("checkpoint") + .and_then(Value::as_str) + .ok_or_else(|| { + "synth_mark_checkpoint: missing string `checkpoint`".to_string() + })?; + vw_lib::write_synth_checkpoint_manifest( + &ws, + std::path::Path::new(checkpoint), + active_variant, + ) + .map_err(|e| format!("writing checkpoint manifest: {e}"))?; + Ok(Value::Null) +} + +/// `synth_needs_update` — inputs `{checkpoint: }`; output +/// is a JSON bool. Delegates to [`vw_lib::synth_needs_update`], +/// which stats the checkpoint against the tracked source set +/// (design VHDL, IP wrappers, synth XDC, workspace htcl, +/// vw.toml, vw.lock). Missing checkpoint → `true`; any source +/// strictly newer than the checkpoint → `true`; otherwise `false`. +/// +/// The active variant is threaded through so a variant-specific +/// design surface (which vw::synth already respects via +/// `vhdl_design_sources`) is used for the mtime scan too — +/// otherwise a variant-owned file change might invalidate a +/// checkpoint that doesn't actually include it. +fn synth_needs_update( + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let obj = args.as_object().ok_or_else(|| { + "synth_needs_update: args must be an object with a `checkpoint` \ + string field" + .to_string() + })?; + let checkpoint = + obj.get("checkpoint") + .and_then(Value::as_str) + .ok_or_else(|| { + "synth_needs_update: missing string `checkpoint`".to_string() + })?; + let needs = vw_lib::synth_needs_update( + &ws, + std::path::Path::new(checkpoint), + active_variant, + ) + .map_err(|e| format!("checking checkpoint freshness: {e}"))?; + Ok(Value::Bool(needs)) +} + +/// `vhdl_dependency_sources` — return every transitive-dep VHDL +/// file, grouped by target library, as a JSON object of the shape +/// `{"library_name": ["/abs/path/a.vhd", "/abs/path/b.vhd"], …}`. +/// Grouped-by-library because the primary consumer is a Vivado +/// `read_vhdl -library $files` loop — the tuple-per-file +/// shape would force the caller to bucket, which we can do here. +async fn vhdl_dependency_sources( + workspace_root: Option<&std::path::Path>, + include_test: bool, + exclude_sim_only: bool, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + // Auto-fetch missing git deps. `workspace_has_unlocked_git_deps` + // is a cheap disk-only check — no network — so the common + // "already up-to-date" path stays cheap. When it returns + // true we invoke the same fetch machinery `vw update` uses; + // downstream enumeration then sees a fully-populated + // `vw.lock` and `~/.vw/deps` layout. + let unlocked = vw_lib::workspace_has_unlocked_git_deps(&ws, include_test) + .map_err(|e| format!("checking lockfile: {e}"))?; + if unlocked { + tracing::info!( + workspace = %ws, + "auto-updating workspace: git deps missing from vw.lock", + ); + // Look up netrc credentials the same way `vw update` does + // so a workspace with private git deps (e.g. an + // organization's internal GitHub repo, gitea, …) doesn't + // 401 when auto-update kicks in. `None` is fine when + // every git URL is public — libgit2 falls back to + // unauthenticated clone. + let creds = + vw_lib::get_access_credentials_for_workspace(&ws, include_test); + vw_lib::update_workspace_with_token(&ws, creds) + .await + .map_err(|e| format!("auto-updating workspace: {e}"))?; + } + let sources = vw_lib::vhdl_dependency_sources_ext( + &ws, + include_test, + exclude_sim_only, + ) + .map_err(|e| format!("enumerating VHDL dep sources: {e}"))?; + let mut by_library: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for src in sources { + by_library + .entry(src.library) + .or_default() + .push(Value::String(src.path.to_string_lossy().into_owned())); + } + let obj: serde_json::Map = by_library + .into_iter() + .map(|(lib, files)| (lib, Value::Array(files))) + .collect(); + Ok(Value::Object(obj)) +} + +/// `vhdl_design_sources` — return every VHDL file under +/// `/hdl/` as a JSON array of absolute-path strings. +/// Empty array when the workspace has no `hdl/` dir yet. +fn vhdl_design_sources( + workspace_root: Option<&std::path::Path>, + variant: Option, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + // When no variant was passed but the workspace declares + // variants, fall back to the workspace default so the flow + // "just works" from `design.htcl` without an explicit + // selector. The design-sources filter uses the resolved + // name to keep only shared + active-variant files. + let resolved = match variant { + Some(name) => Some(name), + None => workspace_default_variant_name(&ws), + }; + let paths = + vw_lib::vhdl_design_sources_for_variant(&ws, resolved.as_deref()) + .map_err(|e| format!("enumerating VHDL design sources: {e}"))?; + Ok(paths_to_json_array(paths)) +} + +/// Look up the workspace's default variant name. Returns `None` +/// when the workspace has no variants OR the variants block is +/// malformed (no default flag on a multi-entry list). Errors +/// are swallowed here — the caller has already produced a +/// diagnostic through the check machinery; we don't want the +/// RPC path to surface the same problem twice. +fn workspace_default_variant_name(ws: &camino::Utf8Path) -> Option { + let cfg = vw_lib::load_workspace_config(ws).ok()?; + cfg.workspace + .default_variant() + .ok() + .flatten() + .map(|v| v.name.clone()) +} + +/// `project_name` — return the `[workspace] name` field of the +/// entry workspace's `vw.toml`, as a JSON string. Errors when no +/// workspace can be discovered; the shim propagates that as a +/// normal Tcl error so htcl callers can branch on it. +/// +/// Used by `vw::synth` and other library procs that want to tag +/// log messages with the project name (`log::info -id [vw::project_name] +/// -msg "…"`) without hard-coding it in every entry file. +fn project_name_value( + workspace_root: Option<&std::path::Path>, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let cfg = vw_lib::load_workspace_config(&ws) + .map_err(|e| format!("failed to load workspace config at {ws}: {e}"))?; + Ok(Value::String(cfg.workspace.name.clone())) +} + +/// `active_variant` — return the name of the variant driving this +/// Vivado session, as a JSON string. Session-scoped precedence: +/// the CLI's `--variant ` selector wins; when unset, the +/// workspace default is used; when the workspace declares no +/// variants at all, the empty string is returned so htcl callers +/// can branch on `[vw::active_variant]` without try/catch. +fn active_variant_value( + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, +) -> Value { + if let Some(name) = active_variant { + return Value::String(name.to_string()); + } + let ws = workspace_root + .and_then(|p| camino::Utf8PathBuf::from_path_buf(p.to_path_buf()).ok()); + let name = ws + .as_deref() + .and_then(workspace_default_variant_name) + .unwrap_or_default(); + Value::String(name) +} + +/// `vhdl_ip_sources` — return every generated IP wrapper under +/// `/target/ip/**/*.vhd` as a JSON array of +/// absolute-path strings. Empty array when nothing has been +/// wrapped yet. +fn vhdl_ip_sources( + workspace_root: Option<&std::path::Path>, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let paths = vw_lib::vhdl_ip_sources(&ws) + .map_err(|e| format!("enumerating VHDL IP sources: {e}"))?; + Ok(paths_to_json_array(paths)) +} + +/// `design_constraints` — return every constraint file under +/// `/constraints/**/*.{xdc,sdc}` as a JSON array of +/// absolute-path strings. Empty array when the workspace has no +/// `constraints/` dir. +fn design_constraints( + workspace_root: Option<&std::path::Path>, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let paths = vw_lib::design_constraints(&ws) + .map_err(|e| format!("enumerating constraint files: {e}"))?; + Ok(paths_to_json_array(paths)) +} + +/// Which phase-scoped constraints dir to enumerate. Mirrors the +/// phase-specific accessors in `vw_lib`; kept as a small internal +/// enum so the dispatch match up top can name each variant without +/// duplicating the workspace-root plumbing three times. +enum ConstraintPhase { + Synth, + Place, + Route, +} + +fn design_phase_constraints( + workspace_root: Option<&std::path::Path>, + phase: ConstraintPhase, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let paths = match phase { + ConstraintPhase::Synth => vw_lib::design_synth_constraints(&ws), + ConstraintPhase::Place => vw_lib::design_place_constraints(&ws), + ConstraintPhase::Route => vw_lib::design_route_constraints(&ws), + } + .map_err(|e| format!("enumerating phase-scoped constraint files: {e}"))?; + Ok(paths_to_json_array(paths)) +} + +/// Pull the `variant` string out of an RPC args object. Missing / +/// null / non-string values return `None` — the handler then +/// falls back to the workspace's default variant. +fn extract_variant(args: &Value) -> Option { + args.as_object() + .and_then(|o| o.get("variant")) + .and_then(Value::as_str) + .map(|s| s.to_string()) +} + +/// Pull the `exclude_sim_only` boolean out of an RPC args object. +/// Missing / null / non-boolean values default to `false` so the +/// legacy call shape (`vw::vhdl_dependency_sources` with no args) +/// stays byte-for-byte compatible. +fn extract_exclude_sim_only(args: &Value) -> bool { + args.as_object() + .and_then(|o| o.get("exclude_sim_only")) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +fn paths_to_json_array(paths: Vec) -> Value { + Value::Array( + paths + .into_iter() + .map(|p| Value::String(p.to_string_lossy().into_owned())) + .collect(), + ) +} + +/// Small helper: workspace-root paths served through the RPC are +/// std `Path`, but `vw_lib` takes `Utf8Path`. Do the conversion in +/// one place and normalize the error to the RPC message shape. +fn workspace_root_or_error( + workspace_root: Option<&std::path::Path>, +) -> Result { + let raw = workspace_root.ok_or_else(|| { + "no workspace root: entry file has no `vw.toml` in its parent chain" + .to_string() + })?; + camino::Utf8PathBuf::from_path_buf(raw.to_path_buf()).map_err(|p| { + format!("workspace root is not valid UTF-8: {}", p.display()) + }) +} + +/// `diff_files` — inputs `{actual: , expected: }`; +/// output is a JSON string carrying the unified diff, or an empty +/// string when the files are byte-equal. Paths are used verbatim +/// (the htcl side already resolves them relative to workspace +/// root before dispatching). +fn diff_files(args: Value) -> Result { + let obj = args.as_object().ok_or_else(|| { + "diff_files: args must be an object with `actual` and \ + `expected` string fields" + .to_string() + })?; + let actual = obj + .get("actual") + .and_then(Value::as_str) + .ok_or_else(|| "diff_files: missing string `actual`".to_string())?; + let expected = obj + .get("expected") + .and_then(Value::as_str) + .ok_or_else(|| "diff_files: missing string `expected`".to_string())?; + let a_bytes = std::fs::read(actual) + .map_err(|e| format!("reading actual `{actual}`: {e}"))?; + let b_bytes = std::fs::read(expected) + .map_err(|e| format!("reading expected `{expected}`: {e}"))?; + if a_bytes == b_bytes { + return Ok(Value::String(String::new())); + } + // Prefer text mode when both files are valid UTF-8 — the + // typical case for VHDL/htcl/SDC/XDC. When either side is + // binary, fall through to a short "binary files differ" + // marker so we don't spew hex. + let (Ok(a_text), Ok(b_text)) = + (std::str::from_utf8(&a_bytes), std::str::from_utf8(&b_bytes)) + else { + return Ok(Value::String(format!( + "binary files differ ({} bytes actual, {} bytes expected)", + a_bytes.len(), + b_bytes.len(), + ))); + }; + let diff = render_unified_diff(a_text, b_text, expected, actual); + Ok(Value::String(diff)) +} + +/// Render a colored unified diff between `expected` and `actual`. +/// The header lines put `expected` on `-` (removed) and `actual` +/// on `+` (added), which matches the "you wanted X but got Y" +/// narrative that assertion messages want. +fn render_unified_diff( + actual: &str, + expected: &str, + expected_path: &str, + actual_path: &str, +) -> String { + use colored::Colorize; + use similar::{ChangeTag, TextDiff}; + let diff = TextDiff::from_lines(expected, actual); + let mut out = String::new(); + out.push_str(&format!("--- {}\n", expected_path).red().to_string()); + out.push_str(&format!("+++ {}\n", actual_path).green().to_string()); + for group in diff.grouped_ops(3) { + // Each group is a run of consecutive ops sharing context — + // similar's own recommended unit for rendering per-hunk + // headers. + for op in &group { + for change in diff.iter_inline_changes(op) { + let sign = match change.tag() { + ChangeTag::Delete => "-", + ChangeTag::Insert => "+", + ChangeTag::Equal => " ", + }; + let mut line = String::new(); + line.push_str(sign); + for (emphasized, value) in change.iter_strings_lossy() { + if emphasized { + // Inline-changed subrun — bold within the + // colored line to draw the eye to the exact + // byte-run that differs. + match change.tag() { + ChangeTag::Delete => { + line.push_str( + &value.on_red().white().bold().to_string(), + ); + } + ChangeTag::Insert => { + line.push_str( + &value + .on_green() + .white() + .bold() + .to_string(), + ); + } + ChangeTag::Equal => line.push_str(&value), + } + } else { + line.push_str(&value); + } + } + // `iter_inline_changes` already emits the trailing + // newline for line-based diffs; only add one if + // it's missing so the last line of a hunk renders + // cleanly. + if !line.ends_with('\n') { + line.push('\n'); + } + let colored_line = match change.tag() { + ChangeTag::Delete => line.red().to_string(), + ChangeTag::Insert => line.green().to_string(), + ChangeTag::Equal => line.dimmed().to_string(), + }; + out.push_str(&colored_line); + } + } + } + out +} diff --git a/vw-vivado/src/lib.rs b/vw-vivado/src/lib.rs new file mode 100644 index 0000000..1c272c8 --- /dev/null +++ b/vw-vivado/src/lib.rs @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Vivado [`EdaBackend`](vw_eda::EdaBackend) implementation. +//! +//! Spawns `vivado -mode tcl` as a long-lived worker, sources the +//! embedded shim TCL file at startup, and exchanges newline-delimited +//! JSON with it over stdio. Resolution order for the `vivado` +//! executable is: `VW_VIVADO` env var, then `PATH` lookup. v0 supports +//! the `eval` op only; structured ops land in phase 4. + +mod handlers; +mod raw_log; +mod rpc; +pub mod stream; +mod worker; + +pub use handlers::{ + make_handler, make_handler_full, make_handler_with_preloaded, + make_handler_with_variant, PreloadedPaths, SharedCriticalWarningCount, + SharedPreload, +}; +pub use raw_log::raw_log_path_for_workspace; +pub use rpc::{FnHandler, RpcHandler}; +pub use stream::{ + severity_of, stream_kind_for, Block, BlockAccumulator, LogLevel, Severity, +}; +pub use worker::{AutoProject, StreamKind, VivadoBackend, VivadoConfig}; diff --git a/vw-vivado/src/raw_log.rs b/vw-vivado/src/raw_log.rs new file mode 100644 index 0000000..5199b4d --- /dev/null +++ b/vw-vivado/src/raw_log.rs @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Helpers for the raw Vivado byte-log — the ground-truth file every +//! `vw run` / `vw repl` session writes so users have an unfiltered +//! record of what Vivado emitted, independent of the block classifier +//! and log-level filters. + +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Compute the raw-log path for a workspace: creates +/// `/target/logs/` if it doesn't exist and returns the +/// timestamped file path inside it. The filename encodes the +/// wall-clock time at the call site as `vivado--.log` +/// so two runs in the same session sort in start order and never +/// collide. +/// +/// Returns `Err` only when the parent directory couldn't be created — +/// the caller can decide whether to abort the session or continue +/// without a raw log (typically the latter, since the log is a +/// diagnostic aid, not a build artifact). +pub fn raw_log_path_for_workspace( + workspace: &Path, +) -> std::io::Result { + let dir = workspace.join("target").join("logs"); + std::fs::create_dir_all(&dir)?; + let name = format!("vivado-{}.log", timestamp_slug()); + Ok(dir.join(name)) +} + +/// Format the current wall-clock time as `YYYYMMDD-HHMMSS`. Manually +/// computed from `UNIX_EPOCH` rather than pulling in `chrono` — the +/// output is one filename component, and vw-vivado has no other +/// reason to depend on a date/time crate. +/// +/// Uses UTC because local-time conversion requires reading `/etc/ +/// localtime`, which fails on some minimal containers, and the log +/// filename doesn't need human-friendly local-time semantics — it +/// only has to be monotonic within a session. +fn timestamp_slug() -> String { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let (year, month, day, hour, minute, second) = split_unix_epoch(now); + format!("{year:04}{month:02}{day:02}-{hour:02}{minute:02}{second:02}") +} + +/// Break a Unix timestamp (seconds since 1970-01-01 UTC) into +/// `(year, month, day, hour, minute, second)`. Deliberately +/// self-contained — the raw-log module has one caller and doesn't +/// warrant a chrono dependency for what amounts to a filename slug. +/// +/// The algorithm is Zeller's congruence-style month/day extraction +/// via the "civil from days" formula popularized by Howard Hinnant +/// (public domain, `date` C++ library). Correct across the Gregorian +/// range vw ever runs in. +fn split_unix_epoch(secs: u64) -> (u64, u32, u32, u32, u32, u32) { + let second = (secs % 60) as u32; + let mins = secs / 60; + let minute = (mins % 60) as u32; + let hours = mins / 60; + let hour = (hours % 24) as u32; + let mut days = (hours / 24) as i64; + // Shift epoch so day 0 is 0000-03-01 (Hinnant's convention: months + // run March→February to make leap-day handling uniform). + days += 719468; + let era = if days >= 0 { days } else { days - 146096 } / 146097; + let doe = (days - era * 146097) as u64; // day of era + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // year of era + let year_shifted = (yoe as i64) + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // day of year + let mp = (5 * doy + 2) / 153; + let day = (doy - (153 * mp + 2) / 5 + 1) as u32; + let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + let year = (year_shifted + if month <= 2 { 1 } else { 0 }) as u64; + (year, month, day, hour, minute, second) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn timestamp_slug_shape() { + let s = timestamp_slug(); + // Format is YYYYMMDD-HHMMSS = 15 chars, digit + hyphen split. + assert_eq!(s.len(), 15, "unexpected slug shape: {s}"); + assert_eq!(s.as_bytes()[8], b'-'); + assert!(s[..8].bytes().all(|b| b.is_ascii_digit())); + assert!(s[9..].bytes().all(|b| b.is_ascii_digit())); + } + + #[test] + fn split_unix_epoch_at_zero_is_1970_01_01() { + assert_eq!(split_unix_epoch(0), (1970, 1, 1, 0, 0, 0)); + } + + #[test] + fn split_unix_epoch_known_timestamps() { + // 2026-06-25 12:34:56 UTC = 1782390896 (verified via + // `date -u -d '2026-06-25 12:34:56' +%s`). + assert_eq!(split_unix_epoch(1782390896), (2026, 6, 25, 12, 34, 56)); + // Leap-day boundary: 2024-02-29 00:00:00 UTC = 1709164800 + assert_eq!(split_unix_epoch(1709164800), (2024, 2, 29, 0, 0, 0)); + // Y2038 boundary + 1 second: 2038-01-19 03:14:08 UTC + assert_eq!(split_unix_epoch(2147483648), (2038, 1, 19, 3, 14, 8)); + } + + #[test] + fn raw_log_path_creates_target_logs_dir() { + let tmp = tempfile::tempdir().unwrap(); + let path = raw_log_path_for_workspace(tmp.path()).unwrap(); + assert!(path.starts_with(tmp.path().join("target").join("logs"))); + // Directory was created as a side effect. + assert!(tmp.path().join("target").join("logs").is_dir()); + // File name matches vivado-.log. + let name = path.file_name().unwrap().to_string_lossy().to_string(); + assert!(name.starts_with("vivado-"), "unexpected name: {name}"); + assert!(name.ends_with(".log"), "unexpected name: {name}"); + } +} diff --git a/vw-vivado/src/rpc.rs b/vw-vivado/src/rpc.rs new file mode 100644 index 0000000..7e010ff --- /dev/null +++ b/vw-vivado/src/rpc.rs @@ -0,0 +1,74 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Shim-initiated RPC — how htcl library procs reach into `vw` +//! (Rust) for values Vivado can't provide on its own. +//! +//! Direction is the mirror of the eval loop: `vw` normally sends +//! requests and the shim answers. Here, the shim sends an +//! [`RpcCall`](vw_eda::protocol::RpcCall) and `vw` answers. The +//! answer is written back as a plain [`Response`](vw_eda::protocol::Response) +//! keyed on the RPC call's id, so no new response type is needed. +//! +//! ## Shape of a handler +//! +//! An [`RpcHandler`] is a `Send + Sync` object with a single async +//! method that looks up a `method` string and computes a JSON +//! payload. Callers construct a concrete handler with whatever +//! Rust-side state they need to serve (workspace root, design +//! source list, dep graph, …) and hand it to +//! [`VivadoConfig::rpc_handler`](crate::VivadoConfig::rpc_handler). +//! +//! Handler methods are named as flat strings (`"workspace_root"`). +//! Namespaces on the htcl side (`vw::workspace_root`) are a +//! call-site convention, not a wire concern. + +use async_trait::async_trait; +use serde_json::Value; +use std::sync::Arc; + +/// Trait implemented by anything that can service RPC calls from +/// the shim. Registered via +/// [`VivadoConfig::rpc_handler`](crate::VivadoConfig::rpc_handler). +/// +/// Unknown methods should return `Err("unknown method: …".into())` — +/// the shim surfaces that verbatim to the caller. +#[async_trait] +pub trait RpcHandler: Send + Sync { + async fn call(&self, method: &str, args: Value) -> Result; +} + +/// Convenience impl so callers can wrap a closure without hand- +/// rolling a struct. `Arc` because the trait bound is +/// `Send + Sync + 'static` and we hold it in an `Arc`. +#[async_trait] +impl RpcHandler for FnHandler +where + F: Fn(String, Value) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + Send, +{ + async fn call(&self, method: &str, args: Value) -> Result { + (self.f)(method.to_string(), args).await + } +} + +/// Type-erased wrapper for closure-based [`RpcHandler`] +/// construction. Use [`FnHandler::new`] rather than constructing +/// directly. +pub struct FnHandler { + f: F, +} + +impl FnHandler { + #[allow(clippy::new_ret_no_self)] // returns a trait-object Arc, not Self + pub fn new(f: F) -> Arc + where + F: Fn(String, Value) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + + Send + + 'static, + { + Arc::new(FnHandler { f }) + } +} diff --git a/vw-vivado/src/stream.rs b/vw-vivado/src/stream.rs new file mode 100644 index 0000000..3d17aca --- /dev/null +++ b/vw-vivado/src/stream.rs @@ -0,0 +1,487 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Block-level segmentation over vw-vivado's classified stream. +//! +//! The [`crate::worker::PtyClassifier`] already tags every chunk it +//! emits with a [`crate::StreamKind`] — `Info`/`Warning`/`Error` for +//! Vivado's severity-prefixed messages (with attached `at : +//! in ::` continuation lines already merged into the same chunk), +//! and `Stdout` for everything else. This module groups consecutive +//! `Stdout` chunks into a single [`Block::None`] so downstream renderers +//! can collapse or dim them as one unit instead of drowning the console +//! in Vivado's tables, section headers, banners, and license chatter. +//! +//! ### The design contract +//! +//! - **Diagnostic block = exactly one classified chunk.** Multi-line +//! diagnostics (severity line + stack frames) are already merged +//! upstream, so we don't have to reconstruct the "obvious +//! continuation" boundary here. +//! - **NONE block = consecutive Stdout chunks.** A run of Stdout chunks +//! feeds into one open NONE block; a Diagnostic chunk flushes it. The +//! accumulator is stateful for exactly this reason. +//! - **Ordering is preserved.** [`BlockAccumulator::push`] returns +//! `Vec` (0-2 elements) rather than a single option, because a +//! Diagnostic arrival both flushes the pending NONE AND emits the +//! diagnostic — the caller sees them in the correct order without +//! having to track state itself. +//! +//! ### Log-level filtering +//! +//! [`LogLevel`] gates which blocks the renderer actually shows. Debug is +//! the escape hatch: it renders raw, no collapse. Info+ suppresses NONE +//! content but keeps it as a rendered "placeholder" (dim in `vw run`, +//! collapsible in the REPL) so users know noise was elided; users pick a +//! level and get exactly the diagnostics at that severity or higher. + +use crate::StreamKind; + +/// Severity ladder used for log-level filtering. Higher = more severe. +/// Vivado does not emit a `DEBUG:`-tagged message of its own; the +/// [`Severity::Debug`] slot exists so [`LogLevel::Debug`] has a proper +/// floor for filtering — at that level every block renders raw. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Severity { + /// Non-diagnostic noise: Vivado tables, section headers, banners, + /// license chatter, `VHDL Output written to …` lines. Rendered + /// dimmed (or collapsed in the REPL) at Info+ so a user can still + /// see something happened; rendered raw at Debug. + None, + /// `INFO: [tag-id]` messages — low-importance advisories. + Info, + /// `WARNING: [tag-id]` messages. + Warning, + /// `CRITICAL WARNING: [tag-id]` messages. Vivado ranks these + /// between WARNING and ERROR; semantically they mean "your run + /// may fail because of this" so downstream rendering treats them + /// like errors. + CriticalWarning, + /// `ERROR: [tag-id]` messages. + Error, +} + +/// The user-facing log level knob. Debug is the escape hatch — at +/// that level even non-diagnostic Stdout renders in full. Info is the +/// default: NONE blocks are elided (dim/collapsed), Info+ diagnostics +/// pass through. Higher levels suppress lower-severity diagnostics. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum LogLevel { + /// Show everything, including raw non-diagnostic output. The + /// classifier's block boundaries still apply for rendering + /// purposes, but nothing is elided. + Debug, + /// Show INFO+ diagnostics normally. Elide/collapse NONE blocks. + #[default] + Info, + /// Show WARNING+ diagnostics. Elide/collapse NONE and INFO. + Warning, + /// Show CRITICAL WARNING+ diagnostics. Elide/collapse NONE, INFO, + /// WARNING. + Critical, + /// Show only ERROR diagnostics. Elide/collapse everything else. + Error, +} + +impl LogLevel { + /// Should a diagnostic at the given severity render at this level? + /// Non-diagnostic ([`Severity::None`]) blocks always return `true` + /// at Debug (raw stream) and `false` otherwise — the renderer is + /// still expected to show a collapsed placeholder for them at + /// Info+, but that's a rendering choice, not a filter decision. + pub fn allows(self, sev: Severity) -> bool { + match self { + LogLevel::Debug => true, + LogLevel::Info => sev >= Severity::Info, + LogLevel::Warning => sev >= Severity::Warning, + LogLevel::Critical => sev >= Severity::CriticalWarning, + LogLevel::Error => sev >= Severity::Error, + } + } + + /// Should a NONE block render as a collapsed placeholder (rather + /// than being elided entirely OR rendered raw)? At Debug it renders + /// raw; at all higher levels it renders as a placeholder. + pub fn collapse_none(self) -> bool { + !matches!(self, LogLevel::Debug) + } + + /// Parse the CLI's `--log-level` argument. Accepts lowercase (the + /// canonical form) plus a few common aliases users type without + /// checking `--help`. + pub fn parse(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "debug" => Ok(LogLevel::Debug), + "info" => Ok(LogLevel::Info), + "warn" | "warning" => Ok(LogLevel::Warning), + "critical" | "crit" | "critical-warning" => Ok(LogLevel::Critical), + "error" | "err" => Ok(LogLevel::Error), + other => Err(format!( + "unknown log level `{other}`; expected one of \ + debug|info|warning|critical|error" + )), + } + } +} + +impl std::fmt::Display for LogLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + LogLevel::Debug => "debug", + LogLevel::Info => "info", + LogLevel::Warning => "warning", + LogLevel::Critical => "critical", + LogLevel::Error => "error", + }; + f.write_str(s) + } +} + +/// One classified block. Ownership: `lines` is the block's content +/// verbatim (trailing `\n` on each line stripped). The renderer +/// re-adds newlines when it writes out. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Block { + /// A run of consecutive non-diagnostic Stdout chunks. Render + /// dimmed (vw run) or collapsed (repl) at Info+; raw at Debug. + None { lines: Vec }, + /// A single classified diagnostic — one severity-tagged line + /// plus any attached `at : in ::` continuation + /// lines (already merged upstream by [`crate::worker::PtyClassifier`]). + Diagnostic { + severity: Severity, + lines: Vec, + }, +} + +impl Block { + pub fn severity(&self) -> Severity { + match self { + Block::None { .. } => Severity::None, + Block::Diagnostic { severity, .. } => *severity, + } + } + + pub fn lines(&self) -> &[String] { + match self { + Block::None { lines } | Block::Diagnostic { lines, .. } => lines, + } + } + + /// Number of source lines in the block, including any continuation + /// lines on a diagnostic. Renderers use this to size the collapsed + /// placeholder ("▶ preview (N lines)"). + pub fn line_count(&self) -> usize { + self.lines().len() + } +} + +/// Map a [`StreamKind`] to a [`Severity`]. `Stdout` → `None` — the +/// classifier's "we don't know what this is" bucket becomes our +/// non-diagnostic tier. +pub fn severity_of(kind: StreamKind) -> Severity { + match kind { + StreamKind::Stdout => Severity::None, + StreamKind::Info => Severity::Info, + StreamKind::Warning => Severity::Warning, + StreamKind::CriticalWarning => Severity::CriticalWarning, + StreamKind::Error => Severity::Error, + } +} + +/// Inverse of [`severity_of`]: pick the [`StreamKind`] a renderer +/// should use to color a block of the given severity. `Severity::None` +/// maps to `StreamKind::Stdout` — the default, unclassified stream. +pub fn stream_kind_for(severity: Severity) -> StreamKind { + match severity { + Severity::None => StreamKind::Stdout, + Severity::Info => StreamKind::Info, + Severity::Warning => StreamKind::Warning, + Severity::CriticalWarning => StreamKind::CriticalWarning, + Severity::Error => StreamKind::Error, + } +} + +/// Streaming block segmenter. Feed classified chunks in with +/// [`push`](Self::push); collect trailing content with +/// [`flush`](Self::flush) when the stream ends. +/// +/// State: the accumulator holds at most one open NONE block. A +/// classified (non-Stdout) chunk flushes it and immediately emits +/// itself as a Diagnostic block, in that order. +#[derive(Debug, Default)] +pub struct BlockAccumulator { + pending_none: Vec, +} + +impl BlockAccumulator { + pub fn new() -> Self { + Self::default() + } + + /// Consume one classified chunk. Returns 0-2 blocks in emission + /// order: a pending-NONE flush (if any) followed by the new + /// Diagnostic. Stdout chunks return `[]` — their content is + /// accumulated for the eventual NONE flush. + /// + /// The `chunk` is split on `\n`, trailing empty lines are dropped + /// so a diagnostic that ends `…in ::proc\n` doesn't leave a + /// spurious empty line in its block. An entirely-empty chunk is + /// a no-op regardless of kind. + pub fn push(&mut self, kind: StreamKind, chunk: &str) -> Vec { + let lines = split_chunk_lines(chunk); + if lines.is_empty() { + return Vec::new(); + } + let severity = severity_of(kind); + if severity == Severity::None { + self.pending_none.extend(lines); + return Vec::new(); + } + let mut out = Vec::with_capacity(2); + if !self.pending_none.is_empty() { + out.push(Block::None { + lines: std::mem::take(&mut self.pending_none), + }); + } + out.push(Block::Diagnostic { severity, lines }); + out + } + + /// Emit any pending NONE block. Call once at end-of-stream (or + /// when a rendering surface needs the trailing noise to settle). + /// Returns `[]` when nothing is pending. + pub fn flush(&mut self) -> Vec { + if self.pending_none.is_empty() { + return Vec::new(); + } + vec![Block::None { + lines: std::mem::take(&mut self.pending_none), + }] + } + + /// Test-only: peek at pending state without emitting. + #[cfg(test)] + fn pending_none(&self) -> &[String] { + &self.pending_none + } +} + +/// Split a chunk into lines the segmenter can accumulate. Drops the +/// single trailing empty element that `split('\n')` produces for a +/// `"a\nb\n"`-shaped chunk. Interior empty lines are preserved — some +/// diagnostic messages include a blank line between the tag and the +/// continuation frames, and renderers want that whitespace back. +fn split_chunk_lines(chunk: &str) -> Vec { + if chunk.is_empty() { + return Vec::new(); + } + let mut lines: Vec = + chunk.split('\n').map(str::to_string).collect(); + if lines.last().map(String::is_empty).unwrap_or(false) { + lines.pop(); + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stdout_chunks_accumulate_into_one_none_block() { + let mut acc = BlockAccumulator::new(); + assert_eq!(acc.push(StreamKind::Stdout, "banner line 1\n"), vec![]); + assert_eq!(acc.push(StreamKind::Stdout, "banner line 2\n"), vec![]); + assert_eq!( + acc.push(StreamKind::Stdout, "banner line 3\nline 4\n"), + vec![] + ); + // Nothing emitted yet — the None block is still open. + assert_eq!(acc.pending_none().len(), 4); + let flushed = acc.flush(); + assert_eq!( + flushed, + vec![Block::None { + lines: vec![ + "banner line 1".into(), + "banner line 2".into(), + "banner line 3".into(), + "line 4".into(), + ] + }] + ); + assert!(acc.pending_none().is_empty()); + } + + #[test] + fn diagnostic_chunk_flushes_pending_none_then_emits_itself() { + let mut acc = BlockAccumulator::new(); + acc.push(StreamKind::Stdout, "table row 1\n"); + acc.push(StreamKind::Stdout, "table row 2\n"); + let out = acc.push( + StreamKind::Warning, + "WARNING: [Synth 8-100] something\n at foo.tcl:12 in ::bar\n", + ); + assert_eq!( + out, + vec![ + Block::None { + lines: vec!["table row 1".into(), "table row 2".into()] + }, + Block::Diagnostic { + severity: Severity::Warning, + lines: vec![ + "WARNING: [Synth 8-100] something".into(), + " at foo.tcl:12 in ::bar".into(), + ] + } + ] + ); + // Pending is now clean. + assert!(acc.pending_none().is_empty()); + } + + #[test] + fn back_to_back_diagnostics_emit_no_none_between_them() { + let mut acc = BlockAccumulator::new(); + let out1 = acc.push(StreamKind::Info, "INFO: [X 1-1] first\n"); + assert_eq!( + out1, + vec![Block::Diagnostic { + severity: Severity::Info, + lines: vec!["INFO: [X 1-1] first".into()] + }] + ); + let out2 = acc.push(StreamKind::Info, "INFO: [X 1-2] second\n"); + assert_eq!( + out2, + vec![Block::Diagnostic { + severity: Severity::Info, + lines: vec!["INFO: [X 1-2] second".into()] + }] + ); + } + + #[test] + fn critical_warning_is_its_own_severity() { + let mut acc = BlockAccumulator::new(); + let out = acc.push( + StreamKind::CriticalWarning, + "CRITICAL WARNING: [Vivado 12-4739] set_clock_groups: …\n", + ); + assert_eq!( + out, + vec![Block::Diagnostic { + severity: Severity::CriticalWarning, + lines: vec![ + "CRITICAL WARNING: [Vivado 12-4739] set_clock_groups: …" + .into() + ] + }] + ); + } + + #[test] + fn empty_and_newline_only_chunks_are_ignored() { + let mut acc = BlockAccumulator::new(); + assert_eq!(acc.push(StreamKind::Stdout, ""), vec![]); + assert_eq!(acc.push(StreamKind::Stdout, "\n"), vec![]); + // The lone "\n" IS a blank line and should accumulate — Vivado + // sometimes emits blank separator lines that a renderer might + // want to preserve as vertical whitespace. Our + // `split_chunk_lines` correctly turns "\n" into one empty + // string line. + assert_eq!(acc.pending_none(), &["".to_string()]); + } + + #[test] + fn trailing_none_after_last_diagnostic_flushes_at_end() { + let mut acc = BlockAccumulator::new(); + acc.push(StreamKind::Error, "ERROR: [X 1-1] boom\n"); + acc.push(StreamKind::Stdout, "postmortem line 1\n"); + acc.push(StreamKind::Stdout, "postmortem line 2\n"); + let out = acc.flush(); + assert_eq!( + out, + vec![Block::None { + lines: vec![ + "postmortem line 1".into(), + "postmortem line 2".into() + ] + }] + ); + } + + #[test] + fn log_level_allows_diagnostic_severities() { + // Debug shows everything. + assert!(LogLevel::Debug.allows(Severity::None)); + assert!(LogLevel::Debug.allows(Severity::Error)); + + // Info hides only NONE. + assert!(!LogLevel::Info.allows(Severity::None)); + assert!(LogLevel::Info.allows(Severity::Info)); + assert!(LogLevel::Info.allows(Severity::Error)); + + // Warning hides NONE + INFO. + assert!(!LogLevel::Warning.allows(Severity::None)); + assert!(!LogLevel::Warning.allows(Severity::Info)); + assert!(LogLevel::Warning.allows(Severity::Warning)); + assert!(LogLevel::Warning.allows(Severity::CriticalWarning)); + + // Critical hides NONE + INFO + WARNING. + assert!(!LogLevel::Critical.allows(Severity::Warning)); + assert!(LogLevel::Critical.allows(Severity::CriticalWarning)); + assert!(LogLevel::Critical.allows(Severity::Error)); + + // Error hides everything but ERROR. + assert!(!LogLevel::Error.allows(Severity::CriticalWarning)); + assert!(LogLevel::Error.allows(Severity::Error)); + } + + #[test] + fn log_level_collapse_none_is_off_only_at_debug() { + assert!(!LogLevel::Debug.collapse_none()); + assert!(LogLevel::Info.collapse_none()); + assert!(LogLevel::Warning.collapse_none()); + assert!(LogLevel::Critical.collapse_none()); + assert!(LogLevel::Error.collapse_none()); + } + + #[test] + fn log_level_parse_accepts_common_forms() { + assert_eq!(LogLevel::parse("debug"), Ok(LogLevel::Debug)); + assert_eq!(LogLevel::parse("INFO"), Ok(LogLevel::Info)); + assert_eq!(LogLevel::parse("warn"), Ok(LogLevel::Warning)); + assert_eq!(LogLevel::parse("Warning"), Ok(LogLevel::Warning)); + assert_eq!(LogLevel::parse("critical"), Ok(LogLevel::Critical)); + assert_eq!(LogLevel::parse("crit"), Ok(LogLevel::Critical)); + assert_eq!(LogLevel::parse("critical-warning"), Ok(LogLevel::Critical)); + assert_eq!(LogLevel::parse("err"), Ok(LogLevel::Error)); + assert_eq!(LogLevel::parse("error"), Ok(LogLevel::Error)); + assert!(LogLevel::parse("").is_err()); + assert!(LogLevel::parse("verbose").is_err()); + } + + #[test] + fn diagnostic_carrying_multiline_message_stays_one_block() { + // The upstream PTY classifier merges the "at foo.tcl:X in ::proc" + // continuation lines into the same chunk it hands us. The + // segmenter should NOT split them across blocks. + let mut acc = BlockAccumulator::new(); + let chunk = "ERROR: [Synth 8-5826] no such design unit 'foo'\n \ + at foo.tcl:1 in ::bar\n at design.htcl:1\n"; + let out = acc.push(StreamKind::Error, chunk); + assert_eq!(out.len(), 1); + let Block::Diagnostic { severity, lines } = &out[0] else { + panic!("expected Diagnostic"); + }; + assert_eq!(*severity, Severity::Error); + assert_eq!(lines.len(), 3); + assert!(lines[0].contains("ERROR:")); + assert!(lines[1].contains("at foo.tcl")); + assert!(lines[2].contains("at design.htcl")); + } +} diff --git a/vw-vivado/src/worker.rs b/vw-vivado/src/worker.rs new file mode 100644 index 0000000..c7ea1b9 --- /dev/null +++ b/vw-vivado/src/worker.rs @@ -0,0 +1,2362 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Vivado worker process: spawn under a PTY, accept the shim's +//! loopback connection, drive the request/response loop. +//! +//! Vivado is spawned with its stdin/stdout/stderr attached to a +//! pseudo-terminal slave (via [`portable_pty`]). The PTY matters: +//! when stdout is a pipe, glibc puts it in full-block-buffering mode +//! and Vivado's banner / source-echo / info messages don't appear +//! until ~4 KB accumulates, which kills the `--verbose` UX. With a +//! PTY Vivado sees a TTY on stdout and switches to line buffering, +//! so output streams as it's produced. `portable_pty` works on Linux +//! and macOS via Unix PTYs, and on Windows via ConPTY. + +use std::fs::File; +use std::io::{BufWriter, Read, Write}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use portable_pty::{ + native_pty_system, Child as PtyChild, CommandBuilder, MasterPty, PtySize, +}; +use tempfile::TempDir; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::tcp::OwnedWriteHalf; +use tokio::net::TcpListener; +use tracing::{debug, warn}; +use vw_eda::{ + BackendError, EdaBackend, EvalOutput, Request, RequestOp, Response, + ResponseResult, WireMessage, +}; + +/// Embedded shim TCL. Written to a temp file at worker startup and +/// passed to `vivado -source`. +const SHIM_TCL: &str = include_str!("../shim/vivado-shim.tcl"); + +/// How long to wait for the shim to connect back to our loopback +/// listener. Vivado's startup takes most of this on a cold cache; on +/// a warm cache it's a few seconds. +const SHIM_CONNECT_TIMEOUT: Duration = Duration::from_secs(180); + +// --- TEMPORARY DIAGNOSTIC --------------------------------------------- +// +// Timing log to disentangle three arrival questions: +// +// 1. When did Vivado *write* the bytes to the PTY? (pump_read / +// pump_send events — captured on the pump thread.) +// 2. When did our select loop *pull* them from the pty_rx +// channel? (pty_rx_recv events.) +// 3. When did the protocol Response arrive relative to those? +// (response_arrival events.) +// +// Answering "is a late error message A) already-written-but-not- +// pumped, or B) not-yet-written-by-Vivado?" needs the delta between +// (1) and (3). Writing to `/tmp/vw-timing.log` so it survives the +// TUI's alternate-screen mode. Remove this block (and its callers) +// once we've settled the timing question. +fn vw_timing_log(event: &str, len: usize, preview: &str) { + use std::io::Write; + use std::sync::OnceLock; + static START: OnceLock = OnceLock::new(); + let start = START.get_or_init(std::time::Instant::now); + let elapsed_us = start.elapsed().as_micros(); + // Only write if the env var opts in — no forced disk I/O in + // normal runs. + static ENABLED: OnceLock = OnceLock::new(); + let enabled = + *ENABLED.get_or_init(|| std::env::var("VW_TIMING_LOG").is_ok()); + if !enabled { + return; + } + static FILE: OnceLock>> = + OnceLock::new(); + let file_slot = FILE.get_or_init(|| { + std::sync::Mutex::new( + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/tmp/vw-timing.log") + .ok(), + ) + }); + if let Ok(mut guard) = file_slot.lock() { + if let Some(f) = guard.as_mut() { + let clean: String = + preview.chars().filter(|c| *c != '\n').take(140).collect(); + let _ = writeln!(f, "{elapsed_us:>10} {event} len={len} {clean}"); + let _ = f.flush(); + } + } +} + +/// Tag attached to each chunk a [`StdoutSink`] receives, so the +/// caller can route it to the right UI lane. The shim's +/// `puts`-interception path always produces [`StreamKind::Stdout`] +/// — user TCL has no way to "label" a write. The PTY-line filter +/// classifies Vivado's standard message format +/// (`ERROR:`/`WARNING:`/`CRITICAL WARNING:`/`INFO:`) into the +/// corresponding kind. +/// +/// A consumer that doesn't care (e.g. `vw run` capturing for +/// stdout pass-through) can ignore the kind and treat every chunk +/// identically; the REPL uses it to colour error/warning lines. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StreamKind { + /// User TCL `puts` output, or any other chunk we don't have a + /// reason to label otherwise. Default. + Stdout, + /// Vivado `INFO:` line — usually low-importance chatter from + /// the message system. + Info, + /// Vivado `WARNING:` line. + Warning, + /// Vivado `CRITICAL WARNING:` line. Semantically means "your + /// run may fail because of this" — Vivado nests this severity + /// between WARNING and ERROR. Distinct from + /// [`StreamKind::Error`] so log-level filtering can treat them + /// separately (`--log-level=error` hides critical warnings; + /// `--log-level=critical` keeps them). + CriticalWarning, + /// Vivado `ERROR:` line. Distinct from the final + /// [`BackendError::Tcl`] returned by `eval` — these are emitted + /// *during* an eval and the final error often refers back to + /// them ("failed due to earlier errors"). + Error, +} + +/// Sink for streamed output during an eval. Called once per chunk +/// the worker observes — from the shim's `puts` interception (Tcl +/// user output) or from the PTY-line filter (Vivado's own message +/// system). The [`StreamKind`] tags the chunk so the caller can +/// route warnings and errors to a more attention-grabbing UI +/// surface than ordinary stdout. +pub type StdoutSink = Box; + +/// Spawn-time configuration for [`VivadoBackend`]. +#[derive(Clone, Default)] +pub struct VivadoConfig { + /// Override the `vivado` executable path. If `None`, resolution + /// order is `$VW_VIVADO`, then a `vivado` lookup on `$PATH`. + pub vivado: Option, + /// Working directory for the spawned process. If `None`, a + /// scratch tempdir is created so Vivado's incidental files don't + /// litter the user's cwd. + pub working_dir: Option, + /// When `true`, forward Vivado's PTY output (banner, source-echo, + /// info messages) as it's produced. When `false` (default), the + /// bytes are read and discarded so they don't pollute either of + /// vw's output streams. User TCL `puts` is always captured per- + /// eval via the shim and streamed in the protocol, independent + /// of this setting. + /// + /// Where the verbose output goes depends on [`verbose_log`](Self::verbose_log): + /// when set, lines stream into that file; when unset, they go + /// to vw's stderr. + pub verbose: bool, + /// When `true`, every Vivado-formatted message + /// (`INFO:`/`WARNING:`/`ERROR:`/`CRITICAL WARNING:`) gets the + /// Tcl call stack appended as `at : in ::proc` + /// continuation lines. When `false` (default), stacks are + /// attached only to WARNINGs and ERRORs — INFO messages render + /// as a single line. INFOs are noisy under heavy Vivado + /// activity (CIPS customization can emit dozens per call) and + /// the stack adds little signal compared to the message text; + /// power users diagnosing why a particular INFO fires can flip + /// this on with `--info-with-stack`. + pub info_with_stack: bool, + /// Optional path to a log file for verbose output. When set, + /// supersedes the default stderr destination — necessary for + /// the REPL, which owns the terminal in alternate-screen mode + /// and would corrupt the TUI rendering if anyone wrote raw + /// bytes to stderr mid-frame. The file is created (or + /// truncated) at spawn time and flushed per-line so it's safe + /// to `tail -f` from another terminal. + pub verbose_log: Option, + /// Optional path to a byte-perfect raw log of Vivado's PTY + /// output. When set, every byte the PTY pump reads is teed to + /// this file BEFORE any line-splitting, classification, or + /// filtering — the file is the ground truth of what Vivado + /// emitted this session. Callers are responsible for making + /// the parent directory exist; the convention is + /// `/target/logs/vivado-.log`, produced + /// by [`raw_log_path_for_workspace`]. Distinct from + /// [`verbose_log`], which only captures unclassified firehose + /// output routed through the classifier. + pub raw_log: Option, + /// Optional RPC handler for shim-initiated calls (`vw::…` + /// procs that reach back into Rust for values Vivado can't + /// provide). Dispatched from the proto-read task per inbound + /// [`vw_eda::protocol::RpcCall`]. When `None`, any RPC call + /// from the shim is answered with "no RPC handler + /// configured". + pub rpc_handler: Option>, + /// When set, spawn creates an in-memory Vivado project with + /// the given (project_name, target_part) as soon as Vivado is + /// ready to accept commands. Eliminates the "no open project" + /// bootstrap problem for `ip::check` / `get_ipdefs` — those + /// commands need a project (the IP catalog is per-part). + /// Composes with `@test(dedicated-eda part=…)` where the test + /// runner overrides the part for isolated tests. + pub auto_project: Option, +} + +/// Auto-created in-memory Vivado project. Materialized inside +/// [`VivadoBackend::spawn`] via +/// `create_project -in_memory -name -part ` as soon +/// as the shim connects. Removes the need for every htcl entry to +/// bootstrap a project by hand and unblocks IP-catalog queries at +/// module load time. +#[derive(Clone, Debug)] +pub struct AutoProject { + /// Project name shown to Vivado. Typically the workspace name. + pub name: String, + /// Full Vivado part specifier + /// (e.g. `xcvm3358-vsvh1747-2M-e-S`). + pub part: String, + /// `None` → `create_project -in_memory` (the legacy path, + /// kept for `vw test` isolation — persisted state would + /// cross-contaminate parallel `@test(dedicated-eda)` runs + /// and dirty the workspace with test scratch). + /// + /// `Some(dir)` → on-disk Vivado project at + /// `//.xpr`. If that `.xpr` already exists + /// the spawn bootstrap runs `open_project` (warm path); if + /// it doesn't exist we `create_project -name -dir + /// ` + immediately `set_property source_mgmt_mode + /// None [current_project]` so Vivado's auto-scan doesn't + /// swallow out-of-fileset files like `vw::make_wrapper`'s + /// `target/ip//wrapper.vhd` into `xil_defaultlib`. + /// Callers are responsible for staleness invalidation (see + /// `vw_lib::project_needs_wipe`) — the worker takes whatever + /// state is on disk as-is. + pub persist_dir: Option, +} + +/// Vivado [`EdaBackend`] implementation. +pub struct VivadoBackend { + child: Option>, + /// Master end of the PTY. Kept alive so the slave (Vivado) doesn't + /// receive EOF on its stdin. + _master: Box, + /// Protocol-socket line reader. Backed by a background task + /// (spawned in [`Self::new`]) that owns the `BufReader` and + /// forwards each newline-terminated frame here. + /// + /// Why the task exists: `BufReader::read_line` is + /// **cancellation-unsafe** per tokio's docs — if it's the + /// event in a `tokio::select!` and another branch fires + /// first, the future is dropped, any bytes already accumulated + /// into the internal buffer are lost, and the BufReader is left + /// in an unspecified state where subsequent reads return + /// incorrect data. Our eval loop races protocol reads against + /// `pty_rx.recv()` in a biased `select!`, so a big-enough + /// response (`props::get` returns ~25 KB Properties reprs) + /// crossing a PTY-line arrival used to corrupt the buffer and + /// produce "malformed message from shim" parse errors + /// truncated mid-JSON. Reading through a channel makes the + /// select's read side cancellation-safe. + proto_read: + tokio::sync::mpsc::UnboundedReceiver>, + _proto_read_task: Option>, + proto_write: OwnedWriteHalf, + /// Shim-initiated RPC handler (see [`crate::RpcHandler`]). + /// Consulted by [`Self::dispatch_rpc_call`] for every inbound + /// [`WireMessage::Rpc`]. `None` (the default) → RPCs return + /// "no RPC handler configured" error. + rpc_handler: Option>, + next_id: AtomicU64, + stdout_pump: Option>, + stdout_sink: Option, + /// Lines the PTY pump has read from Vivado's process stdout, in + /// arrival order. Drained during eval so Vivado's own message + /// system (ERROR/WARNING/CRITICAL WARNING/INFO) reaches the + /// stdout sink alongside user `puts` output — otherwise the + /// "earlier errors" Vivado refers to when failing a command are + /// invisible to the caller. + pty_rx: tokio::sync::mpsc::UnboundedReceiver, + /// Mirrors [`VivadoConfig::verbose`]. When true, PTY lines that + /// don't classify (banner, source-echo, idle chatter) are + /// surfaced — to [`verbose_log`](Self::verbose_log) if set, + /// otherwise to vw's stderr. Classified lines always route + /// through the message filter regardless of verbose. + verbose: bool, + /// Optional log file the verbose firehose streams into. The + /// REPL uses this so verbose output doesn't blow through its + /// TUI alternate screen by hitting stderr. + verbose_log: Option>, + /// Off by default. When true (set via the + /// `VW_TRACE_MESSAGE_SOURCES` env var at spawn time), emit a + /// gray `[vw-pty]` Info line before every classified PTY + /// chunk so the caller can see which path produced it. + /// Useful for diagnosing "where is this warning coming from?" + /// questions; noisy enough that it shouldn't be on by + /// default. + trace_message_sources: bool, + /// Brief-buffer classifier for multi-line PTY warnings. See + /// [`PtyClassifier`] for the merging semantics. + pty_classifier: PtyClassifier, + /// Stack of ready-to-use frame sets sent by the shim via + /// `__VW_CTX_*` PTY markers. Each entry is a set of frames + /// captured at a nesting level: outer entries are older, the + /// top is the innermost currently-executing wrap (a user proc's + /// body, or a wrapped `set_property` / `generate_netlist_ip` + /// call). When a Warning/Error chunk lands without its own + /// trace, the top entry's frames get appended as `\n at + /// ` lines — that's what lets the REPL show "this + /// IP_Flow warning came from configure_cips → + /// create_versal_cips → set_property" even though Vivado's C++ + /// never went through our Tcl stack capture. Nesting matters + /// because user procs call each other and each level's marker + /// wraps the next; a single active slot would clobber outer + /// context when the inner call returned mid-warning-emission. + pty_context_stack: Vec>, + /// Frames currently being assembled between + /// `__VW_CTX_BEGIN__` and `__VW_CTX_READY__`. Pushed onto + /// `pty_context_stack` atomically on READY so a partial + /// marker stream can't leak half-formed traces into emitted + /// warnings. Scalar (not per-nesting-level) because BEGIN and + /// READY are emitted synchronously in a single Tcl step — the + /// shim never emits a nested BEGIN before its outer READY. + building_pty_context: Vec, + _shim_dir: TempDir, + /// Latched when we detect the shim has been torn down (via + /// the `Vivado%` prompt landing in the PTY during an eval — + /// see the shim-died detector in `read_response_for`). + /// Once set, every subsequent `eval` returns a fast-fail + /// error instead of writing the request into an orphaned + /// socket that nobody will ever read. Cleared when the + /// shim is successfully respawned via + /// [`Self::respawn_shim`]. + shim_dead: bool, + /// Absolute path of the shim `.tcl` file inside `_shim_dir`. + /// Needed by [`Self::respawn_shim`] to write + /// `source \n` back to Vivado's interactive PTY prompt + /// after Ctrl-C's `interp cancel -unwind` blew the shim's + /// dispatch loop out from under us. Vivado keeps the same + /// `VW_PROTOCOL_ADDR` env var across shim `source`s, so the + /// respawn re-connects to the listener below. + shim_path: PathBuf, + /// Held-open listener so [`Self::respawn_shim`] can accept a + /// second connection on the same port. Original spawn takes + /// the first accept; if that shim dies to a cancel, the + /// re-sourced shim connects again here. + listener: Option, + _scratch_dir: Option, +} + +impl VivadoBackend { + /// Spawn a Vivado worker under a PTY, wait for the shim to + /// connect back on our loopback listener, and return once we're + /// ready to accept [`EdaBackend::eval`] calls. + pub async fn spawn(config: VivadoConfig) -> Result { + let vivado_path = resolve_vivado(&config)?; + + let shim_dir = tempfile::Builder::new() + .prefix("vw-vivado-shim-") + .tempdir() + .map_err(BackendError::Io)?; + let shim_path = shim_dir.path().join("vivado-shim.tcl"); + tokio::fs::write(&shim_path, SHIM_TCL) + .await + .map_err(BackendError::Io)?; + + let (cwd, scratch_dir) = match &config.working_dir { + Some(dir) => (dir.clone(), None), + None => { + let tmp = tempfile::Builder::new() + .prefix("vw-vivado-cwd-") + .tempdir() + .map_err(BackendError::Io)?; + (tmp.path().to_path_buf(), Some(tmp)) + } + }; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .map_err(BackendError::Io)?; + let local_addr = listener.local_addr().map_err(BackendError::Io)?; + debug!(?vivado_path, ?shim_path, ?cwd, %local_addr, "spawning vivado worker"); + + let pty_system = native_pty_system(); + let pair = pty_system + .openpty(PtySize { + rows: 24, + cols: 80, + pixel_width: 0, + pixel_height: 0, + }) + .map_err(|e| { + BackendError::Worker(format!("openpty failed: {e}")) + })?; + + // `-mode tcl` keeps Vivado alive as a long-running TCL + // interpreter; once the shim's socket loop takes over, Vivado + // never returns to its interactive prompt. + let mut cmd = CommandBuilder::new(&vivado_path); + cmd.arg("-mode"); + cmd.arg("tcl"); + cmd.arg("-nojournal"); + cmd.arg("-nolog"); + cmd.arg("-source"); + cmd.arg(&shim_path); + cmd.env("VW_PROTOCOL_ADDR", local_addr.to_string()); + // Read by the shim's `is_vivado_message` / stack-attach + // logic to decide whether INFO-level messages get stacks + // attached. WARNING / ERROR / CRITICAL always do. + cmd.env( + "VW_INFO_WITH_STACK", + if config.info_with_stack { "1" } else { "0" }, + ); + // Diagnostic knobs — propagate whatever the shell set so + // `VW_TRACE_STACK_CAPTURE=1 vw run …` (or the REPL + // equivalent) reaches the shim. portable-pty inherits by + // default on unix, but pass-through explicitly keeps the + // contract obvious. + if let Ok(v) = std::env::var("VW_TRACE_STACK_CAPTURE") { + cmd.env("VW_TRACE_STACK_CAPTURE", v); + } + cmd.cwd(&cwd); + + let child = pair.slave.spawn_command(cmd).map_err(|e| { + BackendError::Worker(format!( + "failed to spawn vivado at {}: {}", + vivado_path.display(), + e + )) + })?; + // Release our handle to the slave so the master sees EOF + // when the child exits. + drop(pair.slave); + + let reader = pair.master.try_clone_reader().map_err(|e| { + BackendError::Worker(format!("pty reader clone failed: {e}")) + })?; + // Open the raw byte-log if the caller asked for one. + // Parent-directory creation is the caller's job — see + // `raw_log_path_for_workspace`; here we just open (creating + // the file, truncating on repeat) and wrap for buffered + // per-chunk flush. Failure to open bubbles as a spawn + // error rather than silently disabling the log — the + // caller opted in, so a broken opt-in should be loud. + let raw_log: Option> = config + .raw_log + .as_ref() + .map(|p| File::create(p).map(BufWriter::new)) + .transpose() + .map_err(BackendError::Io)?; + let (pty_tx, pty_rx) = tokio::sync::mpsc::unbounded_channel::(); + let stdout_pump = spawn_stdout_pump(reader, pty_tx, raw_log); + + // Wait for the shim to connect. Borrow `&mut listener` so + // it stays alive for [`Self::respawn_shim`] to re-accept + // on the same port after a cancel-induced shim teardown. + let accept_result = + tokio::time::timeout(SHIM_CONNECT_TIMEOUT, listener.accept()).await; + let stream = match accept_result { + Ok(Ok((stream, _peer))) => stream, + Ok(Err(e)) => return Err(BackendError::Io(e)), + Err(_) => { + return Err(BackendError::Worker( + "timed out waiting for shim to connect".into(), + )); + } + }; + stream.set_nodelay(true).map_err(BackendError::Io)?; + debug!("shim connected"); + + let (read_half, write_half) = stream.into_split(); + // Move BufReader::read_line onto a dedicated task so its + // cancellation-unsafe nature can't corrupt state when the + // eval loop's `select!` fires a different branch mid-read. + // See the field-doc on `proto_read` for the failure mode. + let (proto_tx, proto_rx) = tokio::sync::mpsc::unbounded_channel(); + let mut proto_buf = BufReader::new(read_half); + let _proto_read_task = tokio::spawn(async move { + let mut line = String::new(); + loop { + line.clear(); + match proto_buf.read_line(&mut line).await { + Ok(0) => break, + Ok(_) => { + if proto_tx.send(Ok(line.clone())).is_err() { + break; + } + } + Err(e) => { + let _ = proto_tx.send(Err(e)); + break; + } + } + } + }); + let trace_message_sources = std::env::var("VW_TRACE_MESSAGE_SOURCES") + .map(|v| { + let v = v.trim(); + !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false") + }) + .unwrap_or(false); + + let verbose_log = config + .verbose_log + .as_ref() + .map(|p| File::create(p).map(BufWriter::new)) + .transpose() + .map_err(BackendError::Io)?; + + let mut backend = Self { + child: Some(child), + _master: pair.master, + proto_read: proto_rx, + _proto_read_task: Some(_proto_read_task), + proto_write: write_half, + rpc_handler: config.rpc_handler.clone(), + next_id: AtomicU64::new(1), + stdout_pump: Some(stdout_pump), + stdout_sink: None, + pty_rx, + verbose: config.verbose, + verbose_log, + trace_message_sources, + pty_classifier: PtyClassifier::new(PTY_CONTINUATION_WINDOW), + pty_context_stack: Vec::new(), + building_pty_context: Vec::new(), + _shim_dir: shim_dir, + _scratch_dir: scratch_dir, + shim_dead: false, + shim_path: shim_path.clone(), + listener: Some(listener), + }; + // Auto-create (or reopen) the Vivado project if the caller + // specified one. Runs BEFORE we return to the caller so the + // first user eval sees an existing project — `ip::check` / + // `get_ipdefs` / etc. work without special-casing. + // + // Three branches keyed on `persist_dir`: + // * `None` → legacy `create_project -in_memory`. Kept for + // `vw test` isolation. + // * `Some(dir)` with an existing `//.xpr` + // → `open_project` (warm path — Vivado restores the + // full BD/XCI/`ipshared` graph without us hand- + // serializing it). If Vivado rejects the file (version + // mismatch, truncated write, etc.) we log a warning, + // wipe the dir, and fall through to the fresh-create + // branch in the same session so users don't have to + // intervene manually. + // * `Some(dir)` otherwise → `create_project -name + // -dir `. Immediately follow with + // `set_property source_mgmt_mode None + // [current_project]`: default is `All`, which would + // auto-import every `.vhd` Vivado finds under the + // project dir into `xil_defaultlib` — including + // `vw::make_wrapper`'s out-of-fileset + // `target/ip//wrapper.vhd` files. That fights the + // "deliberately do NOT `-import`" model make_wrapper + // relies on. + // + // All branches then force `TARGET_LANGUAGE VHDL` — `vw` is + // VHDL-first, and the default Verilog would silently emit + // `_wrapper.v` files that downstream tools looking for + // `.vhd` fail on with an unhelpful "no wrapper found". + if let Some(ap) = &config.auto_project { + use vw_eda::EdaBackend as _; + let tcl = match &ap.persist_dir { + None => format!( + "create_project -in_memory -name {{{name}}} \ + -part {{{part}}}\n\ + set_property TARGET_LANGUAGE VHDL [current_project]", + name = ap.name, + part = ap.part, + ), + Some(dir) => { + // Vivado's `create_project -dir D -name N` + // places `N.xpr` + `N.srcs` + `N.cache` + + // `N.hw` + `N.ip_user_files` + `N.gen` + // DIRECTLY under D — flat, not nested in + // `D/N/`. Nesting is what we want (clean + // `remove_dir_all` on invalidation, doesn't + // fight sibling artifacts), so we pass `D/N` + // as the `-dir` argument and let Vivado + // scatter its per-project files inside that. + let per_name = dir.join(&ap.name); + let xpr = per_name.join(format!("{}.xpr", ap.name)); + if xpr.exists() { + // Warm path. On failure we `catch` and fall + // through to a fresh create in the same eval + // so the session stays usable even if the + // on-disk project got corrupted. + // + // Intentionally NO `set_property + // source_mgmt_mode None`: the earlier + // migration set it defensively to keep + // `vw::make_wrapper`'s workspace-side + // wrapper (`target/ip//wrapper.vhd`) + // from being auto-imported into + // `xil_defaultlib`, but that's already + // prevented by `-import false` in + // `vw::make_wrapper` itself. With `None`, + // Vivado also stops adding IP-generated + // RTL under `.gen/sources_1/ip//synth/` + // to the synth fileset, so `synth_design` + // fails to find `entity xil_defaultlib.` + // — regressing top-level IPs like + // `primary_clock`. + format!( + "if {{[catch {{open_project {{{xpr}}}}} err]}} {{\n \ + puts \"WARNING: open_project failed ({xpr}): \ + $err — recreating\"\n \ + file delete -force {{{per_name}}}\n \ + file mkdir {{{per_name}}}\n \ + create_project -name {{{name}}} -part {{{part}}} \ + -dir {{{per_name}}}\n \ + }}\n\ + set_property TARGET_LANGUAGE VHDL [current_project]", + xpr = xpr.display(), + per_name = per_name.display(), + name = ap.name, + part = ap.part, + ) + } else { + format!( + "file mkdir {{{per_name}}}\n\ + create_project -name {{{name}}} -part {{{part}}} \ + -dir {{{per_name}}}\n\ + set_property TARGET_LANGUAGE VHDL [current_project]", + per_name = per_name.display(), + name = ap.name, + part = ap.part, + ) + } + } + }; + backend.eval(&tcl).await.map_err(|e| { + BackendError::Worker(format!( + "auto-creating Vivado project (name={}, part={}, \ + persist_dir={:?}): {e}", + ap.name, ap.part, ap.persist_dir, + )) + })?; + } + Ok(backend) + } + + /// Install a sink that's called per streaming chunk as output + /// is produced during eval. With a sink set, chunks are NOT + /// also accumulated into [`EvalOutput::stdout`] — the sink owns + /// the data, and the caller is expected to display or persist + /// it directly. The [`StreamKind`] argument tags each chunk + /// (user `puts` vs. Vivado's WARNING/ERROR/INFO messages) so + /// the caller can route the chunk to the appropriate UI lane. + pub fn set_stdout_sink(&mut self, sink: F) + where + F: FnMut(StreamKind, &str) + Send + 'static, + { + self.stdout_sink = Some(Box::new(sink)); + } + + /// The Vivado child process's OS pid, if the child is still + /// alive. Callers use this to send SIGINT for eval + /// cancellation — see [`Self::interrupt`] for the mechanism + /// and design assumptions. + pub fn child_pid(&self) -> Option { + self.child.as_ref().and_then(|c| c.process_id()) + } + + /// Cancel the current in-flight Tcl eval WITHOUT killing the + /// Vivado session. Sends `SIGINT` to the child; Vivado's Tcl + /// runtime installs a `SIGINT` handler that traps the signal + /// into `interp cancel`, so the mechanism is: + /// + /// 1. Vivado receives `SIGINT` in the middle of executing the + /// current eval body (a `synth_ip`, a long `get_pins + /// -hierarchical`, whatever). + /// 2. Tcl catches the signal, sets the "cancel" flag on the + /// active interpreter, and the currently-executing script + /// returns with `TCL_ERROR` / message `"interrupted"`. + /// 3. Our shim wraps every user eval in `catch`, so the error + /// is caught, wrapped into a protocol response, and sent + /// back over the wire. The eval returns from the caller's + /// perspective as a normal [`vw_eda::BackendError::Tcl`] + /// with the interrupted message. + /// 4. Vivado's outer read loop is untouched — it goes right + /// back to reading the next request from the shim's + /// protocol socket. Project state, IP synth results, all + /// open designs survive. + /// + /// Same semantics as pressing Ctrl-C on an interactive + /// `vivado -mode tcl` console: the running command aborts, + /// the Tcl prompt returns. + /// + /// **Signals the process group, not the process.** Vivado on + /// disk is a bash wrapper (`~/Xilinx/…/bin/vivado`) that runs + /// `loader -exec vivado …` without `exec`, so bash sticks + /// around as the child we spawned, and the actual Vivado + /// runtime is a grandchild via loader. If we `kill(pid, + /// SIGINT)` we hit bash-running-a-script, which normally does + /// NOT forward SIGINT to its children — the signal dies with + /// bash's SIGINT-default handling and Vivado never sees it. + /// + /// portable-pty sets each spawned command up as its own + /// session leader with a fresh PGID = its own pid, and + /// descendants inherit that PGID unless they explicitly call + /// `setpgrp`. So `kill(-pid, SIGINT)` — i.e. signalling the + /// negative-of-pid, which POSIX interprets as "the process + /// group with this ID" — reaches bash, loader, and Vivado + /// all at once. Bash is a no-op recipient (script mode + /// SIGINT); loader relays; Vivado's Tcl runtime catches it + /// and does the `interp cancel`. + /// + /// **Design assumption**: SIGINT-under-PTY behaves the same + /// as SIGINT-on-console. Historically true for Vivado 2020+ + /// (we're on 2025.1). If a future release drops that + /// handling, callers see `interrupt()` return without + /// actually cancelling — the current eval keeps running. We + /// have no clean fallback in that world; the eval would have + /// to be killed via full backend restart. + /// + /// Returns the pid (== pgid) we signalled so callers can log + /// something meaningful ("interrupted eval on pid 12345"). + /// Returns `None` if the child is already gone — in which + /// case there's nothing to interrupt anyway. + /// + /// Unix-only. Windows has no analogue that reliably pumps + /// into the child's Tcl signal handler, and every vw + /// interactive user is on Unix; when Windows matters we'll + /// add a ConPTY-native cancellation path. + #[cfg(unix)] + pub fn interrupt(&self) -> Option { + let pid = self.child_pid()?; + // SAFETY: `libc::kill(-pid, SIGINT)` on a process group + // we spawned. Negative pid selects the pgid. The only + // failure modes documented for the syscall are EPERM + // (we don't own the group — impossible here, we're the + // parent) and ESRCH (the group is empty — handled as a + // silent no-op; race window with child exit is benign). + unsafe { + libc::kill(-(pid as libc::pid_t), libc::SIGINT); + } + Some(pid) + } + + /// Windows stub: no-op. See the unix variant's doc for the + /// eval-cancellation model. + #[cfg(not(unix))] + pub fn interrupt(&self) -> Option { + None + } + + fn alloc_id(&self) -> u64 { + self.next_id.fetch_add(1, Ordering::Relaxed) + } + + async fn write_request( + &mut self, + req: &Request, + ) -> Result<(), BackendError> { + let mut line = serde_json::to_string(req)?; + line.push('\n'); + self.proto_write.write_all(line.as_bytes()).await?; + self.proto_write.flush().await?; + Ok(()) + } + + /// Read messages until we get the response that matches + /// `expected_id`. Stream notifications for the same id are routed + /// to [`Self::stdout_sink`] if set, or accumulated into the + /// returned `String` if not. + /// + /// While waiting, the worker also drains the PTY channel — so + /// Vivado's own `send_msg_id` output (ERROR/WARNING/CRITICAL + /// WARNING/INFO lines printed to the process stdout, not through + /// the shim's `puts` interception) reaches the same sink and + /// gets attributed to the in-flight eval. Without this, the + /// "earlier errors" Vivado refers to when a command fails are + /// invisible to the caller. + async fn read_response_for( + &mut self, + expected_id: u64, + ) -> Result<(Response, String), BackendError> { + let mut accumulated = String::new(); + let mut line = String::new(); + loop { + line.clear(); + // Race the protocol socket against the PTY channel. The + // protocol path eventually terminates the loop (a + // Response arrives); PTY lines are forwarded as + // best-effort context until then. + tokio::select! { + biased; + pty = self.pty_rx.recv() => { + let Some(pty_line) = pty else { + // Pump exited: Vivado's PTY closed. + // Stop trying to drain it but keep waiting + // for a Response on the protocol socket — + // most teardown sequences send the + // shutdown ack before the PTY EOFs. + continue; + }; + vw_timing_log( + "pty_rx_recv", + pty_line.len(), + &pty_line, + ); + // Shim-died detector. When Vivado catches Ctrl-C + // during a long C++ command (`place_design`, + // `phys_opt_design`, ...), its signal handler + // does `interp cancel -unwind` — and -unwind + // rips through EVERYTHING, including the shim's + // `while {1}` dispatch loop AND the `source` + // that loaded it. Vivado then drops to its + // interactive `Vivado%` prompt with no shim + // running, so our protocol socket is orphaned: + // no response will ever come for this id, and + // `read_response_for` would hang forever. + // + // Vivado announces the shim-source unwind with a + // very specific `[Common 17-344] 'source' was + // cancelled` line — precise (per-command + // cancellation like `'opt_design' was cancelled` + // is a different scope; only `'source'` means + // the shim's own `source $shim_path` was + // unwound). Match on that substring rather + // than on the `Vivado% ` prompt itself — the + // prompt has no trailing newline, so the + // line-buffered PTY pump never delivers it + // to us. + if pty_line.contains("'source' was cancelled") + { + // Shim died. Try to re-establish it in + // place — Vivado's C++ state (project, + // IP catalog, loaded designs) survives + // an `interp cancel -unwind`, only the + // Tcl-level shim/user state is lost. + // Fast respawn keeps the Vivado process + // warm (avoids the ~minutes-long + // startup) at the cost of user Tcl + // bindings needing to be reloaded. + self.shim_dead = true; + let respawn = self.respawn_shim().await; + let recovery = match respawn { + Ok(()) => "shim was respawned; \ + Vivado process + project state \ + preserved, but Tcl-level proc \ + bindings and session vars are \ + gone — re-source your entry \ + htcl to restore them", + Err(_) => "shim respawn failed; \ + session is unusable — exit vw \ + and re-launch (`:exit` in the \ + REPL, then `vw repl` / `vw run` \ + again)", + }; + return Err(BackendError::Tcl { + message: format!( + "vivado shim was torn down by \ + Ctrl-C cancellation (Vivado's \ + `interp cancel -unwind` blew \ + through the shim's dispatch \ + loop). {recovery}." + ), + code: Some("VW_SHIM_DEAD".to_string()), + info: None, + stdout: std::mem::take(&mut accumulated), + }); + } + self.handle_pty_line_during_eval( + &pty_line, + &mut accumulated, + ); + continue; + } + read = self.proto_read.recv() => { + // `recv()` on `UnboundedReceiver` is + // cancellation-safe (per tokio's docs on + // mpsc::Receiver::recv), so a losing branch + // in this `select!` just drops a poll — no + // in-flight bytes to corrupt. + let Some(res) = read else { + return Err(BackendError::Worker( + "vivado shim closed protocol socket".into(), + )); + }; + let Ok(l) = res else { + return Err(BackendError::Io(res.unwrap_err())); + }; + line = l; + } + } + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let msg: WireMessage = + serde_json::from_str(trimmed).map_err(|e| { + BackendError::Worker(format!( + "malformed message from shim: {e}; payload={trimmed}" + )) + })?; + match msg { + WireMessage::Stream(s) if s.id == expected_id => { + if let Some(sink) = self.stdout_sink.as_mut() { + // Default: shim-stream chunks are user + // `puts` output (StreamKind::Stdout). But + // our send_msg_id override in the shim + // also emits via this path — those chunks + // start with a Vivado-standard severity + // prefix (`WARNING:`/`ERROR:`/etc.) which + // we re-use the PTY-line classifier to + // detect. + let kind = classify_chunk_for_sink(&s.data); + // Provenance marker (opt-in via + // VW_TRACE_MESSAGE_SOURCES), only on + // classified (non-Stdout) chunks — plain + // user output doesn't benefit from a + // "came via the shim" tag, but a warning + // does so the user can tell it from a + // PTY-routed one. + if self.trace_message_sources + && kind != StreamKind::Stdout + { + sink( + StreamKind::Info, + &format!( + "[vw-shim-stream] \ + classified-as={kind:?}\n" + ), + ); + } + sink(kind, &s.data); + } else { + // Same separator rule as `emit_pty_chunk` — + // if the shim's incoming chunk is a + // classified WARNING/ERROR and the tail of + // `accumulated` isn't newline-terminated, + // inject one so the failure-block renderer + // sees line-bounded messages instead of + // `…clk_in1ERROR:…` smashes. + let kind = classify_chunk_for_sink(&s.data); + if matches!( + kind, + StreamKind::Warning | StreamKind::Error + ) && !accumulated.is_empty() + && !accumulated.ends_with('\n') + { + accumulated.push('\n'); + } + accumulated.push_str(&s.data); + } + } + WireMessage::Stream(s) => { + warn!( + got = s.id, + expected = expected_id, + "stream id mismatch; discarding" + ); + } + WireMessage::Response(r) if r.id == expected_id => { + vw_timing_log( + "response_arrival", + trimmed.len(), + &format!( + "id={} kind={}", + r.id, + match &r.result { + ResponseResult::Ok { .. } => "Ok", + ResponseResult::Err { .. } => "Err", + } + ), + ); + // Force-flush any buffered PTY message — a + // classified line that arrived right before + // the Vivado response would otherwise linger + // in the classifier until the next eval's + // drain. Flushing here means the user always + // sees every classified message attributable + // to the eval before the eval's result. + if let Some((kind, text)) = self.pty_classifier.flush() { + self.emit_pty_chunk(kind, &text, &mut accumulated); + } + return Ok((r, accumulated)); + } + WireMessage::Response(r) => { + warn!( + got = r.id, + expected = expected_id, + "response id mismatch; discarding" + ); + } + WireMessage::Rpc(call) => { + // Shim-initiated RPC (a `vw::…` proc reaching + // back into Rust). Dispatch synchronously here + // — we're the only reader of `proto_read` + // during an eval, and we're the writer of + // `proto_write` too, so replying inline keeps + // ownership simple and avoids a second writer + // task. The htcl caller is blocked on + // `gets $sock` waiting for exactly this + // response; other in-flight RPCs on other + // evals aren't a concern (only one eval runs + // at a time on the shim). + self.dispatch_rpc_call(call).await?; + } + } + } + } + + /// Handle one inbound [`RpcCall`], write the response back to + /// the shim. + async fn dispatch_rpc_call( + &mut self, + call: vw_eda::protocol::RpcCall, + ) -> Result<(), BackendError> { + let result = match &self.rpc_handler { + Some(handler) => handler.call(&call.method, call.args).await, + None => Err(format!( + "no RPC handler configured; can't answer '{}'", + call.method + )), + }; + let resp = match result { + Ok(value) => Response::ok(call.id, value), + Err(msg) => Response::err( + call.id, + vw_eda::protocol::ErrorPayload { + message: msg, + code: None, + info: None, + }, + ), + }; + // Reuse the same write path Rust's own Requests take. + // Writing to `proto_write` while `proto_read` is being + // consumed here (we own both, we're inside `read_ + // response_for`) is safe: TCP is full-duplex and no other + // task holds `proto_write` while `eval` is in progress. + let mut line = serde_json::to_string(&resp)?; + line.push('\n'); + self.proto_write.write_all(line.as_bytes()).await?; + self.proto_write.flush().await?; + Ok(()) + } + + /// Filter a PTY line received during an in-flight eval. Lines + /// matching Vivado's standard message format are forwarded to + /// the stdout sink (or accumulated when there's no sink); + /// everything else (banner, source-echo, idle chatter) is + /// dropped — or stderr-mirrored when `verbose` is set, so a + /// user diagnosing a flaky eval can still get the full firehose. + fn handle_pty_line_during_eval( + &mut self, + line: &str, + accumulated: &mut String, + ) { + if self.consume_ctx_marker(line) { + return; + } + if is_vivado_known_noise(line) { + // Verbose still logs it so the transcript is intact + // when someone's debugging what Vivado emitted. + if self.verbose { + self.write_verbose_line(line); + } + return; + } + let outcome = + self.pty_classifier.handle(line, std::time::Instant::now()); + for (kind, text) in outcome.chunks { + self.emit_pty_chunk(kind, &text, accumulated); + } + if !outcome.absorbed { + // Unclassified PTY output DURING an eval — the tail + // of a multi-line Vivado error (`[BD 41-758] … valid + // clock source:` followed by unindented `/pin` list), + // or arbitrary chatter the user's command triggered. + // Route as Stdout so the user can SEE it in the + // scrollback without it inheriting the previous + // warning/error's styling. Verbose-log too, for + // parity with the previous behavior. + self.emit_pty_chunk(StreamKind::Stdout, line, accumulated); + if self.verbose { + self.write_verbose_line(line); + } + } + } + + /// Recognize one of the `__VW_CTX_*` lines the shim emits + /// around wrapped commands and user proc bodies. Returns + /// `true` if the line was a marker (and should be swallowed); + /// `false` if it's a normal PTY line for the classifier. + /// + /// The marker protocol is stack-based: BEGIN opens a new + /// entry, FRAME lines accumulate into it, READY seals it onto + /// `pty_context_stack`, END pops the top entry. Nested proc + /// calls produce nested BEGIN/END pairs, and the top of the + /// stack — the innermost wrap in flight — is what tags any + /// traceless warning/error that arrives while it's active. + fn consume_ctx_marker(&mut self, line: &str) -> bool { + let stripped = line.trim_end_matches(['\r', '\n']); + match stripped { + "__VW_CTX_BEGIN__" => { + self.building_pty_context.clear(); + true + } + "__VW_CTX_READY__" => { + let frames = std::mem::take(&mut self.building_pty_context); + self.pty_context_stack.push(frames); + true + } + "__VW_CTX_END__" => { + self.pty_context_stack.pop(); + // Defensively clear building too — a stray FRAME + // that arrived after END shouldn't leak into the + // next window. + self.building_pty_context.clear(); + true + } + _ => { + if let Some(frame) = stripped.strip_prefix("__VW_CTX_FRAME__:") + { + self.building_pty_context.push(frame.to_string()); + true + } else { + false + } + } + } + } + + /// Drain whatever PTY lines have queued up between evals. Two + /// classes of line show up here in practice: + /// + /// 1. **Shim startup logs** (`[vw-shim] ...`) — emitted by + /// Vivado-startup-time code before the user's first eval. + /// Routed via the sink as Info so the user can see + /// whether the send_msg_id override installed. + /// 2. **Vivado messages emitted between evals** (e.g., delayed + /// `WARNING:` from a previous eval's async work, or + /// initialization messages fired by Vivado without any + /// eval in flight). Same deal — route to sink so the user + /// sees them in scrollback. + /// + /// Unclassified lines (banner, source-echo, the Vivado prompt) + /// still drop on the floor — or stderr-mirror if `verbose`. + /// Forwarding those would flood scrollback. + /// Post-Err PTY drain — see the call site in [`Self::eval`] for + /// the timing rationale. Polls `pty_rx` with a 10 ms per-wait + /// timeout so a quiescent channel exits fast; caps total wait + /// at 50 ms so a truly stuck Vivado can't hold the REPL longer + /// than that. Each retrieved line is fed through + /// [`Self::handle_pty_line_during_eval`] — same code path as an + /// in-flight eval — so marker BEGIN/END events pop the stack + /// coherently and error messages carry the failed eval's + /// frames when they reach the REPL sink. + /// Recover from a cancelled-shim state. Called by + /// [`Self::read_response_for`] when it sees the `Vivado%` + /// prompt in the PTY — that means Vivado's SIGINT handler + /// did `interp cancel -unwind` and the shim's dispatch loop + /// unwound all the way through the `source` that loaded it, + /// dropping Vivado back to interactive Tcl. + /// + /// Recovery steps: + /// 1. Drop the orphaned protocol socket + its reader task. + /// 2. Write `source \n` to the PTY master — + /// Vivado's interactive prompt sees it and re-runs the + /// shim. `VW_PROTOCOL_ADDR` is still set in the child + /// env, so the re-sourced shim connects back on the + /// same listener. + /// 3. Accept the new connection (with the same connect + /// timeout as the initial spawn) and rewire + /// [`Self::proto_read`] / [`Self::proto_write`]. + /// 4. Clear [`Self::shim_dead`] so subsequent evals can + /// run. + /// + /// Post-condition: Vivado is warm — project, IP catalog, + /// C++-level state all survive — but every Tcl-level user + /// binding (proc definitions from htcl `src`, `set` vars, + /// namespace state) is gone. Callers that had lowered + /// procs into the session need to reload them. + /// + /// Returns `Ok(())` on successful respawn. On failure + /// leaves `shim_dead` latched so future evals fast-fail + /// with a clear error. + async fn respawn_shim(&mut self) -> Result<(), BackendError> { + // Cancel the old reader task so it doesn't fight the + // new one when the old socket goes away. + if let Some(handle) = self._proto_read_task.take() { + handle.abort(); + } + // Drop the old socket ends — closing our side lets any + // buffered writes from the dead shim get discarded, and + // the listener stays open on the same port for accept. + // We can't literally "close" via `drop` here because + // `proto_write` is not Option; replace it below when we + // wire up the new stream. + // Write the `source` line to the PTY. `_master.take_writer()` + // returns a `Box`; each call gives a fresh + // handle so we can grab one just for this write. + let mut writer = self._master.take_writer().map_err(|e| { + BackendError::Worker(format!( + "respawn_shim: cannot obtain PTY writer: {e}" + )) + })?; + let source_cmd = format!("source {}\n", self.shim_path.display()); + use std::io::Write as _; + writer.write_all(source_cmd.as_bytes()).map_err(|e| { + BackendError::Worker(format!("respawn_shim: PTY write failed: {e}")) + })?; + writer.flush().map_err(|e| { + BackendError::Worker(format!("respawn_shim: PTY flush failed: {e}")) + })?; + drop(writer); + // Await the new shim's connection on the retained + // listener. + let listener = self.listener.as_ref().ok_or_else(|| { + BackendError::Worker( + "respawn_shim: no listener retained (backend was \ + constructed without one)" + .into(), + ) + })?; + let accept_result = + tokio::time::timeout(SHIM_CONNECT_TIMEOUT, listener.accept()).await; + let stream = match accept_result { + Ok(Ok((s, _peer))) => s, + Ok(Err(e)) => return Err(BackendError::Io(e)), + Err(_) => { + return Err(BackendError::Worker( + "respawn_shim: timed out waiting for re-sourced \ + shim to connect back" + .into(), + )); + } + }; + stream.set_nodelay(true).map_err(BackendError::Io)?; + let (read_half, write_half) = stream.into_split(); + let (proto_tx, proto_rx) = tokio::sync::mpsc::unbounded_channel(); + let mut proto_buf = BufReader::new(read_half); + let task = tokio::spawn(async move { + let mut line = String::new(); + loop { + line.clear(); + match proto_buf.read_line(&mut line).await { + Ok(0) => break, + Ok(_) => { + if proto_tx.send(Ok(line.clone())).is_err() { + break; + } + } + Err(e) => { + let _ = proto_tx.send(Err(e)); + break; + } + } + } + }); + self.proto_read = proto_rx; + self.proto_write = write_half; + self._proto_read_task = Some(task); + self.shim_dead = false; + Ok(()) + } + + async fn settle_late_pty_after_err(&mut self) { + use std::time::{Duration, Instant}; + let mut sink_void = String::new(); + let hard_deadline = Instant::now() + Duration::from_millis(50); + loop { + let now = Instant::now(); + if now >= hard_deadline { + break; + } + let per_wait = Duration::from_millis(10); + match tokio::time::timeout(per_wait, self.pty_rx.recv()).await { + Ok(Some(line)) => { + self.handle_pty_line_during_eval(&line, &mut sink_void); + } + _ => break, // idle window elapsed or channel closed + } + } + // Flush any classifier-pending message so it emits with + // THIS eval's context, not the next one's. + if let Some((kind, text)) = self.pty_classifier.flush() { + self.emit_pty_chunk(kind, &text, &mut sink_void); + } + } + + fn drain_pty_between_evals(&mut self) { + // Force-flush any pending PTY message from the previous + // eval first — if the eval ended right after a classified + // line and before any continuation could arrive, we want + // it surfaced before whatever the drain finds. + let mut sink_void = String::new(); + if let Some((kind, text)) = self.pty_classifier.flush() { + self.emit_pty_chunk(kind, &text, &mut sink_void); + } + while let Ok(line) = self.pty_rx.try_recv() { + if self.consume_ctx_marker(&line) { + continue; + } + let outcome = + self.pty_classifier.handle(&line, std::time::Instant::now()); + for (kind, text) in outcome.chunks { + self.emit_pty_chunk(kind, &text, &mut sink_void); + } + if !outcome.absorbed && self.verbose { + self.write_verbose_line(&line); + } + } + // Flush again at end of drain — a classified line that + // landed right before the drain stopped might still be + // buffered. No new lines will arrive before the next + // eval's read_response_for, so we'd rather surface this + // now than wait. + if let Some((kind, text)) = self.pty_classifier.flush() { + self.emit_pty_chunk(kind, &text, &mut sink_void); + } + } + + /// Write one verbose-firehose line — to the log file when + /// configured, otherwise to vw's stderr. Used for unclassified + /// PTY lines we'd otherwise discard. Errors silently because + /// dropping a verbose line shouldn't break the eval. + fn write_verbose_line(&mut self, line: &str) { + if let Some(w) = self.verbose_log.as_mut() { + let _ = writeln!(w, "{line}"); + let _ = w.flush(); + } else { + let _ = writeln!(std::io::stderr(), "{line}"); + } + } + + /// Emit one classified PTY chunk: the optional gray provenance + /// marker (when `trace_message_sources` is on) followed by the + /// chunk itself. Used by both the in-eval and between-eval + /// classification paths so the marker / sink-vs-accumulator + /// rule lives in exactly one place. + /// + /// `[vw-*]` self-diagnostic chunks (shim install logs, future + /// internal tracers) are suppressed unless trace is on. Most + /// users don't care that the shim connected to a port and + /// installed an override — that's housekeeping noise. When + /// something goes wrong, set `VW_TRACE_MESSAGE_SOURCES=1` to + /// surface both these chunks AND the per-message provenance + /// markers. + fn emit_pty_chunk( + &mut self, + kind: StreamKind, + text: &str, + accumulated: &mut String, + ) { + if !self.trace_message_sources && is_vw_log_chunk(text) { + return; + } + // Tag warnings/errors that arrived without a trace with the + // innermost active context (the top of `pty_context_stack` + // — frames captured by the shim around the in-flight C++ + // call or user proc body). This is the path that resolves + // "IP_Flow 19-7090" and friends — they go straight from + // Vivado's C++ to the PTY, bypassing every Tcl-side + // stack-capture hook. + let tagged: String; + let payload: &str = if let Some(frames) = self + .pty_context_stack + .last() + .filter(|_| { + matches!(kind, StreamKind::Warning | StreamKind::Error) + && !text.contains("\n at ") + }) + .filter(|f| !f.is_empty()) + { + let trimmed = text.trim_end_matches('\n'); + let mut buf = String::with_capacity(text.len() + 80); + buf.push_str(trimmed); + for frame in frames { + buf.push_str("\n at "); + buf.push_str(frame); + } + // Restore the trailing newline if the caller had one + // — downstream chunk handling assumes line-terminated. + if text.ends_with('\n') { + buf.push('\n'); + } + tagged = buf; + &tagged + } else { + text + }; + if let Some(sink) = self.stdout_sink.as_mut() { + if self.trace_message_sources { + sink( + StreamKind::Info, + &format!("[vw-pty] classified-as={kind:?}\n"), + ); + } + sink(kind, payload); + } else { + // A classified WARNING/ERROR is always the start of a + // new logical message from Vivado. The PTY pump can + // split chunks at arbitrary byte boundaries, so the + // preceding chunk may lack a trailing newline — if we + // just push_str, we get e.g. `/clocky/clk_in1ERROR: [BD + // 41-1031]…` in the failure block. Inject a separator + // so downstream renderers see line-bounded messages. + if matches!(kind, StreamKind::Warning | StreamKind::Error) + && !accumulated.is_empty() + && !accumulated.ends_with('\n') + { + accumulated.push('\n'); + } + accumulated.push_str(payload); + } + } +} + +/// True when the chunk's first non-whitespace content matches one +/// of our `[vw-*]` self-diagnostic prefixes (see [`VW_LOG_PREFIXES`]). +/// Used by [`VivadoBackend::emit_pty_chunk`] to suppress these +/// chunks when trace isn't enabled. +pub(crate) fn is_vw_log_chunk(text: &str) -> bool { + let trimmed = text.trim_start(); + VW_LOG_PREFIXES.iter().any(|p| trimmed.starts_with(p)) +} + +/// True for lines Vivado is known to emit but which carry no signal +/// the user needs at REPL / `vw run` level. Filtered out during +/// eval (verbose mode still records them in the log). Keep this +/// list tight — filter only stuff Vivado has no knob for and no +/// downstream tool cares about. +/// +/// Current entries: +/// +/// - `Wrote : ` — status echo emitted by `save_bd_design`, +/// `write_bd_tcl`, `write_xci`, etc. No Vivado parameter +/// suppresses it (searched `list_param` for wrote / write / +/// save / persist / log / quiet / silent / banner / bd. / +/// verbose / echo / message / command — zero matches). +/// +/// If a genuine Vivado parameter shows up in a later release, drop +/// the corresponding pattern here. +pub(crate) fn is_vivado_known_noise(line: &str) -> bool { + // Vivado uses two spaces between "Wrote" and ":" — match on + // the whole `Wrote : <` prefix rather than just "Wrote" so + // a genuine user `puts "Wrote foo"` doesn't get filtered. + let trimmed = line.trim_start(); + trimmed.starts_with("Wrote : <") +} + +#[async_trait] +impl EdaBackend for VivadoBackend { + fn name(&self) -> &str { + "vivado" + } + + async fn eval(&mut self, tcl: &str) -> Result { + if self.shim_dead { + // Fast-fail after a prior eval detected the shim was + // torn down by cancellation — writing to the orphaned + // socket would either buffer silently or block, and + // reading a response would hang forever. Surface the + // same message the initial detection did so every + // subsequent eval says the same clear thing. + return Err(BackendError::Tcl { + message: "vivado shim is dead (torn down by an \ + earlier Ctrl-C cancellation). The session \ + cannot recover — exit vw and re-launch \ + (`:exit` in the REPL, then `vw repl` / \ + `vw run` again)." + .to_string(), + code: Some("VW_SHIM_DEAD".to_string()), + info: None, + stdout: String::new(), + }); + } + self.drain_pty_between_evals(); + let id = self.alloc_id(); + let req = Request { + id, + op: RequestOp::Eval { tcl: tcl.into() }, + }; + self.write_request(&req).await?; + let (resp, stdout) = self.read_response_for(id).await?; + // Vivado writes the Err response to the protocol socket + // *before* the underlying error text and marker cleanup + // reach the PTY — empirical: 65 μs between an Err response + // arrival and the [BD 41-71] error line landing on our + // pump thread's `read(2)`; another ~180 μs before the + // pump forwards it into `pty_rx`. Returning here without + // draining those bytes leaves them queued in `pty_rx` + // until the NEXT eval's `drain_pty_between_evals`, which + // runs after `pending_eval_index` has advanced — so the + // origin fallback in the REPL tags them with a completely + // unrelated command. Poll briefly for the tail so the + // late messages get emitted with THIS eval's marker + // context still on the stack. Err-only: successful evals + // don't have this misattribution risk (any tail markers + // just adjust the stack idempotently), and running it on + // every eval would add ~1 ms to each of the ~1800 + // auto-load evals. + if matches!(resp.result, ResponseResult::Err { .. }) { + self.settle_late_pty_after_err().await; + } + match resp.result { + ResponseResult::Ok { result, .. } => { + let value = match result { + serde_json::Value::String(s) => s, + other => other.to_string(), + }; + Ok(EvalOutput { value, stdout }) + } + ResponseResult::Err { error, .. } => Err(BackendError::Tcl { + message: error.message, + code: error.code, + info: error.info, + stdout, + }), + } + } + + async fn send( + &mut self, + mut request: Request, + ) -> Result { + self.drain_pty_between_evals(); + if request.id == 0 { + request.id = self.alloc_id(); + } + let id = request.id; + self.write_request(&request).await?; + let (resp, _stdout) = self.read_response_for(id).await?; + Ok(resp) + } + + async fn shutdown(&mut self) -> Result<(), BackendError> { + if self.child.is_none() { + return Ok(()); + } + let id = self.alloc_id(); + let req = Request { + id, + op: RequestOp::Shutdown, + }; + let _ = self.write_request(&req).await; + let _ = self.read_response_for(id).await; + if let Some(mut child) = self.child.take() { + // Vivado's tear-down is slow; bound it. + let waited = tokio::task::spawn_blocking(move || { + let deadline = + std::time::Instant::now() + Duration::from_secs(10); + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) => { + if std::time::Instant::now() >= deadline { + let _ = child.kill(); + return child.wait(); + } + std::thread::sleep(Duration::from_millis(100)); + } + Err(e) => return Err(e), + } + } + }) + .await; + match waited { + Ok(Ok(status)) => debug!(?status, "vivado exited"), + Ok(Err(e)) => return Err(BackendError::Io(e)), + Err(e) => warn!(?e, "vivado wait join error"), + } + } + if let Some(handle) = self.stdout_pump.take() { + let _ = handle.join(); + } + Ok(()) + } +} + +impl Drop for VivadoBackend { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + } + // The pump thread will exit on its own when the PTY master is + // dropped and the read returns EOF. + } +} + +/// Pump Vivado's PTY output in the background. +/// +/// Pump Vivado's process stdout into the worker as a stream of +/// newline-split lines. Runs on a blocking std thread because +/// `portable_pty` only exposes a synchronous `Read`. +/// +/// We always *read* the bytes — otherwise the PTY backpressures and +/// Vivado eventually blocks. Splitting on `\n` here (rather than +/// shipping raw chunks) keeps line semantics consistent for the +/// downstream message-line filter, which works one line at a time. +/// `\r` is stripped — Vivado's PTY output sometimes contains CRLF. +/// +/// The thread exits when the PTY closes (Vivado died) or the +/// receiver is dropped (the worker shut down). +fn spawn_stdout_pump( + mut reader: Box, + tx: tokio::sync::mpsc::UnboundedSender, + mut raw_log: Option>, +) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let mut buf = [0u8; 4096]; + let mut line = String::new(); + loop { + match reader.read(&mut buf) { + Ok(0) => break, + Ok(n) => { + // Raw byte-log tee. Write BEFORE any line + // processing so the log is a byte-perfect + // record of what Vivado emitted — CR/LF + // preserved, non-UTF-8 bytes preserved, tables + // and banners preserved. Flush per chunk so a + // `tail -f` from another terminal stays near + // real-time. Errors are ignored: dropping a log + // write should never take out the eval channel. + if let Some(w) = raw_log.as_mut() { + let _ = w.write_all(&buf[..n]); + let _ = w.flush(); + } + // DIAGNOSTIC (temporary): log every `read(2)` + // completion so we can see when Vivado actually + // wrote bytes to the PTY. Paired with the + // response-arrival and pty_rx.recv() logs in + // `read_response_for`, this tells us whether a + // late message is a pump-forwarding lag or a + // Vivado-flush lag. + let preview: String = std::str::from_utf8(&buf[..n]) + .unwrap_or("") + .chars() + .take(140) + .collect(); + vw_timing_log("pump_read", n, &preview); + for &b in &buf[..n] { + if b == b'\n' { + let send = std::mem::take(&mut line); + vw_timing_log("pump_send", send.len(), &send); + if tx.send(send).is_err() { + return; + } + } else if b != b'\r' { + // Best-effort UTF-8 — replace bytes that + // aren't valid mid-line. Vivado's stdout + // is ASCII in practice. + line.push(b as char); + } + } + } + Err(e) => { + debug!(error = %e, "pty read error"); + break; + } + } + } + // Flush any partial trailing line so EOF doesn't swallow the + // last unterminated message. + if !line.is_empty() { + let _ = tx.send(line); + } + }) +} + +/// Classify `line` as a Vivado standard-format message and +/// translate the prefix into the [`StreamKind`] the sink should +/// receive. Returns `None` for lines that don't match the +/// `common::send_msg_id` prefix set — those are banner / +/// source-echo / idle chatter and don't reach the sink. +/// +/// Conservative on purpose: a false-negative just means a useful +/// line is dropped (recoverable with `verbose=true` for users who +/// need the full firehose), but a false-positive injects banner / +/// source-echo noise into every eval's output, which would degrade +/// the REPL experience for everyone. Leading whitespace is allowed +/// because Vivado occasionally indents within scripted blocks. +/// Classify a multi-line shim-stream chunk for sink routing. The +/// chunk's first line determines its kind: a Vivado-standard +/// severity prefix routes to the matching [`StreamKind`]; +/// anything else falls back to [`StreamKind::Stdout`] (the +/// chunk is treated as user `puts` output). +/// +/// Continuation lines (the `at file:line in proc` frames the +/// send_msg_id override appends) inherit the first-line kind by +/// virtue of being part of the same chunk. The downstream +/// renderer treats each chunk as one scrollback entry. +pub(crate) fn classify_chunk_for_sink(chunk: &str) -> StreamKind { + let first = chunk.lines().next().unwrap_or(""); + classify_vivado_message_line(first).unwrap_or(StreamKind::Stdout) +} + +pub(crate) fn classify_vivado_message_line(line: &str) -> Option { + let l = line.trim_start(); + // `CRITICAL WARNING:` must be checked BEFORE `WARNING:` because + // the latter is a prefix of the former when leading whitespace + // is trimmed. `CRITICAL WARNING:` routes to Error rather than + // Warning — in Vivado's severity hierarchy it's between WARNING + // and ERROR but semantically means "your run may fail because + // of this" (bad connection, missing property, etc.), which + // reads like a hard failure to the user. Rendering it with the + // same red ✗ style as ERROR matches how a reader treats it. + if l.starts_with("CRITICAL WARNING:") { + Some(StreamKind::CriticalWarning) + } else if l.starts_with("ERROR:") { + Some(StreamKind::Error) + } else if l.starts_with("WARNING:") { + Some(StreamKind::Warning) + } else if l.starts_with("INFO:") { + Some(StreamKind::Info) + } else if VW_LOG_PREFIXES.iter().any(|p| l.starts_with(p)) { + // Our own diagnostics — see [`VW_LOG_PREFIXES`] for the + // canonical list. All route as Info: they're gray "where + // did this come from" markers, not warnings. The + // allowlist (rather than a generic `starts_with("[vw-")`) + // prevents a user's `puts "[vw-mystuff] hi"` from getting + // accidentally absorbed. + Some(StreamKind::Info) + } else { + None + } +} + +/// Allowlist of prefixes our shim and worker emit for self- +/// diagnostics. Any line starting with one of these classifies +/// as [`StreamKind::Info`]. +pub(crate) const VW_LOG_PREFIXES: &[&str] = + &["[vw-shim]", "[vw-pty]", "[vw-shim-stream]"]; + +/// Window during which a classified PTY line will absorb a +/// following unclassified line as a continuation. Vivado +/// occasionally emits multi-line messages where the severity +/// prefix only sits on the first line; treating an +/// immediately-following unclassified line as part of the same +/// message renders the warning as one scrollback entry instead +/// of two. +/// +/// 20ms is well above the inter-line latency our PTY pump sees +/// for a single Vivado write (which is sub-millisecond) but +/// well below human reaction time, so a real follow-up message +/// from a *different* call site can't be misattributed. +pub(crate) const PTY_CONTINUATION_WINDOW: std::time::Duration = + std::time::Duration::from_millis(20); + +/// Per-message buffer the worker uses to merge a multi-line PTY +/// warning into one chunk. See [`PtyClassifier`] for the merge +/// semantics. +#[derive(Debug, Clone)] +struct PendingPtyMessage { + kind: StreamKind, + text: String, + arrived_at: std::time::Instant, +} + +/// Outcome of feeding one PTY line through [`PtyClassifier`]. +#[derive(Debug, Default)] +pub(crate) struct ClassifyOutcome { + /// Chunks ready for the sink (or accumulator) in arrival + /// order. At most one *new* classified chunk per call; an + /// additional preceding entry appears only when this call + /// flushed a previously-pending message (either because a + /// new classified line arrived or the window expired). + pub chunks: Vec<(StreamKind, String)>, + /// True when the classifier took responsibility for the + /// input line (stored it as pending, or appended it to a + /// pending message). False when the line was an unclassified + /// non-continuation — caller may stderr-mirror it. + pub absorbed: bool, +} + +/// Brief-buffer classifier for PTY lines. Holds one classified +/// message at a time; an unclassified line arriving within +/// [`PTY_CONTINUATION_WINDOW`] gets folded into it, so a multi- +/// line Vivado warning whose first line carries the severity +/// prefix renders as a single chunk (and thus a single scrollback +/// entry on the App side). +/// +/// Pure / time-injected so it's unit-testable without setting up +/// a worker. +#[derive(Debug)] +pub(crate) struct PtyClassifier { + pending: Option, + window: std::time::Duration, +} + +impl PtyClassifier { + pub fn new(window: std::time::Duration) -> Self { + Self { + pending: None, + window, + } + } + + /// Feed one PTY line. `now` is when the line arrived (taken + /// as a parameter so tests can drive the clock). + pub fn handle( + &mut self, + line: &str, + now: std::time::Instant, + ) -> ClassifyOutcome { + let mut out = ClassifyOutcome::default(); + if let Some(kind) = classify_vivado_message_line(line) { + // A new classified line starts a new pending. Flush + // whatever was pending first — same path as the + // window-expired case below. + if let Some(prev) = self.pending.take() { + out.chunks + .push((prev.kind, with_trailing_newline(&prev.text))); + } + self.pending = Some(PendingPtyMessage { + kind, + text: line.to_string(), + arrived_at: now, + }); + out.absorbed = true; + return out; + } + // Unclassified. Maybe a continuation of the current + // pending warning/error? We only fold for Warning/Error + // kinds because Info messages (`[vw-shim] ...` and + // Vivado `INFO:`) are always single-line in practice — + // and absorbing into them swallowed Vivado's source-echo + // of our shim script (`# catch {...}`, `# while {1} {`, + // etc.) which arrived inside the window during boot. + // Restricting to Warning|Error covers the case we + // actually care about (Vivado occasionally emits multi- + // line WARNING/ERROR text with `\n` between the header + // and a body) without the noise. + if let Some(p) = self.pending.as_mut() { + let merges = matches!( + p.kind, + StreamKind::Warning + | StreamKind::CriticalWarning + | StreamKind::Error + ); + // Only a leading-whitespace non-empty line is a + // real continuation. Vivado indents every body line + // of a multi-line message (e.g. the `.xci` path + // that follows `[Vivado_Tcl 4-393]`), so an + // unindented line — or a blank line acting as a + // separator — signals a new, unrelated message + // (Vivado's `Attempting to get a license…` status + // echoes are the motivating case). Reject them + // even inside the merge window; otherwise those + // status lines glom onto the previous warning and + // inherit its orange/red styling. Non-continuation + // unclassified lines return `absorbed=false` — the + // during-eval caller routes them to Stdout so they + // still surface (e.g. the `/pin` list following + // `[BD 41-758] … clock source:`), just without the + // pending's severity style. + let looks_like_continuation = + line.chars().next().is_some_and(char::is_whitespace); + if merges + && looks_like_continuation + && now.duration_since(p.arrived_at) < self.window + { + p.text.push('\n'); + p.text.push_str(line); + // Refresh the arrival time so a chain of + // continuation lines all qualifies, not just the + // first. + p.arrived_at = now; + out.absorbed = true; + return out; + } + // Either kind doesn't merge, the shape doesn't + // qualify as a continuation, or the window expired — + // flush the pending. The current line itself is not + // absorbed; caller may stderr-mirror it. + let p = self.pending.take().unwrap(); + out.chunks.push((p.kind, with_trailing_newline(&p.text))); + } + out + } + + /// Force-flush any pending message. Called at eval end so a + /// message buffered right before the Vivado response doesn't + /// linger unseen. + pub fn flush(&mut self) -> Option<(StreamKind, String)> { + self.pending + .take() + .map(|p| (p.kind, with_trailing_newline(&p.text))) + } +} + +fn with_trailing_newline(s: &str) -> String { + if s.ends_with('\n') { + s.to_string() + } else { + format!("{s}\n") + } +} + +fn resolve_vivado(config: &VivadoConfig) -> Result { + if let Some(path) = &config.vivado { + return Ok(path.clone()); + } + if let Ok(env) = std::env::var("VW_VIVADO") { + if !env.is_empty() { + return Ok(PathBuf::from(env)); + } + } + if let Some(paths) = std::env::var_os("PATH") { + for dir in std::env::split_paths(&paths) { + let candidate = dir.join("vivado"); + if candidate.is_file() { + return Ok(candidate); + } + } + } + Err(BackendError::Worker( + "could not find `vivado` on PATH; set $VW_VIVADO or pass \ + VivadoConfig::vivado" + .into(), + )) +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use super::{ + classify_chunk_for_sink, classify_vivado_message_line, is_vw_log_chunk, + PtyClassifier, StreamKind, + }; + + fn classifier(window_ms: u64) -> PtyClassifier { + PtyClassifier::new(Duration::from_millis(window_ms)) + } + + #[test] + fn classifier_emits_classified_line_only_on_flush() { + // A classified line gets buffered, not emitted, until + // either a new classified line arrives or flush() is + // called. This is what enables the multi-line-message + // merge that follows. + let mut c = classifier(20); + let t0 = Instant::now(); + let out = c.handle("WARNING: [X 1-1] hi", t0); + assert!(out.chunks.is_empty(), "{:?}", out.chunks); + assert!(out.absorbed); + let flushed = c.flush().expect("pending must flush"); + assert_eq!(flushed.0, StreamKind::Warning); + assert_eq!(flushed.1, "WARNING: [X 1-1] hi\n"); + } + + #[test] + fn classifier_absorbs_indented_continuation_within_window() { + let mut c = classifier(20); + let t0 = Instant::now(); + let out = c.handle("WARNING: [X 1-1] header", t0); + assert!(out.absorbed); + // Real Vivado multi-line messages indent every body line. + let out = c.handle(" second body line", t0 + Duration::from_millis(5)); + assert!(out.chunks.is_empty(), "{:?}", out.chunks); + assert!(out.absorbed); + // A third continuation works too (window refreshes on + // each absorb). + let out = c.handle(" third line", t0 + Duration::from_millis(15)); + assert!(out.chunks.is_empty(), "{:?}", out.chunks); + assert!(out.absorbed); + let (kind, text) = c.flush().unwrap(); + assert_eq!(kind, StreamKind::Warning); + assert_eq!( + text, + "WARNING: [X 1-1] header\n second body line\n third line\n" + ); + } + + /// Regression: an unindented follow-up line (e.g. Vivado's + /// `Attempting to get a license for feature 'Synthesis'…` + /// status echo) must NOT be absorbed as a continuation of the + /// preceding WARNING. Real Vivado multi-line messages indent + /// every body line — an unindented line is a separate message, + /// even when it lands inside the merge window. + #[test] + fn classifier_does_not_absorb_unindented_line_as_continuation() { + let mut c = classifier(20); + let t0 = Instant::now(); + assert!(c.handle("WARNING: [X 1-1] header", t0).absorbed); + // Indented body line: legitimate continuation. + let out = + c.handle(" /path/to/file.xci", t0 + Duration::from_millis(2)); + assert!(out.chunks.is_empty(), "{:?}", out.chunks); + assert!(out.absorbed); + // Unindented follow-up: NOT a continuation. Pending + // flushes, current line is reported as not-absorbed. + let out = c.handle( + "Attempting to get a license for feature 'Synthesis'", + t0 + Duration::from_millis(4), + ); + assert_eq!(out.chunks.len(), 1, "{:?}", out.chunks); + assert_eq!(out.chunks[0].0, StreamKind::Warning); + assert_eq!( + out.chunks[0].1, + "WARNING: [X 1-1] header\n /path/to/file.xci\n" + ); + assert!(!out.absorbed, "unindented follow-up must not be absorbed"); + } + + /// Regression: a blank line between the warning body and the + /// next status message must terminate the pending, not extend + /// it. Blank lines are visual separators, not warning bodies. + #[test] + fn classifier_treats_blank_line_as_message_separator() { + let mut c = classifier(20); + let t0 = Instant::now(); + assert!(c.handle("WARNING: [X 1-1] header", t0).absorbed); + assert!(c.handle(" body", t0 + Duration::from_millis(1)).absorbed); + let out = c.handle("", t0 + Duration::from_millis(2)); + assert_eq!(out.chunks.len(), 1, "{:?}", out.chunks); + assert_eq!(out.chunks[0].1, "WARNING: [X 1-1] header\n body\n"); + assert!(!out.absorbed); + } + + #[test] + fn classifier_flushes_pending_when_window_expires() { + let mut c = classifier(20); + let t0 = Instant::now(); + let out = c.handle("WARNING: [X 1-1] one", t0); + assert!(out.absorbed); + // Unclassified line arrives well past the window: + // pending flushes, line itself is reported as + // not-absorbed so the caller may stderr-mirror it. + let out = c.handle("a", t0 + Duration::from_millis(50)); + assert_eq!(out.chunks.len(), 1); + assert_eq!(out.chunks[0].0, StreamKind::Warning); + assert_eq!(out.chunks[0].1, "WARNING: [X 1-1] one\n"); + assert!(!out.absorbed); + // Nothing further pending. + assert!(c.flush().is_none()); + } + + #[test] + fn classifier_flushes_previous_pending_on_new_classified() { + // Two classified lines back-to-back: the first flushes + // as soon as the second arrives, and the second becomes + // the new pending. + let mut c = classifier(20); + let t0 = Instant::now(); + c.handle("WARNING: [X 1-1] one", t0); + let out = c.handle("ERROR: [Y 1-1] two", t0 + Duration::from_millis(5)); + assert_eq!(out.chunks.len(), 1); + assert_eq!(out.chunks[0].0, StreamKind::Warning); + assert_eq!(out.chunks[0].1, "WARNING: [X 1-1] one\n"); + assert!(out.absorbed); + let (kind, text) = c.flush().unwrap(); + assert_eq!(kind, StreamKind::Error); + assert_eq!(text, "ERROR: [Y 1-1] two\n"); + } + + #[test] + fn vw_log_chunk_detection() { + // Recognized chunks for emit-time suppression. + assert!(is_vw_log_chunk( + "[vw-shim] installed send_msg_id override\n" + )); + assert!(is_vw_log_chunk("[vw-pty] classified-as=Warning\n")); + assert!(is_vw_log_chunk(" [vw-shim-stream] classified-as=Error\n")); + // Real message content stays — never accidentally + // suppress a Vivado warning that mentions our tag in its + // body. + assert!(!is_vw_log_chunk("WARNING: [Common 17-1] no\n")); + assert!(!is_vw_log_chunk( + "INFO: [Common 17-1] something about [vw-shim]\n" + )); + assert!(!is_vw_log_chunk("")); + } + + #[test] + fn classifier_info_kind_does_not_absorb_continuations() { + // Regression guard: Info-kind pending (typically a + // [vw-shim] log line or a Vivado INFO line) must NOT + // absorb subsequent unclassified lines, because those + // are almost always unrelated content arriving in the + // same time window (Vivado source-echo during boot, + // banner lines, etc.). Only Warning/Error kinds merge. + let mut c = classifier(20); + let t0 = Instant::now(); + c.handle("[vw-shim] installed send_msg_id override", t0); + let out = c.handle( + "# catch {::vw::install_send_msg_override}", + t0 + Duration::from_millis(5), + ); + // The pending Info flushed as its own chunk; the source- + // echo line was NOT absorbed. + assert_eq!(out.chunks.len(), 1); + assert_eq!(out.chunks[0].0, StreamKind::Info); + assert_eq!( + out.chunks[0].1, + "[vw-shim] installed send_msg_id override\n" + ); + assert!(!out.absorbed); + } + + #[test] + fn classifier_drops_unclassified_lines_when_no_pending() { + let mut c = classifier(20); + let out = c.handle("plain output, no prefix", Instant::now()); + assert!(out.chunks.is_empty()); + assert!(!out.absorbed); + assert!(c.flush().is_none()); + } + + #[test] + fn classifies_each_standard_prefix_to_its_stream_kind() { + let cases = [ + ( + "ERROR: [Common 17-53] No open project. ...", + StreamKind::Error, + ), + ( + "WARNING: [Coretcl 2-1184] no open project", + StreamKind::Warning, + ), + ( + "CRITICAL WARNING: [Vivado 12-180] ...", + // Critical warnings classify as their own kind so + // log-level filtering can treat them distinctly from + // ERROR. Downstream renderers may still paint them + // with the red ✗ treatment, but the classifier keeps + // the semantic distinction intact. + StreamKind::CriticalWarning, + ), + ( + "INFO: [Vivado 12-3661] auto-pinning enabled", + StreamKind::Info, + ), + ]; + for (line, expected) in cases { + assert_eq!( + classify_vivado_message_line(line), + Some(expected), + "wrong classification for: {line}" + ); + } + } + + #[test] + fn classifies_lines_with_leading_whitespace() { + // Scripted blocks sometimes indent messages — still a + // Vivado-formatted line, still useful to surface. + assert_eq!( + classify_vivado_message_line(" ERROR: [X 1-2] indented"), + Some(StreamKind::Error) + ); + assert_eq!( + classify_vivado_message_line("\tWARNING: [X 1-2] tabbed"), + Some(StreamKind::Warning) + ); + } + + #[test] + fn drops_banner_and_source_echo_and_chatter() { + for line in [ + "", + "Vivado v2024.2 (64-bit)", + "SW Build 5095499 on Wed Nov 13 22:37:05 MST 2024", + "Copyright 1986-2024 Xilinx, Inc.", + "Vivado% ", // The interactive prompt + "source /tmp/vw-vivado-shim/vivado-shim.tcl -notrace", + "create_bd_design metroid", + "errors not at start of line: ERROR: foo", + // Has the substring but not at the start — looks like + // shell or log output, not a message-system line. + "[2024-01-01 12:00] INFO: forwarded by other tool", + ] { + assert_eq!( + classify_vivado_message_line(line), + None, + "should drop: {line:?}" + ); + } + } + + #[test] + fn does_not_match_partial_prefixes() { + // `ERRORS:` would be a hypothetical other label and we + // shouldn't false-positive on it. + assert_eq!(classify_vivado_message_line("ERRORS: bogus prefix"), None); + assert_eq!( + classify_vivado_message_line("INFOMERCIAL: not a message"), + None + ); + } + + #[test] + fn plain_chunk_routes_to_stdout() { + // User `puts hi` produces an ordinary stdout chunk — + // nothing matches the Vivado prefix set so it falls + // through to Stdout. + assert_eq!(classify_chunk_for_sink("hi\n"), StreamKind::Stdout); + assert_eq!( + classify_chunk_for_sink("progress: 42%\n"), + StreamKind::Stdout + ); + assert_eq!(classify_chunk_for_sink(""), StreamKind::Stdout); + } + + #[test] + fn classified_chunk_with_stack_inherits_first_line_kind() { + // The exact shape our send_msg_id override produces: a + // severity-prefixed first line followed by `at ...` + // continuation frames. The whole chunk should route to + // the kind matching the first line, so the warning and + // its stack stay together as one orange (or red) entry. + let warning_with_stack = "WARNING: [Common 17-1496] tclapp out of date\n\ + \x20\x20at /opt/Vivado/foo.tcl:42 in ::tclapp::loader\n\ + \x20\x20at /opt/Vivado/init.tcl:10\n"; + assert_eq!( + classify_chunk_for_sink(warning_with_stack), + StreamKind::Warning + ); + + let error_with_stack = "ERROR: [BD 5-148] no open project\n\ + \x20\x20at /opt/Vivado/bd.tcl:99 in ::bd::create\n"; + assert_eq!( + classify_chunk_for_sink(error_with_stack), + StreamKind::Error + ); + } + + #[test] + fn shim_log_lines_route_as_info() { + // Our shim's own log lines start with `[vw-shim]`. They're + // diagnostic-level info — we don't want them to look like + // a hot error, just a "here's what the worker said" + // notice in the scrollback. + assert_eq!( + classify_vivado_message_line( + "[vw-shim] installed send_msg_id override" + ), + Some(StreamKind::Info) + ); + assert_eq!( + classify_vivado_message_line( + "[vw-shim] ::common::send_msg_id not present; skipping override" + ), + Some(StreamKind::Info) + ); + // The matcher uses an explicit allowlist — see + // VW_LOG_PREFIXES. Each member routes as Info; anything + // outside the list does NOT match, even when it bears + // the `[vw-` shape. + for prefix in super::VW_LOG_PREFIXES { + let line = format!("{prefix} something"); + assert_eq!( + classify_vivado_message_line(&line), + Some(StreamKind::Info), + "should classify our known prefix: {prefix}" + ); + } + // Look-alike inside our namespace that we DIDN'T sanction + // (a future shim subsystem nobody added to the allowlist, + // or a user's puts that happens to bracket `[vw-*]`) is + // rejected — keeps the classifier conservative. + assert_eq!(classify_vivado_message_line("[vw-mystuff] foo"), None); + assert_eq!(classify_vivado_message_line("[other] foo"), None); + } + + #[test] + fn classified_chunk_with_leading_indent_still_routes() { + // `classify_vivado_message_line` tolerates leading + // whitespace; `classify_chunk_for_sink` should inherit + // that — Vivado occasionally indents scripted messages. + let chunk = " WARNING: [X 1-2] indented\n at foo:1\n"; + assert_eq!(classify_chunk_for_sink(chunk), StreamKind::Warning); + } + + /// Regression test for `VivadoBackend::interrupt`'s + /// process-group signalling. + /// + /// The `vivado` on-disk binary is a bash wrapper that runs + /// `loader -exec vivado ...` WITHOUT `exec` — so bash stays + /// as our direct child (which is what `Child::process_id` + /// returns) and the actual Vivado runtime is a grandchild. + /// A naive `kill(pid, SIGINT)` hits bash-running-a-script, + /// which does NOT forward SIGINT to its child; Vivado never + /// sees it. `interrupt` fixes this by signalling the process + /// GROUP (`kill(-pid, SIGINT)`) — under portable-pty every + /// spawned command becomes its own session leader with a + /// fresh PGID equal to its pid, and children inherit that + /// PGID, so the group signal reaches everyone. + /// + /// This test mirrors the shape without needing Vivado: bash + /// spawns a long `sleep` in the background then waits for + /// it. Same lack-of-`exec` as the Vivado wrapper. The group + /// signal must terminate both bash AND the sleep within a + /// small bounded time (a signal that lands takes + /// milliseconds; anything past 3s means we missed the + /// target). + /// + /// If this test starts failing under a future portable-pty + /// release, the fix in `interrupt` needs revisiting — some + /// alternate mechanism (writing ETX to the PTY, using + /// `process_group_leader`, or an out-of-band `interp cancel` + /// through the shim protocol) will be needed instead. + #[cfg(unix)] + #[test] + fn interrupt_reaches_wrapped_grandchild_via_process_group() { + use portable_pty::{native_pty_system, CommandBuilder, PtySize}; + use std::time::{Duration, Instant}; + + let pair = native_pty_system() + .openpty(PtySize { + rows: 24, + cols: 80, + pixel_width: 0, + pixel_height: 0, + }) + .expect("openpty"); + + // Mirror the Vivado wrapper's shape: bash spawns a + // long-running child WITHOUT `exec`, then waits. The + // background `sleep` is the stand-in for Vivado's + // `loader`/runtime; `wait $!` is what keeps bash in the + // foreground so it can propagate the eventual exit. + let mut cmd = CommandBuilder::new("bash"); + cmd.arg("-c"); + cmd.arg("sleep 300 & wait $!"); + let mut child = + pair.slave.spawn_command(cmd).expect("spawn bash wrapper"); + // Match VivadoBackend::spawn: drop the slave so the + // master sees EOF at child exit. + drop(pair.slave); + let pid = child.process_id().expect("child pid"); + + // Give bash a beat to fork off the `sleep` before we + // signal. Without this the test races between spawn and + // signal delivery. + std::thread::sleep(Duration::from_millis(300)); + + // The exact call `VivadoBackend::interrupt` makes. + unsafe { + libc::kill(-(pid as libc::pid_t), libc::SIGINT); + } + + let deadline = Instant::now() + Duration::from_secs(3); + loop { + match child.try_wait() { + Ok(Some(_status)) => return, // pass + Ok(None) => { + if Instant::now() >= deadline { + let _ = child.kill(); + panic!( + "wrapper + grandchild did not exit \ + within 3s of group SIGINT — signalling \ + mechanism is broken" + ); + } + std::thread::sleep(Duration::from_millis(20)); + } + Err(e) => panic!("try_wait error: {e}"), + } + } + } +}