diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..640ea1a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + branches: ["**"] + +env: + CARGO_TERM_COLOR: always + +jobs: + check: + name: Check & Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - name: Clippy + run: cargo clippy --all-targets -- -D warnings + - name: Test + run: cargo test + - name: Test release + run: cargo test --release diff --git a/.github/workflows/crossvalidate.yml b/.github/workflows/crossvalidate.yml new file mode 100644 index 0000000..5af4e56 --- /dev/null +++ b/.github/workflows/crossvalidate.yml @@ -0,0 +1,41 @@ +name: Cross-validate against ape + +# The committed fixtures in tests/fixtures/ let `cargo test` check six trees +# against ape everywhere, with no R needed. This job runs the same comparison +# over 250 freshly generated trees, which is the deeper check but needs an R +# toolchain, so it runs weekly and on demand rather than on every push. + +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + inputs: + trees: + description: "How many trees to generate" + required: false + default: "250" + +jobs: + crossvalidate: + name: distree vs ape + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + packages: | + any::ape + any::phangorn + + - name: Build + run: cargo build --release + + - name: Cross-validate + run: Rscript scripts/crossvalidate.R ${{ inputs.trees || 250 }} target/release/distree diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..f01d774 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,84 @@ +name: Documentation + +# Build the MkDocs Material site and publish it to GitHub Pages (gh-pages +# branch). After the first successful run, set Settings -> Pages -> Source to +# "Deploy from a branch" and pick the gh-pages branch (root). +# +# Deploys on a push to main, so the published site tracks main rather than +# redeploying from every working branch. Pull requests build without deploying, +# which is what catches a dead link before it ships. To publish on demand from +# any branch, use the "Run workflow" button (workflow_dispatch). + +on: + push: + branches: + - main + paths: + - "docs/**" + - "mkdocs.yml" + - "CHANGELOG.md" + - "requirements-docs.txt" + - ".github/workflows/docs.yml" + pull_request: + paths: + - "docs/**" + - "mkdocs.yml" + - "CHANGELOG.md" + - "requirements-docs.txt" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: docs-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install dependencies + run: pip install -r requirements-docs.txt + + # --strict turns broken internal links and missing nav entries into + # errors, so a pull request that breaks one fails here instead of + # shipping a dead link. + - name: Build + run: mkdocs build --strict + + deploy: + name: Deploy + needs: build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Cache MkDocs build + uses: actions/cache@v4 + with: + key: mkdocs-material-${{ github.sha }} + path: .cache + restore-keys: | + mkdocs-material- + + - name: Install dependencies + run: pip install -r requirements-docs.txt + + - name: Build and deploy + run: mkdocs gh-deploy --force --strict diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4f5480b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,66 @@ +name: Build & Release + +on: + push: + tags: ["v*"] + +permissions: + contents: write + +jobs: + build: + name: Build ${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + name: distree-linux-x86_64 + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + name: distree-linux-aarch64 + - target: x86_64-apple-darwin + os: macos-15-intel + name: distree-macos-x86_64 + - target: aarch64-apple-darwin + os: macos-latest + name: distree-macos-aarch64 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - name: Install cross-compilation tools + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config.toml + echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config.toml + - name: Build + run: cargo build --release --target ${{ matrix.target }} + - name: Package + run: | + cp target/${{ matrix.target }}/release/distree ${{ matrix.name }} + chmod +x ${{ matrix.name }} + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.name }} + path: ${{ matrix.name }} + + release: + name: Create Release + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: artifacts + - name: Create release + uses: softprops/action-gh-release@v2 + with: + files: artifacts/**/* + generate_release_notes: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e47dfd9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +target/ + +# MkDocs build output and the Material plugin cache +site/ +.cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2df4505 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,73 @@ +# Changelog + +## [1.0.1] - 2026-04-04 + +### Fixed +- Branch lengths that are not finite are rejected. `A:1e910` parses as infinity rather than failing, and an infinite depth made every distance `inf` with `NaN` down the diagonal, reported with a success exit code. The run also stops when finite depths are large enough that `d_i + d_j` overflows +- A leaf label containing a space is rejected in `--lower` mode: PHYLIP readers treat whitespace as the end of the name, so every row after it landed a column out +- `--lower` with `--lmm` now warns that the omitted diagonal holds each leaf's root-to-tip length rather than zeros +- Parallel computation now speeds the run up instead of slowing it down. One parallel job per row could not pay for synchronising the thread pool, and float formatting sat in the serial write loop; rows are now batched and each worker formats its own. A 20,000-tip matrix went from 10.6 s to 1.6 s, and an 8,000-tip one now scales from 1.26 s on one core to 0.21 s on eight +- Trees prefixed with a `[&R]` / `[&U]` rooting marker, or carrying a comment before a label, no longer fail to parse +- Non-ASCII leaf labels (accents, Greek, CJK) are preserved instead of being mangled into mojibake +- Truncated trees, trailing content, unclosed comments and free-form text are rejected instead of yielding a plausible but wrong matrix +- Files holding more than one tree are rejected instead of silently using the first +- Apostrophes in unquoted labels (`O'Brien`) no longer break parsing, and a doubled quote keeps the whitespace around it +- Deeply nested trees no longer overflow the stack when the parse tree is freed +- `-o FILE` no longer truncates an existing file before the tree has parsed +- Write failures are reported instead of being discarded along with a success exit code; a closed pipe exits quietly +- `--precision` out of range is rejected instead of panicking inside the formatter +- `--threads 0` is rejected instead of quietly starting one thread per core +- Newlines and carriage returns in labels are rejected like tabs, and unlabeled leaves are reported rather than dropped in silence +- `--midpoint` no longer adds a hop to every topological distance crossing the midpoint edge +- Negative patristic distances are reported as computed instead of being rounded up to zero, which claimed distinct taxa were identical +- Midpoint rooting (`--midpoint`) rewritten to fix infinite loop caused by cyclic graph construction +- NHX/bracket comment annotations (e.g., `[&&NHX:S=human]`) no longer crash the parser +- Single-quoted labels (e.g., `'Taxon A'`) now parsed correctly +- Double-quoted labels (e.g., `"Taxon A"`) now parsed correctly +- Whitespace and newlines in Newick strings no longer crash the parser +- Stack overflow on deeply nested trees (>5,000 levels) — parser and flattener rewritten iteratively +- Duplicate leaf names are now detected with a clear error message +- `--lmm --topology` conflict now warns instead of silently choosing LMM +- Topology mode now outputs integers instead of float decimals +- Floating point output uses configurable precision instead of raw representation +- Header row always prints leading tab for R/Python distance matrix compatibility + +### Added +- `--precision` / `-p` flag to control decimal places in output (default: 10) +- `-t` / `--threads` flag to control number of parallel threads +- Stdin support: use `-` as the phylogeny argument to read from stdin +- Warning when no branch lengths are detected in patristic mode +- Warning when negative branch lengths are found in the tree +- `--npy` writes the matrix as a NumPy array of 64-bit floats instead of text: exact, half the size, and 2.6x faster. With `--lower` it writes the condensed vector SciPy reads. Labels go to `.labels.txt` +- `--taxa FILE` restricts the matrix to a list of leaf labels, keeping the distances the full tree gives rather than pruning it +- `--stats` prints a summary to stderr: leaves, nodes, mode, cells, and the minimum, maximum and mean off the diagonal +- Gzipped input is read directly, detected by content rather than by file name, so it works from stdin too +- CITATION.cff with DOI +- MkDocs Material documentation site with a tutorial, published to GitHub Pages +- Comprehensive test suite (96 tests), including randomised checks of midpoint rooting and of MRCA queries against a brute-force walk + +### Changed +- Codebase split into modules: `parser.rs`, `tree.rs`, `lca.rs`, `midpoint.rs` +- Fixed-precision float formatting is done by hand where the rounding is unambiguous and handed to the standard formatter where it is not, which cut a text run by a third to a half with byte-identical output +- Cross-validated against R's ape (`cophenetic.phylo`, `vcv.phylo`) and `phangorn::midpoint` over 250 generated trees; reference matrices for six of them are committed so `cargo test` checks the same thing without R +- The parser and the whole pipeline are fuzzed, in the test suite on every push and with `cargo fuzz` for longer runs +- The tree logic moved into a library target, so it can be fuzzed, documented and used from other Rust code +- LCA binary-lifting table stores plain `usize` rather than `Option`, halving the memory it needs +- Output buffer raised from 8 KB to 1 MB, so a multi-gigabyte matrix is not hundreds of thousands of write syscalls +- The input text and the recursive parse tree are released once the flat node array is built, rather than held to the end of the run +- Version string now derived from Cargo.toml via `env!("CARGO_PKG_VERSION")` +- Removed unused `--format` flag +- Fixed clippy warnings (`&Vec` → `&[Node]`) +- Fixed contributors link in README +- README output examples replaced with the real output of a worked example; the previous ones lost their header row to the code fence and the topological matrix was not a realisable tree +- CI runs on every branch, lints the test code, and runs the test suite in release as well as debug + +## [1.0.0] - 2025-06-01 + +### Added +- Initial release +- Patristic distance matrix extraction +- Topological distance computation +- LMM (var-covar) matrix output +- Midpoint rooting option +- Parallel computation with rayon diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..4e5d87c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,17 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +title: "distree" +version: 1.0.1 +doi: 10.5281/zenodo.16811766 +date-released: 2026-04-04 +url: "https://github.com/PathoGenOmics-Lab/distree" +repository-code: "https://github.com/PathoGenOmics-Lab/distree" +license: GPL-3.0 +type: software +authors: + - family-names: Ruiz-Rodriguez + given-names: Paula + affiliation: "I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain" + - family-names: Coscolla + given-names: Mireia + affiliation: "I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain" diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..b7723e3 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,657 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "distree" +version = "1.0.1" +dependencies = [ + "clap", + "flate2", + "rayon", + "tempfile", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index 72ae370..d3abc4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,13 @@ [package] name = "distree" -version = "1.0.0" +version = "1.0.1" edition = "2021" [dependencies] clap = { version = "4.1", features = ["derive"] } +flate2 = "1.1.9" rayon = "1.5" + +[dev-dependencies] +flate2 = "1.1.9" +tempfile = "3" diff --git a/README.md b/README.md index 4b46bd4..f7f38f1 100644 --- a/README.md +++ b/README.md @@ -10,246 +10,138 @@ [![PGO](https://img.shields.io/badge/PathoGenOmics-lab-red?)](https://github.com/PathoGenOmics-Lab) [![DOI](https://img.shields.io/badge/doi-10.5281%2Fzenodo.16811766-%23ff0077)](https://doi.org/10.5281/zenodo.16811766) -__Paula Ruiz-Rodriguez1__ +__Paula Ruiz-Rodriguez1__ __and Mireia Coscolla1__
- 1. I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain + 1. I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain + +`distree` reads a phylogenetic tree in Newick format and writes the pairwise +distance between every pair of tips: **patristic** (summed branch lengths), +**topological** (edge counts), or the **variance-covariance** matrix a +comparative model wants. Rows stream out as they are computed, so memory grows +with the number of tips rather than with the square of it. + +📖 **Full documentation: ** + +| Page | Description | +|:-----|:------------| +| **[Quick start](https://pathogenomics-lab.github.io/distree/getting-started/quickstart/)** | The handful of commands most runs actually use | +| **[Tutorial](https://pathogenomics-lab.github.io/distree/tutorial/)** | A worked outbreak investigation, tree to transmission clusters | +| **[Input](https://pathogenomics-lab.github.io/distree/guide/input/)** | What Newick is accepted, what is rejected, and why | +| **[Distance modes](https://pathogenomics-lab.github.io/distree/guide/distances/)** | Patristic, topological and var-covar, on one worked example | +| **[Midpoint rooting](https://pathogenomics-lab.github.io/distree/guide/rooting/)** | When the root changes the answer, and when it cannot | +| **[Output](https://pathogenomics-lab.github.io/distree/guide/output/)** | TSV, PHYLIP lower triangle, NumPy arrays, precision | +| **[CLI reference](https://pathogenomics-lab.github.io/distree/guide/cli/)** | Every flag, every default, every message | +| **[How it works](https://pathogenomics-lab.github.io/distree/how-it-works/algorithm/)** | The parser, the LCA structure, the streaming loop | +| **[Performance](https://pathogenomics-lab.github.io/distree/how-it-works/performance/)** | Measured speed, memory and thread scaling | +| **[Recipes](https://pathogenomics-lab.github.io/distree/recipes/)** | PCoA, transmission clusters, PGLS, neighbour-joining | -`distree` is a command-line tool written in Rust that extracts a distance matrix from a phylogenetic tree in Newick format. It is designed to handle large trees with thousands of sequences by using a low-memory, parallelized approach. - -## Features - -* **Patristic distances**: Computes the sum of branch lengths between every pair of leaves (taxa). -* **Topological distances**: Ignores branch lengths and calculates the number of edges (nodes) between leaves. -* **LMM (var-covar) matrix**: Outputs the depth (in branch-length units) of the lowest common ancestor (MRCA) for each pair of leaves. -* **Midpoint rooting**: Optionally re-roots the tree at its midpoint before computing distances. -* **Low memory footprint**: Streams each row to output immediately, without holding the full matrix in RAM. -* **Parallel computation**: Uses multiple CPU cores to compute each row in parallel, reducing runtime for large datasets. - -## Typical Applications - -1. **Comparative Genomics & Evolutionary Studies** - - * **Patristic Distances**: When analyzing evolutionary divergence, the sum of branch lengths between two sequences reflects their accumulated genetic change. This is useful for constructing distance-based phylogenetic trees (e.g., Neighbor-Joining), clustering sequences into operational taxonomic units (OTUs), or computing pairwise divergence metrics for downstream analyses. - * **Topological Distances**: In cases where branch-length estimates are unreliable or not meaningful (e.g., when raw sequence counts are compared), using topological distances (number of nodes between leaves) can provide a rough measure of relatedness. This can be helpful for broad clustering or when using simpler distance-based algorithms that only require tree shape. - -2. **Epidemiology & Public Health** - - * **Rapid Outbreak Tracking**: For pathogens such as bacteria or viruses, building a phylogenetic tree from whole-genome data can be computationally expensive to revisit. Extracting a distance matrix allows quick pairwise comparisons to identify clusters of closely related strains (e.g., potential transmission clusters) without recomputing distances from raw alignments. - * **Contact Tracing & Transmission Networks**: If you have a large outbreak dataset, computing patristic distances between samples quickly helps identify subclusters or “clades” of interest (e.g., to infer likely transmission chains). Similarly, topological distances might approximate epidemiological closeness if branch-length estimates vary widely. - -3. **Microbiome & Environmental Sequencing** - - * **OTU/ASV Clustering**: In 16S rRNA amplicon studies, one often constructs a phylogenetic tree of all amplicon sequence variants (ASVs). A distance matrix (patristic or topological) can feed into beta-diversity metrics (e.g., UniFrac requires branch lengths). Here, `distree` can quickly compute pairwise distances after the tree is built, facilitating ordination (PCoA) or clustering of samples by their phylogenetic composition. - * **Phylogenetic Diversity**: Calculating the sum of branch lengths between taxa supports metrics like Faith’s Phylogenetic Diversity or UniFrac. `distree` can produce the matrix needed for those algorithms without re-traversing the tree multiple times. - -4. **Machine Learning & Dimensionality Reduction** - - * **Input for MDS/t-SNE/UMAP**: Many machine learning or visualization methods (Multidimensional Scaling, t-SNE, UMAP) accept a distance matrix as input. Converting a large phylogeny into a pairwise distance matrix enables embedding taxa into low-dimensional space, highlighting evolutionary relationships or clusters. - * **Distance-Based Feature Engineering**: In trait-based prediction models (e.g., predicting phenotype from genotype), patristic distances can be used as kernel features. `distree` can produce the kernel matrix needed for Gaussian Process Regression or support vector machines (SVM) with a custom kernel. - -5. **Benchmarking & Simulation Studies** - - * **Comparing Tree-Building Methods**: When evaluating different phylogenetic inference methods, it is often necessary to compute distances from each resulting tree. `distree` can generate distance matrices for multiple trees rapidly, enabling head-to-head comparisons. - * **Simulation Validation**: In simulation frameworks (e.g., Seq-Gen), one generates a tree, simulates sequence data, reconstructs a tree, and compares distance matrices. Having a fast, consistent tool to extract matrices accelerates simulation benchmarks. +--- ## Installation -### 🐍 Using conda -``` -conda install -c bioconda distree -``` -### 🐍 Using mamba -``` -mamba install -c bioconda distree -``` -### Compilation -1. Ensure you have [Rust](https://www.rust-lang.org/tools/install) installed. - -2. Clone or download the `distree` repository. - -3. In the project root, run: - - ```bash - cargo build --release - ``` - -4. The optimized binary will be available at: - - ```bash - target/release/distree - ``` - -## Usage - ```bash -Usage: distree [OPTIONS] - -Arguments: - Path to the tree file in Newick format - -Options: - --format Tree file format (only 'newick' is supported) [default: newick] - --midpoint Midpoint-root the tree before computing distances - --lmm Produce the var-covar matrix C (depth of the MRCA) - --topology Ignore branch lengths; use purely topological distances - -o, --output Path to write the TSV output file (defaults to stdout) - -h, --help Print help information - -V, --version Print version information -``` - -### Argument & Option Details - -* ``: Path to the input tree in Newick format. Leaf labels must be unique and not contain tabs or newline characters. - -* `--format ` - - * Currently only `newick` is supported. Future versions may support other formats (e.g., Nexus, PhyloXML). - * This option exists to maintain CLI consistency; it does not change parsing for now. - -* `--midpoint` - - * Before computing any distances, the tree will be re-rooted at its midpoint (the point halfway along the longest path between any two leaves). Useful when no outgroup is known or when you want a balanced root for downstream analyses. - * Use this if your downstream distance metric expects an unrooted, centrically-rooted tree. - -* `--lmm` - - * “LMM” stands for var-covar matrix (matrix C) in Phylogenetic Comparative Methods. Each entry (i, j) equals the depth (distance from root) of the lowest common ancestor of leaf i and leaf j. This matrix is often used in linear mixed models (LAMM, PGLS) to account for phylogenetic covariance. - * When specified, LMM distances override `--topology`. The output is a matrix of MRCA depths, not pairwise distances. - -* `--topology` - - * Ignores branch lengths. Each entry (i, j) equals the number of edges between leaf i and leaf j: - - * Use this mode if branch-lengths are not meaningful or if you only care about tree shape. - -* `-o, --output ` - - * Write the TSV distance matrix to the specified path. If omitted, the matrix is printed to standard output. - * Example: `-o distances.tsv`. - -## Output Format - -The tool outputs a tab-separated values (TSV) matrix. - -1. The first line is a header with leaf labels sorted alphabetically, preceded by an empty cell (for row labels). - -2. Each subsequent line begins with a leaf label (sorted alphabetically), followed by N columns of distances (depending on the chosen mode) to every other leaf in the same sorted order. - -Example (patristic distances): - -```LeafA LeafB LeafC -LeafA 0.000 5.200 3.100 -LeafB 5.200 0.000 4.400 -LeafC 3.100 4.400 0.000 -``` - -Example (topological distances): - -```LeafA LeafB LeafC LeafD -LeafA 0 2 3 1 -LeafB 2 0 1 3 -LeafC 3 1 0 4 -LeafD 1 3 4 0 -``` - -Example (LMM depths): - -```LeafA LeafB LeafC -LeafA 7.000 3.000 5.000 -LeafB 3.000 7.000 2.500 -LeafC 5.000 2.500 7.000 -``` - -## Detailed Use Cases - -### 1. Producing a Patristic Distance Matrix - -**Scenario**: You have a large set of bacterial genomes, build a phylogenetic tree with reliable branch lengths (e.g., using RAxML or IQ-TREE), and now need the pairwise patristic distances to feed into clustering, PCoA, or hierarchical analyses. - -**Command**: +# Bioconda +conda install -c bioconda distree -```bash -./distree tree.nwk -o patristic.tsv +# From source +cargo build --release ``` -**Why**: Downstream tools like SciKit-Learn (for MDS) or R’s `ape::cmdscale()` expect a distance matrix. Patristic distances reflect evolutionary time or change. +Prebuilt binaries for Linux and macOS, x86-64 and arm64, are on the +[releases page](https://github.com/PathoGenOmics-Lab/distree/releases). -### 2. Computing Topological Distances Only - -**Scenario**: You have a phylogenetic tree where branch lengths come from different sources or are not directly comparable (e.g., concatenated multi-locus data). You prefer to measure closeness by number of shared nodes instead. - -**Command**: +## Quick start ```bash -./distree --topology tree.nwk -o topo_distances.tsv -``` - -**Why**: Topological distances emphasize tree structure without scaling by substitution rate or time. Useful when comparing tree shapes or for rapid, coarse clustering when exact branch lengths are noisy. +# Patristic distances, as a tab-separated matrix +distree tree.nwk -o distances.tsv -### 3. Generating an LMM (Var-Covar) Matrix for Comparative Methods +# Six decimals across eight threads, PHYLIP lower triangle +distree tree.nwk --lower -p 6 -t 8 -o distances.phy -**Scenario**: You are performing phylogenetic generalized least squares (PGLS) in R (`caper::pgls` or `nlme::gls`) and need the phylogenetic variance-covariance matrix. Each entry C\[i,j] is the distance from root to MRCA(i, j). +# Edge counts instead of branch lengths +distree tree.nwk --topology -o topology.tsv -**Command**: +# The variance-covariance matrix for PGLS, midpoint-rooted +distree tree.nwk --midpoint --lmm -o varcovar.tsv -```bash -./distree --lmm tree.nwk -o varcovar.tsv -``` - -**Why**: In comparative models, traits shared due to common ancestry introduce covariance. This LMM matrix directly encodes that covariance structure for all taxa. +# A NumPy array: exact, half the size of text, and 2.6x faster +distree tree.nwk --npy -o distances.npy -### 4. Using Midpoint Rooting Before Distance Extraction +# The condensed vector scipy.cluster.hierarchy reads directly +distree tree.nwk --lower --npy -o condensed.npy -**Scenario**: Your input tree is unrooted or rooted arbitrarily (e.g., by outgroup choice). You want a symmetric distance matrix that does not depend on outgroup, so you midpoint-root the tree first. +# Just the samples in a list, from a gzipped tree +distree tree.nwk.gz --taxa cohort.txt --stats -o cohort.tsv -**Command**: - -```bash -./distree --midpoint tree.nwk -o midrooted_distances.tsv +# From stdin +gunzip -c tree.nwk.gz | distree - -o distances.tsv ``` -**Why**: Midpoint rooting places the root at a balanced position. Downstream patristic or LMM calculations become more interpretable if the root is centrally placed. +## What it does -### 5. Downstream Dimensionality Reduction & Visualization +| | | +|:--|:--| +| **Patristic** | The sum of branch lengths on the path between two tips | +| **Topological** | The number of edges on that path, ignoring branch lengths | +| **Variance-covariance** | The root-to-MRCA distance for each pair, the `C` matrix of a PGLS or phylogenetic mixed model | +| **Rooting** | Optional midpoint rooting, which matters for `--lmm` and cannot matter for the other two | +| **Formats** | Square TSV, PHYLIP lower triangle, or a NumPy `.npy` array | +| **Input** | Newick from a file or stdin, plain or gzipped, with quoted labels, polytomies, NHX and BEAST comments, and UTF-8 tip names | +| **Subsets** | `--taxa` restricts the matrix to a list of tips, with the distances the full tree gives | +| **Scale** | Rows stream out as they are computed; memory tracks the tips, not the matrix | +| **Parallelism** | Distances and their formatting both spread across cores | -**Scenario**: You intend to visualize relationships among taxa via MDS or t-SNE. You need a distance matrix as input. +It works on the tree you give it: it does not build one, does not read an +alignment, and does not cluster the matrix for you. It also does not guess. A +truncated tree, a file holding several trees, an unclosed quote or a label with +a tab in it are rejected with the position, rather than turned into a matrix +that looks right and is not. -**Command**: +## Performance -```bash -./distree tree.nwk | Rscript -e "dist <- as.matrix(read.table('file:///dev/stdin', header=TRUE, row.names=1)); mds <- cmdscale(dist); plot(mds)" -``` +Balanced trees, `-p 6 --lower`, release build on an Apple M4 Pro with 14 cores. -**Why**: Feeding the distance matrix directly into R for MDS (multidimensional scaling) or UMAP allows you to view clustering of taxa in two or three dimensions. +| Tips | Cells | Time | Peak memory | +|--:|--:|--:|--:| +| 1,000 | 0.5 M | 0.00 s | 11 MB | +| 8,000 | 32 M | 0.13 s | 32 MB | +| 20,000 | 200 M | 1.02 s | 35 MB | -## Performance and Resource Considerations +Memory flattens out because nothing holds the matrix: past a few thousand tips +it is the LCA table plus one batch of rows. `--npy` is another 2.6x on top, at +full precision and half the file size. Full numbers, thread scaling and the +memory arithmetic are +[here](https://pathogenomics-lab.github.io/distree/how-it-works/performance/). -* **Memory**: `distree` streams one row at a time. At any given moment, only a single vector of length N (number of leaves) resides in memory, plus O(M log M) for LCA structures, where M is the total number of nodes. For trees with tens of thousands of taxa, memory usage remains low. +## Correctness -* **Parallelism**: Each row’s distance computations are parallelized across available CPU cores via Rayon. For N taxa, computing N rows (each of size N) takes O(N^2 / #cores) time. +Distances are cross-validated against R's +[ape](https://cran.r-project.org/package=ape) over generated trees: +`cophenetic.phylo` for patristic, `vcv.phylo` for variance-covariance, +`cophenetic.phylo` over unit branch lengths for edge counts, and +`phangorn::midpoint` for the rooting. All four agree to under 1e-9, which is the +text round-trip, and exactly for edge counts. The parser and the pipeline are +fuzzed. See +[Contributing](https://pathogenomics-lab.github.io/distree/about/contributing/). -* **Disk I/O**: If writing to a file via `--output`, a buffered writer (`BufWriter`) minimizes I/O calls. Streaming directly to stdout also remains efficient. +## Citation -## Troubleshooting and Tips +> Ruiz-Rodriguez P, Coscolla M. *distree: distance matrices from a phylogeny.* +> PathoGenOmics Lab. [doi:10.5281/zenodo.16811766](https://doi.org/10.5281/zenodo.16811766) -* **Invalid Newick**: Ensure your Newick tree is syntactically correct (matching parentheses, semicolon at end). `distree` will error if parsing fails. -* **Whitespace in Labels**: Leaf labels should not contain spaces, tabs, or newline characters, as these break the TSV format. Replace spaces with underscores if needed. -* **Choosing Distance Type**: +Please record the version and the mode you used: a patristic matrix, a +topological one and a variance-covariance matrix are three different objects. - * Use `--lmm` if performing phylogenetic comparative analyses (e.g., trait evolution, PGLS). - * Use default patristic distances for standard evolutionary distance-based methods (e.g., Neighbor-Joining, clustering). - * Use `--topology` for quick, coarse tree-shape comparisons when branch lengths are unreliable. -* **Large Trees**: For very large trees (e.g., >50,000 leaves), ensure you have enough CPU cores. You may run `distree` on a compute node or multi-core server to exploit parallel speedups. - ---- +## License -*distree* is a versatile tool for extracting various phylogenetic distance matrices from large trees. By combining midpoint rooting, multiple distance modes, and parallel streaming, it caters to evolutionary, comparative, and clustering analyses without overwhelming memory. +[GPL-3.0](https://github.com/PathoGenOmics-Lab/distree/blob/main/LICENSE). ---

-✨ [Contributors]((https://github.com/PathoGenOmics-Lab/AMAP/graphs/contributors)) +✨ [Contributors](https://github.com/PathoGenOmics-Lab/distree/graphs/contributors)

@@ -272,7 +164,7 @@ distree is developed with ❤️ by: 🔣 🎨 🔧 - + @@ -285,7 +177,7 @@ distree is developed with ❤️ by: 🧑‍🏫 🔬 📓 - + diff --git a/docs/about/changelog.md b/docs/about/changelog.md new file mode 100644 index 0000000..0536cbb --- /dev/null +++ b/docs/about/changelog.md @@ -0,0 +1,5 @@ +--- +title: Changelog +--- + +--8<-- "CHANGELOG.md" diff --git a/docs/about/citation.md b/docs/about/citation.md new file mode 100644 index 0000000..ba274e8 --- /dev/null +++ b/docs/about/citation.md @@ -0,0 +1,64 @@ +# Citation + +distree is archived on Zenodo. Cite the release you used: + +> Ruiz-Rodriguez P, Coscolla M. *distree: distance matrices from a phylogeny.* +> PathoGenOmics Lab. [doi:10.5281/zenodo.16811766](https://doi.org/10.5281/zenodo.16811766) + +The repository carries a [`CITATION.cff`](https://github.com/PathoGenOmics-Lab/distree/blob/main/CITATION.cff), +so GitHub's "Cite this repository" button and most reference managers will +produce the entry for you. + +Please record the version (`distree --version`) and the mode you used. A +patristic matrix, a topological one and a variance-covariance matrix are three +different objects, and "distances from the tree" does not say which. + +## Authors + +**Paula Ruiz-Rodriguez** and **Mireia Coscolla**, I²SysBio, University of +Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, +Valencia, Spain. + +## Background + +The quantities distree computes are standard, and the terms are worth pinning +down. + +**Patristic distance** is the sum of branch lengths along the path between two +tips. The name is Farris's. + +> Farris JS. *A Method for Computing Wagner Trees.* Systematic Zoology. +> 1970;19(1):83-92. + +**The variance-covariance matrix** that `--lmm` writes is the covariance +structure implied by Brownian-motion trait evolution on a tree, which is what +makes it the `C` of a phylogenetic generalised least squares fit. + +> Felsenstein J. *Phylogenies and the Comparative Method.* The American +> Naturalist. 1985;125(1):1-15. + +> Grafen A. *The phylogenetic regression.* Philosophical Transactions of the +> Royal Society B. 1989;326(1233):119-157. + +**Midpoint rooting** places the root at the middle of the longest tip-to-tip +path. + +> Farris JS. *Estimating Phylogenetic Trees from Distance Matrices.* The +> American Naturalist. 1972;106(951):645-668. + +**Binary lifting** is the technique behind the `O(log n)` ancestor queries. The +sparse-table formulation of the least-common-ancestor problem goes back to: + +> Bender MA, Farach-Colton M. *The LCA Problem Revisited.* LATIN 2000, Lecture +> Notes in Computer Science 1776, pp. 88-94. + +## Formats and downstream tools + +The `--lower` output is PHYLIP's lower-triangular distance format: + +> Felsenstein J. *PHYLIP: Phylogeny Inference Package (Version 3.2).* Cladistics. +> 1989;5:164-166. + +## License + +[GPL-3.0](https://github.com/PathoGenOmics-Lab/distree/blob/main/LICENSE). diff --git a/docs/about/contributing.md b/docs/about/contributing.md new file mode 100644 index 0000000..aec7ecd --- /dev/null +++ b/docs/about/contributing.md @@ -0,0 +1,133 @@ +# Contributing + +Issues and pull requests are welcome at +[PathoGenOmics-Lab/distree](https://github.com/PathoGenOmics-Lab/distree). + +## Reporting a problem + +The most useful bug report is one somebody else can run. Include: + +- the exact command, with every flag; +- the version (`distree --version`) and how it was installed; +- the tree, or a smaller tree that still shows the problem; +- what you expected and what you got. + +The dangerous failures here are the quiet ones. A matrix that is wrong looks +exactly like a matrix that is right, so a report saying "these distances +disagree with `ape::cophenetic` and here are both" is worth more than a stack +trace. + +If the tree carries anything identifiable, do not attach it. Tip labels can be +replaced with `S1`, `S2` and so on without changing the behaviour, and a +generated tree of the same shape usually reproduces the problem. + +## Building and checking + +```bash +cargo build --release +cargo test +cargo test --release +cargo clippy --all-targets -- -D warnings +``` + +CI runs all four on every push and every pull request. A change that fails any +of them will not go in. + +## Tests + +The suite is in three places: + +- **Unit tests** next to the code they cover, in `src/parser.rs`, `src/lca.rs`, + `src/midpoint.rs` and `src/main.rs`. +- **Integration tests** in `tests/integration.rs`, which run the built binary + and check its stdout, its stderr and its exit code. +- **Randomised property checks**, which are where the confidence comes from. + `src/testutil.rs` generates trees with polytomies and zero-length branches + from a seeded xorshift, and the checks assert properties rather than + particular numbers: midpoint rooting preserves every pairwise distance and + leaves a valid tree with the deepest tip half a diameter from the root; the + binary-lifting MRCA agrees with walking up from both nodes, over every pair of + nodes; and a patristic distance is never negative on a tree with non-negative + branch lengths. +- **Cross-validation against ape** in `tests/crossvalidation.rs`, checked + against reference matrices committed under `tests/fixtures/`. +- **Mutation testing** in `tests/torture.rs`, which throws truncated, flipped + and duplicated Newick at the parser and the pipeline. + +### Cross-validating against ape + +Everything above is written against the same understanding of the problem as +the code, so none of it would catch a distance being *defined* wrongly. ape is +the independent check. With R, `ape` and `phangorn` installed: + +```bash +cargo build --release +Rscript scripts/crossvalidate.R 250 target/release/distree +``` + +It compares all four modes against `cophenetic.phylo`, `vcv.phylo`, +`cophenetic.phylo` over unit branch lengths, and `phangorn::midpoint`. The +expected result is agreement to under 1e-9, which is the 12-decimal text +round-trip, and exactly zero for edge counts. + +To refresh the committed fixtures after a deliberate change: + +```bash +Rscript scripts/crossvalidate.R 6 target/release/distree --write-fixtures +``` + +The same comparison runs as a weekly GitHub Actions job, and on demand from the +Actions tab. + +### Fuzzing + +`tests/torture.rs` runs on every push and needs nothing extra. The +coverage-guided version needs a nightly toolchain: + +```bash +cargo +nightly install cargo-fuzz +cargo +nightly fuzz run parse_newick fuzz/seeds +cargo +nightly fuzz run full_pipeline fuzz/seeds +``` + +`parse_newick` throws arbitrary text at the parser; `full_pipeline` takes +everything that parses and pushes it through flattening, rooting and every +distance mode. Both assert only that nothing panics and nothing hangs, which is +a low bar and has already caught a real bug: a branch length of `1e910` parsed +as infinity and turned the matrix into `inf` with `NaN` down the diagonal. + +A parser change wants a case in `src/parser.rs` for what should now parse **and** +one for what should still be rejected. The failures worth guarding are the ones +where a malformed tree produced a plausible matrix instead of an error. + +## Writing a fix + +A few conventions the codebase follows: + +- **Refuse rather than guess.** If the input is ambiguous or truncated, error + with the position. A matrix that looks right and is not costs more than a + failed run. +- **Warnings go to stderr**, never into the matrix, so a piped run stays clean. +- **Keep the tree traversals iterative.** The parser, the flattener, the LCA + build and even `RawNode`'s `Drop` are loops with explicit stacks, because a + ladder-shaped tree hundreds of thousands of levels deep is a real input. +- **Do not put work in the serial write loop.** Whatever a worker can do to its + own row, it should, including formatting the numbers. + +## Documentation + +This site is MkDocs Material, and the pages live in `docs/`: + +```bash +pip install -r requirements-docs.txt +mkdocs serve +``` + +```bash +mkdocs build --strict +``` + +`--strict` turns a broken internal link or a dead anchor into a failed build, +which is what CI runs. Numbers quoted in the docs, in the benchmark tables above +all, should be measured rather than estimated, and the page should say which +when it is not obvious. diff --git a/docs/assets/distree_logo.svg b/docs/assets/distree_logo.svg new file mode 100644 index 0000000..c6192cd --- /dev/null +++ b/docs/assets/distree_logo.svg @@ -0,0 +1,187 @@ + + + + diff --git a/docs/assets/distree_wordmark.svg b/docs/assets/distree_wordmark.svg new file mode 100644 index 0000000..3d087f2 --- /dev/null +++ b/docs/assets/distree_wordmark.svg @@ -0,0 +1,99 @@ + + + + diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 0000000..72fc621 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,80 @@ +# Installation + +distree is a single self-contained binary. It has no runtime dependencies, no +interpreter, and no configuration file. + +## From Bioconda + +=== "conda" + + ```bash + conda install -c bioconda distree + ``` + +=== "mamba" + + ```bash + mamba install -c bioconda distree + ``` + +=== "pixi" + + ```bash + pixi add --channel bioconda distree + ``` + +## From a release binary + +Each release publishes binaries for Linux and macOS, on both x86-64 and arm64, +on the [releases page](https://github.com/PathoGenOmics-Lab/distree/releases). +Download the one for your platform, make it executable, and put it on your +`PATH`: + +```bash +curl -LO https://github.com/PathoGenOmics-Lab/distree/releases/latest/download/distree-linux-x86_64 +chmod +x distree-linux-x86_64 +mv distree-linux-x86_64 ~/.local/bin/distree +``` + +The four assets are `distree-linux-x86_64`, `distree-linux-aarch64`, +`distree-macos-x86_64` and `distree-macos-aarch64`. + +!!! note "macOS Gatekeeper" + + The binaries are not notarised, so macOS quarantines a downloaded one. Clear + the flag with `xattr -d com.apple.quarantine distree` before the first run. + +## From source + +Any stable Rust toolchain will do; [rustup](https://rustup.rs) is the usual way +to get one. + +```bash +git clone https://github.com/PathoGenOmics-Lab/distree.git +cd distree +cargo build --release +``` + +The binary lands at `target/release/distree`. Copy it wherever you keep local +tools, or run it in place. + +## Checking the install + +```bash +distree --version +echo '((A:1.0,B:2.0):0.5,C:3.0);' | distree - +``` + +``` + A B C +A 0.0000000000 3.0000000000 4.5000000000 +B 3.0000000000 0.0000000000 5.5000000000 +C 4.5000000000 5.5000000000 0.0000000000 +``` + +A tab-separated square matrix, tips sorted alphabetically, with an empty cell in +the top-left corner so the header lines up over the data columns. + +## Next + +[Quick start](quickstart.md) covers the commands most runs actually use. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 0000000..cad3569 --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,149 @@ +# Quick start + +Every example on this page uses the same four-tip tree, so the matrices can be +compared directly: + +``` +((LeafA:1.95,LeafB:3.25):0.35,(LeafC:0.80,LeafD:1.20):0.50); +``` + +## The default: patristic distances + +```bash +distree tree.nwk -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 0.000 | 5.200 | 3.600 | 4.000 | +| **LeafB** | 5.200 | 0.000 | 4.900 | 5.300 | +| **LeafC** | 3.600 | 4.900 | 0.000 | 2.000 | +| **LeafD** | 4.000 | 5.300 | 2.000 | 0.000 | + +
+ +Each cell is the sum of the branch lengths on the path between the two tips. +LeafA to LeafB is `1.95 + 3.25 = 5.20`; LeafA to LeafC crosses the root, +`1.95 + 0.35 + 0.50 + 0.80 = 3.60`. + +`-p` sets the decimal places. The default is 10, which is more than most branch +lengths carry any information at; 6 is a reasonable working value and 3 is +enough to read by eye. + +## Reading from stdin + +`-` reads the tree from standard input, so distree drops into a pipeline: + +```bash +gunzip -c tree.nwk.gz | distree - -o distances.tsv +``` + +## Ignoring branch lengths + +```bash +distree tree.nwk --topology +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 0 | 2 | 4 | 4 | +| **LeafB** | 2 | 0 | 4 | 4 | +| **LeafC** | 4 | 4 | 0 | 2 | +| **LeafD** | 4 | 4 | 2 | 0 | + +
+ +The number of edges between the two tips, as integers. Useful when the branch +lengths come from different sources, or when only the shape of the tree is +meant to matter. + +## The variance-covariance matrix + +```bash +distree tree.nwk --lmm -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.300 | 0.350 | 0.000 | 0.000 | +| **LeafB** | 0.350 | 3.600 | 0.000 | 0.000 | +| **LeafC** | 0.000 | 0.000 | 1.300 | 0.500 | +| **LeafD** | 0.000 | 0.000 | 0.500 | 1.700 | + +
+ +Entry `(i, j)` is the distance from the root down to the most recent common +ancestor of `i` and `j`: how much evolutionary history the two tips share. The +diagonal is each tip's own root-to-tip length, and any pair whose ancestor is +the root scores 0. This is the `C` matrix that PGLS and phylogenetic mixed +models expect, and it is the one mode where [where you root the +tree](../guide/rooting.md) changes the answer. + +## PHYLIP lower triangle + +```bash +distree tree.nwk --lower -p 4 +``` + +``` +4 +LeafA +LeafB 5.2000 +LeafC 3.6000 4.9000 +LeafD 4.0000 5.3000 2.0000 +``` + +A taxa count, then one row per taxon holding its label and the distances to +every taxon above it. Half the cells and no diagonal, which is what PHYLIP, +Mash and most distance-matrix readers want. + +## Midpoint rooting + +```bash +distree tree.nwk --midpoint --lmm -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.550 | 0.000 | 0.600 | 0.600 | +| **LeafB** | 0.000 | 2.650 | 0.000 | 0.000 | +| **LeafC** | 0.600 | 0.000 | 2.250 | 1.450 | +| **LeafD** | 0.600 | 0.000 | 1.450 | 2.650 | + +
+ +The longest path in this tree runs from LeafB to LeafD and is 5.30 long, so the +new root goes 2.65 from each: partway along LeafB's own branch. Every tip is now +at most 2.65 from the root, and the shared history of the other three tips has +been reorganised around it. + +!!! tip "It cannot change a patristic matrix" + + Rooting moves the root, not the tips. The path between two tips is the same + path whatever you call the top of the tree, so `--midpoint` leaves a + patristic or topological matrix exactly as it was. Reach for it when you are + producing `--lmm`. See [Midpoint rooting](../guide/rooting.md). + +## Controlling threads and output + +```bash +distree tree.nwk -t 8 -o distances.tsv +``` + +`-t` caps the thread count; without it distree uses every core. `-o` writes to a +file instead of stdout, and is not created until the tree has parsed, so a +failed run leaves whatever was there before untouched. + +## Next + +- [Input](../guide/input.md), for what distree accepts and what it turns down. +- [Distance modes](../guide/distances.md), for the three modes in full. +- [Recipes](../recipes.md), for feeding the matrix to R, Python or PHYLIP. diff --git a/docs/guide/cli.md b/docs/guide/cli.md new file mode 100644 index 0000000..05022f2 --- /dev/null +++ b/docs/guide/cli.md @@ -0,0 +1,166 @@ +# CLI reference + +``` +Usage: distree [OPTIONS] +``` + +## Argument + +`` + +: Path to the tree file in Newick format, plain or gzipped. Use `-` to read + from stdin. The file must hold exactly one tree, and its tip labels must be + unique. See [Input](input.md). + +## Options + +| Flag | Default | Effect | +|:--|:--|:--| +| `--midpoint` | off | Midpoint-root the tree before computing distances. Ignored with `--topology`. See [Midpoint rooting](rooting.md) | +| `--lmm` | off | Write the variance-covariance matrix: each cell is the root-to-MRCA distance. Takes precedence over `--topology` | +| `--topology` | off | Ignore branch lengths and count edges. Values are written as integers | +| `--lower` | off | Write a PHYLIP lower triangle: a taxa count, then one row per taxon with no diagonal. With `--npy`, the condensed vector instead | +| `--npy` | off | Write a NumPy `.npy` array of 64-bit floats instead of text. Requires `-o`; labels go to `.labels.txt` | +| `--taxa FILE` | all tips | Restrict the matrix to the labels listed in `FILE`, one per line | +| `--stats` | off | Print a summary of the run to stderr | +| `-o, --output FILE` | stdout | Write the matrix to a file. Not created until the tree has parsed | +| `-p, --precision N` | `10` | Decimal places, from 0 to 30. Ignored by `--topology` and by `--npy` | +| `-t, --threads N` | all cores | Threads for the parallel row computation. Must be at least 1 | +| `-h, --help` | | Print help | +| `-V, --version` | | Print version | + +Without `--lmm` or `--topology`, distree computes patristic distances. + +## Selecting a subset + +```bash +distree tree.nwk --taxa keep.txt -o subset.tsv +``` + +`keep.txt` holds one leaf label per line. Blank lines and lines starting with +`#` are skipped, and duplicates are collapsed. Order does not matter: the matrix +comes out alphabetical as always. + +A label that is not a leaf of the tree is an error naming it, rather than a +matrix quietly smaller than the list. + +The distances themselves are the ones from the full tree. A subset filters the +output, it does not prune the tree, so the path between two tips is the same +whether or not the tips in between are in the matrix. That is the opposite of +what pruning the tree first would give you. + +## The run summary + +```bash +distree tree.nwk --stats -o distances.tsv +``` + +``` +--- Statistics --- +Leaves in matrix: 7 +Nodes in tree: 13 +Mode: patristic distance +Cells written: 49 +Minimum: 0.0000004534 +Maximum: 0.0000612030 +Mean: 0.0000358688 +Time: 0.001s +``` + +It goes to stderr, so a piped run is unaffected. The minimum, maximum and mean +are taken off the diagonal, which is zero by construction in the distance modes +and would only pull the summary towards it. When `--taxa` is in use, the summary +reports both counts. + +The numbers are accumulated by the workers as they compute, so the summary costs +one pass and nothing measurable. + +## How the modes interact + +| Combination | Result | +|:--|:--| +| `--lmm --topology` | `--lmm` wins, with a warning. They ask for different things | +| `--midpoint --topology` | The rooting is skipped, with a warning. Edge counts do not depend on the root, and the inserted node would add a hop | +| `--midpoint --lmm` | Both apply. This is the combination `--midpoint` exists for | +| `--midpoint` alone | Applies, and changes nothing: a patristic matrix is the same under any rooting | +| `--lower` with any mode | Applies to all three | +| `-p` with `--topology` | Ignored; edge counts are integers | +| `--npy` without `-o` | An error. Binary output needs a file | +| `--npy` with `--lower` | Writes the condensed vector SciPy reads, not PHYLIP's triangle | +| `--npy` with `-p` | The precision is ignored; a 64-bit float is exact | +| `--npy` with `--topology` | Works, with a warning that integers are stored as floats | +| `--taxa` with any mode | Applies to all three, and to both output formats | + +## Exit codes + +| Code | Meaning | +|:--|:--| +| 0 | Success, including a downstream pipe closing early | +| 1 | Failure. The message is on stderr and begins with `Error:` | + +## Messages + +Everything below goes to stderr, never into the matrix. + +### Errors + +| Message | Cause | +|:--|:--| +| `Cannot open '': ...` | The tree file is missing or unreadable | +| `... is gzip-compressed but could not be read: ...` | The gzip stream is truncated or corrupt | +| `... is not valid UTF-8 text` | Not a text file, and not gzip either | +| `Empty input: no Newick tree found.` | The file is empty or only whitespace | +| `Branch length '' at position N is not a finite number.` | An exponent past the range of a 64-bit float, which would make every distance infinite | +| `Branch lengths are too large to compute distances with` | The depths are finite but summing two of them overflows | +| ` of the labels in '' are not leaves of the tree` | `--taxa` names something the tree does not have | +| `Cannot read the taxa list '': ...` | `--taxa` file missing or unreadable | +| `--npy writes binary, so it needs a file` | `--npy` without `-o` | +| `Failed to parse Newick tree: ...` | Malformed tree. The rest of the message names the position | +| `No labeled leaves found in the tree.` | The tree parsed but no tip carries a label | +| `Duplicate leaf name '' found.` | Two tips share a label; the matrix could not be indexed | +| `Leaf name '' contains a tab character / a newline / a carriage return` | The label would split a row | +| `--precision must be between 0 and 30, got N.` | `-p` out of range | +| `--threads must be at least 1.` | `-t 0` | +| `Cannot write to '': ...` | The output path is not writable | +| `Failed to write '': ...` | The write failed partway, typically a full disk | +| `Failed to initialize thread pool: ...` | Rayon could not start the requested threads | + +### Warnings + +| Message | Meaning | +|:--|:--| +| `negative branch lengths detected` | Some distances may come out negative, and `--midpoint` cannot locate the diameter reliably | +| `no branch lengths detected, all patristic distances will be zero` | A cladogram. Use `--topology` | +| `N leaf/leaves have no label and were excluded from the matrix` | Unlabelled tips cannot be named in a row, so they are left out | +| `--lmm and --topology are mutually exclusive. Using --lmm.` | Both were passed | +| `--midpoint is ignored in --topology mode.` | Both were passed | +| `--lower omits the diagonal, which in --lmm mode holds each leaf's root-to-tip length` | The lower triangle drops data that is not zeros under `--lmm` | +| `--npy writes 64-bit floats, so --topology edge counts are stored as floats` | Both were passed | + +## Examples + +```bash +# Patristic distances to a file, six decimals +distree tree.nwk -p 6 -o distances.tsv + +# PHYLIP lower triangle for neighbor +distree tree.nwk --lower -p 6 -o infile + +# Variance-covariance matrix for PGLS, midpoint-rooted +distree tree.nwk --midpoint --lmm -p 8 -o varcovar.tsv + +# Edge counts from a cladogram +distree cladogram.nwk --topology -o topo.tsv + +# A gzipped tree, capped at 4 threads +distree tree.nwk.gz -t 4 -o distances.tsv + +# Just the 200 samples in a list, as a NumPy array +distree tree.nwk --taxa cohort.txt --npy -o cohort.npy + +# The condensed vector scipy.cluster.hierarchy takes directly +distree tree.nwk --lower --npy -o condensed.npy + +# Just the header, to check the tip ordering +distree tree.nwk | head -1 | tr '\t' '\n' | tail -n +2 +``` diff --git a/docs/guide/distances.md b/docs/guide/distances.md new file mode 100644 index 0000000..f73e518 --- /dev/null +++ b/docs/guide/distances.md @@ -0,0 +1,161 @@ +# Distance modes + +distree computes three different things, and they answer three different +questions. All of them come from the same primitive: for a pair of tips `i` and +`j`, find their most recent common ancestor `m`, then read off depths that were +precomputed in a single pass. + +| Mode | Flag | Cell `(i, j)` | +|:--|:--|:--| +| Patristic | *(default)* | `depth(i) + depth(j) - 2 · depth(m)` | +| Topological | `--topology` | `hops(i) + hops(j) - 2 · hops(m)` | +| Variance-covariance | `--lmm` | `depth(m)` | + +`depth` is the summed branch length from the root; `hops` is the number of edges +from the root. Every mode is one MRCA query plus three array reads, which is why +the cost of a cell does not grow with how far apart the tips are. + +The examples below all use: + +``` +((LeafA:1.95,LeafB:3.25):0.35,(LeafC:0.80,LeafD:1.20):0.50); +``` + +## Patristic + +```bash +distree tree.nwk -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 0.000 | 5.200 | 3.600 | 4.000 | +| **LeafB** | 5.200 | 0.000 | 4.900 | 5.300 | +| **LeafC** | 3.600 | 4.900 | 0.000 | 2.000 | +| **LeafD** | 4.000 | 5.300 | 2.000 | 0.000 | + +
+ +The sum of the branch lengths along the path between the two tips: how much +change has accumulated between them along the tree. This is the mode nearly +every downstream use wants, from transmission clustering to ordination. + +If the tree was built with a substitution model, a patristic distance is in +substitutions per site. Multiply by the alignment length for a SNP-scale +distance: + +```bash +distree tree.nwk -p 10 | awk 'NR==1 {print; next} {printf "%s", $1; + for (i=2; i<=NF; i++) printf "\t%.1f", $i * 4411532; print ""}' +``` + +The diagonal is exactly `0.0` and the matrix is exactly symmetric, both by +construction rather than by rounding. + +!!! note "Negative distances" + + With non-negative branch lengths, the subtraction cannot come out negative: + `depth` accumulates from the root, so `depth(i)` and `depth(j)` are both at + least `depth(m)`, and the arithmetic preserves that. A negative cell + therefore means the tree has a negative branch length, which + neighbour-joining produces routinely. distree reports it rather than + rounding it up to zero, since a zero would claim two distinct taxa are the + same sample. + +## Topological + +```bash +distree tree.nwk --topology +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 0 | 2 | 4 | 4 | +| **LeafB** | 2 | 0 | 4 | 4 | +| **LeafC** | 4 | 4 | 0 | 2 | +| **LeafD** | 4 | 4 | 2 | 0 | + +
+ +The number of edges on the path, written as integers with no decimal point and +no `-p`. LeafA and LeafB are siblings, so 2. LeafA to LeafC goes up two and back +down two, so 4. + +Use it when the branch lengths are not comparable across the tree, when they +come from concatenated loci with different rates, or when only the shape of the +tree is meant to matter. A cladogram with no lengths at all is exactly this +case: patristic mode would return an all-zero matrix and warns as much. + +Two things follow from counting edges rather than lengths. A polytomy shortens +distances, because the tips under it are one hop apart rather than several. And +the count does not depend on where the tree is rooted, which is why +[`--midpoint` is ignored here](rooting.md#why-midpoint-is-ignored-in-topology-mode). + +## Variance-covariance + +```bash +distree tree.nwk --lmm -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.300 | 0.350 | 0.000 | 0.000 | +| **LeafB** | 0.350 | 3.600 | 0.000 | 0.000 | +| **LeafC** | 0.000 | 0.000 | 1.300 | 0.500 | +| **LeafD** | 0.000 | 0.000 | 0.500 | 1.700 | + +
+ +Entry `(i, j)` is the distance from the root down to the MRCA of the two tips: +the length of the evolutionary history they share before they diverge. Under a +Brownian-motion model of trait evolution, that is exactly the covariance between +their trait values, which is what makes this the `C` matrix of a phylogenetic +generalised least squares fit or a phylogenetic mixed model. + +Reading the matrix above: + +- `C[LeafA][LeafA] = 2.300` is LeafA's own root-to-tip length, `0.35 + 1.95`. + The diagonal is always the tip's total path from the root, because a tip's + MRCA with itself is itself. +- `C[LeafA][LeafB] = 0.350` is the depth of the node the two share. +- `C[LeafA][LeafC] = 0.000`, because their MRCA is the root, and they share no + history at all. + +Three consequences are worth keeping in mind: + +- **The matrix is not a distance.** The diagonal is not zero and larger values + mean more similarity, not less. Do not feed it to something expecting + distances. +- **Rooting changes it.** `depth` is measured from the root, so where the root + sits is part of the answer. This is the only mode where `--midpoint` does + anything. +- **An ultrametric tree gives a constant diagonal.** If every tip is the same + distance from the root, every `C[i][i]` is that distance, which is what most + comparative methods assume. + +Patristic and variance-covariance are two views of the same tree and convert +into each other: `d(i,j) = C[i][i] + C[j][j] - 2·C[i][j]`. Check it on the +matrices above: `2.300 + 3.600 - 2 × 0.350 = 5.200`, which is what the patristic +matrix says for LeafA and LeafB. + +## Choosing + +| If you are | Use | +|:--|:--| +| Clustering isolates by genetic distance | Patristic | +| Running PCoA, MDS, t-SNE or UMAP on the tree | Patristic | +| Building a distance-based tree from an existing one | Patristic, with [`--lower`](output.md#phylip-lower-triangle) | +| Fitting PGLS, `caper::pgls`, `nlme::gls` or a phylogenetic mixed model | `--lmm` | +| Comparing tree shapes, or working from a cladogram | `--topology` | +| Working with branch lengths from incomparable sources | `--topology` | + +## Next + +- [Midpoint rooting](rooting.md), for when the root is part of the answer. +- [Output](output.md), for the formats these matrices come out in. diff --git a/docs/guide/input.md b/docs/guide/input.md new file mode 100644 index 0000000..26f73b8 --- /dev/null +++ b/docs/guide/input.md @@ -0,0 +1,166 @@ +# Input + +distree reads one Newick tree, from a file or from stdin, plain or gzipped: + +```bash +distree tree.nwk +distree tree.nwk.gz +gunzip -c tree.nwk.gz | distree - +``` + +Compression is detected from the file's leading bytes rather than its name, so +it works from stdin and on a `.gz` file that was renamed. + +## What is accepted + +**Branch lengths** in any form Rust's float parser takes, including scientific +notation and negatives: `A:0.1`, `B:1.5e-3`, `C:2.0E+1`, `D:-0.5`. A missing +length is 0. + +**Polytomies.** `(A:1,B:2,C:3);` is a node with three children, and nothing in +distree assumes a bifurcating tree. + +**Internal node labels**, quoted or not, with or without a length: +`((A:1,B:2)clade:0.5,C:3);`. Bootstrap values live in this position too and are +read as labels, which is to say ignored. + +**Quoted tip labels**, single or double, so a label may hold spaces: +`('Taxon A':1.0,"Taxon B":2.0);`. Per the Newick convention a doubled quote +inside a quoted label is one literal quote, so `'it''s'` is the label `it's`. + +**Comments** in square brackets, before or after a label, nested, and at the +start of the file: + +``` +[&R] ((A:0.1[&&NHX:S=human],B:0.2[&rate=0.5]):0.3,C:0.4); +``` + +The `[&R]` / `[&U]` rooting marker that IQ-TREE, MrBayes and BEAST write is a +comment like any other. So is NHX and BEAST per-branch metadata, wherever it +sits relative to the label. All of it is skipped; the labels and the branch +lengths survive. + +**Non-ASCII labels.** Accents, Greek letters and CJK text pass through +unchanged, so a label keeps matching the sample name it came from. + +**Whitespace and newlines** anywhere outside a quoted label, which is how most +tools wrap a long tree over several lines. + +**A missing trailing semicolon**, which some tools omit. + +## What is rejected + +The guiding rule is that distree would rather stop than hand back a matrix that +looks right and is not. Each of these fails with a message naming the offending +position, and nothing is written to `-o`. + +| Input | Message | +|:--|:--| +| Empty file | `Empty input: no Newick tree found.` | +| `((A:1,B:2),C:3` | `Unexpected end of input: 1 unclosed '(' remain. The tree is truncated.` | +| `(A:1,B:2);(C:1,D:2);` | `the file appears to hold more than one tree` | +| `(A:1,B:2);garbage` | `Unexpected content after the end of the tree` | +| `('unclosed:1,B:2);` | `Unclosed quote starting at position 1.` | +| `(A:1,B:2)[oops;` | `Unclosed comment starting at position 9: no matching ']'.` | +| `(A:,B:2);` | `Expected a numeric branch length` | +| `(A:1e910,B:2);` | `Branch length '1e910' ... is not a finite number.` | +| `(A:1,A:2);` | `Duplicate leaf name 'A' found. Leaf names must be unique.` | +| A label holding a tab, newline or carriage return | `contains a tab character, which would corrupt the output by splitting the row` | +| A tree with no labelled tips | `No labeled leaves found in the tree.` | + +!!! warning "One tree per file" + + A file of bootstrap replicates or a posterior sample holds many trees. + distree computes a matrix for one tree, and earlier versions silently used + the first one. It now refuses, because a matrix labelled with your dataset's + name that describes replicate 1 of 1,000 is worse than an error. Split the + file first: + + ```bash + split -l 1 posterior.trees tree_ + for t in tree_*; do distree "$t" -o "${t}.tsv"; done + ``` + +!!! warning "Branch lengths at the edge of what a float can hold" + + An exponent past the range of a 64-bit float, `A:1e910`, does not fail to + parse: Rust returns infinity for it. An infinite depth then makes every + distance infinite and the diagonal `NaN`, because `inf - inf` is `NaN`, and + the whole matrix used to come out that way with a success exit code. Such a + branch length is now rejected outright. + + A distance is `d_i + d_j - 2·d_m`, so the same overflow can happen with + every individual length finite, if the depths are past half of the float + range. The run stops for that too, saying to rescale the tree. Ordinary + large values are unaffected: `1e100` still works. + +!!! question "Truncated trees" + + A tree cut short by an interrupted download or a full disk used to parse: + the open parentheses were closed at end of input, and every internal node + left dangling silently lost its branch length. The matrix came out looking + entirely reasonable. That is now an error naming how many parentheses were + still open. + +## Warnings + +These do not stop the run. They go to stderr, so they stay out of a piped +matrix. + +**Negative branch lengths.** + +``` +Warning: negative branch lengths detected in the tree. Some distances may come +out negative, and --midpoint cannot locate the diameter reliably. +``` + +Neighbour-joining and BioNJ produce these routinely. distree reports the +distances as computed rather than clamping them to zero, since a clamp would +claim two distinct taxa sit on top of each other. Midpoint rooting, on the other +hand, finds the longest path by a two-pass sweep that assumes non-negative +lengths, so `--midpoint` on such a tree is not reliable. + +**No branch lengths at all**, in patristic mode: + +``` +Warning: no branch lengths detected, all patristic distances will be zero. +Consider --topology. +``` + +A cladogram has topology and nothing else. `--topology` is the mode that reads +it. + +**Unlabelled tips.** + +``` +Warning: 1 leaf/leaves have no label and were excluded from the matrix. +``` + +A tip with no label cannot be named in a row or a column, so it is left out. The +matrix is correct for the tips that remain, but it is smaller than the tree, and +the warning is there so that is not a surprise. + +**Conflicting modes.** `--lmm` and `--topology` ask for different things; +passing both uses `--lmm` and says so. `--midpoint` with `--topology` is ignored, +for the reason in [Midpoint rooting](rooting.md). + +## Label rules + +Tip labels become row and column headers in a TSV, which puts two requirements +on them. + +**They must be unique.** Two tips called `A` would give two identical rows with +no way to tell which is which, so a duplicate is an error rather than a warning. + +**They must not hold a tab, a newline or a carriage return.** Any of the three +splits a row and leaves a file whose row count no longer matches its header. +A quoted label can hold them, so this is checked rather than assumed. Replace +them with underscores before running distree. + +Spaces are fine, as long as the label is quoted in the Newick. So is anything +else UTF-8 can express. + +## Next + +- [Distance modes](distances.md), for what distree does with the tree. +- [Output](output.md), for the shape of what comes back. diff --git a/docs/guide/output.md b/docs/guide/output.md new file mode 100644 index 0000000..5383fa1 --- /dev/null +++ b/docs/guide/output.md @@ -0,0 +1,194 @@ +# Output + +The matrix goes to stdout unless `-o` names a file. Warnings go to stderr, so a +piped run stays clean: + +```bash +distree tree.nwk | head -1 +distree tree.nwk -o distances.tsv +``` + +## The square TSV + +The default layout is a full tab-separated matrix: + +``` + LeafA LeafB LeafC LeafD +LeafA 0.000 5.200 3.600 4.000 +LeafB 5.200 0.000 4.900 5.300 +LeafC 3.600 4.900 0.000 2.000 +LeafD 4.000 5.300 2.000 0.000 +``` + +- The first line is a header of tip labels, preceded by an **empty cell** so the + labels sit over the data columns rather than one to the left. This is what + `read.table(header=TRUE, row.names=1)` and `pandas.read_csv(index_col=0)` + expect. +- Every following line begins with a tip label and holds one value per tip. +- **Tips are sorted alphabetically**, by byte order, in both the rows and the + columns. The order is the same in every run and does not depend on how the + tree was written, so two matrices from two trees over the same tips line up + cell for cell. +- The matrix is symmetric and the diagonal is exactly zero, except under + [`--lmm`](distances.md#variance-covariance), where the diagonal is each tip's + root-to-tip length. + +Reading it back: + +=== "R" + + ```r + m <- as.matrix(read.table("distances.tsv", header = TRUE, + row.names = 1, sep = "\t", check.names = FALSE)) + ``` + + `check.names = FALSE` keeps labels like `Sample-1` and `2024_isolate` intact; + without it R rewrites them. + +=== "Python" + + ```python + import pandas as pd + m = pd.read_csv("distances.tsv", sep="\t", index_col=0) + ``` + +=== "Shell" + + ```bash + # the distance between two named tips + awk -F'\t' -v a=LeafA -v b=LeafC ' + NR==1 { for (i=2; i<=NF; i++) if ($i==b) col=i; next } + $1==a { print $col }' distances.tsv + ``` + +## PHYLIP lower triangle + +```bash +distree tree.nwk --lower -p 4 +``` + +``` +4 +LeafA +LeafB 5.2000 +LeafC 3.6000 4.9000 +LeafD 4.0000 5.3000 2.0000 +``` + +A taxa count on the first line, then one row per taxon holding its label and the +distances to every taxon **above** it in the ordering. No diagonal, and none of +the mirrored half. + +This is the format PHYLIP's `neighbor` and `fitch` read, and what Mash and a +number of clustering tools expect. It also halves the output: a 50,000-tip tree +is 2.5 billion cells as a square matrix and 1.25 billion as a lower triangle. + +The first data row is the label alone, with no values after it, which is correct +and is what those readers expect. + +!!! note "Relaxed PHYLIP" + + Fields are tab-separated and labels are written in full. Strict PHYLIP pads + every name to exactly ten characters and truncates anything longer, which + would collide two tips whose names share a prefix. distree writes the + relaxed form that whitespace-splitting readers, including modern PHYLIP + builds, accept. + +## NumPy arrays + +```bash +distree tree.nwk --npy -o distances.npy +``` + +Text is an expensive way to move a large matrix. At twelve decimals a cell is +fourteen bytes and producing them is most of the run; as a 64-bit float it is +eight bytes, exact, and costs nothing to write. On an 8,000-tip tree: + +| | Time | Size | +|:--|--:|--:| +| `--lower -p 12` | 0.26 s | 458 MB | +| `--lower --npy` | 0.10 s | 244 MB | +| `-p 12` | 0.56 s | 916 MB | +| `--npy` | 0.19 s | 488 MB | + +`.npy` has nowhere to put labels, so they go to `.labels.txt` in row +order. That is why `--npy` needs `-o` rather than writing to stdout. + +```python +import numpy as np +m = np.load("distances.npy") +labels = open("distances.npy.labels.txt").read().split() +``` + +With `--lower`, the array is the **condensed vector** rather than PHYLIP's +lower triangle, because those are two different triangles: SciPy reads the +upper one row by row. It goes straight into `scipy`: + +```python +import numpy as np +from scipy.cluster.hierarchy import linkage, fcluster +from scipy.spatial.distance import squareform + +v = np.load("condensed.npy") # distree tree.nwk --lower --npy -o condensed.npy +z = linkage(v, method="single") # takes the condensed form directly +full = squareform(v) # or expand it to the square matrix +``` + +`-p` has no effect here; a 64-bit float carries the value exactly. + +## Precision + +```bash +distree tree.nwk -p 6 +``` + +`-p` sets the number of decimal places, defaulting to 10 and capped at 30. It +applies to patristic and `--lmm`; `--topology` writes integers and ignores it. + +A 64-bit float carries about 17 significant decimal digits, so anything past +that is zero padding, and the cap is there to keep an accidental `-p 50000000` +from being an error inside the formatter rather than an error about the flag. + +Precision is also the main lever on output size. At `p=10` each cell is 12 to 14 +bytes; at `p=6` it is 8 to 10. Across a 20,000-tip square matrix that is the +difference between roughly 5 GB and 3.5 GB. + +## Writing to a file + +```bash +distree tree.nwk -o distances.tsv +``` + +The file is not created until the tree has parsed, midpoint rooting has run and +the tip labels have been checked, so a run that fails on a malformed tree leaves +whatever was at that path untouched. + +Write errors are reported. A full disk fails with the path in the message and a +non-zero exit code, rather than leaving a truncated matrix behind and reporting +success. + +## Piping + +`distree tree.nwk | head` closes the pipe early, which is a normal way to look +at the first few rows. distree treats that as the reader having seen enough and +exits cleanly rather than reporting a broken pipe. + +Because rows are written as they are computed, the first row of a very large +matrix appears long before the last one. A pipeline that consumes rows as they +arrive does not wait for the whole matrix: + +```bash +distree big.nwk -p 6 | awk -F'\t' 'NR>1 { for (i=2; i<=NF; i++) if ($i+0 < 1e-5 && i-1 != NR-1) print $1, i }' +``` + +## Exit codes + +| Code | Meaning | +|:--|:--| +| 0 | The matrix was written and flushed. Also the code when a downstream pipe closed early | +| 1 | Anything went wrong: an unreadable file, a malformed tree, a bad flag, a failed write. The message is on stderr and begins with `Error:` | + +## Next + +- [Recipes](../recipes.md), for what to do with the matrix once you have it. +- [CLI reference](cli.md), for every flag in one table. diff --git a/docs/guide/rooting.md b/docs/guide/rooting.md new file mode 100644 index 0000000..f6228a6 --- /dev/null +++ b/docs/guide/rooting.md @@ -0,0 +1,129 @@ +# Midpoint rooting + +```bash +distree tree.nwk --midpoint --lmm -p 3 +``` + +`--midpoint` re-roots the tree at the middle of its longest tip-to-tip path +before any distance is computed. It exists for one mode. + +## When rooting matters + +A [patristic](distances.md#patristic) distance is the length of the path between +two tips. A [topological](distances.md#topological) distance is the number of +edges on that path. Neither depends on where you call the top of the tree: it is +the same path either way. Re-rooting a tree cannot change either matrix, and +distree's own tests assert exactly that. + +A [variance-covariance](distances.md#variance-covariance) entry is the distance +from **the root** down to the MRCA of two tips. Move the root and every entry +moves. That makes `--lmm` the mode where rooting is a modelling decision rather +than a formality, and `--midpoint` the flag that makes it explicitly. + +## What it does + +The longest path between any two tips is the tree's diameter. Midpoint rooting +puts the new root exactly halfway along it, so the two deepest tips end up +equidistant from the root, and no tip is further from the root than half the +diameter. + +On the tree used throughout this guide: + +``` +((LeafA:1.95,LeafB:3.25):0.35,(LeafC:0.80,LeafD:1.20):0.50); +``` + +the diameter runs LeafB to LeafD and measures `3.25 + 0.35 + 0.50 + 1.20 = 5.30`. +Half of that is 2.65, which falls **inside LeafB's own branch**, 2.65 up from +LeafB and 0.60 below the node it hangs from. distree inserts a new root there, +splitting the 3.25 branch into 2.65 and 0.60, and reverses the parent-child +links from that point up to the old root so the tree hangs correctly from its +new top. + +The `--lmm` matrix before and after: + +
+ +| Original root | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.300 | 0.350 | 0.000 | 0.000 | +| **LeafB** | 0.350 | 3.600 | 0.000 | 0.000 | +| **LeafC** | 0.000 | 0.000 | 1.300 | 0.500 | +| **LeafD** | 0.000 | 0.000 | 0.500 | 1.700 | + +
+ +
+ +| Midpoint root | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.550 | 0.000 | 0.600 | 0.600 | +| **LeafB** | 0.000 | 2.650 | 0.000 | 0.000 | +| **LeafC** | 0.600 | 0.000 | 2.250 | 1.450 | +| **LeafD** | 0.600 | 0.000 | 1.450 | 2.650 | + +
+ +The whole structure has changed. LeafB now splits off at the root and shares +nothing with anyone, while LeafA, LeafC and LeafD share 0.600 of history that +the original rooting attributed differently. The deepest tips, LeafB and LeafD, +sit at 2.650, which is half the diameter, as they should. + +The patristic matrix, meanwhile, is byte-identical with and without the flag. + +## Why midpoint is ignored in topology mode + +Adding `--midpoint` to `--topology` used to inflate the answer. The new root is +a **new node**, inserted partway along an existing edge, and it splits that edge +into two. Every pair of tips whose path crosses that edge therefore gained one +hop: + +``` +((A:1,B:1):1,(C:1,D:1):1); +``` + +| | Without `--midpoint` | With `--midpoint` | +|:--|--:|--:| +| A to C | 4 | 5 | +| A to B | 2 | 2 | + +The 5 was an artefact of the extra node, not a property of the tree. Since edge +counts cannot depend on the root in the first place, there is nothing +`--midpoint` could correctly contribute here, so distree skips the rooting and +says so: + +``` +Warning: --midpoint is ignored in --topology mode. Edge counts do not depend on +the root, and the node inserted at the midpoint would add 1 to every distance +crossing that edge. +``` + +The same insertion happens in patristic mode, but there it is harmless: the two +pieces of the split edge sum to the original length, so no path changes. + +## Caveats + +!!! warning "Negative branch lengths" + + distree finds the diameter with the standard two-pass sweep: walk to the + furthest tip from an arbitrary tip, then to the furthest tip from that one. + The argument that this finds the true diameter assumes non-negative edge + weights, and a neighbour-joining tree does not satisfy it. Midpoint rooting + such a tree may pick the wrong path, and the run warns about the negative + lengths for that reason among others. + +**A zero-length diameter leaves the tree alone.** If every branch is zero there +is no midpoint to find, and distree returns the tree as it was. + +**Only tips count.** The diameter is measured tip to tip, matching what `ape` +and `phytools` do. A root with a single child is a degree-one node in the +unrooted tree, but it is not a tip and does not extend the diameter. + +**The new root is unlabelled** and adds one node to the tree. That is invisible +in the output, which only ever has one row per labelled tip. + +## Next + +- [Distance modes](distances.md), for what rooting does and does not affect. +- [How it works](../how-it-works/algorithm.md#midpoint-rooting), for the + re-rooting procedure itself. diff --git a/docs/how-it-works/algorithm.md b/docs/how-it-works/algorithm.md new file mode 100644 index 0000000..4d0226d --- /dev/null +++ b/docs/how-it-works/algorithm.md @@ -0,0 +1,126 @@ +# How it works + +A run has five stages. The first four touch the tree once each; the last one +does all the work. + +```mermaid +flowchart LR + A[Read Newick] --> B[Parse to a flat node array] + B --> C{--midpoint?} + C -- yes --> D[Re-root at the diameter midpoint] + C -- no --> E + D --> E[Build the LCA structure] + E --> F[Stream one row at a time] +``` + +## Parsing + +The Newick string is read once to strip whitespace outside quoted labels, then +scanned byte by byte. Every structural character in Newick is ASCII, so a +byte-wise scan is safe on UTF-8 input: no byte of a multi-byte sequence can be +mistaken for a delimiter. Label bytes are collected and decoded as UTF-8 in one +step, which is what keeps accented and CJK tip names intact. + +The parser is **iterative**, with an explicit stack of partially built nodes +rather than a recursive descent, so a tree nested hundreds of thousands of +levels deep parses without touching the call stack. The same applies to freeing +the parse tree: the compiler's own drop glue would recurse once per level, so +`RawNode` dismantles its children in a loop instead. + +Parsing is strict about structure. Reaching the end of input with parentheses +still open is an error, not an implicit close, and anything after the tree other +than a semicolon and comments is an error too. Both cases used to produce a +matrix rather than a message. See [Input](../guide/input.md#what-is-rejected). + +The recursive parse tree is then flattened, again iteratively, into a +`Vec` where a node is a label, a branch length, a parent index and a list +of child indices. Everything after this point is index arithmetic over flat +arrays. + +## Midpoint rooting + +Only when `--midpoint` is passed, and only outside `--topology`. + +The diameter is found with the standard two-pass sweep: from an arbitrary tip, +walk the tree to the furthest tip `a`; from `a`, walk to the furthest tip `b`. +The second sweep records the path it took, so `a` to `b` can be reconstructed +without a third pass. + +distree then walks that path accumulating length until it passes half the +diameter, which locates the edge the midpoint falls on. It inserts a new node +there, splitting the edge into the two pieces either side of the midpoint, +attaches both ends to it, and reverses the parent-child links from the old +parent up to the old root, carrying each branch length down to the node that is +now the child. The result is the same tree hanging from a different point. + +The tests check this on 300 generated trees, including polytomies and +zero-length branches, against three properties: every pairwise patristic +distance survives, the result is still a tree with nothing orphaned and no +cycle, and the deepest tip sits exactly half a diameter from the new root. + +## The LCA structure + +This is where the run buys its speed. Walking the tree for each of `N²` pairs +would cost `O(N² · depth)`. Instead one depth-first pass records, for every node: + +- `depth_len`, its summed branch length from the root; +- `depth_top`, its edge count from the root; +- `up[0]`, its parent. + +The parent table is then doubled `log₂ M` times, so that `up[k][u]` is the +`2^k`-th ancestor of `u`. Finding the most recent common ancestor of two nodes +becomes: lift the deeper one by the difference in `depth_top`, one power of two +per set bit; then, from the largest jump down to the smallest, move both up +together whenever their ancestors at that level still differ. They meet one step +above where they stop. That is `O(log M)` per query, with no allocation. + +The table is stored as one flat `Vec` with `usize::MAX` standing in for +"no ancestor". An `Option` would be the obvious type, but it has no spare +niche and so costs 16 bytes per entry instead of 8, across `M log M` entries. + +The queries are checked against a brute-force walk from both nodes to the root, +over every pair of nodes in 200 generated trees. + +## Computing a cell + +With the structure in place, every mode is one MRCA query and three array reads: + +| Mode | Cell `(i, j)` | +|:--|:--| +| Patristic | `depth_len[i] + depth_len[j] - 2 · depth_len[m]` | +| Topological | `depth_top[i] + depth_top[j] - 2 · depth_top[m]` | +| Variance-covariance | `depth_len[m]` | + +No square root, no allocation, no tree walk. The cost of a cell does not depend +on how far apart the two tips are. + +## Streaming the matrix + +Tips are collected, checked for duplicates and for characters that would break a +row, and sorted by label. Then, for each row in turn: + +1. the row's cells are computed in parallel across the thread pool, into a + buffer that is reused between rows; +2. the row is written; +3. the next row starts. + +At no point does the full matrix exist. Peak memory is the tree plus the LCA +table plus one row, which is why a matrix far larger than RAM can be written to +a file or straight into a pipeline. See +[Performance](performance.md) for the arithmetic. + +Rayon splits each row across the available cores, and `-t` caps how many. The +row order and the cell order are fixed regardless of thread count, so two runs +of the same tree produce byte-identical output. + +## Why the output is sorted + +Tips are ordered by label, not by their position in the tree. That costs one +sort and buys three things: the ordering does not depend on how the Newick was +written, two matrices over the same tips line up cell for cell, and a run is +reproducible without recording anything about the input's layout. + +## Next + +- [Performance](performance.md), for what all this costs. +- [Distance modes](../guide/distances.md), for what the numbers mean. diff --git a/docs/how-it-works/performance.md b/docs/how-it-works/performance.md new file mode 100644 index 0000000..12d360d --- /dev/null +++ b/docs/how-it-works/performance.md @@ -0,0 +1,164 @@ +# Performance + +## Complexity + +For a tree with `N` labelled tips and `M` nodes: + +| Stage | Time | Memory | +|:--|:--|:--| +| Parse and flatten | `O(M)` | `O(M)` | +| Midpoint rooting | `O(M)` | `O(M)` | +| LCA structure | `O(M log M)` | `O(M log M)` | +| The matrix | `O(N² log M)` | `O(1)` beyond the buffer | + +Everything before the matrix is linear or near-linear and, in practice, free. +On a 100,000-tip tree, parsing and building the LCA structure takes 0.05 s. The +run is the matrix, and the matrix is quadratic in the number of tips no matter +what: `N²` cells have to be computed and written. + +The `log M` in the cell cost is the MRCA query. It is a handful of array reads +against a table that fits in cache for anything under a few thousand nodes, so +the constant is small; the format of the number costs more than finding it. + +## Measured + +Balanced binary trees, `-p 6 --lower`, output discarded, release build on an +Apple M4 Pro with 14 cores: + +| Tips | Cells | Time | Peak memory | +|--:|--:|--:|--:| +| 1,000 | 0.5 M | 0.00 s | 11 MB | +| 2,000 | 2.0 M | 0.01 s | 24 MB | +| 4,000 | 8.0 M | 0.03 s | 35 MB | +| 8,000 | 32 M | 0.13 s | 32 MB | +| 20,000 | 200 M | 1.02 s | 35 MB | + +Time is quadratic in the tips, as it must be. Memory is not: it flattens out +around 35 MB, because past a few thousand tips it is the fixed batch buffer plus +the LCA table rather than anything that grows with the matrix. + +## Threads + +`-t` caps the thread count; the default is every core. On the 8,000-tip tree: + +| `-t` | Time | Speedup | +|--:|--:|--:| +| 1 | 0.56 s | 1.0x | +| 2 | 0.31 s | 1.8x | +| 4 | 0.18 s | 3.1x | +| 8 | 0.12 s | 4.7x | +| 14 | 0.13 s | 4.3x | + +Each worker computes and formats whole rows, so the parallel section covers both +halves of the cost and the curve holds up until it runs into memory bandwidth +and the single writer. + +!!! note "This was not always true" + + Before version 1.0.1, each row was its own parallel job. A cell is one MRCA + query and three array reads, so a row of a small matrix could not pay for + synchronising the thread pool, and the same 8,000-tip run took 2.52 s on 14 + cores against 1.52 s on one. Formatting the floats sat in the serial write + loop on top of that. Both are fixed; rows are now batched, and each worker + formats its own. + +Set `-t` when you are sharing a machine, or when the run is part of a pipeline +that is already parallel. Leaving it unset is right for a dedicated node. + +## Where the time goes + +For a text run, roughly half of it used to be turning floats into decimal +digits, which is more expensive than computing the distance being printed. +`write!("{:.p$}")` expands the float to its exact decimal form and goes through +`core::fmt`; distree instead scales by a power of ten and emits the digits +directly, which is about seven times faster at that step and cut the whole run +by a third to a half: + +| | Before | After | +|:--|--:|--:| +| `--lower -p 6` | 0.22 s | 0.13 s | +| `--lower -p 10` | 0.24 s | 0.15 s | +| `-p 10` square | 0.47 s | 0.29 s | + +The shortcut is only taken where the answer is not in doubt. Multiplying by +`10^p` rounds once, and where that could carry the product across the nearest +`.5` boundary the value goes to the standard formatter instead, so the output is +byte-identical either way. About one value in a thousand takes that path. + +If none of the time should go on formatting at all, [`--npy`](../guide/output.md#numpy-arrays) +writes raw 64-bit floats and skips it entirely: another 2.6x, at full precision +and half the file size. + +## Memory + +Nothing in distree holds the matrix. The peak is: + +``` + LCA table M × (⌈log₂ M⌉ + 1) × 8 bytes + Node depths M × 16 bytes + The tree roughly M × 72 bytes, plus the labels + Batch buffer about 15 MB, fixed + Output buffer 1 MB +``` + +For a bifurcating tree, `M ≈ 2N`. Worked through: + +| Tips | Nodes | LCA table | Total, roughly | +|--:|--:|--:|--:| +| 10,000 | 20,000 | 2.6 MB | 35 MB | +| 100,000 | 200,000 | 30 MB | 100 MB | +| 1,000,000 | 2,000,000 | 350 MB | 600 MB | + +The 100,000-tip figure is measured; the million-tip one is the arithmetic. + +The LCA table is the term that grows, and it is stored as plain `usize` with a +sentinel rather than `Option`, which halves it. The batch buffer holds +about a million cells' worth of formatted rows regardless of tree size, which is +what keeps the total flat as the matrix grows. + +## Output size + +The output is usually larger than anything in memory, and precision is the lever +on it. At `-p 10` a cell is 12 to 14 bytes; at `-p 6` it is 8 to 10. + +| Tips | Square, `-p 10` | Square, `-p 6` | `--lower`, `-p 6` | +|--:|--:|--:|--:| +| 1,000 | 13 MB | 9 MB | 4.5 MB | +| 10,000 | 1.3 GB | 900 MB | 450 MB | +| 50,000 | 33 GB | 22 GB | 11 GB | + +Three things follow. Use `--lower` when the reader accepts it, which halves the +file. Do not ask for more decimals than the branch lengths carry: `-p 6` against +the default `-p 10` is a third off the file for no loss on any realistic tree. +And if the reader is Python, [`--npy`](../guide/output.md#numpy-arrays) is 8 +bytes a cell at full precision, which beats text at any setting. + +Rows are written as they are computed, so a pipeline that consumes them as they +arrive never holds the whole matrix either: + +```bash +distree big.nwk -p 6 | gzip > distances.tsv.gz +``` + +## Where the ceiling is + +The quadratic term decides. Around 50,000 tips a square matrix is tens of +gigabytes and the run is minutes; at 100,000 it is hundreds of gigabytes and +the file, not distree, is the problem. + +If the tips are more than that, the question is usually not really "what is the +whole matrix". Reduce the tree first, with something like +[axetree](https://github.com/PathoGenOmics-Lab/axetree), and compute the matrix +over the subset. + +## Reproducibility + +The row order is the tip labels sorted, and the cell order within a row is the +same sort, neither of which depends on the thread count or on how the Newick was +written. Two runs of the same tree produce byte-identical output, on any number +of cores. + +## Next + +- [How it works](algorithm.md), for what these stages actually do. +- [Output](../guide/output.md), for the formats and the precision flag. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..365796b --- /dev/null +++ b/docs/index.md @@ -0,0 +1,110 @@ +--- +hide: + - navigation +--- + +
+ +![distree](assets/distree_wordmark.svg){ .hero-logo } + +# distree + +Pairwise distance matrices from a phylogeny, in Rust. + +
+ +Once a tree is built, most of the questions that follow are pairwise. Which +isolates sit within five substitutions of each other. What the ordination looks +like. What covariance structure a comparative model should assume. All of them +want the same thing first: the distance between every pair of tips. + +Computing that naively means walking the tree once per pair, and holding an +N × N matrix while you do it. distree instead spends `O(M log M)` up front on a +binary-lifting structure that answers *"where do these two tips meet?"* in +`O(log M)` time, then streams the matrix out one row at a time. A row is +computed in parallel across cores and written before the next one starts, so +memory tracks the number of tips, not the square of it. + +
+ +- :material-download: **[Install](getting-started/installation.md)** + + Bioconda, a prebuilt binary, or `cargo build --release`. No runtime + dependencies. + +- :material-rocket-launch: **[Quick start](getting-started/quickstart.md)** + + The handful of commands that cover most of what people want a matrix for. + +- :material-file-tree: **[Input](guide/input.md)** + + What Newick distree accepts, what it rejects, and why it rejects rather + than guesses. + +- :material-ruler: **[Distance modes](guide/distances.md)** + + Patristic, topological and variance-covariance, on one worked example. + +- :material-format-vertical-align-center: **[Midpoint rooting](guide/rooting.md)** + + When the root changes the answer, and when it cannot. + +- :material-table: **[Output](guide/output.md)** + + The TSV layout, PHYLIP lower triangle, precision, and reading it back in. + +- :material-school: **[Tutorial](tutorial.md)** + + A worked outbreak investigation, from the tree to transmission clusters. + +- :material-cog: **[How it works](how-it-works/algorithm.md)** + + The parser, the LCA structure, and the streaming loop. + +- :material-speedometer: **[Performance](how-it-works/performance.md)** + + What the run costs in time and memory, and where the ceiling is. + +
+ +## In one command + +```bash +distree tree.nwk --lower -p 6 -t 8 -o distances.phy +``` + +Patristic distances for every pair of tips, as a PHYLIP lower triangle, six +decimal places, across eight threads. Swap `--topology` for edge counts, +`--lmm` for the variance-covariance matrix a comparative model wants, and drop +`--lower` for a full square TSV that `read.table` and `pandas` open directly. + +## What it does + +| | | +|:--|:--| +| **Patristic** | The sum of branch lengths on the path between two tips | +| **Topological** | The number of edges on that path, ignoring branch lengths | +| **Variance-covariance** | The root-to-MRCA distance for each pair, the `C` matrix of a PGLS or a phylogenetic mixed model | +| **Rooting** | Optional midpoint rooting, which matters for `--lmm` and cannot matter for the other two | +| **Formats** | Square TSV with labels, a PHYLIP lower triangle, or a NumPy array | +| **Input** | Newick from a file or stdin, plain or gzipped, with quoted labels, polytomies, NHX and BEAST comments, and UTF-8 tip names | +| **Subsets** | `--taxa` restricts the matrix to a list of tips, with the distances the full tree gives | +| **Scale** | Memory grows with the number of tips, not with the matrix; rows stream out as they are computed | +| **Parallelism** | Each row is computed across all cores, or as many as `-t` allows | + +## What it does not do + +distree works on the tree you give it. It does not build one, it does not read +an alignment, and it does not cluster the matrix for you. + +It also does not guess. A truncated tree, a file holding several trees, an +unclosed quote or a label with a tab in it are all rejected with a message +naming the position, rather than turned into a matrix that looks right and is +not. The one case that is reported and carried through rather than refused is a +negative branch length, because neighbour-joining trees legitimately have them +and the resulting negative distances are information, not noise. + +## Citation + +If you use distree, please cite the archived release. See +[Citation](about/citation.md) for the DOI and the version to record. diff --git a/docs/recipes.md b/docs/recipes.md new file mode 100644 index 0000000..9256282 --- /dev/null +++ b/docs/recipes.md @@ -0,0 +1,279 @@ +# Recipes + +Short, complete examples of what people actually do with the matrix. + +## Ordination: PCoA and MDS + +=== "R" + + ```r + m <- as.matrix(read.table("distances.tsv", header = TRUE, + row.names = 1, sep = "\t", check.names = FALSE)) + fit <- cmdscale(as.dist(m), k = 2, eig = TRUE) + plot(fit$points, pch = 19, xlab = "PCo1", ylab = "PCo2") + text(fit$points, labels = rownames(m), pos = 3, cex = 0.6) + + # variance explained by the first two axes + round(100 * fit$eig[1:2] / sum(fit$eig[fit$eig > 0]), 1) + ``` + +=== "Python" + + ```python + import pandas as pd + from sklearn.manifold import MDS + + m = pd.read_csv("distances.tsv", sep="\t", index_col=0) + coords = MDS(n_components=2, dissimilarity="precomputed", + random_state=0, normalized_stress="auto").fit_transform(m.values) + ``` + + For UMAP, pass the same matrix with `metric="precomputed"`. + +Generate it with plenty of precision; ordination is sensitive to the small +distances: + +```bash +distree tree.nwk -p 10 -o distances.tsv +``` + +## Transmission clusters by a SNP threshold + +A patristic distance from a tree built on an alignment is in substitutions per +site. Multiply by the alignment length to get SNP-scale distances, then cut the +matrix at your threshold: + +=== "R" + + ```r + m <- as.matrix(read.table("distances.tsv", header = TRUE, + row.names = 1, sep = "\t", check.names = FALSE)) + snps <- m * 4411532 # H37Rv genome length + clusters <- cutree(hclust(as.dist(snps), method = "single"), h = 5) + split(names(clusters), clusters) + ``` + +=== "Python" + + ```python + import pandas as pd + from scipy.cluster.hierarchy import linkage, fcluster + from scipy.spatial.distance import squareform + + m = pd.read_csv("distances.tsv", sep="\t", index_col=0) * 4411532 + z = linkage(squareform(m.values, checks=False), method="single") + labels = fcluster(z, t=5, criterion="distance") + ``` + +Single linkage is the right choice for a transmission threshold: it clusters +samples that are within the threshold **of someone** in the cluster, which is +what a chain of transmission looks like. + +## PGLS and phylogenetic mixed models + +`--lmm` writes the `C` matrix these models want. Root the tree deliberately +first, because [rooting is part of the answer](guide/rooting.md): + +```bash +distree tree.nwk --midpoint --lmm -p 10 -o varcovar.tsv +``` + +```r +C <- as.matrix(read.table("varcovar.tsv", header = TRUE, + row.names = 1, sep = "\t", check.names = FALSE)) +traits <- read.csv("traits.csv", row.names = 1) +C <- C[rownames(traits), rownames(traits)] # align the orderings + +library(nlme) +fit <- gls(y ~ x, data = traits, + correlation = corSymm(C[lower.tri(C)] / max(C), fixed = TRUE)) +``` + +Two things to check before fitting. The row and column order is alphabetical by +tip label, which is almost certainly not the order of your trait table, so +reindex rather than assume. And a non-constant diagonal means the tree is not +ultrametric, which several comparative methods assume. + +## Neighbour-joining from an existing tree + +PHYLIP's `neighbor` reads the lower triangle directly: + +```bash +distree tree.nwk --lower -p 6 -o infile +printf 'L\nY\n' | neighbor +``` + +The `L` selects lower-triangular input; `neighbor` writes `outtree` and +`outfile`. + +## The closest relative of every sample + +```bash +distree tree.nwk -p 8 | awk -F'\t' ' + NR == 1 { for (i = 2; i <= NF; i++) name[i] = $i; next } + { + best = ""; bestd = 1e308 + for (i = 2; i <= NF; i++) if (i - 1 != NR - 1 && $i + 0 < bestd) { bestd = $i + 0; best = name[i] } + printf "%s\t%s\t%s\n", $1, best, bestd + }' +``` + +Skipping `i - 1 == NR - 1` skips the diagonal, which is otherwise always the +closest. + +## Comparing two trees over the same tips + +Because the tip ordering is alphabetical in every run, two matrices over the +same tip set line up cell for cell: + +```bash +distree iqtree.nwk -p 10 --lower -o a.phy +distree raxml.nwk -p 10 --lower -o b.phy +``` + +```python +import numpy as np, pandas as pd +from scipy.stats import pearsonr + +def lower(path): + vals = [] + with open(path) as fh: + next(fh) # taxa count + for line in fh: + vals += [float(v) for v in line.split("\t")[1:]] + return np.array(vals) + +a, b = lower("a.phy"), lower("b.phy") +print(pearsonr(a, b)) +``` + +A high correlation with a slope away from 1 means the two trees agree on the +shape and disagree on the rate. + +## Cladograms and unreliable branch lengths + +A tree with no branch lengths gives an all-zero patristic matrix, and says so. +Count edges instead: + +```bash +distree cladogram.nwk --topology -o topo.tsv +``` + +The same applies when the lengths exist but are not comparable across the tree, +for instance from concatenated loci with different rates. + +## Straight into SciPy, without the text + +For anything Python is going to read, `--npy` skips the text round trip +entirely: exact 64-bit floats, half the file, and none of the formatting cost. + +```bash +distree tree.nwk --lower --npy -o condensed.npy +``` + +```python +import numpy as np +from scipy.cluster.hierarchy import linkage, fcluster + +v = np.load("condensed.npy") # the condensed form scipy wants +labels = open("condensed.npy.labels.txt").read().split() + +z = linkage(v, method="single") +clusters = fcluster(z, t=12 / 4411532, criterion="distance") +``` + +Drop `--lower` for the square matrix, which is what `MDS(dissimilarity= +"precomputed")` and `umap.UMAP(metric="precomputed")` take. + +## One cohort out of a big tree + +```bash +distree big.nwk --taxa cohort.txt -p 10 -o cohort.tsv +``` + +`cohort.txt` is one label per line, with `#` comments allowed. The distances are +the ones from the full tree, which is the point: the path between two isolates +does not change because the isolates between them were left out of the matrix. +Pruning the tree first would give you different numbers. + +Building the list from a metadata table: + +```bash +awk -F'\t' 'NR > 1 && $3 == "Valencia" { print $1 }' metadata.tsv > cohort.txt +distree big.nwk --taxa cohort.txt --stats -o valencia.tsv +``` + +`--stats` then reports both counts, so it is obvious how much of the tree the +cohort was. + +## Very large trees + +Rows are written as they are computed, so nothing needs the whole matrix in one +piece. Compress on the way out: + +```bash +distree big.nwk -p 6 --lower | gzip > distances.phy.gz +``` + +The input can be gzipped too, and does not need decompressing first: + +```bash +distree big.nwk.gz --lower --npy -o distances.npy +``` + +Or filter as it streams, keeping only the pairs that matter: + +```bash +distree big.nwk -p 8 --lower | awk -F'\t' ' + NR == 1 { next } + { for (i = 2; i <= NF; i++) if ($i + 0 < 1e-5) print $1, i - 1, $i }' +``` + +Past 50,000 tips or so the file becomes the constraint rather than the +computation. See [Performance](how-it-works/performance.md#where-the-ceiling-is). + +## Batch runs + +One matrix per tree, in parallel, one core each: + +```bash +ls trees/*.nwk | xargs -P 8 -I{} sh -c \ + 'distree "$1" -t 1 -p 6 --lower -o "${1%.nwk}.phy"' _ {} +``` + +`-t 1` matters here: without it every distree would try to use every core and +they would fight each other. + +## Checking a run at a glance + +```bash +distree tree.nwk --stats -o distances.tsv +``` + +``` +--- Statistics --- +Leaves in matrix: 1284 +Nodes in tree: 2567 +Mode: patristic distance +Cells written: 1648656 +Minimum: 0.0000002267 +Maximum: 0.0004821553 +Mean: 0.0001839204 +Time: 0.042s +``` + +Worth a look before anything downstream. A leaf count that is not what you +expected usually means the tree was not the one you meant; a maximum in the +wrong order of magnitude usually means the branch lengths are not in the units +you assumed. + +## Splitting a multi-tree file + +distree takes one tree per file, and refuses a file holding several rather than +silently using the first: + +```bash +split -l 1 -d posterior.trees rep_ --additional-suffix=.nwk +ls rep_*.nwk | xargs -P 8 -I{} sh -c \ + 'distree "$1" -t 1 -p 6 --lower -o "${1%.nwk}.phy"' _ {} +``` diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 0000000..e539f3c --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,139 @@ +/* ========================================================================== + distree theme + The sage green of the mark for the chrome, a warm clay for anything the + reader can act on, so a page has exactly two colours that mean something. + ========================================================================== */ + +[data-md-color-primary="custom"] { + --md-primary-fg-color: #40695e; + --md-primary-fg-color--light: #568b7d; + --md-primary-fg-color--dark: #2f4f47; +} + +[data-md-color-accent="custom"] { + --md-accent-fg-color: #b25f3c; +} + +[data-md-color-scheme="slate"] { + --md-default-bg-color: hsl(168, 8%, 9%); + --md-code-bg-color: hsl(168, 7%, 14%); +} +[data-md-color-scheme="slate"][data-md-color-accent="custom"] { + --md-accent-fg-color: #e39a63; +} + +/* The mark is wide rather than square, so it needs width to stay legible next + to the site name. */ +.md-header__button.md-logo img { + width: auto; + height: 1.5rem; +} + +/* ========================================================================== + Typography + Material's defaults are a touch loose for a reference site: tighten the + headings and give sections more air above than below, so a long page reads + as blocks rather than an even stream. + ========================================================================== */ + +.md-typeset h1 { + font-weight: 700; + letter-spacing: -0.02em; + color: var(--md-default-fg-color); +} + +.md-typeset h2 { + font-weight: 650; + letter-spacing: -0.01em; + margin-top: 2.4em; + padding-bottom: 0.3rem; + border-bottom: 1px solid var(--md-default-fg-color--lightest); +} + +.md-typeset h3 { + font-weight: 600; + margin-top: 1.8em; +} + +/* ========================================================================== + Tables + Most of this site is flag tables and small matrices, and Material lets them + stretch to the text column and wrap every cell. Let them scroll instead, and + keep the header in place while you read down a long one. + ========================================================================== */ + +.md-typeset table:not([class]) { + font-size: 0.72rem; + display: table; +} + +.md-typeset table:not([class]) th { + position: sticky; + top: 0; + background: var(--md-default-bg-color); + font-weight: 600; + white-space: nowrap; +} + +.md-typeset table:not([class]) td code { + white-space: nowrap; +} + +/* A distance matrix is a grid of numbers: right-aligning it makes the decimal + points line up, which is the only way to read one by eye. */ +.md-typeset .matrix table:not([class]) td:not(:first-child), +.md-typeset .matrix table:not([class]) th:not(:first-child) { + text-align: right; + font-variant-numeric: tabular-nums; +} + +/* ========================================================================== + Home page + ========================================================================== */ + +.hero { + text-align: center; + margin: 0.4rem 0 2.2rem; +} + +.hero .hero-logo { + width: 260px; + max-width: 80%; + margin: 0 auto; +} + +.md-typeset .hero h1 { + margin: 0.8rem 0 0; + font-size: 2rem; +} + +.md-typeset .hero p { + max-width: 34rem; + margin: 0.9rem auto 0; + font-size: 0.85rem; + color: var(--md-default-fg-color--light); +} + +/* Material's own card grid, kept a little tighter than the default. */ +.md-typeset .grid.cards > ul > li { + border-radius: 0.4rem; + padding: 0.8rem 1rem; +} + +.md-typeset .grid.cards > ul > li:hover { + border-color: var(--md-accent-fg-color); +} + +.md-typeset .grid.cards > ul > li > hr { + margin: 0.6rem 0; +} + +.md-typeset .grid.cards .twemoji, +.md-typeset .grid.cards svg { + color: var(--md-primary-fg-color); + vertical-align: -0.15em; +} + +[data-md-color-scheme="slate"] .md-typeset .grid.cards svg { + color: var(--md-primary-fg-color--light); +} diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 0000000..629290e --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,334 @@ +# Tutorial + +A worked outbreak investigation, start to finish. Seven *Mycobacterium +tuberculosis* isolates, one tree, and the question every such dataset opens +with: **which of these are transmission links, and which are unrelated cases +that happen to be in the same clinic?** + +Everything here runs in a few seconds and needs nothing but distree, so you can +paste along. R and Python appear in tabs; pick whichever you use. + +## 1. The tree + +Save this as `outbreak.nwk`. It is what a maximum-likelihood tool would hand +you: branch lengths in **substitutions per site**, in scientific notation. + +``` +((((TB_001:2.266786e-07,TB_002:2.266786e-07):2.266786e-07,TB_003:1.586750e-06):8.613788e-06,(TB_004:3.400179e-07,TB_005:3.400179e-07):9.860520e-06):1.360072e-05,(TB_006:1.813429e-05,TB_007:2.153447e-05):1.586750e-05); +``` + +```bash +distree --version +``` + +## 2. The first matrix + +```bash +distree outbreak.nwk -p 9 +``` + +``` + TB_001 TB_002 TB_003 TB_004 TB_005 TB_006 TB_007 +TB_001 0.000000000 0.000000453 0.000002040 0.000019268 0.000019268 0.000056670 0.000060070 +TB_002 0.000000453 0.000000000 0.000002040 0.000019268 0.000019268 0.000056670 0.000060070 +TB_003 0.000002040 0.000002040 0.000000000 0.000020401 0.000020401 0.000057803 0.000061203 +TB_004 0.000019268 0.000019268 0.000020401 0.000000000 0.000000680 0.000057803 0.000061203 +TB_005 0.000019268 0.000019268 0.000020401 0.000000680 0.000000000 0.000057803 0.000061203 +TB_006 0.000056670 0.000056670 0.000057803 0.000057803 0.000057803 0.000000000 0.000039669 +TB_007 0.000060070 0.000060070 0.000061203 0.000061203 0.000061203 0.000039669 0.000000000 +``` + +That is the whole tool, really. Each cell is the sum of the branch lengths on +the path between two tips: a **patristic distance**. The rest of this page is +about reading it. + +Some structure is already visible. TB_001 and TB_002 are an order of magnitude +closer to each other than to anything else, TB_004 and TB_005 likewise, and +TB_006 and TB_007 sit far from everyone. + +!!! note "Why the empty first cell" + + The header line starts with a tab, so the labels sit over the data columns + rather than one to the left. That is what `read.table(row.names = 1)` and + `pandas.read_csv(index_col=0)` expect, and it is why the matrix loads + without any fiddling in [step 4](#4-load-it). + +## 3. Substitutions to SNPs + +Nobody thinks in substitutions per site. Multiply by the length of the reference +the tree was built against, and the numbers become countable differences. +H37Rv is 4,411,532 bases: + +```bash +distree outbreak.nwk -p 12 | awk 'NR == 1 { print; next } + { printf "%s", $1; for (i = 2; i <= NF; i++) printf "\t%.0f", $i * 4411532; print "" }' +``` + +``` + TB_001 TB_002 TB_003 TB_004 TB_005 TB_006 TB_007 +TB_001 0 2 9 85 85 250 265 +TB_002 2 0 9 85 85 250 265 +TB_003 9 9 0 90 90 255 270 +TB_004 85 85 90 0 3 255 270 +TB_005 85 85 90 3 0 255 270 +TB_006 250 250 255 255 255 0 175 +TB_007 265 265 270 270 270 175 0 +``` + +Now it reads. TB_001 and TB_002 differ by **2 SNPs**. TB_003 is **9** from both. +TB_004 and TB_005 are **3** apart but **85** from the first group. TB_006 and +TB_007 are hundreds of SNPs from everything, including each other. + +!!! warning "Use enough precision before you scale" + + `-p 12` above, not the default `-p 10`. Multiplying by four million + multiplies the rounding too: at `-p 6` a distance of 2 SNPs would round to 0 + before you ever saw it. Scale from the most precise output you can, and + round at the end. + +## 4. Load it + +```bash +distree outbreak.nwk -p 12 -o distances.tsv +``` + +=== "R" + + ```r + m <- as.matrix(read.table("distances.tsv", header = TRUE, row.names = 1, + sep = "\t", check.names = FALSE)) + snps <- m * 4411532 + round(snps) + ``` + + `check.names = FALSE` matters. Without it R rewrites labels like `TB-001` + or `2024_isolate` and they stop matching your metadata. + +=== "Python" + + ```python + import pandas as pd + + m = pd.read_csv("distances.tsv", sep="\t", index_col=0) + snps = m * 4411532 + print(snps.round()) + ``` + +## 5. Find the clusters + +Public health uses SNP thresholds. Two are standard for *M. tuberculosis*: +**5 SNPs** for recent transmission, and **12 SNPs** for a plausible +epidemiological link.[^walker] Single linkage is the right rule, because it +groups isolates that are within the threshold of *someone* in the group, which +is what a chain of transmission looks like. + +=== "R" + + ```r + clusters <- function(h) split(rownames(snps), + cutree(hclust(as.dist(snps), method = "single"), h = h)) + clusters(5) + clusters(12) + ``` + +=== "Python" + + ```python + import collections + from scipy.cluster.hierarchy import linkage, fcluster + from scipy.spatial.distance import squareform + + z = linkage(squareform(snps.values, checks=False), method="single") + + def clusters(t): + out = collections.defaultdict(list) + for name, c in zip(snps.index, fcluster(z, t=t, criterion="distance")): + out[c].append(name) + return list(out.values()) + + print(clusters(5)) + print(clusters(12)) + ``` + +``` +threshold 5 SNPs: TB_001,TB_002 | TB_003 | TB_004,TB_005 | TB_006 | TB_007 +threshold 12 SNPs: TB_001,TB_002,TB_003 | TB_004,TB_005 | TB_006 | TB_007 +``` + +**The threshold is the finding.** TB_003 is 9 SNPs away: outside recent +transmission, inside a plausible link. Whether it belongs in the cluster is an +epidemiological question, not a computational one, and the matrix will not +answer it for you. Report which threshold you used. + +Note also what did *not* change. TB_004 and TB_005 are a pair at either +threshold, and TB_006 and TB_007 are singletons at either. Findings that survive +both thresholds are the ones to lead with. + +## 6. Who is closest to whom + +Before any clustering, the cheapest useful question is each isolate's nearest +neighbour: + +```bash +distree outbreak.nwk -p 12 | awk -F'\t' ' + NR == 1 { for (i = 2; i <= NF; i++) name[i] = $i; next } + { best = ""; bestd = 1e308 + for (i = 2; i <= NF; i++) + if (i - 1 != NR - 1 && $i + 0 < bestd) { bestd = $i + 0; best = name[i] } + printf "%s\t%s\t%.0f\n", $1, best, bestd * 4411532 }' +``` + +``` +TB_001 TB_002 2 +TB_002 TB_001 2 +TB_003 TB_001 9 +TB_004 TB_005 3 +TB_005 TB_004 3 +TB_006 TB_007 175 +TB_007 TB_006 175 +``` + +The `i - 1 != NR - 1` skips the diagonal, which is otherwise always the closest. + +The last two rows are the interesting ones. TB_006 and TB_007 are each other's +nearest neighbours and still 175 SNPs apart, which is not a transmission link; +it just means nobody closer was sampled. + +## 7. When branch lengths are not the point + +Sometimes the tree's branch lengths are not comparable, or there are none at +all. `--topology` counts edges instead: + +```bash +distree outbreak.nwk --topology +``` + +``` + TB_001 TB_002 TB_003 TB_004 TB_005 TB_006 TB_007 +TB_001 0 2 3 5 5 6 6 +TB_002 2 0 3 5 5 6 6 +TB_003 3 3 0 4 4 5 5 +TB_004 5 5 4 0 2 5 5 +TB_005 5 5 4 2 0 5 5 +TB_006 6 6 5 5 5 0 2 +TB_007 6 6 5 5 5 2 0 +``` + +Compare it with the SNP matrix and the difference in what they measure is +plain. Topologically TB_006 and TB_007 are 2 apart, the same as TB_001 and +TB_002; in SNPs they are 175 apart against 2. Edge counts describe the shape of +the tree, not the divergence across it, so use them for tree-shape questions +and never as a proxy for relatedness when you have usable branch lengths. + +## 8. The comparative-analysis matrix + +A different question: you have a trait for each isolate and want to test whether +it associates with something, without the shared ancestry inflating your +significance. That needs the variance-covariance matrix, and here rooting is +part of the answer, so root deliberately: + +```bash +distree outbreak.nwk --midpoint --lmm -p 8 -o varcovar.tsv +``` + +``` + TB_001 TB_002 TB_003 TB_004 TB_005 TB_006 TB_007 +TB_001 0.00002947 0.00002924 0.00002901 0.00002040 0.00002040 0.00000000 0.00000000 +TB_002 0.00002924 0.00002947 0.00002901 0.00002040 0.00002040 0.00000000 0.00000000 +... +``` + +Entry `(i, j)` is how much evolutionary history the two isolates share: the +distance from the root down to their common ancestor. The diagonal is each +isolate's own root-to-tip length. Pairs whose ancestor is the root share +nothing and score 0. + +Three things to check before you fit anything with it: + +- **It is not a distance matrix.** The diagonal is not zero, and bigger means + more similar. Do not feed it to something expecting distances. +- **Reindex it.** Rows and columns are in alphabetical label order, which is + almost certainly not the order of your trait table. +- **Root on purpose.** `--midpoint` is one defensible choice; an outgroup is + another. Unlike the patristic matrix, this one changes if you change your + mind. See [Midpoint rooting](guide/rooting.md). + +## 9. Making it bigger + +Seven isolates is a tutorial. Seven thousand is a Tuesday, and two flags matter +then. + +```bash +distree big.nwk --lower -p 9 -t 8 -o distances.phy +``` + +`--lower` writes the PHYLIP lower triangle, halving the file: + +``` +7 +TB_001 +TB_002 0.000000453 +TB_003 0.000002040 0.000002040 +TB_004 0.000019268 0.000019268 0.000020401 +TB_005 0.000019268 0.000019268 0.000020401 0.000000680 +TB_006 0.000056670 0.000056670 0.000057803 0.000057803 0.000057803 +TB_007 0.000060070 0.000060070 0.000061203 0.000061203 0.000061203 0.000039669 +``` + +`-t` caps the threads; leave it off on a machine of your own and set it when +you are sharing one. + +Rows are written as they are computed, so a large run streams rather than +building the whole matrix first. Compress on the way out, or filter as it goes: + +```bash +distree big.nwk -p 12 --lower | gzip > distances.phy.gz +``` + +For what a run costs at various sizes, see +[Performance](how-it-works/performance.md). + +## 10. When it refuses + +distree would rather stop than hand back a matrix that looks right and is not. +The three you are most likely to meet: + +``` +Error: /path/tree.nwk is gzip-compressed. distree reads plain text; decompress it on the way in: + gunzip -c /path/tree.nwk | distree - +``` + +``` +Error: Failed to parse Newick tree: Unexpected content at position 84: the file appears to +hold more than one tree. distree processes a single Newick tree; split the file first. +``` + +``` +Error: Duplicate leaf name 'TB_001' found. Leaf names must be unique. +``` + +A duplicate label is worth pausing over. It usually means the same sample went +into the tree twice under two accessions, and no matrix can tell the two rows +apart afterwards. The full list is in [Input](guide/input.md#what-is-rejected). + +## What to take away + +- The **patristic** matrix is what almost every downstream question wants, and + it does not care how the tree is rooted. +- **Scale it once**, from high precision, and round at the end. +- **The threshold is a decision**, not an output. State it. +- **`--lmm` is a different object**: a covariance, not a distance, and rooting + changes it. +- **`--topology` measures shape**, not divergence. + +## Next + +- [Recipes](recipes.md) for PCoA, PGLS and neighbour-joining, at the same length + as this section but without the story. +- [Distance modes](guide/distances.md) for the three modes in full. +- [CLI reference](guide/cli.md) for every flag. + +[^walker]: + Walker TM, Ip CLC, Harrell RH, et al. *Whole-genome sequencing to delineate + Mycobacterium tuberculosis outbreaks: a retrospective observational study.* + The Lancet Infectious Diseases. 2013;13(2):137-146. diff --git a/fuzz/.gitignore b/fuzz/.gitignore new file mode 100644 index 0000000..fe68c97 --- /dev/null +++ b/fuzz/.gitignore @@ -0,0 +1,4 @@ +target/ +corpus/ +artifacts/ +coverage/ diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock new file mode 100644 index 0000000..da9473a --- /dev/null +++ b/fuzz/Cargo.lock @@ -0,0 +1,325 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "distree" +version = "1.0.1" +dependencies = [ + "clap", + "rayon", +] + +[[package]] +name = "distree-fuzz" +version = "0.0.0" +dependencies = [ + "distree", + "libfuzzer-sys", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 0000000..7fbc40c --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "distree-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.distree] +path = ".." + +# Detached from the parent workspace so `cargo build` at the repo root does not +# try to build a nightly-only crate. +[workspace] + +[[bin]] +name = "parse_newick" +path = "fuzz_targets/parse_newick.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "full_pipeline" +path = "fuzz_targets/full_pipeline.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/full_pipeline.rs b/fuzz/fuzz_targets/full_pipeline.rs new file mode 100644 index 0000000..a66b8c3 --- /dev/null +++ b/fuzz/fuzz_targets/full_pipeline.rs @@ -0,0 +1,66 @@ +#![no_main] + +//! Run whatever parses all the way through to a distance. +//! +//! The parser rejecting bad input is only half of it. A tree that parses can +//! still be a shape nothing downstream expects, and flattening, midpoint +//! rooting and the binary-lifting queries all index into flat arrays. This +//! target takes every input the parser accepts and pushes it through the rest +//! of the pipeline, in every mode, with and without rooting. + +use libfuzzer_sys::fuzz_target; + +use distree::lca::build_lca_structure; +use distree::midpoint::midpoint_root; +use distree::parser::{flatten_raw, parse_newick}; +use distree::tree::Node; +use distree::{compute_distance, DistMode}; + +fuzz_target!(|data: &[u8]| { + let Ok(text) = std::str::from_utf8(data) else { + return; + }; + if text.len() > 16 * 1024 { + return; + } + let Ok(raw) = parse_newick(text) else { + return; + }; + + for root_it in [false, true] { + let mut nodes: Vec = Vec::new(); + let mut root = flatten_raw(&raw, None, &mut nodes); + + // Midpoint rooting is undefined with negative branch lengths, and the + // binary warns rather than refusing, so keep those out of this arm. + if root_it { + if nodes.iter().any(|n| n.length < 0.0 || !n.length.is_finite()) { + continue; + } + root = midpoint_root(root, &mut nodes); + } + + let lca = build_lca_structure(root, &nodes); + + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + // The matrix is quadratic; a few hundred leaves is plenty to exercise + // the query paths without starving the fuzzer of executions. + let sample = leaves.len().min(24); + for &i in leaves.iter().take(sample) { + for &j in leaves.iter().take(sample) { + for mode in [DistMode::Patristic, DistMode::Topology, DistMode::Lmm] { + let d = compute_distance(i, j, mode, &lca); + if i == j && mode != DistMode::Lmm { + assert_eq!(d, 0.0, "a leaf must be exactly 0 from itself"); + } + } + } + } + } +}); diff --git a/fuzz/fuzz_targets/parse_newick.rs b/fuzz/fuzz_targets/parse_newick.rs new file mode 100644 index 0000000..2625b8b --- /dev/null +++ b/fuzz/fuzz_targets/parse_newick.rs @@ -0,0 +1,28 @@ +#![no_main] + +//! Throw arbitrary text at the Newick parser. +//! +//! The parser is hand-written, scans bytes, and is deliberately strict, which +//! is exactly the combination that hides an index that runs off the end or a +//! loop that fails to advance. Any input at all must come back as `Ok` or +//! `Err`, never a panic and never a hang. + +use libfuzzer_sys::fuzz_target; + +use distree::parser::parse_newick; + +fuzz_target!(|data: &[u8]| { + // The parser takes &str; anything that is not UTF-8 is rejected before it + // ever gets there, so there is nothing to learn from feeding it here. + let Ok(text) = std::str::from_utf8(data) else { + return; + }; + + // Very long inputs only slow the fuzzer down; the interesting behaviour is + // in the structure, not the size. + if text.len() > 64 * 1024 { + return; + } + + let _ = parse_newick(text); +}); diff --git a/fuzz/seeds/01.nwk b/fuzz/seeds/01.nwk new file mode 100644 index 0000000..ddcd782 --- /dev/null +++ b/fuzz/seeds/01.nwk @@ -0,0 +1 @@ +(A:1.0,B:2.0); \ No newline at end of file diff --git a/fuzz/seeds/02.nwk b/fuzz/seeds/02.nwk new file mode 100644 index 0000000..5e03591 --- /dev/null +++ b/fuzz/seeds/02.nwk @@ -0,0 +1 @@ +((A:1.0,B:2.0):0.5,C:3.0); \ No newline at end of file diff --git a/fuzz/seeds/03.nwk b/fuzz/seeds/03.nwk new file mode 100644 index 0000000..83128a8 --- /dev/null +++ b/fuzz/seeds/03.nwk @@ -0,0 +1 @@ +(A:1,B:2,C:3); \ No newline at end of file diff --git a/fuzz/seeds/04.nwk b/fuzz/seeds/04.nwk new file mode 100644 index 0000000..c782586 --- /dev/null +++ b/fuzz/seeds/04.nwk @@ -0,0 +1 @@ +[&R] ((A:0.1[&&NHX:S=human],B:0.2):0.3,C:0.4); \ No newline at end of file diff --git a/fuzz/seeds/05.nwk b/fuzz/seeds/05.nwk new file mode 100644 index 0000000..e79adbe --- /dev/null +++ b/fuzz/seeds/05.nwk @@ -0,0 +1 @@ +('Taxon A':1.0,"Taxon B":2.0)'my clade':0.5; \ No newline at end of file diff --git a/fuzz/seeds/06.nwk b/fuzz/seeds/06.nwk new file mode 100644 index 0000000..b739e14 --- /dev/null +++ b/fuzz/seeds/06.nwk @@ -0,0 +1 @@ +(A:1.5e-3,B:2.0E+1,(C:0.0,D:-0.5):1e10); \ No newline at end of file diff --git a/fuzz/seeds/07.nwk b/fuzz/seeds/07.nwk new file mode 100644 index 0000000..4cd4be6 --- /dev/null +++ b/fuzz/seeds/07.nwk @@ -0,0 +1 @@ +((((A:1):1):1):1); \ No newline at end of file diff --git a/fuzz/seeds/08.nwk b/fuzz/seeds/08.nwk new file mode 100644 index 0000000..444840b --- /dev/null +++ b/fuzz/seeds/08.nwk @@ -0,0 +1 @@ +('it''s':1,B:2); \ No newline at end of file diff --git a/fuzz/seeds/09.nwk b/fuzz/seeds/09.nwk new file mode 100644 index 0000000..0fb07c9 --- /dev/null +++ b/fuzz/seeds/09.nwk @@ -0,0 +1 @@ +(A,B,(C,D)); \ No newline at end of file diff --git a/fuzz/seeds/10.nwk b/fuzz/seeds/10.nwk new file mode 100644 index 0000000..2ca2e8a --- /dev/null +++ b/fuzz/seeds/10.nwk @@ -0,0 +1 @@ +(A:1.0,B:2.0) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..fede502 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,145 @@ +site_name: distree +site_description: >- + Pairwise distance matrices from a phylogeny. distree reads a Newick tree and + writes patristic, topological or variance-covariance distances between every + pair of leaves, one row at a time, in parallel and without holding the matrix + in memory. +site_author: PathoGenOmics Lab +site_url: https://pathogenomics-lab.github.io/distree/ + +repo_name: PathoGenOmics-Lab/distree +repo_url: https://github.com/PathoGenOmics-Lab/distree +edit_uri: edit/main/docs/ +copyright: Copyright © PathoGenOmics Lab · GPL-3.0 + +theme: + name: material + logo: assets/distree_logo.svg + favicon: assets/distree_logo.svg + language: en + font: + text: Inter + code: JetBrains Mono + icon: + repo: fontawesome/brands/github + edit: material/pencil + admonition: + note: material/dna + palette: + - media: "(prefers-color-scheme)" + primary: custom + accent: custom + toggle: + icon: material/brightness-auto + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: custom + accent: custom + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: custom + accent: custom + toggle: + icon: material/brightness-4 + name: Switch to system preference + features: + - navigation.tabs + - navigation.tabs.sticky + - navigation.sections + - navigation.top + - navigation.footer + - navigation.indexes + - navigation.tracking + - toc.follow + - search.suggest + - search.highlight + - search.share + - content.code.copy + - content.code.annotate + - content.tabs.link + - content.tooltips + +# Broken cross-references and dead anchors become warnings, which `mkdocs build +# --strict` in CI turns into a failed build. +validation: + omitted_files: warn + absolute_links: warn + unrecognized_links: warn + anchors: warn + +extra_css: + - stylesheets/extra.css + +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - tables + - toc: + permalink: true + toc_depth: 3 + - pymdownx.betterem + - pymdownx.caret + - pymdownx.keys + - pymdownx.mark + - pymdownx.tilde + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets: + base_path: + - "." + check_paths: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + +plugins: + - search + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/PathoGenOmics-Lab/distree + name: distree on GitHub + generator: false + +nav: + - Home: index.md + - Getting started: + - getting-started/installation.md + - getting-started/quickstart.md + - Tutorial: tutorial.md + - User guide: + - guide/input.md + - guide/distances.md + - guide/rooting.md + - guide/output.md + - guide/cli.md + - How it works: + - how-it-works/algorithm.md + - how-it-works/performance.md + - Recipes: recipes.md + - About: + - about/changelog.md + - about/citation.md + - about/contributing.md diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..422f4b7 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,4 @@ +# Documentation site (MkDocs Material). Build locally with: +# pip install -r requirements-docs.txt +# mkdocs serve +mkdocs-material>=9.6,<10 diff --git a/scripts/crossvalidate.R b/scripts/crossvalidate.R new file mode 100755 index 0000000..c12979a --- /dev/null +++ b/scripts/crossvalidate.R @@ -0,0 +1,141 @@ +#!/usr/bin/env Rscript + +# Cross-validate distree against ape. +# +# The Rust test suite is self-consistent: it checks that midpoint rooting +# preserves distances, that the binary-lifting MRCA agrees with walking up from +# both nodes, and that a patristic distance is never negative. None of that +# would catch a systematic error in what the distances are *defined* to be, +# because every check is written against the same understanding as the code. +# +# ape is that independent understanding. cophenetic.phylo is the reference +# patristic matrix, vcv.phylo the reference variance-covariance matrix, and +# phangorn::midpoint the reference midpoint rooting. +# +# Usage: +# Rscript scripts/crossvalidate.R [n_trees] [path/to/distree] +# +# Add --write-fixtures to regenerate tests/fixtures/, which is what lets the +# Rust suite check the same thing in CI without needing R. + +suppressMessages({ + library(ape) + library(phangorn) +}) + +args <- commandArgs(trailingOnly = TRUE) +write_fx <- "--write-fixtures" %in% args +args <- args[args != "--write-fixtures"] +n_trees <- if (length(args) >= 1) as.integer(args[1]) else 200 +distree <- if (length(args) >= 2) args[2] else "target/release/distree" + +if (!file.exists(distree)) { + stop("distree binary not found at '", distree, "'. Run: cargo build --release") +} + +fixture_dir <- "tests/fixtures" +if (write_fx) dir.create(fixture_dir, showWarnings = FALSE, recursive = TRUE) + +# distree sorts labels by byte order; radix sorting is the locale-independent +# match for that. Anything else and the two matrices are compared transposed. +byte_sort <- function(x) sort(x, method = "radix") + +# Read a distree square TSV back into a matrix. +read_distree <- function(path) { + m <- as.matrix(read.table(path, header = TRUE, row.names = 1, + sep = "\t", check.names = FALSE)) + colnames(m) <- rownames(m) + m +} + +run_distree <- function(tree, flags) { + nwk <- tempfile(fileext = ".nwk") + out <- tempfile(fileext = ".tsv") + write.tree(tree, file = nwk) + status <- system2(distree, c(shQuote(nwk), flags, "-p", "12", "-o", shQuote(out)), + stdout = NULL, stderr = NULL) + if (status != 0) stop("distree exited ", status, " for flags: ", paste(flags, collapse = " ")) + m <- read_distree(out) + unlink(c(nwk, out)) + m +} + +# Largest absolute difference, after lining both matrices up by label. +deviation <- function(got, want) { + labs <- byte_sort(rownames(got)) + max(abs(got[labs, labs] - want[labs, labs])) +} + +set.seed(20260726) +worst <- list(patristic = 0, topology = 0, lmm = 0, midpoint_lmm = 0) +fixtures <- list() + +for (i in seq_len(n_trees)) { + n <- sample(3:40, 1) + + # A spread of shapes: random topologies with random lengths, ultrametric + # coalescent trees, and trees with polytomies collapsed into them. + tree <- switch(sample(1:3, 1), + rtree(n), + rcoal(n), + di2multi(rtree(n), tol = 0.05) + ) + # ape allows negative lengths through rtree only rarely; keep them out, since + # a negative branch makes midpoint rooting undefined for both tools. + tree$edge.length <- abs(tree$edge.length) + + # --- patristic --------------------------------------------------------- + d <- deviation(run_distree(tree, character(0)), cophenetic(tree)) + worst$patristic <- max(worst$patristic, d) + + # --- topological ------------------------------------------------------- + unit <- tree + unit$edge.length <- rep(1, nrow(unit$edge)) + d <- deviation(run_distree(tree, "--topology"), cophenetic(unit)) + worst$topology <- max(worst$topology, d) + + # --- variance-covariance ---------------------------------------------- + d <- deviation(run_distree(tree, "--lmm"), vcv(tree)) + worst$lmm <- max(worst$lmm, d) + + # --- midpoint rooting, seen through the var-covar matrix --------------- + # This is the only mode where rooting changes the answer, so it is the one + # that tests whether the two tools put the root in the same place. + rooted <- tryCatch(midpoint(tree), error = function(e) NULL) + if (!is.null(rooted)) { + d <- deviation(run_distree(tree, "--midpoint --lmm"), vcv(rooted)) + worst$midpoint_lmm <- max(worst$midpoint_lmm, d) + } + + if (write_fx && i <= 6) { + stem <- sprintf("%s/tree%02d", fixture_dir, i) + write.tree(tree, file = paste0(stem, ".nwk")) + for (mode in c("patristic", "topology", "lmm")) { + ref <- switch(mode, + patristic = cophenetic(tree), + topology = cophenetic(unit), + lmm = vcv(tree) + ) + labs <- byte_sort(rownames(ref)) + ref <- ref[labs, labs] + con <- file(sprintf("%s.%s.tsv", stem, mode), "w") + writeLines(paste0("\t", paste(labs, collapse = "\t")), con) + for (r in labs) { + writeLines(paste(c(r, sprintf("%.12f", ref[r, ])), collapse = "\t"), con) + } + close(con) + } + fixtures <- c(fixtures, stem) + } +} + +cat("Cross-validated", n_trees, "trees against ape", as.character(packageVersion("ape")), "\n\n") +cat(sprintf(" %-14s max |distree - ape| = %.3e\n", names(worst), unlist(worst)), sep = "") + +tolerance <- 1e-9 +if (max(unlist(worst)) > tolerance) { + cat("\nFAIL: deviation above", tolerance, "\n") + quit(status = 1) +} +cat("\nAll modes agree to within", tolerance, "\n") +if (write_fx) cat("Wrote", length(fixtures), "fixtures to", fixture_dir, "\n") diff --git a/src/lca.rs b/src/lca.rs new file mode 100644 index 0000000..2491274 --- /dev/null +++ b/src/lca.rs @@ -0,0 +1,232 @@ +use crate::tree::Node; + +/// Marks "no such ancestor" in the binary-lifting table. +/// +/// `Option` would be the obvious type, but it has no spare niche and so +/// costs 16 bytes per entry against 8 for a bare `usize`. The table holds +/// n·log n entries, which on a million-leaf tree is the difference between +/// roughly 700 MB and 350 MB. +const NO_ANCESTOR: usize = usize::MAX; + +/// Precomputed data for O(log n) LCA queries using binary lifting. +pub struct LcaData { + /// Binary lifting table, stored flat: `up[k * n + u]` is the 2^k-th + /// ancestor of node `u`, or [`NO_ANCESTOR`] if `u` has none. + up: Vec, + /// Node count, and therefore the stride of a row of `up`. + n: usize, + /// Number of levels held in `up`. + levels: usize, + /// Cumulative branch-length distance from the root to each node. + pub depth_len: Vec, + /// Topological depth (number of edges) from the root to each node. + pub depth_top: Vec, +} + +/// Build the [`LcaData`] binary-lifting structure rooted at `root_idx`. +/// +/// Runs in O(n log n) time and O(n log n) space. +pub fn build_lca_structure(root_idx: usize, nodes: &[Node]) -> LcaData { + let n = nodes.len(); + let levels = if n <= 1 { 1 } else { ((n as f64).log2().ceil() as usize) + 1 }; + let mut depth_len = vec![0.0; n]; + let mut depth_top = vec![0; n]; + let mut up: Vec = vec![NO_ANCESTOR; levels * n]; + + { + let mut stack = vec![root_idx]; + depth_len[root_idx] = 0.0; + depth_top[root_idx] = 0; + + while let Some(u) = stack.pop() { + for &v in &nodes[u].children { + up[v] = u; + depth_len[v] = depth_len[u] + nodes[v].length; + depth_top[v] = depth_top[u] + 1; + stack.push(v); + } + } + } + + for k in 1..levels { + for u in 0..n { + let mid = up[(k - 1) * n + u]; + up[k * n + u] = if mid == NO_ANCESTOR { + NO_ANCESTOR + } else { + up[(k - 1) * n + mid] + }; + } + } + + LcaData { + up, + n, + levels, + depth_len, + depth_top, + } +} + +impl LcaData { + /// The 2^k-th ancestor of `u`, or [`NO_ANCESTOR`]. + #[inline] + fn ancestor(&self, k: usize, u: usize) -> usize { + self.up[k * self.n + u] + } + + /// Return the index of the Most Recent Common Ancestor (MRCA) of nodes `u` and `v`. + /// + /// Uses binary lifting for O(log n) queries. + pub fn mrca(&self, mut u: usize, mut v: usize) -> usize { + if u == v { + return u; + } + if self.depth_top[u] < self.depth_top[v] { + std::mem::swap(&mut u, &mut v); + } + // Lift the deeper node until both sit at the same depth + let mut diff = self.depth_top[u] - self.depth_top[v]; + let mut k = 0; + while diff > 0 { + if (diff & 1) == 1 { + u = self.ancestor(k, u); + assert_ne!( + u, NO_ANCESTOR, + "LCA: depth table disagrees with the tree; tree may be malformed" + ); + } + diff >>= 1; + k += 1; + } + if u == v { + return u; + } + // Climb in step as long as the ancestors differ; they meet one above + for k in (0..self.levels).rev() { + let au = self.ancestor(k, u); + let av = self.ancestor(k, v); + if au != av && au != NO_ANCESTOR && av != NO_ANCESTOR { + u = au; + v = av; + } + } + let m = self.ancestor(0, u); + assert_ne!( + m, NO_ANCESTOR, + "LCA: could not find common ancestor; tree may be malformed" + ); + m + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::{flatten_raw, parse_newick}; + use crate::testutil::{random_newick, Rng}; + use std::collections::HashSet; + + fn make(newick: &str) -> (Vec, usize) { + let raw = parse_newick(newick).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + (nodes, root) + } + + #[test] + fn test_mrca_siblings() { + // (A:1,B:2); — MRCA(A,B) = root + let (nodes, root) = make("(A:1.0,B:2.0);"); + let lca = build_lca_structure(root, &nodes); + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + let b = nodes.iter().position(|n| n.name.as_deref() == Some("B")).unwrap(); + assert_eq!(lca.mrca(a, b), root); + } + + #[test] + fn test_mrca_deeper() { + // ((A:1,B:2):3,C:4); — MRCA(A,B) = inner node, MRCA(A,C) = root + let (nodes, root) = make("((A:1.0,B:2.0):3.0,C:4.0);"); + let lca = build_lca_structure(root, &nodes); + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + let b = nodes.iter().position(|n| n.name.as_deref() == Some("B")).unwrap(); + let c = nodes.iter().position(|n| n.name.as_deref() == Some("C")).unwrap(); + let ab = lca.mrca(a, b); + assert_ne!(ab, root, "MRCA(A,B) should be the inner node, not root"); + assert_eq!(lca.mrca(a, c), root); + // depth of inner node = 3.0 + assert!((lca.depth_len[ab] - 3.0).abs() < 1e-10); + } + + #[test] + fn test_mrca_self() { + // MRCA(x, x) must be x itself + let (nodes, root) = make("(A:1.0,B:2.0);"); + let lca = build_lca_structure(root, &nodes); + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + assert_eq!(lca.mrca(a, a), a); + assert_eq!(lca.mrca(root, root), root); + } + + /// Walk to the root from `u`, then from `v` until the paths meet. + fn mrca_by_walking(nodes: &[Node], u: usize, v: usize) -> usize { + let mut seen = HashSet::new(); + let mut cur = Some(u); + while let Some(x) = cur { + seen.insert(x); + cur = nodes[x].parent; + } + let mut cur = Some(v); + while let Some(x) = cur { + if seen.contains(&x) { + return x; + } + cur = nodes[x].parent; + } + panic!("no common ancestor between {} and {}", u, v); + } + + #[test] + fn test_mrca_matches_walking_up_on_random_trees() { + let mut rng = Rng::new(0x2545_F491_4F6C_DD1D); + + for case in 0..200 { + let n_leaves = 2 + rng.below(30); + let newick = random_newick(&mut rng, n_leaves); + let raw = parse_newick(&newick).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + let lca = build_lca_structure(root, &nodes); + + for u in 0..nodes.len() { + for v in 0..nodes.len() { + let expected = mrca_by_walking(&nodes, u, v); + assert_eq!( + lca.mrca(u, v), + expected, + "case {}: mrca({}, {}) in {}", + case, + u, + v, + newick + ); + } + } + } + } + + #[test] + fn test_depth_top() { + // ((A,B),C); — A and B are at depth 2, C at depth 1 + let (nodes, root) = make("((A:1,B:1):1,C:1);"); + let lca = build_lca_structure(root, &nodes); + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + let b = nodes.iter().position(|n| n.name.as_deref() == Some("B")).unwrap(); + let c = nodes.iter().position(|n| n.name.as_deref() == Some("C")).unwrap(); + assert_eq!(lca.depth_top[root], 0); + assert_eq!(lca.depth_top[c], 1); + assert_eq!(lca.depth_top[a], 2); + assert_eq!(lca.depth_top[b], 2); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..272c584 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,277 @@ +//! Distance matrices from a phylogeny. +//! +//! The tree side of `distree`: parsing Newick, flattening it into an +//! index-addressed array, answering most-recent-common-ancestor queries in +//! logarithmic time, midpoint rooting, and turning a pair of leaves into a +//! distance. The binary in `main.rs` is the command line over the top of it. + +pub mod lca; +pub mod midpoint; +pub mod parser; +#[cfg(test)] +mod testutil; +pub mod tree; + +/// Distance mode to compute. +/// +/// Selects how pairwise leaf distances are calculated from the tree. +#[derive(Clone, Copy, PartialEq)] +pub enum DistMode { + Patristic, + Topology, + Lmm, +} + +/// Compute the distance between two leaves according to `mode`. +/// +/// Returns the patristic distance, topological hop count, or LMM covariance depth. +#[inline] +pub fn compute_distance( + leaf_i: usize, + leaf_j: usize, + mode: DistMode, + lca_data: &lca::LcaData, +) -> f64 { + let m = lca_data.mrca(leaf_i, leaf_j); + match mode { + DistMode::Lmm => lca_data.depth_len[m], + DistMode::Topology => { + let d_i = lca_data.depth_top[leaf_i]; + let d_j = lca_data.depth_top[leaf_j]; + let d_m = lca_data.depth_top[m]; + ((d_i + d_j).saturating_sub(2 * d_m)) as f64 + } + DistMode::Patristic => { + let d_i = lca_data.depth_len[leaf_i]; + let d_j = lca_data.depth_len[leaf_j]; + let d_m = lca_data.depth_len[m]; + // Not clamped to zero. Rounding cannot make this negative: + // depth_len accumulates lengths from the root, so with + // non-negative branches d_i and d_j are both >= d_m, 2*d_m is + // exact, and rounding a non-negative exact result to nearest keeps + // it non-negative. The only way to get a negative value here is a + // negative branch length, which the tree is warned about, and + // rounding that up to zero would silently claim two distinct taxa + // sit on top of each other. + d_i + d_j - 2.0 * d_m + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lca::build_lca_structure; + use crate::parser::{flatten_raw, parse_newick}; + use crate::tree::Node; + use std::collections::HashSet; + + fn build_tree(newick: &str) -> (Vec, usize) { + let raw = parse_newick(newick).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + (nodes, root) + } + + fn get_leaf(nodes: &[Node], name: &str) -> usize { + nodes + .iter() + .position(|n| n.name.as_deref() == Some(name)) + .unwrap() + } + + fn patristic(nodes: &[Node], root: usize, a: &str, b: &str) -> f64 { + let lca = build_lca_structure(root, nodes); + let ai = get_leaf(nodes, a); + let bi = get_leaf(nodes, b); + compute_distance(ai, bi, DistMode::Patristic, &lca) + } + + #[test] + fn test_simple_patristic_distances() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + assert!((patristic(&nodes, root, "A", "B") - 3.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.5).abs() < 1e-10); + assert!((patristic(&nodes, root, "B", "C") - 5.5).abs() < 1e-10); + } + + #[test] + fn test_topology_distances() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let c = get_leaf(&nodes, "C"); + let d = compute_distance(a, c, DistMode::Topology, &lca); + assert_eq!(d as i64, 3); + } + + #[test] + fn test_lmm_matrix() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let b = get_leaf(&nodes, "B"); + let c = get_leaf(&nodes, "C"); + // MRCA(A,B) depth = 0.5 + assert!((compute_distance(a, b, DistMode::Lmm, &lca) - 0.5).abs() < 1e-10); + // MRCA(A,C) depth = 0.0 (root) + assert!(compute_distance(a, c, DistMode::Lmm, &lca).abs() < 1e-10); + } + + #[test] + fn test_duplicate_leaves_detected() { + let (nodes, _root) = build_tree("(A:1.0,A:2.0);"); + let leaf_indices: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + let mut seen = HashSet::new(); + let has_dup = leaf_indices + .iter() + .any(|&i| !seen.insert(nodes[i].name.as_ref().unwrap().as_str())); + assert!(has_dup); + } + + #[test] + fn test_negative_branch_length_detected() { + let (nodes, _root) = build_tree("(A:-0.5,B:2.0);"); + assert!(nodes.iter().any(|n| n.length < 0.0)); + } + + #[test] + fn test_trifurcation() { + let (nodes, root) = build_tree("(A:1.0,B:2.0,C:3.0);"); + assert_eq!(nodes[root].children.len(), 3); + assert!((patristic(&nodes, root, "A", "B") - 3.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.0).abs() < 1e-10); + } + + #[test] + fn test_no_branch_lengths_all_zero() { + let (nodes, _root) = build_tree("(A,B,(C,D));"); + assert!(nodes.iter().all(|n| n.length == 0.0)); + } + + #[test] + fn test_single_leaf() { + let (nodes, _root) = build_tree("(A:1.0);"); + assert_eq!( + nodes + .iter() + .filter(|n| n.children.is_empty() && n.name.is_some()) + .count(), + 1 + ); + } + + #[test] + fn test_large_symmetric_tree() { + // ((A:1,B:1):1,(C:1,D:1):1) — all pairwise distances known + let (nodes, root) = build_tree("((A:1,B:1):1,(C:1,D:1):1);"); + assert!((patristic(&nodes, root, "A", "B") - 2.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "C", "D") - 2.0).abs() < 1e-10); + } + + #[test] + fn test_zero_distance_same_leaf() { + let (nodes, root) = build_tree("(A:1.0,B:2.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + assert!(compute_distance(a, a, DistMode::Patristic, &lca).abs() < 1e-10); + } + + #[test] + fn test_lower_triangle_row_counts() { + // In lower-triangle mode, row i has exactly i distance columns + let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); + let _lca = build_lca_structure(root, &nodes); + let mut leaf_pairs: Vec<(String, usize)> = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, n)| (n.name.clone().unwrap(), i)) + .collect(); + leaf_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + let sorted: Vec = leaf_pairs.iter().map(|(_, i)| *i).collect(); + + // Row 0 (A): 0 distance cols, Row 1 (B): 1 distance col, Row 2 (C): 2 distance cols + for (row_i, _) in sorted.iter().enumerate() { + let col_end = row_i; // lower triangle: number of distance values + assert_eq!(col_end, row_i); + } + } + + #[test] + fn test_patristic_clamped_nonnegative() { + // Self-distance must be exactly 0.0 (no negative FP artifacts) + let (nodes, root) = build_tree("(A:1.0000000000001,B:1.0000000000002);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let d = compute_distance(a, a, DistMode::Patristic, &lca); + assert_eq!(d, 0.0, "Self-distance must be exactly 0.0, got {}", d); + } + + #[test] + fn test_patristic_never_negative_on_random_trees() { + // Rounding must not produce a negative distance on any tree with + // non-negative branch lengths, and a leaf must be exactly 0 from itself + let mut rng = crate::testutil::Rng::new(0x5DEE_CE66_D000_0005); + + for _ in 0..200 { + let n_leaves = 2 + rng.below(30); + let newick = crate::testutil::random_newick(&mut rng, n_leaves); + let (nodes, root) = build_tree(&newick); + let lca = build_lca_structure(root, &nodes); + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + for &i in &leaves { + assert_eq!( + compute_distance(i, i, DistMode::Patristic, &lca), + 0.0, + "a leaf must be exactly 0.0 from itself in {}", + newick + ); + for &j in &leaves { + let d = compute_distance(i, j, DistMode::Patristic, &lca); + assert!(d >= 0.0, "negative distance {} in {}", d, newick); + } + } + } + } + + #[test] + fn test_negative_branch_length_gives_negative_distance() { + // A tree with negative branches is warned about; reporting 0.0 would + // claim two distinct taxa sit on top of each other + let (nodes, root) = build_tree("(A:-2.0,B:0.5);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let b = get_leaf(&nodes, "B"); + let d = compute_distance(a, b, DistMode::Patristic, &lca); + assert!((d - (-1.5)).abs() < 1e-12, "expected -1.5, got {}", d); + } + + #[test] + fn test_mode_conflict() { + // Just test the logic: LMM should be distinct from topology + let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let c = get_leaf(&nodes, "C"); + let d_pat = compute_distance(a, c, DistMode::Patristic, &lca); + let d_top = compute_distance(a, c, DistMode::Topology, &lca); + let d_lmm = compute_distance(a, c, DistMode::Lmm, &lca); + // All three should give different values for this tree + assert!((d_pat - 8.0).abs() < 1e-10); // 1+3+4 + assert_eq!(d_top as i64, 3); // 2 edges from A + 1 from C + assert!(d_lmm.abs() < 1e-10); // MRCA(A,C) = root, depth 0 + } +} diff --git a/src/main.rs b/src/main.rs index eff1520..adc6420 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,60 +1,62 @@ use clap::{Arg, ArgAction, Command}; +use flate2::read::GzDecoder; use rayon::prelude::*; +use std::collections::HashSet; use std::fs::File; use std::io::{self, BufWriter, Read, Write}; -use std::str::Chars; -use std::iter::Peekable; -use std::num::ParseFloatError; - -/// Our internal representation of a tree node: -/// - `name`: if this is a leaf, its label; otherwise `None`. -/// - `length`: branch length from its parent (0.0 if unspecified). -/// - `parent`: index of the parent node in `nodes`, or `None` if this is the root. -/// - `children`: list of indices of child nodes. -struct Node { - name: Option, - length: f64, - parent: Option, - children: Vec, -} - -/// Precomputed data for LCA (binary lifting): -/// - `up[k][u]`: the 2^k-th ancestor of node `u`, or `None` if above the root. -/// - `depth_len[u]`: distance (sum of branch lengths) from node `u` up to the root. -/// - `depth_top[u]`: number of edges from node `u` up to the root. -struct LcaData { - up: Vec>>, - depth_len: Vec, - depth_top: Vec, -} - -/// A temporary structure used during Newick parsing, -/// before flattening into `Vec`. -struct RawNode { - name: Option, - length: f64, - children: Vec, +use std::process::ExitCode; +use std::time::Instant; + +use distree::lca::build_lca_structure; +use distree::midpoint::midpoint_root; +use distree::parser::{flatten_raw, parse_newick}; +use distree::tree::Node; +use distree::{compute_distance, DistMode}; + +/// Largest accepted `--precision`. +/// +/// An f64 carries about 17 significant decimal digits, so anything past this +/// is zero padding. The cap also keeps the formatting machinery inside the +/// range it accepts: `{:.prec$}` panics outright on very large values. +const MAX_PRECISION: usize = 30; + +/// Target number of cells per parallel batch, which caps the batch buffer at +/// 8 MB and gives each fork/join enough work to be worth its cost. +const BATCH_CELLS: usize = 1 << 20; + +/// Output buffer size. The default 8 KB turns a multi-gigabyte matrix into +/// hundreds of thousands of write syscalls. +const WRITE_BUFFER: usize = 1 << 20; + +fn main() -> ExitCode { + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + // A closed downstream pipe ("distree tree.nwk | head") is how a + // reader says it has seen enough, not a failure to report. + if let Some(io_err) = e.downcast_ref::() { + if io_err.kind() == io::ErrorKind::BrokenPipe { + return ExitCode::SUCCESS; + } + } + eprintln!("Error: {}", e); + ExitCode::FAILURE + } + } } -fn main() -> io::Result<()> { - // ========== 1. Parse command-line arguments ========== +fn run() -> Result<(), Box> { + let started = Instant::now(); let matches = Command::new("distree") - .version("1.0.0") + .version(env!("CARGO_PKG_VERSION")) .author("Paula Ruiz-Rodriguez") .about("Extracts a distance matrix from a phylogeny (parallel, low-memory)") .arg( Arg::new("phylogeny") - .help("Path to the tree file in Newick format") + .help("Path to the tree file in Newick format (use '-' for stdin)") .required(true) .index(1), ) - .arg( - Arg::new("format") - .long("format") - .help("Tree file format (only 'newick' is supported)") - .default_value("newick") - .value_parser(["newick"]), - ) .arg( Arg::new("midpoint") .long("midpoint") @@ -80,475 +82,818 @@ fn main() -> io::Result<()> { .help("Path to write the TSV output file (defaults to stdout)") .value_name("FILE"), ) + .arg( + Arg::new("precision") + .long("precision") + .short('p') + .help("Number of decimal places for output values") + .default_value("10") + .value_parser(clap::value_parser!(usize)), + ) + .arg( + Arg::new("threads") + .long("threads") + .short('t') + .help("Number of threads for parallel computation (default: all cores)") + .value_parser(clap::value_parser!(usize)), + ) + .arg( + Arg::new("lower") + .long("lower") + .help("Output PHYLIP lower-triangle format (taxa count header, row labels, no diagonal)") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new("taxa") + .long("taxa") + .help("Restrict the matrix to the leaf labels listed in FILE, one per line") + .value_name("FILE"), + ) + .arg( + Arg::new("stats") + .long("stats") + .help("Print a summary of the run to stderr") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new("npy") + .long("npy") + .help("Write a NumPy .npy array instead of text (requires -o; labels go to .labels.txt)") + .action(ArgAction::SetTrue), + ) .get_matches(); let tree_path = matches .get_one::("phylogeny") - .expect("Tree file path is required") + .ok_or("Tree file path is required")? .to_string(); - let _format = matches.get_one::("format").unwrap().as_str(); // only "newick" - let do_midpoint = *matches.get_one::("midpoint").unwrap(); - let do_lmm = *matches.get_one::("lmm").unwrap(); - let do_topology = *matches.get_one::("topology").unwrap(); + let do_midpoint = *matches.get_one::("midpoint").unwrap_or(&false); + let do_lmm = *matches.get_one::("lmm").unwrap_or(&false); + let do_topology = *matches.get_one::("topology").unwrap_or(&false); + let do_lower = *matches.get_one::("lower").unwrap_or(&false); let output_path = matches.get_one::("output"); + let precision = *matches.get_one::("precision").unwrap_or(&10); - // Prepare writer: either a file or stdout - let mut writer: Box = if let Some(path) = output_path { - Box::new(BufWriter::new(File::create(path)?)) + let do_npy = *matches.get_one::("npy").unwrap_or(&false); + + if do_npy && output_path.is_none() { + return Err( + "--npy writes binary, so it needs a file: add -o FILE. The labels go to \ + FILE.labels.txt beside it." + .into(), + ); + } + if do_npy && do_topology && !do_lmm { + // Edge counts are integers; an f64 array of them is honest but odd, so + // say what is happening rather than let it surprise anyone. + eprintln!("Warning: --npy writes 64-bit floats, so --topology edge counts are stored as floats."); + } + + if precision > MAX_PRECISION { + return Err(format!( + "--precision must be between 0 and {}, got {}. A 64-bit float holds about \ + 17 significant digits, so more decimals only add zeros.", + MAX_PRECISION, precision + ) + .into()); + } + + // Determine distance mode + let mode = if do_lmm { + if do_topology { + eprintln!("Warning: --lmm and --topology are mutually exclusive. Using --lmm."); + } + DistMode::Lmm + } else if do_topology { + DistMode::Topology } else { - Box::new(io::stdout()) + DistMode::Patristic }; - // ========== 2. Read the entire Newick file into a String ========== - let mut newick_str = String::new(); - { - let mut f = File::open(&tree_path)?; - f.read_to_string(&mut newick_str)?; - } - - // ========== 3. Parse the Newick string with our minimal parser ========== - let mut chars = newick_str.trim().chars().peekable(); - let raw_root = parse_subtree(&mut chars) - .expect("Failed to parse Newick tree"); - // After the tree, there may be a trailing semicolon - if let Some(&c) = chars.peek() { - if c == ';' { - chars.next(); + // Configure thread pool + if let Some(&num_threads) = matches.get_one::("threads") { + if num_threads == 0 { + // Rayon reads 0 as "pick the default", which silently ignores what + // the user asked for. + return Err("--threads must be at least 1. Omit the flag to use all cores.".into()); } + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global() + .map_err(|e| format!("Failed to initialize thread pool: {}", e))?; + } + + // Read input from file or stdin + let source = if tree_path == "-" { "standard input" } else { tree_path.as_str() }; + let mut raw_input: Vec = Vec::new(); + if tree_path == "-" { + io::stdin().read_to_end(&mut raw_input)?; + } else { + File::open(&tree_path) + .map_err(|e| format!("Cannot open '{}': {}", tree_path, e))? + .read_to_end(&mut raw_input)?; + } + + // Large trees are usually shipped compressed, so read gzip directly rather + // than making the caller pipe it through gunzip. Detected by magic bytes + // rather than by extension, so it works from stdin too. + if raw_input.starts_with(&[0x1f, 0x8b]) { + let mut decoded = Vec::new(); + GzDecoder::new(&raw_input[..]) + .read_to_end(&mut decoded) + .map_err(|e| format!("{} is gzip-compressed but could not be read: {}", source, e))?; + raw_input = decoded; } - // ========== 4. Flatten RawNode into Vec ========== + let newick_str = String::from_utf8(raw_input).map_err(|_| { + format!( + "{} is not valid UTF-8 text, so it cannot be a Newick tree.", + source + ) + })?; + + // Parse the Newick string + let raw_root = parse_newick(&newick_str) + .map_err(|e| format!("Failed to parse Newick tree: {}", e))?; + + // Flatten into Vec let mut nodes: Vec = Vec::new(); - let root_idx = flatten_raw(&raw_root, None, &mut nodes); + let mut root_idx = flatten_raw(&raw_root, None, &mut nodes); + + // Nothing below reads the input text or the recursive parse tree, and on a + // large tree they are a second copy of it. Holding them to the end of the + // run would carry that alongside the LCA table, which is the peak. + drop(raw_root); + drop(newick_str); + + // Warn about negative branch lengths + if nodes.iter().any(|n| n.length < 0.0) { + eprintln!( + "Warning: negative branch lengths detected in the tree. Some distances may come \ + out negative, and --midpoint cannot locate the diameter reliably." + ); + } - // ========== 5. Midpoint-root optional ========== + // Midpoint-root if requested. + // + // Topological distance counts edges and does not depend on where the tree + // is rooted, so rooting can only distort it: the node inserted at the + // midpoint splits one edge in two and adds 1 to every pair whose path + // crosses it. Skip the rooting rather than report those inflated counts. if do_midpoint { - midpoint_root(root_idx, &mut nodes); + if mode == DistMode::Topology { + eprintln!( + "Warning: --midpoint is ignored in --topology mode. Edge counts do not depend \ + on the root, and the node inserted at the midpoint would add 1 to every \ + distance crossing that edge." + ); + } else { + root_idx = midpoint_root(root_idx, &mut nodes); + } } - // ========== 6. Build the LCA data structure ========== + // Build LCA let lca_data = build_lca_structure(root_idx, &nodes); - // ========== 7. Collect all leaves (taxa) ========== - let mut leaf_indices: Vec = Vec::new(); - for (i, nd) in nodes.iter().enumerate() { - if nd.children.is_empty() && nd.name.is_some() { - leaf_indices.push(i); - } + // A distance is d_i + d_j - 2*d_m, so a root-to-tip depth past half of + // f64's range overflows that sum to infinity, and inf - inf is NaN. The + // matrix comes out as inf with NaN down the diagonal, which is not a + // failure any downstream tool would notice. + let deepest = lca_data.depth_len.iter().copied().fold(0.0_f64, f64::max); + if !(deepest * 2.0).is_finite() { + return Err(format!( + "Branch lengths are too large to compute distances with: the deepest leaf is {:e} \ + from the root, and summing two such depths overflows a 64-bit float. Rescale the \ + tree.", + deepest + ) + .into()); } + + // Collect leaves + let leaf_indices: Vec = nodes + .iter() + .enumerate() + .filter(|(_, nd)| nd.children.is_empty() && nd.name.is_some()) + .map(|(i, _)| i) + .collect(); + if leaf_indices.is_empty() { - eprintln!("No labeled leaves found in the tree. Exiting."); - std::process::exit(1); + return Err("No labeled leaves found in the tree.".into()); + } + + // Leaves without a label cannot be named in the matrix, so they are left + // out entirely. Say so rather than quietly returning a smaller matrix. + let unlabeled_leaves = nodes + .iter() + .filter(|nd| nd.children.is_empty() && nd.name.is_none()) + .count(); + if unlabeled_leaves > 0 { + eprintln!( + "Warning: {} leaf/leaves have no label and were excluded from the matrix.", + unlabeled_leaves + ); + } + + // Check for duplicate leaf names and for characters that would break the + // row/column structure of the output + { + let mut seen = HashSet::with_capacity(leaf_indices.len()); + for &i in &leaf_indices { + let name = nodes[i].name.as_ref().expect("leaf has name"); + if !seen.insert(name.as_str()) { + return Err(format!( + "Duplicate leaf name '{}' found. Leaf names must be unique.", + name + ) + .into()); + } + if let Some(bad) = name.chars().find(|c| matches!(c, '\t' | '\n' | '\r')) { + let what = match bad { + '\t' => "a tab character", + '\n' => "a newline", + _ => "a carriage return", + }; + return Err(format!( + "Leaf name '{}' contains {}, which would corrupt the output by splitting \ + the row. Use an underscore or rename the leaf.", + name.escape_debug(), + what + ) + .into()); + } + // A PHYLIP row is "namevalues", so a reader that splits + // on whitespace reads "Taxon A4.5" as three fields and every + // row after it lands one column out. The square TSV is delimited by + // tabs alone and does not have the problem. + if do_lower && name.chars().any(char::is_whitespace) { + return Err(format!( + "Leaf name '{}' contains whitespace, which PHYLIP readers treat as the \ + end of the name, so --lower would produce a file they misread. Use an \ + underscore, or drop --lower for the square TSV.", + name + ) + .into()); + } + } } - // Build Vec<(label, index)> and sort by label + // Sort by label let mut leaf_label_pairs: Vec<(String, usize)> = leaf_indices .iter() - .map(|&i| (nodes[i].name.clone().unwrap(), i)) + .map(|&i| (nodes[i].name.clone().expect("leaf has name"), i)) .collect(); - leaf_label_pairs.sort_by(|a, b| a.0.cmp(&b.0)); - let sorted_labels: Vec = leaf_label_pairs.iter().map(|(lab, _)| lab.clone()).collect(); + // Restrict to the requested subset, if any. The distances themselves are + // unaffected: the path between two leaves does not depend on which other + // leaves are in the matrix, so this filters the output rather than pruning + // the tree. + if let Some(path) = matches.get_one::("taxa") { + let wanted = read_taxa_list(path)?; + let present: HashSet<&str> = leaf_label_pairs.iter().map(|(l, _)| l.as_str()).collect(); + + let missing: Vec<&str> = wanted + .iter() + .filter(|w| !present.contains(w.as_str())) + .map(String::as_str) + .collect(); + if !missing.is_empty() { + // Silently dropping these would give a matrix smaller than asked + // for, which is the kind of thing nobody notices until much later. + let shown = missing.iter().take(5).cloned().collect::>().join(", "); + return Err(format!( + "{} of the {} labels in '{}' are not leaves of the tree: {}{}", + missing.len(), + wanted.len(), + path, + shown, + if missing.len() > 5 { ", ..." } else { "" } + ) + .into()); + } + + let keep: HashSet<&str> = wanted.iter().map(String::as_str).collect(); + leaf_label_pairs.retain(|(label, _)| keep.contains(label.as_str())); + if leaf_label_pairs.is_empty() { + return Err(format!("'{}' selected no leaves.", path).into()); + } + } + + leaf_label_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + + let sorted_labels: Vec<&str> = leaf_label_pairs + .iter() + .map(|(lab, _)| lab.as_str()) + .collect(); let sorted_leaf_indices: Vec = leaf_label_pairs.iter().map(|(_, idx)| *idx).collect(); let n_leaves = sorted_leaf_indices.len(); - // ========== 8. Print the TSV header row ========== - writer.write_all(b"\t")?; - for (i, lab) in sorted_labels.iter().enumerate() { - writer.write_all(lab.as_bytes())?; - if i + 1 < n_leaves { - writer.write_all(b"\t")?; - } + // Warn if no branch lengths and not topology mode + if mode == DistMode::Patristic && nodes.iter().all(|n| n.length == 0.0) { + eprintln!( + "Warning: no branch lengths detected, all patristic distances will be zero. Consider --topology." + ); } - writer.write_all(b"\n")?; - - // ========== 9. For each leaf (row), compute distances to all leaves in parallel ========== - for (row_i, &leaf_i) in sorted_leaf_indices.iter().enumerate() { - let this_row: Vec = sorted_leaf_indices - .par_iter() - .map(|&leaf_j| { - if do_lmm { - // LMM: depth of MRCA in branch-length units - let m = lca_data.mrca(leaf_i, leaf_j); - lca_data.depth_len[m] - } else if do_topology { - // Topological distance = edge count - let m = lca_data.mrca(leaf_i, leaf_j); - let d_i = lca_data.depth_top[leaf_i]; - let d_j = lca_data.depth_top[leaf_j]; - let d_m = lca_data.depth_top[m]; - ((d_i + d_j).saturating_sub(2 * d_m)) as f64 - } else { - // Patristic distance = sum of branch lengths - let m = lca_data.mrca(leaf_i, leaf_j); - let d_i = lca_data.depth_len[leaf_i]; - let d_j = lca_data.depth_len[leaf_j]; - let d_m = lca_data.depth_len[m]; - d_i + d_j - 2.0 * d_m - } - }) - .collect(); - writer.write_all(sorted_labels[row_i].as_bytes())?; - for dist in this_row.iter() { - writer.write_all(b"\t")?; - writer.write_all(format!("{}", dist).as_bytes())?; + // Open the output only once the tree is known to be usable: creating it + // earlier truncated the previous results before a parse error could be + // reported, leaving the user with an empty file and nothing to fall back on. + let mut writer: Box = if let Some(path) = output_path { + Box::new(BufWriter::with_capacity( + WRITE_BUFFER, + File::create(path).map_err(|e| format!("Cannot write to '{}': {}", path, e))?, + )) + } else { + Box::new(BufWriter::with_capacity(WRITE_BUFFER, io::stdout())) + }; + + // The lower triangle has no diagonal. For a distance matrix that loses + // nothing, since it is all zeros, but an LMM diagonal is each tip's own + // root-to-tip length: real data, and what the off-diagonal covariances have + // to be read against. + if do_lower && mode == DistMode::Lmm { + eprintln!( + "Warning: --lower omits the diagonal, which in --lmm mode holds each leaf's \ + root-to-tip length rather than zeros. Drop --lower to keep it." + ); + } + + // Print header + if do_npy { + // NumPy needs the shape up front, which is known, so the array still + // streams out a row at a time behind this. + let shape = if do_lower { + // The condensed form scipy.spatial.distance.squareform expects + format!("({},)", n_leaves * (n_leaves - 1) / 2) + } else { + format!("({}, {})", n_leaves, n_leaves) + }; + writer.write_all(&npy_header(&shape))?; + + // .npy has nowhere to put the labels, so they go beside it in the + // order of the rows. + let labels_path = format!("{}.labels.txt", output_path.expect("--npy requires -o")); + let mut labels = String::with_capacity(n_leaves * 12); + for lab in &sorted_labels { + labels.push_str(lab); + labels.push('\n'); + } + std::fs::write(&labels_path, labels) + .map_err(|e| format!("Cannot write the labels to '{}': {}", labels_path, e))?; + } else if do_lower { + // PHYLIP format: first line is the number of taxa + writeln!(writer, "{}", n_leaves)?; + } else { + writer.write_all(b"\t")?; + for (i, lab) in sorted_labels.iter().enumerate() { + writer.write_all(lab.as_bytes())?; + if i + 1 < n_leaves { + writer.write_all(b"\t")?; + } } writer.write_all(b"\n")?; } - Ok(()) -} - -/// Recursively parse a Newick subtree and return a `RawNode`. -fn parse_subtree(chars: &mut Peekable) -> Result { - let mut node = RawNode { - name: None, - length: 0.0, - children: Vec::new(), + // Compute and print the distance matrix, a batch of rows at a time. + // + // The batch is sized so each one carries roughly BATCH_CELLS cells of work, + // which bounds the buffer at 8 MB and, more to the point, keeps the + // fork/join out of the inner loop. A cell is an MRCA query and three array + // reads, so a single row of a small matrix is nowhere near enough work to + // pay for synchronising a thread pool: on an 8,000-leaf tree, one job per + // row ran *slower* on 14 cores than on one. + // Each worker formats its rows as it computes them, so the writer only ever + // hands finished bytes to the operating system. Formatting a float to a + // fixed number of decimals costs more than the distance it prints, so + // leaving it in the serial write loop capped the whole run: at six decimals + // it was about two thirds of the work no number of cores could touch. + let batch_rows = (BATCH_CELLS / n_leaves).clamp(1, n_leaves); + let mut batch: Vec<(Vec, Stats)> = vec![(Vec::new(), Stats::default()); batch_rows]; + + // Which columns a row carries. The two triangular forms are not the same + // triangle: PHYLIP wants row i to hold columns 0..i, while the condensed + // vector SciPy reads wants columns i+1..n, which concatenated over the rows + // is its (0,1), (0,2), ... (1,2), ... ordering. Emitting the PHYLIP order + // into a .npy would hand squareform the right values in the wrong places. + let row_range = |row_i: usize| -> std::ops::Range { + match (do_lower, do_npy) { + (false, _) => 0..n_leaves, + (true, false) => 0..row_i, + (true, true) => (row_i + 1).min(n_leaves)..n_leaves, + } }; - - // If the next character is '(', this is an internal node with children - if let Some(&c) = chars.peek() { - if c == '(' { - // Consume '(' - chars.next(); - - // Parse each child until we see a ')' - loop { - let child = parse_subtree(chars)?; - node.children.push(child); - match chars.peek() { - Some(',') => { - chars.next(); - continue; - } - Some(')') => { - chars.next(); - break; + let mut totals = Stats::default(); + + let mut first_row = 0; + while first_row < n_leaves { + let rows = (n_leaves - first_row).min(batch_rows); + + batch[..rows] + .par_iter_mut() + .enumerate() + .for_each(|(k, (out, stats))| { + let row_i = first_row + k; + let leaf_i = sorted_leaf_indices[row_i]; + out.clear(); + if !do_npy { + out.extend_from_slice(sorted_labels[row_i].as_bytes()); + } + *stats = Stats::default(); + let cols = row_range(row_i); + let first_col = cols.start; + for (offset, &leaf_j) in sorted_leaf_indices[cols].iter().enumerate() { + let col = first_col + offset; + let dist = compute_distance(leaf_i, leaf_j, mode, &lca_data); + if do_npy { + out.extend_from_slice(&dist.to_le_bytes()); + } else { + out.push(b'\t'); + format_distance(out, dist, mode, precision); } - Some(other) => { - return Err(format!("Expected ',' or ')', found '{}'", other)); + // The diagonal is zero by construction in the distance + // modes and the leaf's own depth under --lmm, so it would + // only drag the summary around. + if col != row_i { + stats.add(dist); } - None => return Err("Unexpected end of input after '('".to_string()), } - } - - // After ')', we may have an internal node label - if let Some(&c2) = chars.peek() { - if c2 != ':' && c2 != ',' && c2 != ')' && c2 != ';' { - let name = parse_label(chars); - if !name.is_empty() { - node.name = Some(name); - } + if !do_npy { + out.push(b'\n'); } - } - - // Then optionally a colon followed by a branch length - if let Some(&':') = chars.peek() { - chars.next(); - let length = parse_length(chars)?; - node.length = length; - } + }); - return Ok(node); + for (row, stats) in &batch[..rows] { + writer.write_all(row)?; + totals.merge(stats); } + + first_row += rows; } - // Otherwise, it's a leaf: parse the label - let name = parse_label(chars); - if !name.is_empty() { - node.name = Some(name); + if *matches.get_one::("stats").unwrap_or(&false) { + let unit = match mode { + DistMode::Patristic => "patristic distance", + DistMode::Topology => "edge count", + DistMode::Lmm => "root-to-MRCA depth", + }; + eprintln!("--- Statistics ---"); + eprintln!("Leaves in matrix: {}", n_leaves); + if n_leaves != leaf_indices.len() { + eprintln!("Leaves in tree: {}", leaf_indices.len()); + } + eprintln!("Nodes in tree: {}", nodes.len()); + eprintln!("Mode: {}", unit); + if do_midpoint && mode != DistMode::Topology { + eprintln!("Rooting: midpoint"); + } + eprintln!("Cells written: {}", totals.count + n_leaves as u64 * u64::from(!do_lower)); + if totals.count > 0 { + // Off the diagonal, which is zero by construction in the distance + // modes and would only pull the summary towards it. + if mode == DistMode::Topology { + eprintln!("Minimum: {}", totals.min as i64); + eprintln!("Maximum: {}", totals.max as i64); + eprintln!("Mean: {:.3}", totals.mean()); + } else { + eprintln!("Minimum: {:.*}", precision, totals.min); + eprintln!("Maximum: {:.*}", precision, totals.max); + eprintln!("Mean: {:.*}", precision, totals.mean()); + } + } + eprintln!("Time: {:.3}s", started.elapsed().as_secs_f64()); } - // Optionally, a colon followed by a branch length - if let Some(&':') = chars.peek() { - chars.next(); - let length = parse_length(chars)?; - node.length = length; + // BufWriter flushes on drop but discards whatever error it hits, so a full + // disk or a failing filesystem produced a truncated matrix and exit code 0. + if let Err(e) = writer.flush() { + if e.kind() == io::ErrorKind::BrokenPipe { + return Ok(()); + } + return Err(match output_path { + Some(path) => format!("Failed to write '{}': {}", path, e).into(), + None => format!("Failed to write to stdout: {}", e).into(), + }); } - Ok(node) + Ok(()) } -/// Parse a node label (until we hit ':', ',', ')', ';', or whitespace). -fn parse_label(chars: &mut Peekable) -> String { - let mut label = String::new(); - while let Some(&c) = chars.peek() { - if c == ':' || c == ',' || c == ')' || c == ';' || c.is_whitespace() { - break; - } - label.push(c); - chars.next(); - } - label +/// Build the header of a NumPy `.npy` file (format 1.0) for a little-endian +/// f64 array of the given shape. +/// +/// Magic, version, then a two-byte header length and an ASCII dict, the whole +/// thing padded to a multiple of 64 bytes so the array data lands aligned. +fn npy_header(shape: &str) -> Vec { + let dict = format!( + "{{'descr': ') -> Result { - let mut numstr = String::new(); - while let Some(&c) = chars.peek() { - if c.is_ascii_digit() || c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-' { - numstr.push(c); - chars.next(); - } else { - break; - } - } - numstr - .parse::() - .map_err(|e: ParseFloatError| format!("Failed to parse branch length '{}': {}", numstr, e)) +/// Running summary of the off-diagonal cells, for `--stats`. +/// +/// Accumulated per row by the worker that computed it, then merged in the +/// writing loop, so the numbers cost one pass and no second traversal. +#[derive(Clone, Copy)] +struct Stats { + min: f64, + max: f64, + sum: f64, + count: u64, } -/// Recursively flatten a `RawNode` into a flat `Vec`, returning the index of the newly-added node. -fn flatten_raw(raw: &RawNode, parent: Option, nodes: &mut Vec) -> usize { - let idx = nodes.len(); - nodes.push(Node { - name: raw.name.clone(), - length: raw.length, - parent, - children: Vec::new(), - }); - if let Some(p) = parent { - nodes[p].children.push(idx); - } - for child in &raw.children { - flatten_raw(child, Some(idx), nodes); - } - idx +impl Default for Stats { + fn default() -> Self { + Stats { min: f64::INFINITY, max: f64::NEG_INFINITY, sum: 0.0, count: 0 } + } } -/// Midpoint-rooting: find two leaves at the ends of the diameter (maximum branch-length distance), -/// reconstruct that path, locate its midpoint, and insert a new root node there. -fn midpoint_root(root_idx: usize, nodes: &mut Vec) { - // Helper: from `start`, do a DFS to find the farthest leaf and its distance. - fn farthest_from(start: usize, nodes: &Vec) -> (usize, f64, Vec>) { - let mut best_leaf = start; - let mut best_dist = 0.0; - let mut parent_trace = vec![None; nodes.len()]; - let mut visited = vec![false; nodes.len()]; - let mut stack = vec![(start, 0.0)]; - visited[start] = true; - - while let Some((u, dist_u)) = stack.pop() { - if nodes[u].children.is_empty() { - // If it's a leaf, check distance - if dist_u > best_dist { - best_dist = dist_u; - best_leaf = u; - } - } - // Visit the parent, if any - if let Some(p) = nodes[u].parent { - if !visited[p] { - visited[p] = true; - parent_trace[p] = Some(u); - stack.push((p, dist_u + nodes[u].length)); - } - } - // Visit children - for &v in &nodes[u].children { - if !visited[v] { - visited[v] = true; - parent_trace[v] = Some(u); - stack.push((v, dist_u + nodes[v].length)); - } - } +impl Stats { + #[inline] + fn add(&mut self, value: f64) { + if value < self.min { + self.min = value; + } + if value > self.max { + self.max = value; } - (best_leaf, best_dist, parent_trace) + self.sum += value; + self.count += 1; } - // 1) Find any leaf to start (descend to a leaf if root isn't one) - let mut any_leaf = root_idx; - if !nodes[any_leaf].children.is_empty() { - let mut cur = any_leaf; - while !nodes[cur].children.is_empty() { - cur = nodes[cur].children[0]; + fn merge(&mut self, other: &Stats) { + if other.count == 0 { + return; } - any_leaf = cur; + if other.min < self.min { + self.min = other.min; + } + if other.max > self.max { + self.max = other.max; + } + self.sum += other.sum; + self.count += other.count; } - // 2) From any_leaf, find leaf_a (one endpoint of the diameter) - let (leaf_a, _, _) = farthest_from(any_leaf, nodes); - // 3) From leaf_a, find leaf_b and the total distance dist_ab - let (leaf_b, dist_ab, parent_trace_b) = farthest_from(leaf_a, nodes); + fn mean(&self) -> f64 { + if self.count == 0 { + f64::NAN + } else { + self.sum / self.count as f64 + } + } +} - // Reconstruct the path from leaf_b back to leaf_a - let mut path = Vec::new(); - { - let mut cur = leaf_b; - loop { - path.push(cur); - if cur == leaf_a { - break; - } - cur = parent_trace_b[cur].unwrap(); +/// Read the leaf labels for `--taxa`: one per line, blanks and `#` comments +/// skipped, duplicates collapsed. +fn read_taxa_list(path: &str) -> Result, Box> { + let text = std::fs::read_to_string(path) + .map_err(|e| format!("Cannot read the taxa list '{}': {}", path, e))?; + + let mut seen = HashSet::new(); + let mut wanted = Vec::new(); + for line in text.lines() { + // Trailing \r survives a file written on Windows and would stop every + // label matching. + let label = line.trim_end_matches(['\r', '\n']).trim(); + if label.is_empty() || label.starts_with('#') { + continue; + } + if seen.insert(label.to_string()) { + wanted.push(label.to_string()); } } - // The midpoint lies at dist_ab/2 along that path (starting from leaf_b) - let half = dist_ab / 2.0; - let mut accum = 0.0; - let mut midpoint_node = path[0]; - - for i in 0..path.len() - 1 { - let u = path[i]; - let v = path[i + 1]; - // Edge length between u ↔ v - let edge_len = if nodes[v].children.contains(&u) { - nodes[u].length - } else { - nodes[v].length - }; - if accum + edge_len >= half { - // The midpoint falls on this edge (u–v) - let dist_into_edge = half - accum; - // Determine which is parent and which is child in the current tree - let (parent_node, child_node, parent_to_child_len) = - if nodes[v].parent == Some(u) { - (u, v, nodes[v].length) - } else { - (v, u, nodes[u].length) - }; - let new_root_idx = nodes.len(); - let dist_to_old_parent = parent_to_child_len - dist_into_edge; - let dist_to_old_child = dist_into_edge; - - // Create the new root node R - nodes.push(Node { - name: None, - length: 0.0, - parent: None, - children: Vec::new(), - }); - // 1) Detach child_node from parent_node, attach child_node → R - let old_parent_of_child = nodes[child_node].parent.take(); - assert_eq!(old_parent_of_child, Some(parent_node)); - nodes[child_node].parent = Some(new_root_idx); - - // 2) In parent_node.children, replace child_node with new_root_idx - if let Some(pos) = nodes[parent_node] - .children - .iter() - .position(|&x| x == child_node) - { - nodes[parent_node].children[pos] = new_root_idx; - } + if wanted.is_empty() { + return Err(format!("The taxa list '{}' is empty.", path).into()); + } + Ok(wanted) +} - // 3) Now set R.children = [child_node], and R.parent = Some(parent_node) - nodes[new_root_idx].parent = Some(parent_node); - nodes[new_root_idx].children.push(child_node); - - // 4) Adjust branch lengths: - // - child_node.length = dist_to_old_child - // - R.length = dist_to_old_parent - nodes[child_node].length = dist_to_old_child; - nodes[new_root_idx].length = dist_to_old_parent; - - // 5) Insert parent_node above R - let old_parent_of_parent = nodes[parent_node].parent.take(); - nodes[parent_node].parent = Some(new_root_idx); - nodes[new_root_idx].children.push(parent_node); - if let Some(grand) = old_parent_of_parent { - // If parent_node wasn't originally the root, replace in its old parent's children - if let Some(pos2) = nodes[grand].children.iter().position(|&x| x == parent_node) { - nodes[grand].children[pos2] = new_root_idx; - } - } - // Finally, R must become the new root - nodes[new_root_idx].parent = None; +/// Powers of ten that are exact as f64, which is all of them up to 1e22. +const POW10: [f64; 16] = [ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, +]; + +/// Write `value` with `precision` decimals, or return false and write nothing. +/// +/// `write!("{:.p$}")` goes through `core::fmt` and an exact decimal expansion +/// of the float, which is correct and slow: it was around half the cost of a +/// text run. Scaling by a power of ten and emitting the digits by hand is +/// roughly seven times faster. +/// +/// It is only safe where the answer is not in doubt. Multiplying by 10^p +/// rounds once, by at most `|scaled| * 2^-53`, and if that is enough to carry +/// the product across the nearest `.5` boundary then which way the exact value +/// rounds is unknowable from here. Those cases return false and go to the +/// formatter, which is why this can be fast without being approximate. +#[inline] +fn write_fixed_fast(out: &mut Vec, value: f64, precision: usize) -> bool { + if precision >= POW10.len() || !value.is_finite() { + return false; + } + let scaled = value * POW10[precision]; + // Past 2^53 the integers are no longer consecutive, so the digits below + // would be invented. + if scaled.abs() >= 9.007_199_254_740_992e15 { + return false; + } - midpoint_node = new_root_idx; - break; - } - accum += edge_len; - midpoint_node = v; + let distance_from_tie = ((scaled - scaled.floor()) - 0.5).abs(); + let rounding_error = scaled.abs() * f64::EPSILON + f64::MIN_POSITIVE; + if distance_from_tie <= rounding_error { + return false; } - // If for some reason we never split (degenerate case), force midpoint_node to be root - if let Some(p) = nodes[midpoint_node].parent { - if let Some(pos) = nodes[p].children.iter().position(|&x| x == midpoint_node) { - nodes[p].children.remove(pos); - } - nodes[midpoint_node].parent = None; + let rounded = scaled.round() as i64; + // The sign comes from the input rather than the rounded integer, so a small + // negative value that rounds to zero still prints as "-0.00", matching the + // standard formatter. + let negative = value.is_sign_negative(); + + let mut buf = [0u8; 32]; + let mut at = buf.len(); + let mut digits = rounded.unsigned_abs(); + for _ in 0..precision { + at -= 1; + buf[at] = b'0' + (digits % 10) as u8; + digits /= 10; + } + if precision > 0 { + at -= 1; + buf[at] = b'.'; + } + if digits == 0 { + at -= 1; + buf[at] = b'0'; + } + while digits > 0 { + at -= 1; + buf[at] = b'0' + (digits % 10) as u8; + digits /= 10; + } + if negative { + at -= 1; + buf[at] = b'-'; } + out.extend_from_slice(&buf[at..]); + true } -/// Build the `LcaData` for binary lifting: -/// 1) Do a single DFS from `root_idx` to fill `depth_len`, `depth_top`, and `up[0][u]`. -/// 2) Fill `up[k][u] = up[k-1][ up[k-1][u] ]` for k = 1..⌈log₂(n)⌉. -fn build_lca_structure(root_idx: usize, nodes: &Vec) -> LcaData { - let n = nodes.len(); - let mut depth_len = vec![0.0; n]; - let mut depth_top = vec![0; n]; - let max_log = ((n as f64).log2().ceil() as usize) + 1; - let mut up: Vec>> = vec![vec![None; n]; max_log]; - - // 1) DFS to set depth and immediate parent - { - let mut stack = vec![root_idx]; - depth_len[root_idx] = 0.0; - depth_top[root_idx] = 0; - up[0][root_idx] = None; - - while let Some(u) = stack.pop() { - for &v in &nodes[u].children { - up[0][v] = Some(u); - depth_len[v] = depth_len[u] + nodes[v].length; - depth_top[v] = depth_top[u] + 1; - stack.push(v); - } - } +/// Append a single distance value to a row buffer. +/// +/// Topology mode outputs integers; patristic and LMM use `precision` decimal +/// places. Writing into a `Vec` cannot fail, which is what lets this run +/// inside the parallel row builder. +#[inline] +fn format_distance(out: &mut Vec, dist: f64, mode: DistMode, precision: usize) { + if mode == DistMode::Topology { + write!(out, "{}", dist as i64).expect("writing to a Vec cannot fail"); + return; } + if write_fixed_fast(out, dist, precision) { + return; + } + write!(out, "{:.prec$}", dist, prec = precision).expect("writing to a Vec cannot fail"); +} - // 2) Build all 2^k ancestors - for k in 1..max_log { - for u in 0..n { - up[k][u] = up[k - 1][u].and_then(|mid| up[k - 1][mid]); +#[cfg(test)] +mod tests { + use super::*; + + /// xorshift64, so a failure is reproducible. + struct Rng(u64); + + impl Rng { + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x } - } - LcaData { - up, - depth_len, - depth_top, + fn unit(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 + } } -} -impl LcaData { - /// Return the index of the lowest common ancestor (LCA) of nodes `u` and `v` in O(log n). - fn mrca(&self, mut u: usize, mut v: usize) -> usize { - if u == v { - return u; + /// The fast path either produces exactly what the standard formatter would, + /// or declines. Anything else is a silently wrong number in the matrix. + fn assert_agrees(value: f64, precision: usize) { + let mut fast = Vec::new(); + if !write_fixed_fast(&mut fast, value, precision) { + return; // declined, so the formatter handles it } - // 1) Lift the deeper node up until both are at the same depth_top - if self.depth_top[u] < self.depth_top[v] { - std::mem::swap(&mut u, &mut v); + let slow = format!("{:.prec$}", value, prec = precision); + assert_eq!( + String::from_utf8(fast).unwrap(), + slow, + "fast path disagrees for {:e} at {} decimals", + value, + precision + ); + } + + #[test] + fn test_fast_formatter_matches_std_on_random_values() { + let mut rng = Rng(0x9E37_79B9_7F4A_7C15); + for _ in 0..200_000 { + let bits = rng.next_u64(); + let magnitude = (bits % 24) as i32 - 12; + let sign = if bits & 1 == 0 { 1.0 } else { -1.0 }; + let value = rng.unit() * 10f64.powi(magnitude) * sign; + assert_agrees(value, (rng.next_u64() % 16) as usize); } - let diff = self.depth_top[u] - self.depth_top[v]; - let mut x = diff; - let mut k = 0; - while x > 0 { - if (x & 1) == 1 { - u = self.up[k][u].unwrap(); + } + + #[test] + fn test_fast_formatter_matches_std_near_rounding_boundaries() { + // Exact halves are where a scaled-integer shortcut goes wrong, so walk + // straight down the middle of them at every precision. + for precision in 0..16usize { + let step = 10f64.powi(-(precision as i32)); + for i in 0..4_000u64 { + for delta in [0.0, 0.5, -0.5, 0.499_999_999, 0.500_000_001, 1.0 / 3.0] { + assert_agrees((i as f64 + delta) * step, precision); + assert_agrees(-(i as f64 + delta) * step, precision); + } } - x >>= 1; - k += 1; } - if u == v { - return u; + } + + #[test] + fn test_fast_formatter_matches_std_on_distance_shaped_values() { + // What distree actually emits: small positive distances, at the + // precisions people use. + let mut rng = Rng(0x2545_F491_4F6C_DD1D); + for _ in 0..100_000 { + let value = rng.unit() * 1e-4; + for precision in [0usize, 3, 6, 9, 10, 12, 15] { + assert_agrees(value, precision); + assert_agrees(value * 4_411_532.0, precision); + } } - // 2) Lift both in powers of two until their parents differ - for k in (0..self.up.len()).rev() { - if let (Some(au), Some(av)) = (self.up[k][u], self.up[k][v]) { - if au != av { - u = au; - v = av; - } + } + + #[test] + fn test_fast_formatter_edge_cases() { + let awkward = [ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + -0.5, + 0.05, + 0.005, + f64::MIN_POSITIVE, + -f64::MIN_POSITIVE, + 5e-324, // the smallest subnormal + 1e-300, + 1e300, + 9.007_199_254_740_992e15, // 2^53 + 9.007_199_254_740_991e15, + 0.1, + 0.2, + 0.3, + 1.0 / 3.0, + 2.0 / 3.0, + 1e15, + 1e16, + 123_456.789_012_345, + ]; + for &value in &awkward { + for precision in 0..=MAX_PRECISION { + assert_agrees(value, precision); } } - // Now u and v have the same parent, which is the LCA - self.up[0][u].unwrap() + } + + #[test] + fn test_fast_formatter_declines_rather_than_guesses() { + // Anything it cannot do exactly must be refused, not approximated + let mut out = Vec::new(); + assert!(!write_fixed_fast(&mut out, 1.0, 16), "precision past the table"); + assert!(!write_fixed_fast(&mut out, f64::NAN, 6), "NaN"); + assert!(!write_fixed_fast(&mut out, f64::INFINITY, 6), "infinity"); + assert!(!write_fixed_fast(&mut out, 1e300, 6), "past 2^53 once scaled"); + assert!(out.is_empty(), "a declined value must write nothing"); } } diff --git a/src/midpoint.rs b/src/midpoint.rs new file mode 100644 index 0000000..62d6bc5 --- /dev/null +++ b/src/midpoint.rs @@ -0,0 +1,402 @@ +use crate::tree::Node; + +/// Midpoint-root the tree: find the diameter, split the midpoint edge, re-root there. +/// Returns the index of the new root. +pub fn midpoint_root(root_idx: usize, nodes: &mut Vec) -> usize { + // BFS/DFS on the unrooted tree to find the farthest leaf from `start`. + fn farthest_from(start: usize, nodes: &[Node]) -> (usize, f64, Vec>) { + let mut best_leaf = start; + let mut best_dist = 0.0; + let mut parent_trace: Vec> = vec![None; nodes.len()]; + let mut visited = vec![false; nodes.len()]; + let mut stack = vec![(start, 0.0)]; + visited[start] = true; + + while let Some((u, dist_u)) = stack.pop() { + if nodes[u].children.is_empty() && u != start && dist_u > best_dist { + best_dist = dist_u; + best_leaf = u; + } + if let Some(p) = nodes[u].parent { + if !visited[p] { + visited[p] = true; + parent_trace[p] = Some(u); + stack.push((p, dist_u + nodes[u].length)); + } + } + for &v in &nodes[u].children { + if !visited[v] { + visited[v] = true; + parent_trace[v] = Some(u); + stack.push((v, dist_u + nodes[v].length)); + } + } + } + (best_leaf, best_dist, parent_trace) + } + + // 1) Find diameter endpoints + let mut any_leaf = root_idx; + while !nodes[any_leaf].children.is_empty() { + any_leaf = nodes[any_leaf].children[0]; + } + + let (leaf_a, _, _) = farthest_from(any_leaf, nodes); + let (leaf_b, dist_ab, parent_trace) = farthest_from(leaf_a, nodes); + + if dist_ab == 0.0 { + return root_idx; + } + + // 2) Reconstruct path from leaf_b back to leaf_a + let mut path = Vec::new(); + { + let mut cur = leaf_b; + loop { + path.push(cur); + if cur == leaf_a { + break; + } + cur = parent_trace[cur] + .expect("midpoint: path from leaf_b to leaf_a is disconnected; tree may be malformed"); + } + } + + // 3) Walk the path and find the edge where the midpoint falls + let half = dist_ab / 2.0; + let mut accum = 0.0; + + for i in 0..path.len() - 1 { + let u = path[i]; // closer to leaf_b + let v = path[i + 1]; // closer to leaf_a + + let edge_len = if nodes[u].parent == Some(v) { + nodes[u].length + } else { + nodes[v].length + }; + + if accum + edge_len >= half { + let dist_to_u = half - accum; + let dist_to_v = edge_len - dist_to_u; + + // Determine parent-child in the current tree + let (parent_side, child_side, len_child, len_parent) = + if nodes[u].parent == Some(v) { + // v is parent of u + (v, u, dist_to_u, dist_to_v) + } else { + // u is parent of v + (u, v, dist_to_v, dist_to_u) + }; + + // Insert new root node on this edge + let new_root = nodes.len(); + nodes.push(Node { + name: None, + length: 0.0, + parent: None, + children: Vec::new(), + }); + + // Detach child_side from parent_side + nodes[parent_side].children.retain(|&x| x != child_side); + + // Attach child_side to new_root + nodes[child_side].parent = Some(new_root); + nodes[child_side].length = len_child; + nodes[new_root].children.push(child_side); + + // Collect path from parent_side to old root + let mut path_to_old_root = vec![]; + { + let mut c = parent_side; + loop { + path_to_old_root.push(c); + if let Some(p) = nodes[c].parent { + c = p; + } else { + break; + } + } + } + + // Attach parent_side to new_root + let old_len = nodes[parent_side].length; + nodes[parent_side].parent = Some(new_root); + nodes[parent_side].length = len_parent; + nodes[new_root].children.push(parent_side); + + // Reverse parent-child relationships from parent_side up to old root + let mut prev_old_length = old_len; + for j in 1..path_to_old_root.len() { + let child_now = path_to_old_root[j]; + let parent_now = path_to_old_root[j - 1]; + + // Remove parent_now from child_now's children + nodes[child_now].children.retain(|&x| x != parent_now); + // Add child_now as child of parent_now + nodes[parent_now].children.push(child_now); + // Set child_now's parent + nodes[child_now].parent = Some(parent_now); + + // Swap branch lengths + std::mem::swap(&mut nodes[child_now].length, &mut prev_old_length); + } + + return new_root; + } + accum += edge_len; + } + + root_idx +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::{flatten_raw, parse_newick}; + use crate::lca::build_lca_structure; + use crate::testutil::{random_newick, Rng}; + + fn patristic_distance( + leaf_i: usize, + leaf_j: usize, + lca: &crate::lca::LcaData, + ) -> f64 { + let m = lca.mrca(leaf_i, leaf_j); + lca.depth_len[leaf_i] + lca.depth_len[leaf_j] - 2.0 * lca.depth_len[m] + } + + /// Every node reachable exactly once, every child pointing back at its parent. + fn assert_tree_is_consistent(nodes: &[Node], root: usize) { + assert!(nodes[root].parent.is_none(), "the root must have no parent"); + let mut visited = vec![false; nodes.len()]; + let mut stack = vec![root]; + visited[root] = true; + while let Some(u) = stack.pop() { + for &v in &nodes[u].children { + assert_eq!( + nodes[v].parent, + Some(u), + "child {} does not point back at parent {}", + v, + u + ); + assert!(!visited[v], "node {} reached twice: the graph has a cycle", v); + visited[v] = true; + stack.push(v); + } + } + for (i, seen) in visited.iter().enumerate() { + assert!(seen, "node {} is unreachable from the root", i); + } + } + + fn leaf_indices(nodes: &[Node]) -> Vec { + nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect() + } + + #[test] + fn test_midpoint_random_trees() { + let mut rng = Rng::new(0x9E37_79B9_7F4A_7C15); + + for case in 0..300 { + let n_leaves = 2 + rng.below(24); + let newick = random_newick(&mut rng, n_leaves); + + let raw = parse_newick(&newick).unwrap(); + let mut before = Vec::new(); + let root_before = flatten_raw(&raw, None, &mut before); + let lca_before = build_lca_structure(root_before, &before); + let leaves_before = leaf_indices(&before); + + // Names in flatten order, so the two trees can be lined up by label + let names: Vec = leaves_before + .iter() + .map(|&i| before[i].name.clone().unwrap()) + .collect(); + + let mut diameter: f64 = 0.0; + let mut dist_before = vec![vec![0.0; leaves_before.len()]; leaves_before.len()]; + for (a, &i) in leaves_before.iter().enumerate() { + for (b, &j) in leaves_before.iter().enumerate() { + let d = patristic_distance(i, j, &lca_before); + dist_before[a][b] = d; + diameter = diameter.max(d); + } + } + + let raw2 = parse_newick(&newick).unwrap(); + let mut after = Vec::new(); + let root_after = flatten_raw(&raw2, None, &mut after); + let new_root = midpoint_root(root_after, &mut after); + + assert_tree_is_consistent(&after, new_root); + + let lca_after = build_lca_structure(new_root, &after); + let leaves_after: Vec = names + .iter() + .map(|name| { + after + .iter() + .position(|n| n.name.as_deref() == Some(name.as_str())) + .unwrap_or_else(|| panic!("case {}: leaf {} vanished", case, name)) + }) + .collect(); + + for a in 0..leaves_after.len() { + for b in 0..leaves_after.len() { + let d = patristic_distance(leaves_after[a], leaves_after[b], &lca_after); + assert!( + (d - dist_before[a][b]).abs() < 1e-9, + "case {}: rooting changed d({},{}) from {} to {}\n{}", + case, + names[a], + names[b], + dist_before[a][b], + d, + newick + ); + } + } + + // The defining property: the two longest root-to-tip paths are + // equal, so the deepest tip sits exactly half a diameter away. + let deepest = leaves_after + .iter() + .map(|&i| lca_after.depth_len[i]) + .fold(0.0_f64, f64::max); + assert!( + (deepest - diameter / 2.0).abs() < 1e-9, + "case {}: deepest tip is {} from the root, expected {} (diameter {})\n{}", + case, + deepest, + diameter / 2.0, + diameter, + newick + ); + } + } + + #[test] + fn test_midpoint_simple() { + // ((A:1,B:1):1,(C:1,D:1):1); + // Diameter = 4 (e.g. A→root→D), midpoint at distance 2 + let input = "((A:1,B:1):1,(C:1,D:1):1);"; + let raw = parse_newick(input).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + let new_root = midpoint_root(root, &mut nodes); + + // Verify the tree is valid: new root has no parent + assert!(nodes[new_root].parent.is_none()); + + // Build LCA and check distances are preserved + let lca = build_lca_structure(new_root, &nodes); + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + // A-B should be 2.0, A-C should be 4.0, etc. + for &i in &leaves { + for &j in &leaves { + let d = patristic_distance(i, j, &lca); + assert!(d >= 0.0, "Negative distance between leaves"); + } + } + + // Check specific: A to B = 2.0 + let a = leaves.iter().find(|&&i| nodes[i].name.as_deref() == Some("A")).unwrap(); + let b = leaves.iter().find(|&&i| nodes[i].name.as_deref() == Some("B")).unwrap(); + let c = leaves.iter().find(|&&i| nodes[i].name.as_deref() == Some("C")).unwrap(); + assert!((patristic_distance(*a, *b, &lca) - 2.0).abs() < 1e-10); + assert!((patristic_distance(*a, *c, &lca) - 4.0).abs() < 1e-10); + } + + #[test] + fn test_midpoint_asymmetric() { + // (A:1.0,B:3.0); + // Diameter = 4, midpoint at distance 2 from each + let input = "(A:1.0,B:3.0);"; + let raw = parse_newick(input).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + let new_root = midpoint_root(root, &mut nodes); + + assert!(nodes[new_root].parent.is_none()); + let lca = build_lca_structure(new_root, &nodes); + + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + let b = nodes.iter().position(|n| n.name.as_deref() == Some("B")).unwrap(); + assert!((patristic_distance(a, b, &lca) - 4.0).abs() < 1e-10); + } + + #[test] + fn test_midpoint_preserves_distances() { + let input = "(((A:0.5,B:0.3):0.4,C:0.9):0.1,D:1.2);"; + let raw = parse_newick(input).unwrap(); + + // Compute distances before midpoint rooting + let mut nodes_before = Vec::new(); + let root_before = flatten_raw(&raw, None, &mut nodes_before); + let lca_before = build_lca_structure(root_before, &nodes_before); + + let leaf_names = ["A", "B", "C", "D"]; + let leaves_before: Vec = leaf_names + .iter() + .map(|name| { + nodes_before + .iter() + .position(|n| n.name.as_deref() == Some(name)) + .unwrap() + }) + .collect(); + + let mut dists_before = vec![vec![0.0; 4]; 4]; + for i in 0..4 { + for j in 0..4 { + dists_before[i][j] = patristic_distance(leaves_before[i], leaves_before[j], &lca_before); + } + } + + // Now midpoint root + let raw2 = parse_newick(input).unwrap(); + let mut nodes_after = Vec::new(); + let root_after = flatten_raw(&raw2, None, &mut nodes_after); + let new_root = midpoint_root(root_after, &mut nodes_after); + let lca_after = build_lca_structure(new_root, &nodes_after); + + let leaves_after: Vec = leaf_names + .iter() + .map(|name| { + nodes_after + .iter() + .position(|n| n.name.as_deref() == Some(name)) + .unwrap() + }) + .collect(); + + for i in 0..4 { + for j in 0..4 { + let d = patristic_distance(leaves_after[i], leaves_after[j], &lca_after); + assert!( + (d - dists_before[i][j]).abs() < 1e-10, + "Distance mismatch for {}-{}: before={}, after={}", + leaf_names[i], + leaf_names[j], + dists_before[i][j], + d + ); + } + } + } +} diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000..d0968f1 --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,706 @@ +use std::num::ParseFloatError; + +use crate::tree::Node; + +/// Temporary recursive structure used during Newick parsing. +/// Converted to a flat `Vec` by [`flatten_raw`] after parsing. +pub struct RawNode { + pub name: Option, + pub length: f64, + pub children: Vec, +} + +impl Drop for RawNode { + /// Dismantle the subtree iteratively. + /// + /// The compiler-generated drop glue recurses once per level, which + /// overflows the stack on the deeply nested trees the parser and the + /// flattener were made iterative to support. + fn drop(&mut self) { + let mut pending = std::mem::take(&mut self.children); + while let Some(mut node) = pending.pop() { + pending.append(&mut node.children); + } + } +} + +/// Parse a complete Newick tree from a string. +/// +/// Strips whitespace outside of quoted labels before parsing. +/// Accepts trees with or without a trailing semicolon. +/// Returns an error on malformed input or empty input. +pub fn parse_newick(input: &str) -> Result { + let trimmed = input.trim(); + if trimmed.is_empty() { + return Err("Empty input: no Newick tree found.".to_string()); + } + // Pre-process: strip whitespace outside of single/double quotes + let cleaned = strip_whitespace_outside_quotes(trimmed); + let bytes = cleaned.as_bytes(); + let mut pos = 0; + let root = parse_subtree_iterative(bytes, &mut pos)?; + + // Only an optional ';' and trailing comments may follow the tree. Anything + // else means the input is not a single well-formed tree, and silently + // ignoring it would hand back a matrix for something the user never asked + // for. + skip_ignorable_bytes(bytes, &mut pos)?; + let mut ended_with_semicolon = false; + if pos < bytes.len() && bytes[pos] == b';' { + pos += 1; + ended_with_semicolon = true; + } + skip_ignorable_bytes(bytes, &mut pos)?; + if pos < bytes.len() { + if ended_with_semicolon && bytes[pos] == b'(' { + return Err(format!( + "Unexpected content at position {}: the file appears to hold more than one tree. \ + distree processes a single Newick tree; split the file first.", + pos + )); + } + return Err(format!( + "Unexpected content after the end of the tree at position {}: '{}'.", + pos, + bytes[pos] as char + )); + } + + Ok(root) +} + +/// Strip whitespace characters outside of quoted regions. +/// +/// Works on raw bytes, which is safe for UTF-8 input: every byte of a +/// multi-byte sequence has the high bit set and therefore never matches an +/// ASCII quote or whitespace byte. Only whole ASCII bytes are dropped, so the +/// remaining bytes are still valid UTF-8. +fn strip_whitespace_outside_quotes(s: &str) -> String { + let src = s.as_bytes(); + let mut out: Vec = Vec::with_capacity(src.len()); + // Last non-whitespace byte emitted. A quote only opens a quoted label + // where a label may begin: at the very start of the tree, or right after + // '(', ',' or ')'. Anywhere else an apostrophe is an ordinary character of + // an unquoted label ("O'Brien") or of a comment ("[don't]"). + let mut previous: Option = None; + let mut i = 0; + + while i < src.len() { + let b = src[i]; + let opens_label = matches!(b, b'\'' | b'"') + && matches!(previous, None | Some(b'(') | Some(b',') | Some(b')')); + + if opens_label { + // Copy the quoted label verbatim so its whitespace survives, + // treating a doubled quote as an escaped literal rather than as a + // close followed by a re-open. + out.push(b); + i += 1; + while i < src.len() { + if src[i] == b { + if src.get(i + 1) == Some(&b) { + out.extend_from_slice(&[b, b]); + i += 2; + continue; + } + out.push(b); + i += 1; + break; + } + out.push(src[i]); + i += 1; + } + previous = Some(b); + } else if b.is_ascii_whitespace() { + i += 1; + } else { + out.push(b); + previous = Some(b); + i += 1; + } + } + + String::from_utf8(out).expect("dropping ASCII whitespace preserves UTF-8 validity") +} + +/// Iterative Newick parser — no stack overflow on deep trees. +fn parse_subtree_iterative(bytes: &[u8], pos: &mut usize) -> Result { + // We use an explicit stack to avoid recursion. + // Each frame represents a node being constructed. + struct Frame { + node: RawNode, + } + + let mut stack: Vec = Vec::new(); + + loop { + // A subtree may be preceded by comments: the rooting marker some + // programs emit ("[&R] (A,B);") or per-branch metadata ("(A,[&x=1]B)"). + skip_ignorable_bytes(bytes, pos)?; + + // Decide what to parse at current position + let current_node = if *pos < bytes.len() && bytes[*pos] == b'(' { + // Internal node: push frame, advance past '(' + *pos += 1; + skip_whitespace_bytes(bytes, pos); + stack.push(Frame { + node: RawNode { name: None, length: 0.0, children: Vec::new() }, + }); + continue; // loop back to parse first child + } else { + // Leaf node + let name = parse_label_bytes(bytes, pos)?; + skip_comments_bytes(bytes, pos)?; + let length = parse_optional_length(bytes, pos)?; + skip_comments_bytes(bytes, pos)?; + let mut leaf = RawNode { name: None, length, children: Vec::new() }; + if !name.is_empty() { + leaf.name = Some(name); + } + leaf + }; + + // Now we have a completed node. Feed it back up the stack. + let mut completed = current_node; + loop { + if let Some(frame) = stack.last_mut() { + frame.node.children.push(completed); + // Check what comes next: ',' means more children, ')' means done + if *pos < bytes.len() && bytes[*pos] == b',' { + *pos += 1; + skip_whitespace_bytes(bytes, pos); + break; // break inner loop, continue outer to parse next child + } else if *pos < bytes.len() && bytes[*pos] == b')' { + *pos += 1; + // Internal node complete — read its label and length + skip_comments_bytes(bytes, pos)?; + let name = parse_label_bytes(bytes, pos)?; + skip_comments_bytes(bytes, pos)?; + let length = parse_optional_length(bytes, pos)?; + skip_comments_bytes(bytes, pos)?; + + let mut frame = stack.pop().unwrap(); + if !name.is_empty() { + frame.node.name = Some(name); + } + frame.node.length = length; + completed = frame.node; + // continue inner loop to feed this up further + } else if *pos >= bytes.len() { + // End of input while still inside parentheses: the tree is + // truncated. Closing it silently would emit a matrix whose + // branch lengths are quietly wrong. + return Err(format!( + "Unexpected end of input: {} unclosed '(' remain. The tree is truncated.", + stack.len() + )); + } else { + let ch = bytes[*pos] as char; + return Err(format!("Expected ',' or ')', found '{}' at position {}", ch, *pos)); + } + } else { + // Stack empty — this is the root + return Ok(completed); + } + } + } +} + +fn skip_whitespace_bytes(bytes: &[u8], pos: &mut usize) { + while *pos < bytes.len() && bytes[*pos].is_ascii_whitespace() { + *pos += 1; + } +} + +/// Skip any run of whitespace and '[...]' comments. +fn skip_ignorable_bytes(bytes: &[u8], pos: &mut usize) -> Result<(), String> { + loop { + let before = *pos; + skip_whitespace_bytes(bytes, pos); + skip_comments_bytes(bytes, pos)?; + if *pos == before { + return Ok(()); + } + } +} + +/// Parse a node label from bytes. Supports single-quoted and double-quoted labels. +/// Returns an error if a quoted label is not properly closed. +fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> Result { + if *pos >= bytes.len() { + return Ok(String::new()); + } + + // Handle quoted labels (single or double quotes) + // Per the Newick spec, quotes inside a quoted label are escaped by doubling: + // 'it''s a name' → it's a name + let ch = bytes[*pos]; + if ch == b'\'' || ch == b'"' { + let open_pos = *pos; + *pos += 1; // consume opening quote + let mut label: Vec = Vec::new(); + let mut closed = false; + while *pos < bytes.len() { + if bytes[*pos] == ch { + // Check for escaped quote (doubled) + if *pos + 1 < bytes.len() && bytes[*pos + 1] == ch { + label.push(ch); + *pos += 2; // skip both quotes + } else { + *pos += 1; // consume closing quote + closed = true; + break; + } + } else { + label.push(bytes[*pos]); + *pos += 1; + } + } + if !closed { + return Err(format!( + "Unclosed quote starting at position {}.", + open_pos + )); + } + return label_from_utf8(label, open_pos); + } + + // Unquoted label: read until delimiter + let start = *pos; + while *pos < bytes.len() { + let c = bytes[*pos]; + if c == b':' + || c == b',' + || c == b')' + || c == b'(' + || c == b';' + || c == b'[' + || c.is_ascii_whitespace() + { + break; + } + *pos += 1; + } + label_from_utf8(bytes[start..*pos].to_vec(), start) +} + +/// Turn the raw bytes of a label into a `String`. +/// +/// Labels are scanned byte-wise because every Newick delimiter is ASCII, but +/// the bytes in between may encode any UTF-8 text and must be decoded as such +/// rather than reinterpreted one byte at a time. +fn label_from_utf8(bytes: Vec, start: usize) -> Result { + String::from_utf8(bytes).map_err(|_| { + format!( + "Label starting at position {} is not valid UTF-8.", + start + ) + }) +} + +/// Skip '[...]' comment blocks (NHX annotations, BEAST metadata, etc.). +/// Handles nested brackets. Errors if a comment is never closed, since the +/// rest of the tree would otherwise be swallowed as comment text. +fn skip_comments_bytes(bytes: &[u8], pos: &mut usize) -> Result<(), String> { + while *pos < bytes.len() && bytes[*pos] == b'[' { + let open_pos = *pos; + *pos += 1; + let mut depth: usize = 1; + while *pos < bytes.len() && depth > 0 { + match bytes[*pos] { + b'[' => depth += 1, + b']' => depth -= 1, + _ => {} + } + *pos += 1; + } + if depth > 0 { + return Err(format!( + "Unclosed comment starting at position {}: no matching ']'.", + open_pos + )); + } + } + Ok(()) +} + +/// Parse optional ":length" — returns 0.0 if no colon present. +fn parse_optional_length(bytes: &[u8], pos: &mut usize) -> Result { + if *pos < bytes.len() && bytes[*pos] == b':' { + *pos += 1; + parse_length_bytes(bytes, pos) + } else { + Ok(0.0) + } +} + +/// Parse a floating-point branch length (supports scientific notation and negative values). +/// Returns an error if no valid numeric characters follow the colon. +fn parse_length_bytes(bytes: &[u8], pos: &mut usize) -> Result { + let start = *pos; + while *pos < bytes.len() { + let c = bytes[*pos]; + if c.is_ascii_digit() || c == b'.' || c == b'e' || c == b'E' || c == b'+' || c == b'-' { + *pos += 1; + } else { + break; + } + } + if start == *pos { + return Err(format!( + "Expected a numeric branch length at position {}, found '{}'.", + pos, + bytes.get(*pos).map(|&b| b as char).unwrap_or('\0') + )); + } + let numstr = std::str::from_utf8(&bytes[start..*pos]) + .map_err(|_| "Invalid UTF-8 in branch length".to_string())?; + let value = numstr + .parse::() + .map_err(|e: ParseFloatError| { + format!("Failed to parse branch length '{}': {}", numstr, e) + })?; + // Rust parses an exponent past f64's range as infinity rather than as an + // error, and an infinite branch length turns the whole matrix into inf with + // NaN down the diagonal, since inf - inf is NaN. + if !value.is_finite() { + return Err(format!( + "Branch length '{}' at position {} is not a finite number.", + numstr, start + )); + } + Ok(value) +} + +/// Flatten a `RawNode` tree into a flat `Vec` iteratively (no stack overflow). +/// +/// Returns the index of the root node in `nodes`. +/// Parent/child relationships are set up correctly for LCA queries. +pub fn flatten_raw(raw: &RawNode, parent: Option, nodes: &mut Vec) -> usize { + // Stack of (raw_node_ref, parent_index) + let mut stack: Vec<(&RawNode, Option)> = vec![(raw, parent)]; + let root_idx = nodes.len(); + + while let Some((raw_node, par)) = stack.pop() { + let idx = nodes.len(); + nodes.push(Node { + name: raw_node.name.clone(), + length: raw_node.length, + parent: par, + children: Vec::new(), + }); + if let Some(p) = par { + nodes[p].children.push(idx); + } + // Push children in reverse order so they're processed left-to-right + for child in raw_node.children.iter().rev() { + stack.push((child, Some(idx))); + } + } + + root_idx +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_simple_tree() { + let raw = parse_newick("(A:1.0,B:2.0);").unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[0].name.as_deref(), Some("A")); + assert_eq!(raw.children[0].length, 1.0); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + assert_eq!(raw.children[1].length, 2.0); + } + + #[test] + fn test_parse_internal_labels() { + let raw = parse_newick("((A:1.0,B:2.0)inner:0.5,C:3.0);").unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[0].name.as_deref(), Some("inner")); + assert_eq!(raw.children[0].length, 0.5); + assert_eq!(raw.children[0].children.len(), 2); + } + + #[test] + fn test_parse_no_branch_lengths() { + let raw = parse_newick("(A,B,(C,D));").unwrap(); + assert_eq!(raw.children.len(), 3); + assert_eq!(raw.children[0].length, 0.0); + assert_eq!(raw.children[2].children.len(), 2); + } + + #[test] + fn test_parse_nhx_comments() { + let raw = parse_newick("((A:0.1[&&NHX:S=human],B:0.2[&&NHX:S=mouse]):0.3,C:0.4);").unwrap(); + assert_eq!(raw.children.len(), 2); + let inner = &raw.children[0]; + assert_eq!(inner.children[0].name.as_deref(), Some("A")); + assert_eq!(inner.children[0].length, 0.1); + assert_eq!(inner.children[1].name.as_deref(), Some("B")); + assert_eq!(inner.children[1].length, 0.2); + assert_eq!(inner.length, 0.3); + } + + #[test] + fn test_parse_quoted_labels_single() { + let raw = parse_newick("('Taxon A':1.0,'Taxon B':2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("Taxon A")); + assert_eq!(raw.children[1].name.as_deref(), Some("Taxon B")); + } + + #[test] + fn test_parse_quoted_labels_double() { + let raw = parse_newick(r#"("Taxon A":1.0,"Taxon B":2.0);"#).unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("Taxon A")); + assert_eq!(raw.children[1].name.as_deref(), Some("Taxon B")); + } + + #[test] + fn test_non_ascii_labels_preserved() { + let raw = parse_newick("((Señor_Ñu:1.0,'β strain':2.0):0.5,日本株:3.0);").unwrap(); + let inner = &raw.children[0]; + assert_eq!(inner.children[0].name.as_deref(), Some("Señor_Ñu")); + assert_eq!(inner.children[1].name.as_deref(), Some("β strain")); + assert_eq!(raw.children[1].name.as_deref(), Some("日本株")); + } + + #[test] + fn test_parse_scientific_notation() { + let raw = parse_newick("(A:1.5e-3,B:2.0E+1);").unwrap(); + assert!((raw.children[0].length - 0.0015).abs() < 1e-10); + assert!((raw.children[1].length - 20.0).abs() < 1e-10); + } + + #[test] + fn test_flatten() { + let raw = parse_newick("((A:1.0,B:2.0):0.5,C:3.0);").unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + assert_eq!(root, 0); + assert_eq!(nodes.len(), 5); + assert!(nodes[0].parent.is_none()); + assert_eq!(nodes[0].children.len(), 2); + } + + #[test] + fn test_bracket_in_label_position() { + let raw = parse_newick("((A:0.1[comment],B:0.2):0.3[more],C:0.4[x]);").unwrap(); + assert_eq!(raw.children[0].children[0].name.as_deref(), Some("A")); + assert_eq!(raw.children[1].name.as_deref(), Some("C")); + } + + #[test] + fn test_whitespace_in_newick() { + let raw = parse_newick("( A:1.0 , B:2.0 ) ;").unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[0].name.as_deref(), Some("A")); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + } + + #[test] + fn test_newlines_in_newick() { + let raw = parse_newick("((A:1.0,\nB:2.0):0.5,\nC:3.0);\n").unwrap(); + assert_eq!(raw.children.len(), 2); + } + + #[test] + fn test_deep_tree_no_stack_overflow() { + // Build a caterpillar tree with 5,000 levels using push (no recursive format!) + let mut tree = String::with_capacity(200_000); + for _ in 0..5_000 { + tree.push('('); + } + tree.push_str("A:0.1"); + for i in 1..=5_000 { + tree.push_str(&format!(",T{}:0.1):0.01", i)); + } + tree.push(';'); + let raw = parse_newick(&tree).unwrap(); + // Should not stack overflow; just verify it parsed + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + assert!(nodes.len() > 5_000); + assert!(nodes[root].parent.is_none()); + } + + #[test] + fn test_leading_rooting_comment() { + // IQ-TREE, MrBayes and BEAST prefix the tree with a rooting marker + let raw = parse_newick("[&R] ((A:1.0,B:2.0):0.5,C:3.0);").unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[1].name.as_deref(), Some("C")); + assert_eq!(raw.children[0].children[0].name.as_deref(), Some("A")); + } + + #[test] + fn test_comment_before_label() { + let raw = parse_newick("(A:1.0,[&x=1]B:2.0);").unwrap(); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + assert_eq!(raw.children[1].length, 2.0); + } + + #[test] + fn test_comment_before_subtree() { + let raw = parse_newick("([&clade=1](A:1.0,B:2.0):0.5,C:3.0);").unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[0].children.len(), 2); + assert_eq!(raw.children[0].length, 0.5); + } + + #[test] + fn test_deeply_nested_tree_drops_without_overflow() { + // A ladder 200,000 levels deep: parsing, flattening and *dropping* + // the parse tree all have to stay iterative + let depth = 200_000; + let mut tree = String::with_capacity(2 * depth + 8); + for _ in 0..depth { + tree.push('('); + } + tree.push_str("A:0.1"); + for _ in 0..depth { + tree.push(')'); + } + tree.push(';'); + + let raw = parse_newick(&tree).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + assert_eq!(nodes.len(), depth + 1); + assert!(nodes[root].parent.is_none()); + drop(raw); + } + + #[test] + fn test_nested_brackets() { + let raw = parse_newick("((A:0.1[&rate=0.5[inner]],B:0.2):0.3,C:0.4);").unwrap(); + assert_eq!(raw.children[0].children[0].name.as_deref(), Some("A")); + } + + #[test] + fn test_negative_branch_length_parsed() { + let raw = parse_newick("(A:-0.5,B:2.0);").unwrap(); + assert!((raw.children[0].length - (-0.5)).abs() < 1e-10); + } + + #[test] + fn test_empty_input_error() { + assert!(parse_newick("").is_err()); + assert!(parse_newick(" ").is_err()); + assert!(parse_newick("\n\t").is_err()); + } + + #[test] + fn test_empty_branch_length_error() { + // ":" followed by non-numeric should error, not panic + let result = parse_newick("(A:,B:2.0);"); + assert!(result.is_err(), "Expected error for empty branch length"); + } + + #[test] + fn test_escaped_single_quotes_in_labels() { + // Newick spec: doubled quotes inside a quoted label are a single literal quote + // 'it''s' -> it's + let raw = parse_newick(r"('it''s':1.0,B:2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("it's")); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + } + + #[test] + fn test_escaped_double_quotes_in_labels() { + // "a ""name""" -> a "name" + let input = r#"("a ""name""":1.0,B:2.0);"#; + let raw = parse_newick(input).unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some(r#"a "name""#)); + } + + #[test] + fn test_apostrophe_in_unquoted_label() { + // An apostrophe mid-label must not be read as an opening quote + let raw = parse_newick("(O'Brien:1.0 , B:2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("O'Brien")); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + } + + #[test] + fn test_apostrophe_in_comment() { + let raw = parse_newick("(A:1.0[don't panic],B:2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("A")); + assert_eq!(raw.children[1].length, 2.0); + } + + #[test] + fn test_escaped_quote_keeps_inner_whitespace() { + // The doubled quote must not end the label, or the space after it + // would be stripped as if it were outside the quotes + let raw = parse_newick(r"('it''s a name':1.0,B:2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("it's a name")); + } + + #[test] + fn test_quoted_internal_node_label() { + let raw = parse_newick("((A:1.0,B:2.0)'my clade':0.5,C:3.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("my clade")); + } + + #[test] + fn test_no_trailing_semicolon() { + // Trees without semicolons are valid in many tools + let result = parse_newick("(A:1.0,B:2.0)"); + // Should parse successfully + assert!(result.is_ok()); + let raw = result.unwrap(); + assert_eq!(raw.children.len(), 2); + } + + #[test] + fn test_unclosed_quote_error() { + let result = parse_newick("('unclosed:1.0,B:2.0);"); + assert!(result.is_err(), "Unclosed quote should error"); + let msg = result.err().expect("should be Err"); + assert!(msg.contains("Unclosed quote"), "Error message: {}", msg); + } + + #[test] + fn test_unmatched_paren_error() { + // A truncated tree must be rejected: closing it silently would give a + // matrix whose branch lengths are wrong without any warning. + let err = parse_newick("((A:1.0,B:2.0)").err().expect("truncated tree should error"); + assert!(err.contains("truncated"), "Error message: {}", err); + assert!(parse_newick("((A:1.0,B:2.0):0.5,(C:1.0,D:2.0").is_err()); + } + + #[test] + fn test_trailing_content_error() { + let err = parse_newick("(A:1.0,B:2.0);garbage").err().expect("trailing junk should error"); + assert!(err.contains("Unexpected content"), "Error message: {}", err); + } + + #[test] + fn test_multiple_trees_error() { + let err = parse_newick("(A:1,B:2);\n(C:1,D:2);\n") + .err().expect("a multi-tree file should error"); + assert!(err.contains("more than one tree"), "Error message: {}", err); + } + + #[test] + fn test_free_text_is_not_a_leaf() { + // Plain prose used to parse as one enormous leaf label + assert!(parse_newick("not a tree at all (((").is_err()); + } + + #[test] + fn test_unclosed_comment_error() { + let err = parse_newick("((A:1.0,B:2.0):0.5,C:3.0)[unclosed;") + .err().expect("unclosed comment should error"); + assert!(err.contains("Unclosed comment"), "Error message: {}", err); + } + + #[test] + fn test_trailing_comment_allowed() { + let raw = parse_newick("(A:1.0,B:2.0); [ generated by iqtree ]").unwrap(); + assert_eq!(raw.children.len(), 2); + } +} diff --git a/src/testutil.rs b/src/testutil.rs new file mode 100644 index 0000000..7cb58ad --- /dev/null +++ b/src/testutil.rs @@ -0,0 +1,51 @@ +//! Helpers shared by the test modules: a deterministic PRNG and a random tree +//! generator, so the randomised checks stay reproducible without pulling in a +//! dependency. + +/// xorshift64. +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: u64) -> Self { + Rng(seed) + } + + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + /// A number in `0..n`. + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n as u64) as usize + } + + /// A branch length, zero one time in seven: collapsing weakly supported + /// branches leaves plenty of zero-length ones in real trees. + pub fn length(&mut self) -> f64 { + if self.below(7) == 0 { + return 0.0; + } + 0.01 + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 * 2.0 + } +} + +/// Join random groups of 2 or 3 until one tree remains, so the sample covers +/// polytomies as well as strictly bifurcating trees. +pub fn random_newick(rng: &mut Rng, n_leaves: usize) -> String { + let mut pool: Vec = (0..n_leaves) + .map(|i| format!("L{}:{:.6}", i, rng.length())) + .collect(); + while pool.len() > 1 { + let arity = if pool.len() > 2 && rng.below(4) == 0 { 3 } else { 2 }; + let children: Vec = (0..arity) + .map(|_| pool.swap_remove(rng.below(pool.len()))) + .collect(); + pool.push(format!("({}):{:.6}", children.join(","), rng.length())); + } + format!("{};", pool.pop().unwrap()) +} diff --git a/src/tree.rs b/src/tree.rs new file mode 100644 index 0000000..4d27ec0 --- /dev/null +++ b/src/tree.rs @@ -0,0 +1,13 @@ +/// Internal representation of a phylogenetic tree node stored in a flat `Vec`. +/// +/// Indices into the enclosing `Vec` are used instead of pointers. +pub struct Node { + /// Leaf label, or `None` for internal nodes. + pub name: Option, + /// Branch length from this node to its parent (0.0 if absent or for the root). + pub length: f64, + /// Index of the parent node, or `None` for the root. + pub parent: Option, + /// Indices of child nodes in DFS left-to-right order. + pub children: Vec, +} diff --git a/tests/crossvalidation.rs b/tests/crossvalidation.rs new file mode 100644 index 0000000..e668a64 --- /dev/null +++ b/tests/crossvalidation.rs @@ -0,0 +1,177 @@ +//! Check distree's output against reference matrices computed by ape. +//! +//! The rest of the suite is self-consistent: it asserts that midpoint rooting +//! preserves distances, that the binary-lifting MRCA agrees with walking up +//! from both nodes, that a patristic distance is never negative. All of that is +//! written against the same understanding of the problem as the code, so none +//! of it would catch a systematic error in what a distance is defined to be. +//! +//! The fixtures here come from somewhere else. `tests/fixtures/*.nwk` are +//! random trees, and the matrices beside them were computed by R's ape package +//! (`cophenetic.phylo` for patristic, `vcv.phylo` for the variance-covariance +//! matrix, and `cophenetic.phylo` over unit branch lengths for edge counts). +//! +//! Regenerate them, and cross-validate over hundreds more trees than are +//! committed here, with: +//! +//! ```text +//! Rscript scripts/crossvalidate.R 250 target/release/distree +//! Rscript scripts/crossvalidate.R 6 target/release/distree --write-fixtures +//! ``` + +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::Command; + +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_distree") +} + +fn fixture_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures") +} + +/// Parse a square TSV into labels and a row-major matrix. +fn parse_matrix(text: &str) -> (Vec, HashMap<(String, String), f64>) { + let mut lines = text.lines(); + let header: Vec = lines + .next() + .expect("matrix has a header") + .split('\t') + .skip(1) // leading empty cell + .map(str::to_string) + .collect(); + + let mut cells = HashMap::new(); + for line in lines { + if line.is_empty() { + continue; + } + let mut fields = line.split('\t'); + let row = fields.next().expect("row label").to_string(); + for (col, value) in header.iter().zip(fields) { + let v: f64 = value + .parse() + .unwrap_or_else(|_| panic!("cell '{}' is not a number", value)); + cells.insert((row.clone(), col.clone()), v); + } + } + (header, cells) +} + +fn run_distree(tree: &PathBuf, flags: &[&str]) -> String { + let out = Command::new(bin()) + .arg(tree) + .args(flags) + .args(["-p", "12"]) + .output() + .expect("failed to run distree"); + assert!( + out.status.success(), + "distree failed on {:?} with {:?}: {}", + tree, + flags, + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("distree writes UTF-8") +} + +/// Compare distree's matrix for `flags` against the fixture for `mode`. +fn check_mode(stem: &str, mode: &str, flags: &[&str], tolerance: f64) { + let dir = fixture_dir(); + let tree = dir.join(format!("{}.nwk", stem)); + let expected_path = dir.join(format!("{}.{}.tsv", stem, mode)); + + let expected_text = std::fs::read_to_string(&expected_path) + .unwrap_or_else(|e| panic!("cannot read {:?}: {}", expected_path, e)); + + let (want_labels, want) = parse_matrix(&expected_text); + let (got_labels, got) = parse_matrix(&run_distree(&tree, flags)); + + assert_eq!( + got_labels, want_labels, + "{} {}: label ordering differs from ape's", + stem, mode + ); + + let mut worst = 0.0_f64; + let mut worst_at = (String::new(), String::new()); + for a in &want_labels { + for b in &want_labels { + let key = (a.clone(), b.clone()); + let g = got[&key]; + let w = want[&key]; + let diff = (g - w).abs(); + if diff > worst { + worst = diff; + worst_at = key; + } + } + } + + assert!( + worst <= tolerance, + "{} {}: distree and ape disagree by {:.3e} at ({}, {}); tolerance is {:.0e}", + stem, + mode, + worst, + worst_at.0, + worst_at.1, + tolerance + ); +} + +fn stems() -> Vec { + let mut found: Vec = std::fs::read_dir(fixture_dir()) + .expect("tests/fixtures exists") + .filter_map(|e| { + let name = e.ok()?.file_name().into_string().ok()?; + name.strip_suffix(".nwk").map(str::to_string) + }) + .collect(); + found.sort(); + assert!(!found.is_empty(), "no fixture trees found"); + found +} + +/// Both tools write 12 decimals, so a value in the hundreds carries about +/// 1e-10 of round-trip error. Anything larger is a real disagreement. +const TOLERANCE: f64 = 1e-9; + +#[test] +fn test_patristic_matches_ape_cophenetic() { + for stem in stems() { + check_mode(&stem, "patristic", &[], TOLERANCE); + } +} + +#[test] +fn test_lmm_matches_ape_vcv() { + for stem in stems() { + check_mode(&stem, "lmm", &["--lmm"], TOLERANCE); + } +} + +#[test] +fn test_topology_matches_ape_unit_branch_cophenetic() { + for stem in stems() { + // Edge counts are integers on both sides, so they must agree exactly + check_mode(&stem, "topology", &["--topology"], 0.0); + } +} + +#[test] +fn test_midpoint_leaves_patristic_alone() { + // ape has no opinion to check here, but the fixtures are a broader set of + // shapes than the hand-written trees, and rooting must not move a + // patristic distance on any of them. + for stem in stems() { + let tree = fixture_dir().join(format!("{}.nwk", stem)); + assert_eq!( + run_distree(&tree, &[]), + run_distree(&tree, &["--midpoint"]), + "{}: --midpoint changed the patristic matrix", + stem + ); + } +} diff --git a/tests/fixtures/tree01.lmm.tsv b/tests/fixtures/tree01.lmm.tsv new file mode 100644 index 0000000..ee8a15c --- /dev/null +++ b/tests/fixtures/tree01.lmm.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 4.044311133508 2.127465543815 0.000000000000 2.127465543815 2.127465543815 0.000000000000 2.127465543815 0.000000000000 0.000000000000 0.000000000000 2.127465543815 2.127465543815 +t10 2.127465543815 4.044311133508 0.000000000000 3.278214901797 3.278214901797 0.000000000000 3.966945735241 0.000000000000 0.000000000000 0.000000000000 3.278214901797 3.278214901797 +t11 0.000000000000 0.000000000000 4.044311133508 0.000000000000 0.000000000000 3.694086227346 0.000000000000 4.029185174483 3.694086227346 3.694086227346 0.000000000000 0.000000000000 +t12 2.127465543815 3.278214901797 0.000000000000 4.044311133508 4.009939549891 0.000000000000 3.278214901797 0.000000000000 0.000000000000 0.000000000000 3.966888006743 3.722208144128 +t2 2.127465543815 3.278214901797 0.000000000000 4.009939549891 4.044311133508 0.000000000000 3.278214901797 0.000000000000 0.000000000000 0.000000000000 3.966888006743 3.722208144128 +t3 0.000000000000 0.000000000000 3.694086227346 0.000000000000 0.000000000000 4.044311133508 0.000000000000 3.694086227346 4.017223405830 4.026706611511 0.000000000000 0.000000000000 +t4 2.127465543815 3.966945735241 0.000000000000 3.278214901797 3.278214901797 0.000000000000 4.044311133508 0.000000000000 0.000000000000 0.000000000000 3.278214901797 3.278214901797 +t5 0.000000000000 0.000000000000 4.029185174483 0.000000000000 0.000000000000 3.694086227346 0.000000000000 4.044311133508 3.694086227346 3.694086227346 0.000000000000 0.000000000000 +t6 0.000000000000 0.000000000000 3.694086227346 0.000000000000 0.000000000000 4.017223405830 0.000000000000 3.694086227346 4.044311133508 4.017223405830 0.000000000000 0.000000000000 +t7 0.000000000000 0.000000000000 3.694086227346 0.000000000000 0.000000000000 4.026706611511 0.000000000000 3.694086227346 4.017223405830 4.044311133508 0.000000000000 0.000000000000 +t8 2.127465543815 3.278214901797 0.000000000000 3.966888006743 3.966888006743 0.000000000000 3.278214901797 0.000000000000 0.000000000000 0.000000000000 4.044311133508 3.722208144128 +t9 2.127465543815 3.278214901797 0.000000000000 3.722208144128 3.722208144128 0.000000000000 3.278214901797 0.000000000000 0.000000000000 0.000000000000 3.722208144128 4.044311133508 diff --git a/tests/fixtures/tree01.nwk b/tests/fixtures/tree01.nwk new file mode 100644 index 0000000..3441485 --- /dev/null +++ b/tests/fixtures/tree01.nwk @@ -0,0 +1 @@ +(((t5:0.01512595902,t11:0.01512595902):0.3350989471,(t6:0.02708772768,(t3:0.017604522,t7:0.017604522):0.009483205681):0.3231371785):3.694086227,(t1:1.91684559,((t10:0.07736539827,t4:0.07736539827):0.6887308334,(t9:0.3221029894,(t8:0.07742312677,(t2:0.03437158362,t12:0.03437158362):0.04305154315):0.2446798626):0.4439932423):1.150749358):2.127465544); diff --git a/tests/fixtures/tree01.patristic.tsv b/tests/fixtures/tree01.patristic.tsv new file mode 100644 index 0000000..ade6ed4 --- /dev/null +++ b/tests/fixtures/tree01.patristic.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 3.833691179386 8.088622267016 3.833691179386 3.833691179386 8.088622267016 3.833691179386 8.088622267016 8.088622267016 8.088622267016 3.833691179386 3.833691179386 +t10 3.833691179386 0.000000000000 8.088622267016 1.532192463422 1.532192463422 8.088622267016 0.154730796533 8.088622267016 8.088622267016 8.088622267016 1.532192463422 1.532192463422 +t11 8.088622267016 8.088622267016 0.000000000000 8.088622267016 8.088622267016 0.700449812324 8.088622267016 0.030251918049 0.700449812324 0.700449812324 8.088622267016 8.088622267016 +t12 3.833691179386 1.532192463422 8.088622267016 0.000000000000 0.068743167234 8.088622267016 1.532192463422 8.088622267016 8.088622267016 8.088622267016 0.154846253531 0.644205978759 +t2 3.833691179386 1.532192463422 8.088622267016 0.068743167234 0.000000000000 8.088622267016 1.532192463422 8.088622267016 8.088622267016 8.088622267016 0.154846253531 0.644205978759 +t3 8.088622267016 8.088622267016 0.700449812324 8.088622267016 8.088622267016 0.000000000000 8.088622267016 0.700449812324 0.054175455356 0.035209043994 8.088622267016 8.088622267016 +t4 3.833691179386 0.154730796533 8.088622267016 1.532192463422 1.532192463422 8.088622267016 0.000000000000 8.088622267016 8.088622267016 8.088622267016 1.532192463422 1.532192463422 +t5 8.088622267016 8.088622267016 0.030251918049 8.088622267016 8.088622267016 0.700449812324 8.088622267016 0.000000000000 0.700449812324 0.700449812324 8.088622267016 8.088622267016 +t6 8.088622267016 8.088622267016 0.700449812324 8.088622267016 8.088622267016 0.054175455356 8.088622267016 0.700449812324 0.000000000000 0.054175455356 8.088622267016 8.088622267016 +t7 8.088622267016 8.088622267016 0.700449812324 8.088622267016 8.088622267016 0.035209043994 8.088622267016 0.700449812324 0.054175455356 0.000000000000 8.088622267016 8.088622267016 +t8 3.833691179386 1.532192463422 8.088622267016 0.154846253531 0.154846253531 8.088622267016 1.532192463422 8.088622267016 8.088622267016 8.088622267016 0.000000000000 0.644205978759 +t9 3.833691179386 1.532192463422 8.088622267016 0.644205978759 0.644205978759 8.088622267016 1.532192463422 8.088622267016 8.088622267016 8.088622267016 0.644205978759 0.000000000000 diff --git a/tests/fixtures/tree01.topology.tsv b/tests/fixtures/tree01.topology.tsv new file mode 100644 index 0000000..f584631 --- /dev/null +++ b/tests/fixtures/tree01.topology.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 4.000000000000 5.000000000000 6.000000000000 6.000000000000 6.000000000000 4.000000000000 5.000000000000 5.000000000000 6.000000000000 5.000000000000 4.000000000000 +t10 4.000000000000 0.000000000000 7.000000000000 6.000000000000 6.000000000000 8.000000000000 2.000000000000 7.000000000000 7.000000000000 8.000000000000 5.000000000000 4.000000000000 +t11 5.000000000000 7.000000000000 0.000000000000 9.000000000000 9.000000000000 5.000000000000 7.000000000000 2.000000000000 4.000000000000 5.000000000000 8.000000000000 7.000000000000 +t12 6.000000000000 6.000000000000 9.000000000000 0.000000000000 2.000000000000 10.000000000000 6.000000000000 9.000000000000 9.000000000000 10.000000000000 3.000000000000 4.000000000000 +t2 6.000000000000 6.000000000000 9.000000000000 2.000000000000 0.000000000000 10.000000000000 6.000000000000 9.000000000000 9.000000000000 10.000000000000 3.000000000000 4.000000000000 +t3 6.000000000000 8.000000000000 5.000000000000 10.000000000000 10.000000000000 0.000000000000 8.000000000000 5.000000000000 3.000000000000 2.000000000000 9.000000000000 8.000000000000 +t4 4.000000000000 2.000000000000 7.000000000000 6.000000000000 6.000000000000 8.000000000000 0.000000000000 7.000000000000 7.000000000000 8.000000000000 5.000000000000 4.000000000000 +t5 5.000000000000 7.000000000000 2.000000000000 9.000000000000 9.000000000000 5.000000000000 7.000000000000 0.000000000000 4.000000000000 5.000000000000 8.000000000000 7.000000000000 +t6 5.000000000000 7.000000000000 4.000000000000 9.000000000000 9.000000000000 3.000000000000 7.000000000000 4.000000000000 0.000000000000 3.000000000000 8.000000000000 7.000000000000 +t7 6.000000000000 8.000000000000 5.000000000000 10.000000000000 10.000000000000 2.000000000000 8.000000000000 5.000000000000 3.000000000000 0.000000000000 9.000000000000 8.000000000000 +t8 5.000000000000 5.000000000000 8.000000000000 3.000000000000 3.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 0.000000000000 3.000000000000 +t9 4.000000000000 4.000000000000 7.000000000000 4.000000000000 4.000000000000 8.000000000000 4.000000000000 7.000000000000 7.000000000000 8.000000000000 3.000000000000 0.000000000000 diff --git a/tests/fixtures/tree02.lmm.tsv b/tests/fixtures/tree02.lmm.tsv new file mode 100644 index 0000000..b562307 --- /dev/null +++ b/tests/fixtures/tree02.lmm.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 3.256266693119 0.065225892933 0.065225892933 0.629955607234 0.861331883352 0.065225892933 2.431582783815 1.467767681926 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 1.467767681926 +t10 0.065225892933 2.384166998556 0.826933017233 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 1.803952952381 0.065225892933 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.826933017233 0.000000000000 0.065225892933 +t11 0.065225892933 0.826933017233 3.142999361502 0.065225892933 0.065225892933 2.553210664308 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 0.065225892933 1.580081571126 0.065225892933 1.580081571126 0.000000000000 0.065225892933 +t12 0.629955607234 0.065225892933 0.065225892933 2.301803021459 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 0.635071521392 0.635071521392 1.744033653056 0.065225892933 1.260509673739 0.065225892933 0.000000000000 0.629955607234 +t13 0.861331883352 0.065225892933 0.065225892933 0.629955607234 1.754404090578 0.065225892933 0.861331883352 0.861331883352 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 0.861331883352 +t14 0.065225892933 0.826933017233 2.553210664308 0.065225892933 0.065225892933 3.477648017928 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 0.065225892933 1.580081571126 0.065225892933 1.580081571126 0.000000000000 0.065225892933 +t15 2.431582783815 0.065225892933 0.065225892933 0.629955607234 0.861331883352 0.065225892933 2.762291881256 1.467767681926 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 1.467767681926 +t16 1.467767681926 0.065225892933 0.065225892933 0.629955607234 0.861331883352 0.065225892933 1.467767681926 2.570631016977 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 2.250247935066 +t17 0.065225892933 1.803952952381 0.826933017233 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 2.413126796251 0.065225892933 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.826933017233 0.000000000000 0.065225892933 +t2 0.629955607234 0.065225892933 0.065225892933 0.635071521392 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 1.348461325280 0.702091647778 0.635071521392 0.065225892933 0.635071521392 0.065225892933 0.000000000000 0.629955607234 +t3 0.629955607234 0.065225892933 0.065225892933 0.635071521392 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 0.702091647778 1.225792548154 0.635071521392 0.065225892933 0.635071521392 0.065225892933 0.000000000000 0.629955607234 +t4 0.629955607234 0.065225892933 0.065225892933 1.744033653056 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 0.635071521392 0.635071521392 1.780867669731 0.065225892933 1.260509673739 0.065225892933 0.000000000000 0.629955607234 +t5 0.065225892933 0.826933017233 1.580081571126 0.065225892933 0.065225892933 1.580081571126 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 0.065225892933 3.408075469313 0.065225892933 2.512174850330 0.000000000000 0.065225892933 +t6 0.629955607234 0.065225892933 0.065225892933 1.260509673739 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 0.635071521392 0.635071521392 1.260509673739 0.065225892933 2.177918948466 0.065225892933 0.000000000000 0.629955607234 +t7 0.065225892933 0.826933017233 1.580081571126 0.065225892933 0.065225892933 1.580081571126 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 0.065225892933 2.512174850330 0.065225892933 2.690017238259 0.000000000000 0.065225892933 +t8 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.633634719998 0.000000000000 +t9 1.467767681926 0.065225892933 0.065225892933 0.629955607234 0.861331883352 0.065225892933 1.467767681926 2.250247935066 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 3.018237155862 diff --git a/tests/fixtures/tree02.nwk b/tests/fixtures/tree02.nwk new file mode 100644 index 0000000..2abdb8e --- /dev/null +++ b/tests/fixtures/tree02.nwk @@ -0,0 +1 @@ +(t8:0.63363472,((((t14:0.9244373536,t11:0.5897886972):0.9731290932,(t5:0.895900619,t7:0.1778423879):0.9320932792):0.7531485539,(t17:0.6091738439,t10:0.5802140462):0.9770199351):0.7617071243,(((t6:0.9174092747,(t4:0.03683401668,t12:0.5577693684):0.4835239793):0.6254381523,(t2:0.6463696775,t3:0.5237009004):0.06702012639):0.005115914159,(t13:0.8930722072,((t1:0.8246839093,t15:0.3307090974):0.9638151019,(t16:0.3203830819,t9:0.7679892208):0.7824802531):0.6064357986):0.2313762761):0.5647297143):0.06522589293); diff --git a/tests/fixtures/tree02.patristic.tsv b/tests/fixtures/tree02.patristic.tsv new file mode 100644 index 0000000..96f4fc0 --- /dev/null +++ b/tests/fixtures/tree02.patristic.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 5.509981905809 6.268814268755 4.298158500111 3.288007016992 6.603462925181 1.155393006746 2.891362346243 5.538941703504 3.344816803932 3.222148026805 3.777223148383 6.533890376566 4.174274427118 5.815832145512 3.889901413117 3.338968485128 +t10 5.509981905809 0.000000000000 3.873300325591 4.555518234149 4.008119303267 4.207948982017 5.016007093946 4.824346229667 1.189387890045 3.602176537970 3.479507760843 4.034582882421 4.138376433402 4.431634161156 3.420318202348 3.017801718554 5.271952368552 +t11 6.268814268755 3.873300325591 0.000000000000 5.314350597095 4.766951666214 1.514226050815 5.774839456892 5.583178592613 3.902260123286 4.361008900916 4.238340123789 4.793415245367 3.390911688562 5.190466524102 2.672853457509 3.776634081500 6.030784731498 +t12 4.298158500111 4.555518234149 5.314350597095 0.000000000000 2.796295897570 5.648999253521 3.804183688248 3.612522823969 4.584478031844 2.380121303955 2.257452526828 0.594603385078 5.579426704906 1.958702622447 4.861368473852 2.935437741457 4.060128962854 +t13 3.288007016992 4.008119303267 4.766951666214 2.796295897570 0.000000000000 5.101600322640 2.794032205129 2.602371340850 4.037079100963 1.842954201391 1.720285424264 2.275360545842 5.032027774025 2.672411824577 4.313969542971 2.388038810575 3.049977479735 +t14 6.603462925181 4.207948982017 1.514226050815 5.648999253521 5.101600322640 0.000000000000 6.109488113318 5.917827249039 4.236908779712 4.695657557342 4.572988780215 5.128063901793 3.725560344988 5.525115180528 3.007502113935 4.111282737926 6.365433387924 +t15 1.155393006746 5.016007093946 5.774839456892 3.804183688248 2.794032205129 6.109488113318 0.000000000000 2.397387534380 5.044966891641 2.850841992069 2.728173214942 3.283248336520 6.039915564703 3.680299615255 5.321857333649 3.395926601253 2.844993673265 +t16 2.891362346243 4.824346229667 5.583178592613 3.612522823969 2.602371340850 5.917827249039 2.397387534380 0.000000000000 4.853306027362 2.659181127790 2.536512350664 3.091587472241 5.848254700424 3.488638750976 5.130196469370 3.204265736975 1.088372302707 +t17 5.538941703504 1.189387890045 3.902260123286 4.584478031844 4.037079100963 4.236908779712 5.044966891641 4.853306027362 0.000000000000 3.631136335665 3.508467558539 4.063542680116 4.167336231098 4.460593958851 3.449278000044 3.046761516249 5.300912166247 +t2 3.344816803932 3.602176537970 4.361008900916 2.380121303955 1.842954201391 4.695657557342 2.850841992069 2.659181127790 3.631136335665 0.000000000000 1.170070577879 1.859185952228 4.626085008727 2.256237230962 3.908026777674 1.982096045278 3.106787266675 +t3 3.222148026805 3.479507760843 4.238340123789 2.257452526828 1.720285424264 4.572988780215 2.728173214942 2.536512350664 3.508467558539 1.170070577879 0.000000000000 1.736517175101 4.503416231601 2.133568453835 3.785358000547 1.859427268151 2.984118489549 +t4 3.777223148383 4.034582882421 4.793415245367 0.594603385078 2.275360545842 5.128063901793 3.283248336520 3.091587472241 4.063542680116 1.859185952228 1.736517175101 0.000000000000 5.058491353178 1.437767270720 4.340433122125 2.414502389729 3.539193611126 +t5 6.533890376566 4.138376433402 3.390911688562 5.579426704906 5.032027774025 3.725560344988 6.039915564703 5.848254700424 4.167336231098 4.626085008727 4.503416231601 5.058491353178 0.000000000000 5.455542631913 1.073743006913 4.041710189311 6.295860839309 +t6 4.174274427118 4.431634161156 5.190466524102 1.958702622447 2.672411824577 5.525115180528 3.680299615255 3.488638750976 4.460593958851 2.256237230962 2.133568453835 1.437767270720 5.455542631913 0.000000000000 4.737484400859 2.811553668464 3.936244889861 +t7 5.815832145512 3.420318202348 2.672853457509 4.861368473852 4.313969542971 3.007502113935 5.321857333649 5.130196469370 3.449278000044 3.908026777674 3.785358000547 4.340433122125 1.073743006913 4.737484400859 0.000000000000 3.323651958257 5.577802608255 +t8 3.889901413117 3.017801718554 3.776634081500 2.935437741457 2.388038810575 4.111282737926 3.395926601253 3.204265736975 3.046761516249 1.982096045278 1.859427268151 2.414502389729 4.041710189311 2.811553668464 3.323651958257 0.000000000000 3.651871875860 +t9 3.338968485128 5.271952368552 6.030784731498 4.060128962854 3.049977479735 6.365433387924 2.844993673265 1.088372302707 5.300912166247 3.106787266675 2.984118489549 3.539193611126 6.295860839309 3.936244889861 5.577802608255 3.651871875860 0.000000000000 diff --git a/tests/fixtures/tree02.topology.tsv b/tests/fixtures/tree02.topology.tsv new file mode 100644 index 0000000..3eedd60 --- /dev/null +++ b/tests/fixtures/tree02.topology.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 8.000000000000 9.000000000000 8.000000000000 4.000000000000 9.000000000000 2.000000000000 4.000000000000 8.000000000000 7.000000000000 7.000000000000 8.000000000000 9.000000000000 7.000000000000 9.000000000000 7.000000000000 4.000000000000 +t10 8.000000000000 0.000000000000 5.000000000000 8.000000000000 6.000000000000 5.000000000000 8.000000000000 8.000000000000 2.000000000000 7.000000000000 7.000000000000 8.000000000000 5.000000000000 7.000000000000 5.000000000000 5.000000000000 8.000000000000 +t11 9.000000000000 5.000000000000 0.000000000000 9.000000000000 7.000000000000 2.000000000000 9.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 4.000000000000 8.000000000000 4.000000000000 6.000000000000 9.000000000000 +t12 8.000000000000 8.000000000000 9.000000000000 0.000000000000 6.000000000000 9.000000000000 8.000000000000 8.000000000000 8.000000000000 5.000000000000 5.000000000000 2.000000000000 9.000000000000 3.000000000000 9.000000000000 7.000000000000 8.000000000000 +t13 4.000000000000 6.000000000000 7.000000000000 6.000000000000 0.000000000000 7.000000000000 4.000000000000 4.000000000000 6.000000000000 5.000000000000 5.000000000000 6.000000000000 7.000000000000 5.000000000000 7.000000000000 5.000000000000 4.000000000000 +t14 9.000000000000 5.000000000000 2.000000000000 9.000000000000 7.000000000000 0.000000000000 9.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 4.000000000000 8.000000000000 4.000000000000 6.000000000000 9.000000000000 +t15 2.000000000000 8.000000000000 9.000000000000 8.000000000000 4.000000000000 9.000000000000 0.000000000000 4.000000000000 8.000000000000 7.000000000000 7.000000000000 8.000000000000 9.000000000000 7.000000000000 9.000000000000 7.000000000000 4.000000000000 +t16 4.000000000000 8.000000000000 9.000000000000 8.000000000000 4.000000000000 9.000000000000 4.000000000000 0.000000000000 8.000000000000 7.000000000000 7.000000000000 8.000000000000 9.000000000000 7.000000000000 9.000000000000 7.000000000000 2.000000000000 +t17 8.000000000000 2.000000000000 5.000000000000 8.000000000000 6.000000000000 5.000000000000 8.000000000000 8.000000000000 0.000000000000 7.000000000000 7.000000000000 8.000000000000 5.000000000000 7.000000000000 5.000000000000 5.000000000000 8.000000000000 +t2 7.000000000000 7.000000000000 8.000000000000 5.000000000000 5.000000000000 8.000000000000 7.000000000000 7.000000000000 7.000000000000 0.000000000000 2.000000000000 5.000000000000 8.000000000000 4.000000000000 8.000000000000 6.000000000000 7.000000000000 +t3 7.000000000000 7.000000000000 8.000000000000 5.000000000000 5.000000000000 8.000000000000 7.000000000000 7.000000000000 7.000000000000 2.000000000000 0.000000000000 5.000000000000 8.000000000000 4.000000000000 8.000000000000 6.000000000000 7.000000000000 +t4 8.000000000000 8.000000000000 9.000000000000 2.000000000000 6.000000000000 9.000000000000 8.000000000000 8.000000000000 8.000000000000 5.000000000000 5.000000000000 0.000000000000 9.000000000000 3.000000000000 9.000000000000 7.000000000000 8.000000000000 +t5 9.000000000000 5.000000000000 4.000000000000 9.000000000000 7.000000000000 4.000000000000 9.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 0.000000000000 8.000000000000 2.000000000000 6.000000000000 9.000000000000 +t6 7.000000000000 7.000000000000 8.000000000000 3.000000000000 5.000000000000 8.000000000000 7.000000000000 7.000000000000 7.000000000000 4.000000000000 4.000000000000 3.000000000000 8.000000000000 0.000000000000 8.000000000000 6.000000000000 7.000000000000 +t7 9.000000000000 5.000000000000 4.000000000000 9.000000000000 7.000000000000 4.000000000000 9.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 2.000000000000 8.000000000000 0.000000000000 6.000000000000 9.000000000000 +t8 7.000000000000 5.000000000000 6.000000000000 7.000000000000 5.000000000000 6.000000000000 7.000000000000 7.000000000000 5.000000000000 6.000000000000 6.000000000000 7.000000000000 6.000000000000 6.000000000000 6.000000000000 0.000000000000 7.000000000000 +t9 4.000000000000 8.000000000000 9.000000000000 8.000000000000 4.000000000000 9.000000000000 4.000000000000 2.000000000000 8.000000000000 7.000000000000 7.000000000000 8.000000000000 9.000000000000 7.000000000000 9.000000000000 7.000000000000 0.000000000000 diff --git a/tests/fixtures/tree03.lmm.tsv b/tests/fixtures/tree03.lmm.tsv new file mode 100644 index 0000000..762bb3c --- /dev/null +++ b/tests/fixtures/tree03.lmm.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 2.475192304933 1.022800418548 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.780427609337 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.022800418548 0.000000000000 1.022800418548 1.022800418548 0.645073785447 +t10 1.022800418548 3.779524728190 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.022800418548 0.000000000000 0.000000000000 0.000000000000 0.000000000000 2.107599585783 0.000000000000 2.910439446801 1.322111306246 0.645073785447 +t11 0.000000000000 0.000000000000 1.865152974846 0.107218391960 0.000000000000 0.257950400701 0.107218391960 0.000000000000 0.107218391960 0.257950400701 0.107218391960 0.996794874081 0.000000000000 0.107218391960 0.000000000000 0.000000000000 0.000000000000 +t12 0.000000000000 0.000000000000 0.107218391960 1.954871815396 0.000000000000 0.107218391960 1.401385987177 0.000000000000 0.475520007778 0.107218391960 1.835114753572 0.107218391960 0.000000000000 0.475520007778 0.000000000000 0.000000000000 0.000000000000 +t13 0.645073785447 0.645073785447 0.000000000000 0.000000000000 1.762679928914 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.645073785447 0.645073785447 1.450053893495 +t14 0.000000000000 0.000000000000 0.257950400701 0.107218391960 0.000000000000 0.697857100284 0.107218391960 0.000000000000 0.107218391960 0.429025762947 0.107218391960 0.257950400701 0.000000000000 0.107218391960 0.000000000000 0.000000000000 0.000000000000 +t15 0.000000000000 0.000000000000 0.107218391960 1.401385987177 0.000000000000 0.107218391960 2.149279252626 0.000000000000 0.475520007778 0.107218391960 1.401385987177 0.107218391960 0.000000000000 0.475520007778 0.000000000000 0.000000000000 0.000000000000 +t16 1.780427609337 1.022800418548 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 2.489130385220 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.022800418548 0.000000000000 1.022800418548 1.022800418548 0.645073785447 +t17 0.000000000000 0.000000000000 0.107218391960 0.475520007778 0.000000000000 0.107218391960 0.475520007778 0.000000000000 0.884996788111 0.107218391960 0.475520007778 0.107218391960 0.000000000000 0.748974374263 0.000000000000 0.000000000000 0.000000000000 +t2 0.000000000000 0.000000000000 0.257950400701 0.107218391960 0.000000000000 0.429025762947 0.107218391960 0.000000000000 0.107218391960 1.048476446653 0.107218391960 0.257950400701 0.000000000000 0.107218391960 0.000000000000 0.000000000000 0.000000000000 +t3 0.000000000000 0.000000000000 0.107218391960 1.835114753572 0.000000000000 0.107218391960 1.401385987177 0.000000000000 0.475520007778 0.107218391960 2.190816687187 0.107218391960 0.000000000000 0.475520007778 0.000000000000 0.000000000000 0.000000000000 +t4 0.000000000000 0.000000000000 0.996794874081 0.107218391960 0.000000000000 0.257950400701 0.107218391960 0.000000000000 0.107218391960 0.257950400701 0.107218391960 1.964608333772 0.000000000000 0.107218391960 0.000000000000 0.000000000000 0.000000000000 +t5 1.022800418548 2.107599585783 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.022800418548 0.000000000000 0.000000000000 0.000000000000 0.000000000000 3.003212999320 0.000000000000 2.107599585783 1.322111306246 0.645073785447 +t6 0.000000000000 0.000000000000 0.107218391960 0.475520007778 0.000000000000 0.107218391960 0.475520007778 0.000000000000 0.748974374263 0.107218391960 0.475520007778 0.107218391960 0.000000000000 1.257501092739 0.000000000000 0.000000000000 0.000000000000 +t7 1.022800418548 2.910439446801 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.022800418548 0.000000000000 0.000000000000 0.000000000000 0.000000000000 2.107599585783 0.000000000000 3.337722870987 1.322111306246 0.645073785447 +t8 1.022800418548 1.322111306246 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.022800418548 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.322111306246 0.000000000000 1.322111306246 1.673777386779 0.645073785447 +t9 0.645073785447 0.645073785447 0.000000000000 0.000000000000 1.450053893495 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.645073785447 0.645073785447 1.854699239135 diff --git a/tests/fixtures/tree03.nwk b/tests/fixtures/tree03.nwk new file mode 100644 index 0000000..26b5b97 --- /dev/null +++ b/tests/fixtures/tree03.nwk @@ -0,0 +1 @@ +((((t11:0.8683581008,t4:0.9678134597):0.7388444734,(t2:0.6194506837,t14:0.2688313373):0.1710753622):0.1507320087,((t6:0.5085267185,t17:0.1360224138):0.2734543665,((t12:0.1197570618,t3:0.3557019336):0.4337287664,t15:0.7478932654):0.9258659794):0.3683016158):0.107218392,(((t1:0.6947646956,t16:0.7087027759):0.7576271908,(t8:0.3516660805,((t7:0.4272834242,t10:0.8690852814):0.802839861,t5:0.8956134135):0.7854882795):0.2993108877):0.3777266331,(t9:0.4046453456,t13:0.3126260354):0.804980108):0.6450737854); diff --git a/tests/fixtures/tree03.patristic.tsv b/tests/fixtures/tree03.patristic.tsv new file mode 100644 index 0000000..9e184dd --- /dev/null +++ b/tests/fixtures/tree03.patristic.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 4.209116196027 4.340345279779 4.430064120330 2.947724662954 3.173049405217 4.624471557559 1.403467471479 3.360189093044 3.523668751586 4.666008992121 4.439800638705 3.432804467157 3.732693397673 3.767314338824 2.103368854616 3.039743973175 +t10 4.209116196027 0.000000000000 5.644677703036 5.734396543587 4.252057086211 4.477381828474 5.928803980816 4.223054276314 4.664521516301 4.828001174843 5.970341415377 5.744133061962 2.567538555944 5.037025820930 1.296368705574 2.809079502476 4.344076396432 +t11 4.340345279779 5.644677703036 0.000000000000 3.605588006321 3.627832903760 2.047109273728 3.799995443551 4.354283360066 2.535712979035 2.397728620097 3.841532878112 1.836171560455 4.868365974165 2.908217283664 5.202875845833 3.538930361625 3.719852213981 +t12 4.430064120330 5.734396543587 3.605588006321 0.000000000000 3.717551744310 2.438292131759 1.301379093667 4.444002200617 1.888828587951 2.788911478128 0.475458995439 3.705043365248 4.958084814716 2.261332892580 5.292594686383 3.628649202175 3.809571054531 +t13 2.947724662954 4.252057086211 3.627832903760 3.717551744310 0.000000000000 2.460537029197 3.911959181540 2.961662743241 2.647676717024 2.811156375566 3.953496616101 3.727288262686 3.475745357340 3.020181021653 3.810255229007 2.146309744800 0.717271381058 +t14 3.173049405217 4.477381828474 2.047109273728 2.438292131759 2.460537029197 0.000000000000 2.632699568989 3.186987485504 1.368417104473 0.888282021042 2.674237003550 2.146564632654 3.701070099603 1.740921409102 4.035579971271 2.371634487063 2.552556339419 +t15 4.624471557559 5.928803980816 3.799995443551 1.301379093667 3.911959181540 2.632699568989 0.000000000000 4.638409637846 2.083236025181 2.983318915358 1.537323965458 3.899450802477 5.152492251946 2.455740329809 5.487002123613 3.823056639405 4.003978491761 +t16 1.403467471479 4.223054276314 4.354283360066 4.444002200617 2.961662743241 3.186987485504 4.638409637846 0.000000000000 3.374127173331 3.537606831873 4.679947072407 4.453738718992 3.446742547443 3.746631477959 3.781252419110 2.117306934902 3.053682053462 +t17 3.360189093044 4.664521516301 2.535712979035 1.888828587951 2.647676717024 1.368417104473 2.083236025181 3.374127173331 0.000000000000 1.719036450842 2.124773459742 2.635168337962 3.888209787430 0.644549132325 4.222719659097 2.558774174890 2.739696027245 +t2 3.523668751586 4.828001174843 2.397728620097 2.788911478128 2.811156375566 0.888282021042 2.983318915358 3.537606831873 1.719036450842 0.000000000000 3.024856349919 2.497183979023 4.051689445972 2.091540755471 4.386199317640 2.722253833432 2.903175685788 +t3 4.666008992121 5.970341415377 3.841532878112 0.475458995439 3.953496616101 2.674237003550 1.537323965458 4.679947072407 2.124773459742 3.024856349919 0.000000000000 3.940988237038 5.194029686507 2.497277764371 5.528539558174 3.864594073966 4.045515926322 +t4 4.439800638705 5.744133061962 1.836171560455 3.705043365248 3.727288262686 2.146564632654 3.899450802477 4.453738718992 2.635168337962 2.497183979023 3.940988237038 0.000000000000 4.967821333092 3.007672642590 5.302331204759 3.638385720551 3.819307572907 +t5 3.432804467157 2.567538555944 4.868365974165 4.958084814716 3.475745357340 3.701070099603 5.152492251946 3.446742547443 3.888209787430 4.051689445972 5.194029686507 4.967821333092 0.000000000000 4.260714092059 2.125736698741 2.032767773606 3.567764667561 +t6 3.732693397673 5.037025820930 2.908217283664 2.261332892580 3.020181021653 1.740921409102 2.455740329809 3.746631477959 0.644549132325 2.091540755471 2.497277764371 3.007672642590 4.260714092059 0.000000000000 4.595223963726 2.931278479518 3.112200331874 +t7 3.767314338824 1.296368705574 5.202875845833 5.292594686383 3.810255229007 4.035579971271 5.487002123613 3.781252419110 4.222719659097 4.386199317640 5.528539558174 5.302331204759 2.125736698741 4.595223963726 0.000000000000 2.367277645273 3.902274539229 +t8 2.103368854616 2.809079502476 3.538930361625 3.628649202175 2.146309744800 2.371634487063 3.823056639405 2.117306934902 2.558774174890 2.722253833432 3.864594073966 3.638385720551 2.032767773606 2.931278479518 2.367277645273 0.000000000000 2.238329055021 +t9 3.039743973175 4.344076396432 3.719852213981 3.809571054531 0.717271381058 2.552556339419 4.003978491761 3.053682053462 2.739696027245 2.903175685788 4.045515926322 3.819307572907 3.567764667561 3.112200331874 3.902274539229 2.238329055021 0.000000000000 diff --git a/tests/fixtures/tree03.topology.tsv b/tests/fixtures/tree03.topology.tsv new file mode 100644 index 0000000..042bcbe --- /dev/null +++ b/tests/fixtures/tree03.topology.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 6.000000000000 8.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 2.000000000000 8.000000000000 8.000000000000 9.000000000000 8.000000000000 5.000000000000 8.000000000000 6.000000000000 4.000000000000 5.000000000000 +t10 6.000000000000 0.000000000000 10.000000000000 11.000000000000 7.000000000000 10.000000000000 10.000000000000 6.000000000000 10.000000000000 10.000000000000 11.000000000000 10.000000000000 3.000000000000 10.000000000000 2.000000000000 4.000000000000 7.000000000000 +t11 8.000000000000 10.000000000000 0.000000000000 7.000000000000 7.000000000000 4.000000000000 6.000000000000 8.000000000000 6.000000000000 4.000000000000 7.000000000000 2.000000000000 9.000000000000 6.000000000000 10.000000000000 8.000000000000 7.000000000000 +t12 9.000000000000 11.000000000000 7.000000000000 0.000000000000 8.000000000000 7.000000000000 3.000000000000 9.000000000000 5.000000000000 7.000000000000 2.000000000000 7.000000000000 10.000000000000 5.000000000000 11.000000000000 9.000000000000 8.000000000000 +t13 5.000000000000 7.000000000000 7.000000000000 8.000000000000 0.000000000000 7.000000000000 7.000000000000 5.000000000000 7.000000000000 7.000000000000 8.000000000000 7.000000000000 6.000000000000 7.000000000000 7.000000000000 5.000000000000 2.000000000000 +t14 8.000000000000 10.000000000000 4.000000000000 7.000000000000 7.000000000000 0.000000000000 6.000000000000 8.000000000000 6.000000000000 2.000000000000 7.000000000000 4.000000000000 9.000000000000 6.000000000000 10.000000000000 8.000000000000 7.000000000000 +t15 8.000000000000 10.000000000000 6.000000000000 3.000000000000 7.000000000000 6.000000000000 0.000000000000 8.000000000000 4.000000000000 6.000000000000 3.000000000000 6.000000000000 9.000000000000 4.000000000000 10.000000000000 8.000000000000 7.000000000000 +t16 2.000000000000 6.000000000000 8.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 0.000000000000 8.000000000000 8.000000000000 9.000000000000 8.000000000000 5.000000000000 8.000000000000 6.000000000000 4.000000000000 5.000000000000 +t17 8.000000000000 10.000000000000 6.000000000000 5.000000000000 7.000000000000 6.000000000000 4.000000000000 8.000000000000 0.000000000000 6.000000000000 5.000000000000 6.000000000000 9.000000000000 2.000000000000 10.000000000000 8.000000000000 7.000000000000 +t2 8.000000000000 10.000000000000 4.000000000000 7.000000000000 7.000000000000 2.000000000000 6.000000000000 8.000000000000 6.000000000000 0.000000000000 7.000000000000 4.000000000000 9.000000000000 6.000000000000 10.000000000000 8.000000000000 7.000000000000 +t3 9.000000000000 11.000000000000 7.000000000000 2.000000000000 8.000000000000 7.000000000000 3.000000000000 9.000000000000 5.000000000000 7.000000000000 0.000000000000 7.000000000000 10.000000000000 5.000000000000 11.000000000000 9.000000000000 8.000000000000 +t4 8.000000000000 10.000000000000 2.000000000000 7.000000000000 7.000000000000 4.000000000000 6.000000000000 8.000000000000 6.000000000000 4.000000000000 7.000000000000 0.000000000000 9.000000000000 6.000000000000 10.000000000000 8.000000000000 7.000000000000 +t5 5.000000000000 3.000000000000 9.000000000000 10.000000000000 6.000000000000 9.000000000000 9.000000000000 5.000000000000 9.000000000000 9.000000000000 10.000000000000 9.000000000000 0.000000000000 9.000000000000 3.000000000000 3.000000000000 6.000000000000 +t6 8.000000000000 10.000000000000 6.000000000000 5.000000000000 7.000000000000 6.000000000000 4.000000000000 8.000000000000 2.000000000000 6.000000000000 5.000000000000 6.000000000000 9.000000000000 0.000000000000 10.000000000000 8.000000000000 7.000000000000 +t7 6.000000000000 2.000000000000 10.000000000000 11.000000000000 7.000000000000 10.000000000000 10.000000000000 6.000000000000 10.000000000000 10.000000000000 11.000000000000 10.000000000000 3.000000000000 10.000000000000 0.000000000000 4.000000000000 7.000000000000 +t8 4.000000000000 4.000000000000 8.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 4.000000000000 8.000000000000 8.000000000000 9.000000000000 8.000000000000 3.000000000000 8.000000000000 4.000000000000 0.000000000000 5.000000000000 +t9 5.000000000000 7.000000000000 7.000000000000 8.000000000000 2.000000000000 7.000000000000 7.000000000000 5.000000000000 7.000000000000 7.000000000000 8.000000000000 7.000000000000 6.000000000000 7.000000000000 7.000000000000 5.000000000000 0.000000000000 diff --git a/tests/fixtures/tree04.lmm.tsv b/tests/fixtures/tree04.lmm.tsv new file mode 100644 index 0000000..8ef2f84 --- /dev/null +++ b/tests/fixtures/tree04.lmm.tsv @@ -0,0 +1,19 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t18 t2 t3 t4 t5 t6 t7 t8 t9 +t1 1.826454006135 1.434058769606 1.323048059829 0.000000000000 0.000000000000 0.055704348022 0.055704348022 1.323048059829 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.960756540997 0.000000000000 0.960756540997 0.000000000000 +t10 1.434058769606 1.785461900057 1.323048059829 0.000000000000 0.000000000000 0.055704348022 0.055704348022 1.323048059829 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.960756540997 0.000000000000 0.960756540997 0.000000000000 +t11 1.323048059829 1.323048059829 1.958333726740 0.000000000000 0.000000000000 0.055704348022 0.055704348022 1.952479454223 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.960756540997 0.000000000000 0.960756540997 0.000000000000 +t12 0.000000000000 0.000000000000 0.000000000000 1.834822334116 0.517228086945 0.000000000000 0.000000000000 0.000000000000 0.517228086945 1.042361159576 0.492893308168 0.492893308168 0.492893308168 0.517228086945 0.000000000000 0.517228086945 0.000000000000 0.492893308168 +t13 0.000000000000 0.000000000000 0.000000000000 0.517228086945 3.687702669296 0.000000000000 0.000000000000 0.000000000000 1.509061724180 0.517228086945 0.492893308168 0.492893308168 0.492893308168 2.833347449545 0.000000000000 2.441255136626 0.000000000000 0.492893308168 +t14 0.055704348022 0.055704348022 0.055704348022 0.000000000000 0.000000000000 0.435066005681 0.104274603771 0.055704348022 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.055704348022 0.000000000000 0.055704348022 0.000000000000 +t15 0.055704348022 0.055704348022 0.055704348022 0.000000000000 0.000000000000 0.104274603771 0.457994617289 0.055704348022 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.055704348022 0.000000000000 0.055704348022 0.000000000000 +t16 1.323048059829 1.323048059829 1.952479454223 0.000000000000 0.000000000000 0.055704348022 0.055704348022 2.269571904792 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.960756540997 0.000000000000 0.960756540997 0.000000000000 +t17 0.000000000000 0.000000000000 0.000000000000 0.517228086945 1.509061724180 0.000000000000 0.000000000000 0.000000000000 1.601748950779 0.517228086945 0.492893308168 0.492893308168 0.492893308168 1.509061724180 0.000000000000 1.509061724180 0.000000000000 0.492893308168 +t18 0.000000000000 0.000000000000 0.000000000000 1.042361159576 0.517228086945 0.000000000000 0.000000000000 0.000000000000 0.517228086945 1.766559675103 0.492893308168 0.492893308168 0.492893308168 0.517228086945 0.000000000000 0.517228086945 0.000000000000 0.492893308168 +t2 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 1.831984268734 1.537219957914 0.760391999735 0.492893308168 0.000000000000 0.492893308168 0.000000000000 0.760391999735 +t3 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 1.537219957914 2.051610622788 0.760391999735 0.492893308168 0.000000000000 0.492893308168 0.000000000000 0.760391999735 +t4 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.760391999735 0.760391999735 1.824602956185 0.492893308168 0.000000000000 0.492893308168 0.000000000000 0.883239736082 +t5 0.000000000000 0.000000000000 0.000000000000 0.517228086945 2.833347449545 0.000000000000 0.000000000000 0.000000000000 1.509061724180 0.517228086945 0.492893308168 0.492893308168 0.492893308168 3.014986074297 0.000000000000 2.441255136626 0.000000000000 0.492893308168 +t6 0.960756540997 0.960756540997 0.960756540997 0.000000000000 0.000000000000 0.055704348022 0.055704348022 0.960756540997 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 2.235513422871 0.000000000000 1.956952624256 0.000000000000 +t7 0.000000000000 0.000000000000 0.000000000000 0.517228086945 2.441255136626 0.000000000000 0.000000000000 0.000000000000 1.509061724180 0.517228086945 0.492893308168 0.492893308168 0.492893308168 2.441255136626 0.000000000000 3.185927398736 0.000000000000 0.492893308168 +t8 0.960756540997 0.960756540997 0.960756540997 0.000000000000 0.000000000000 0.055704348022 0.055704348022 0.960756540997 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.956952624256 0.000000000000 2.867384099402 0.000000000000 +t9 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.760391999735 0.760391999735 0.883239736082 0.492893308168 0.000000000000 0.492893308168 0.000000000000 0.979572141776 diff --git a/tests/fixtures/tree04.nwk b/tests/fixtures/tree04.nwk new file mode 100644 index 0000000..c5d1741 --- /dev/null +++ b/tests/fixtures/tree04.nwk @@ -0,0 +1 @@ +((((t17:0.0926872266,((t5:0.1816386248,t13:0.8543552198):0.3920923129,t7:0.7446722621):0.9321934124):0.9918336372,(t18:0.7241985155,t12:0.7924611745):0.5251330726):0.02433477878,((t4:0.9413632201,t9:0.09633240569):0.1228477363,(t2:0.2947643108,t3:0.5143906649):0.7768279582):0.2674986916):0.4928933082,(((t6:0.2785607986,t8:0.9104314751):0.9961960833,((t11:0.005854272516,t16:0.3170924506):0.6294313944,(t1:0.3923952365,t10:0.3514031305):0.1110107098):0.3622915188):0.905052193,(t15:0.3537200135,t14:0.3307914019):0.04857025575):0.05570434802); diff --git a/tests/fixtures/tree04.patristic.tsv b/tests/fixtures/tree04.patristic.tsv new file mode 100644 index 0000000..e88876d --- /dev/null +++ b/tests/fixtures/tree04.patristic.tsv @@ -0,0 +1,19 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t18 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 0.743798366981 1.138691613218 3.661276340252 5.514156675432 2.150111315772 2.173039927380 1.449929791270 3.428202956915 3.593013681239 3.658438274870 3.878064628923 3.651056962321 4.841440080432 2.140454347013 5.012381404871 2.772325023543 2.806026147911 +t10 0.743798366981 0.000000000000 1.097699507140 3.620284234174 5.473164569354 2.109119209694 2.132047821302 1.408937685192 3.387210850837 3.552021575160 3.617446168792 3.837072522845 3.610064856242 4.800447974354 2.099462240934 4.971389298793 2.731332917465 2.765034041833 +t11 1.138691613218 1.097699507140 0.000000000000 3.793156060856 5.646036396036 2.281991036376 2.304919647984 0.322946723085 3.560082677519 3.724893401843 3.790317995474 4.009944349527 3.782936682925 4.973319801036 2.272334067617 5.144261125475 2.904204744147 2.937905868515 +t12 3.661276340252 3.620284234174 3.793156060856 0.000000000000 4.488068829523 2.269888339797 2.292816951405 4.104394238908 2.402115111006 1.516659690067 2.681019986514 2.900646340568 2.673638673965 3.815352234524 4.070335756987 3.986293558963 4.702206433518 1.828607859556 +t13 5.514156675432 5.473164569354 5.646036396036 4.488068829523 0.000000000000 4.122768674977 4.145697286585 5.957274574088 2.271328171715 4.419806170510 4.533900321694 4.753526675748 4.526519009145 1.035993844504 5.923216092167 1.991119794780 6.555086768698 3.681488194736 +t14 2.150111315772 2.109119209694 2.281991036376 2.269888339797 4.122768674977 0.000000000000 0.684511415428 2.593229214428 2.036814956460 2.201625680784 2.267050274415 2.486676628469 2.259668961866 3.450052079977 2.559170732507 3.620993404416 3.191041409038 1.414638147457 +t15 2.173039927380 2.132047821302 2.304919647984 2.292816951405 4.145697286585 0.684511415428 0.000000000000 2.616157826036 2.059743568068 2.224554292392 2.289978886023 2.509605240077 2.282597573474 3.472980691586 2.582099344116 3.643922016025 3.213970020646 1.437566759065 +t16 1.449929791270 1.408937685192 0.322946723085 4.104394238908 5.957274574088 2.593229214428 2.616157826036 0.000000000000 3.871320855571 4.036131579895 4.101556173526 4.321182527579 4.094174860977 5.284557979088 2.583572245669 5.455499303527 3.215442922199 3.249144046567 +t17 3.428202956915 3.387210850837 3.560082677519 2.402115111006 2.271328171715 2.036814956460 2.059743568068 3.871320855571 0.000000000000 2.333852451993 2.447946603177 2.667572957231 2.440565290628 1.598611576715 3.837262373650 1.769552901154 4.469133050181 1.595534476219 +t18 3.593013681239 3.552021575160 3.724893401843 1.516659690067 4.419806170510 2.201625680784 2.224554292392 4.036131579895 2.333852451993 0.000000000000 2.612757327501 2.832383681554 2.605376014952 3.747089575510 4.002073097974 3.918030899949 4.633943774505 1.760345200542 +t2 3.658438274870 3.617446168792 3.790317995474 2.681019986514 4.533900321694 2.267050274415 2.289978886023 4.101556173526 2.447946603177 2.612757327501 0.000000000000 0.809154975694 2.135803225450 3.861183726694 4.067497691605 4.032125051133 4.699368368136 1.290772411041 +t3 3.878064628923 3.837072522845 4.009944349527 2.900646340568 4.753526675748 2.486676628469 2.509605240077 4.321182527579 2.667572957231 2.832383681554 0.809154975694 0.000000000000 2.355429579504 4.080810080748 4.287124045659 4.251751405187 4.918994722189 1.510398765095 +t4 3.651056962321 3.610064856242 3.782936682925 2.673638673965 4.526519009145 2.259668961866 2.282597573474 4.094174860977 2.440565290628 2.605376014952 2.135803225450 2.355429579504 0.000000000000 3.853802414145 4.060116379056 4.024743738584 4.691987055587 1.037695625797 +t5 4.841440080432 4.800447974354 4.973319801036 3.815352234524 1.035993844504 3.450052079977 3.472980691586 5.284557979088 1.598611576715 3.747089575510 3.861183726694 4.080810080748 3.853802414145 0.000000000000 5.250499497168 1.318403199781 5.882370173698 3.008771599736 +t6 2.140454347013 2.099462240934 2.272334067617 4.070335756987 5.923216092167 2.559170732507 2.582099344116 2.583572245669 3.837262373650 4.002073097974 4.067497691605 4.287124045659 4.060116379056 5.250499497168 0.000000000000 5.421440821607 1.188992273761 3.215085564647 +t7 5.012381404871 4.971389298793 5.144261125475 3.986293558963 1.991119794780 3.620993404416 3.643922016025 5.455499303527 1.769552901154 3.918030899949 4.032125051133 4.251751405187 4.024743738584 1.318403199781 5.421440821607 0.000000000000 6.053311498137 3.179712924175 +t8 2.772325023543 2.731332917465 2.904204744147 4.702206433518 6.555086768698 3.191041409038 3.213970020646 3.215442922199 4.469133050181 4.633943774505 4.699368368136 4.918994722189 4.691987055587 5.882370173698 1.188992273761 6.053311498137 0.000000000000 3.846956241177 +t9 2.806026147911 2.765034041833 2.937905868515 1.828607859556 3.681488194736 1.414638147457 1.437566759065 3.249144046567 1.595534476219 1.760345200542 1.290772411041 1.510398765095 1.037695625797 3.008771599736 3.215085564647 3.179712924175 3.846956241177 0.000000000000 diff --git a/tests/fixtures/tree04.topology.tsv b/tests/fixtures/tree04.topology.tsv new file mode 100644 index 0000000..93d57a3 --- /dev/null +++ b/tests/fixtures/tree04.topology.tsv @@ -0,0 +1,19 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t18 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 2.000000000000 4.000000000000 9.000000000000 11.000000000000 6.000000000000 6.000000000000 4.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 11.000000000000 5.000000000000 10.000000000000 5.000000000000 9.000000000000 +t10 2.000000000000 0.000000000000 4.000000000000 9.000000000000 11.000000000000 6.000000000000 6.000000000000 4.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 11.000000000000 5.000000000000 10.000000000000 5.000000000000 9.000000000000 +t11 4.000000000000 4.000000000000 0.000000000000 9.000000000000 11.000000000000 6.000000000000 6.000000000000 2.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 11.000000000000 5.000000000000 10.000000000000 5.000000000000 9.000000000000 +t12 9.000000000000 9.000000000000 9.000000000000 0.000000000000 6.000000000000 7.000000000000 7.000000000000 9.000000000000 4.000000000000 2.000000000000 6.000000000000 6.000000000000 6.000000000000 6.000000000000 8.000000000000 5.000000000000 8.000000000000 6.000000000000 +t13 11.000000000000 11.000000000000 11.000000000000 6.000000000000 0.000000000000 9.000000000000 9.000000000000 11.000000000000 4.000000000000 6.000000000000 8.000000000000 8.000000000000 8.000000000000 2.000000000000 10.000000000000 3.000000000000 10.000000000000 8.000000000000 +t14 6.000000000000 6.000000000000 6.000000000000 7.000000000000 9.000000000000 0.000000000000 2.000000000000 6.000000000000 7.000000000000 7.000000000000 7.000000000000 7.000000000000 7.000000000000 9.000000000000 5.000000000000 8.000000000000 5.000000000000 7.000000000000 +t15 6.000000000000 6.000000000000 6.000000000000 7.000000000000 9.000000000000 2.000000000000 0.000000000000 6.000000000000 7.000000000000 7.000000000000 7.000000000000 7.000000000000 7.000000000000 9.000000000000 5.000000000000 8.000000000000 5.000000000000 7.000000000000 +t16 4.000000000000 4.000000000000 2.000000000000 9.000000000000 11.000000000000 6.000000000000 6.000000000000 0.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 11.000000000000 5.000000000000 10.000000000000 5.000000000000 9.000000000000 +t17 9.000000000000 9.000000000000 9.000000000000 4.000000000000 4.000000000000 7.000000000000 7.000000000000 9.000000000000 0.000000000000 4.000000000000 6.000000000000 6.000000000000 6.000000000000 4.000000000000 8.000000000000 3.000000000000 8.000000000000 6.000000000000 +t18 9.000000000000 9.000000000000 9.000000000000 2.000000000000 6.000000000000 7.000000000000 7.000000000000 9.000000000000 4.000000000000 0.000000000000 6.000000000000 6.000000000000 6.000000000000 6.000000000000 8.000000000000 5.000000000000 8.000000000000 6.000000000000 +t2 9.000000000000 9.000000000000 9.000000000000 6.000000000000 8.000000000000 7.000000000000 7.000000000000 9.000000000000 6.000000000000 6.000000000000 0.000000000000 2.000000000000 4.000000000000 8.000000000000 8.000000000000 7.000000000000 8.000000000000 4.000000000000 +t3 9.000000000000 9.000000000000 9.000000000000 6.000000000000 8.000000000000 7.000000000000 7.000000000000 9.000000000000 6.000000000000 6.000000000000 2.000000000000 0.000000000000 4.000000000000 8.000000000000 8.000000000000 7.000000000000 8.000000000000 4.000000000000 +t4 9.000000000000 9.000000000000 9.000000000000 6.000000000000 8.000000000000 7.000000000000 7.000000000000 9.000000000000 6.000000000000 6.000000000000 4.000000000000 4.000000000000 0.000000000000 8.000000000000 8.000000000000 7.000000000000 8.000000000000 2.000000000000 +t5 11.000000000000 11.000000000000 11.000000000000 6.000000000000 2.000000000000 9.000000000000 9.000000000000 11.000000000000 4.000000000000 6.000000000000 8.000000000000 8.000000000000 8.000000000000 0.000000000000 10.000000000000 3.000000000000 10.000000000000 8.000000000000 +t6 5.000000000000 5.000000000000 5.000000000000 8.000000000000 10.000000000000 5.000000000000 5.000000000000 5.000000000000 8.000000000000 8.000000000000 8.000000000000 8.000000000000 8.000000000000 10.000000000000 0.000000000000 9.000000000000 2.000000000000 8.000000000000 +t7 10.000000000000 10.000000000000 10.000000000000 5.000000000000 3.000000000000 8.000000000000 8.000000000000 10.000000000000 3.000000000000 5.000000000000 7.000000000000 7.000000000000 7.000000000000 3.000000000000 9.000000000000 0.000000000000 9.000000000000 7.000000000000 +t8 5.000000000000 5.000000000000 5.000000000000 8.000000000000 10.000000000000 5.000000000000 5.000000000000 5.000000000000 8.000000000000 8.000000000000 8.000000000000 8.000000000000 8.000000000000 10.000000000000 2.000000000000 9.000000000000 0.000000000000 8.000000000000 +t9 9.000000000000 9.000000000000 9.000000000000 6.000000000000 8.000000000000 7.000000000000 7.000000000000 9.000000000000 6.000000000000 6.000000000000 4.000000000000 4.000000000000 2.000000000000 8.000000000000 8.000000000000 7.000000000000 8.000000000000 0.000000000000 diff --git a/tests/fixtures/tree05.lmm.tsv b/tests/fixtures/tree05.lmm.tsv new file mode 100644 index 0000000..6be3e5c --- /dev/null +++ b/tests/fixtures/tree05.lmm.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 1.412819813471 0.000000000000 0.000000000000 0.963843473466 0.000000000000 1.018107262440 0.000000000000 0.000000000000 0.000000000000 1.018107262440 0.000000000000 0.000000000000 +t10 0.000000000000 0.682040252956 0.121796532068 0.000000000000 0.121796532068 0.000000000000 0.121796532068 0.121796532068 0.457185057225 0.000000000000 0.121796532068 0.121796532068 +t11 0.000000000000 0.121796532068 2.139318238711 0.000000000000 1.104086857522 0.000000000000 0.996908592293 1.723483712412 0.121796532068 0.000000000000 1.723483712412 0.996908592293 +t12 0.963843473466 0.000000000000 0.000000000000 1.052241293481 0.000000000000 0.963843473466 0.000000000000 0.000000000000 0.000000000000 0.963843473466 0.000000000000 0.000000000000 +t2 0.000000000000 0.121796532068 1.104086857522 0.000000000000 1.268882896286 0.000000000000 0.996908592293 1.104086857522 0.121796532068 0.000000000000 1.104086857522 0.996908592293 +t3 1.018107262440 0.000000000000 0.000000000000 0.963843473466 0.000000000000 1.379060709616 0.000000000000 0.000000000000 0.000000000000 1.259154318599 0.000000000000 0.000000000000 +t4 0.000000000000 0.121796532068 0.996908592293 0.000000000000 0.996908592293 0.000000000000 2.172244624468 0.996908592293 0.121796532068 0.000000000000 0.996908592293 1.303926248802 +t5 0.000000000000 0.121796532068 1.723483712412 0.000000000000 1.104086857522 0.000000000000 0.996908592293 2.642430364620 0.121796532068 0.000000000000 2.313111583935 0.996908592293 +t6 0.000000000000 0.457185057225 0.121796532068 0.000000000000 0.121796532068 0.000000000000 0.121796532068 0.121796532068 0.909587969305 0.000000000000 0.121796532068 0.121796532068 +t7 1.018107262440 0.000000000000 0.000000000000 0.963843473466 0.000000000000 1.259154318599 0.000000000000 0.000000000000 0.000000000000 1.890060830163 0.000000000000 0.000000000000 +t8 0.000000000000 0.121796532068 1.723483712412 0.000000000000 1.104086857522 0.000000000000 0.996908592293 2.313111583935 0.121796532068 0.000000000000 3.174793167273 0.996908592293 +t9 0.000000000000 0.121796532068 0.996908592293 0.000000000000 0.996908592293 0.000000000000 1.303926248802 0.996908592293 0.121796532068 0.000000000000 0.996908592293 1.668768494390 diff --git a/tests/fixtures/tree05.nwk b/tests/fixtures/tree05.nwk new file mode 100644 index 0000000..07c4e33 --- /dev/null +++ b/tests/fixtures/tree05.nwk @@ -0,0 +1 @@ +(((t1:0.394712551,(t3:0.119906391,t7:0.6309065116):0.2410470562):0.05426378897,t12:0.08839782001):0.9638434735,(((t4:0.8683183757,t9:0.3648422456):0.3070176565,(t2:0.1647960388,(t11:0.4158345263,(t5:0.3293187807,t8:0.8616815833):0.5896278715):0.6193968549):0.1071782652):0.8751120602,(t6:0.4524029121,t10:0.2248551957):0.3353885252):0.1217965321); diff --git a/tests/fixtures/tree05.patristic.tsv b/tests/fixtures/tree05.patristic.tsv new file mode 100644 index 0000000..8f489a6 --- /dev/null +++ b/tests/fixtures/tree05.patristic.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 2.094860066427 3.552138052182 0.537374160020 2.681702709757 0.755665998207 3.585064437939 4.055250178091 2.322407782776 1.266666118754 4.587612980744 3.081588307861 +t10 2.094860066427 0.000000000000 2.577765427530 1.734281546436 1.707330085104 2.061100962572 2.610691813286 3.080877553439 0.677258107811 2.572101083118 3.613240356091 2.107215683209 +t11 3.552138052182 2.577765427530 0.000000000000 3.191559532192 1.200027419953 3.518378948327 2.317745678592 1.334781178506 2.805313143879 4.029379068874 1.867143981159 1.814269548515 +t12 0.537374160020 1.734281546436 3.191559532192 0.000000000000 2.321124189766 0.503615056165 3.224485917948 3.694671658101 1.961829262786 1.014615176711 4.227034460753 2.721009787871 +t2 2.681702709757 1.707330085104 1.200027419953 2.321124189766 0.000000000000 2.647943605902 1.447310336167 1.703139545862 1.934877801454 3.158943726448 2.235502348514 0.943834206089 +t3 0.755665998207 2.061100962572 3.518378948327 0.503615056165 2.647943605902 0.000000000000 3.551305334084 4.021491074236 2.288648678921 0.750812902581 4.553853876889 3.047829204006 +t4 3.585064437939 2.610691813286 2.317745678592 3.224485917948 1.447310336167 3.551305334084 0.000000000000 2.820857804501 2.838239529636 4.062305454630 3.353220607154 1.233160621254 +t5 4.055250178091 3.080877553439 1.334781178506 3.694671658101 1.703139545862 4.021491074236 2.820857804501 0.000000000000 3.308425269788 4.532491194783 1.191000364022 2.317381674424 +t6 2.322407782776 0.677258107811 2.805313143879 1.961829262786 1.934877801454 2.288648678921 2.838239529636 3.308425269788 0.000000000000 2.799648799468 3.840788072441 2.334763399558 +t7 1.266666118754 2.572101083118 4.029379068874 1.014615176711 3.158943726448 0.750812902581 4.062305454630 4.532491194783 2.799648799468 0.000000000000 5.064853997435 3.558829324553 +t8 4.587612980744 3.613240356091 1.867143981159 4.227034460753 2.235502348514 4.553853876889 3.353220607154 1.191000364022 3.840788072441 5.064853997435 0.000000000000 2.849744477076 +t9 3.081588307861 2.107215683209 1.814269548515 2.721009787871 0.943834206089 3.047829204006 1.233160621254 2.317381674424 2.334763399558 3.558829324553 2.849744477076 0.000000000000 diff --git a/tests/fixtures/tree05.topology.tsv b/tests/fixtures/tree05.topology.tsv new file mode 100644 index 0000000..4028630 --- /dev/null +++ b/tests/fixtures/tree05.topology.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 6.000000000000 8.000000000000 3.000000000000 7.000000000000 3.000000000000 7.000000000000 9.000000000000 6.000000000000 3.000000000000 9.000000000000 7.000000000000 +t10 6.000000000000 0.000000000000 6.000000000000 5.000000000000 5.000000000000 7.000000000000 5.000000000000 7.000000000000 2.000000000000 7.000000000000 7.000000000000 5.000000000000 +t11 8.000000000000 6.000000000000 0.000000000000 7.000000000000 3.000000000000 9.000000000000 5.000000000000 3.000000000000 6.000000000000 9.000000000000 3.000000000000 5.000000000000 +t12 3.000000000000 5.000000000000 7.000000000000 0.000000000000 6.000000000000 4.000000000000 6.000000000000 8.000000000000 5.000000000000 4.000000000000 8.000000000000 6.000000000000 +t2 7.000000000000 5.000000000000 3.000000000000 6.000000000000 0.000000000000 8.000000000000 4.000000000000 4.000000000000 5.000000000000 8.000000000000 4.000000000000 4.000000000000 +t3 3.000000000000 7.000000000000 9.000000000000 4.000000000000 8.000000000000 0.000000000000 8.000000000000 10.000000000000 7.000000000000 2.000000000000 10.000000000000 8.000000000000 +t4 7.000000000000 5.000000000000 5.000000000000 6.000000000000 4.000000000000 8.000000000000 0.000000000000 6.000000000000 5.000000000000 8.000000000000 6.000000000000 2.000000000000 +t5 9.000000000000 7.000000000000 3.000000000000 8.000000000000 4.000000000000 10.000000000000 6.000000000000 0.000000000000 7.000000000000 10.000000000000 2.000000000000 6.000000000000 +t6 6.000000000000 2.000000000000 6.000000000000 5.000000000000 5.000000000000 7.000000000000 5.000000000000 7.000000000000 0.000000000000 7.000000000000 7.000000000000 5.000000000000 +t7 3.000000000000 7.000000000000 9.000000000000 4.000000000000 8.000000000000 2.000000000000 8.000000000000 10.000000000000 7.000000000000 0.000000000000 10.000000000000 8.000000000000 +t8 9.000000000000 7.000000000000 3.000000000000 8.000000000000 4.000000000000 10.000000000000 6.000000000000 2.000000000000 7.000000000000 10.000000000000 0.000000000000 6.000000000000 +t9 7.000000000000 5.000000000000 5.000000000000 6.000000000000 4.000000000000 8.000000000000 2.000000000000 6.000000000000 5.000000000000 8.000000000000 6.000000000000 0.000000000000 diff --git a/tests/fixtures/tree06.lmm.tsv b/tests/fixtures/tree06.lmm.tsv new file mode 100644 index 0000000..780dfec --- /dev/null +++ b/tests/fixtures/tree06.lmm.tsv @@ -0,0 +1,10 @@ + t1 t2 t3 t4 t5 t6 t7 t8 t9 +t1 2.987882709131 0.844335314119 1.340232875198 0.844335314119 2.126695765881 0.000000000000 0.000000000000 1.340232875198 0.000000000000 +t2 0.844335314119 1.951782006305 0.844335314119 1.352037202567 0.844335314119 0.000000000000 0.000000000000 0.844335314119 0.000000000000 +t3 1.340232875198 0.844335314119 2.459578693379 0.844335314119 1.340232875198 0.000000000000 0.000000000000 1.621566965710 0.000000000000 +t4 0.844335314119 1.352037202567 0.844335314119 2.301177664427 0.844335314119 0.000000000000 0.000000000000 0.844335314119 0.000000000000 +t5 2.126695765881 0.844335314119 1.340232875198 0.844335314119 2.127780230949 0.000000000000 0.000000000000 1.340232875198 0.000000000000 +t6 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.198197783902 0.203751023160 0.000000000000 0.114089301787 +t7 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.203751023160 0.346611253452 0.000000000000 0.114089301787 +t8 1.340232875198 0.844335314119 1.621566965710 0.844335314119 1.340232875198 0.000000000000 0.000000000000 2.261270712363 0.000000000000 +t9 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.114089301787 0.114089301787 0.000000000000 0.766413139878 diff --git a/tests/fixtures/tree06.nwk b/tests/fixtures/tree06.nwk new file mode 100644 index 0000000..896fc35 --- /dev/null +++ b/tests/fixtures/tree06.nwk @@ -0,0 +1 @@ +(((t2:0.5997448037,t4:0.9491404619):0.5077018884,((t5:0.001084465068,t1:0.8611869432):0.7864628907,(t3:0.8380117277,t8:0.6397037467):0.2813340905):0.4958975611):0.8443353141,((t7:0.1428602303,t6:0.9944467607):0.08966172137,t9:0.6523238381):0.1140893018); diff --git a/tests/fixtures/tree06.patristic.tsv b/tests/fixtures/tree06.patristic.tsv new file mode 100644 index 0000000..ad66b40 --- /dev/null +++ b/tests/fixtures/tree06.patristic.tsv @@ -0,0 +1,10 @@ + t1 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 3.250994087197 2.766995652113 3.600389745319 0.862271408318 4.186080493033 3.334493962582 2.568687671097 3.754295849008 +t2 3.250994087197 0.000000000000 2.722690071445 1.548885265598 2.390891609015 3.149979790207 2.298393259756 2.524382090429 2.718195146183 +t3 2.766995652113 2.722690071445 0.000000000000 3.072085729567 1.906893173931 3.657776477281 2.806189946830 1.477715474321 3.225991833257 +t4 3.600389745319 1.548885265598 3.072085729567 0.000000000000 2.740287267137 3.499375448329 2.647788917879 2.873777748551 3.067590804305 +t5 0.862271408318 2.390891609015 1.906893173931 2.740287267137 0.000000000000 3.325978014851 2.474391484400 1.708585192915 2.894193370827 +t6 4.186080493033 3.149979790207 3.657776477281 3.499375448329 3.325978014851 0.000000000000 1.137306991033 3.459468496265 1.736432320205 +t7 3.334493962582 2.298393259756 2.806189946830 2.647788917879 2.474391484400 1.137306991033 0.000000000000 2.607881965814 0.884845789755 +t8 2.568687671097 2.524382090429 1.477715474321 2.873777748551 1.708585192915 3.459468496265 2.607881965814 0.000000000000 3.027683852240 +t9 3.754295849008 2.718195146183 3.225991833257 3.067590804305 2.894193370827 1.736432320205 0.884845789755 3.027683852240 0.000000000000 diff --git a/tests/fixtures/tree06.topology.tsv b/tests/fixtures/tree06.topology.tsv new file mode 100644 index 0000000..d70e425 --- /dev/null +++ b/tests/fixtures/tree06.topology.tsv @@ -0,0 +1,10 @@ + t1 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 5.000000000000 4.000000000000 5.000000000000 2.000000000000 7.000000000000 7.000000000000 4.000000000000 6.000000000000 +t2 5.000000000000 0.000000000000 5.000000000000 2.000000000000 5.000000000000 6.000000000000 6.000000000000 5.000000000000 5.000000000000 +t3 4.000000000000 5.000000000000 0.000000000000 5.000000000000 4.000000000000 7.000000000000 7.000000000000 2.000000000000 6.000000000000 +t4 5.000000000000 2.000000000000 5.000000000000 0.000000000000 5.000000000000 6.000000000000 6.000000000000 5.000000000000 5.000000000000 +t5 2.000000000000 5.000000000000 4.000000000000 5.000000000000 0.000000000000 7.000000000000 7.000000000000 4.000000000000 6.000000000000 +t6 7.000000000000 6.000000000000 7.000000000000 6.000000000000 7.000000000000 0.000000000000 2.000000000000 7.000000000000 3.000000000000 +t7 7.000000000000 6.000000000000 7.000000000000 6.000000000000 7.000000000000 2.000000000000 0.000000000000 7.000000000000 3.000000000000 +t8 4.000000000000 5.000000000000 2.000000000000 5.000000000000 4.000000000000 7.000000000000 7.000000000000 0.000000000000 6.000000000000 +t9 6.000000000000 5.000000000000 6.000000000000 5.000000000000 6.000000000000 3.000000000000 3.000000000000 6.000000000000 0.000000000000 diff --git a/tests/integration.rs b/tests/integration.rs new file mode 100644 index 0000000..3c5e72b --- /dev/null +++ b/tests/integration.rs @@ -0,0 +1,492 @@ +/// Integration tests: run the `distree` binary and check stdout/exit codes. +use std::io::Write; +use std::process::{Command, Stdio}; + +/// Path to the binary under test. +/// +/// Cargo sets this to the build actually being tested. Pointing at +/// target/debug by hand meant `cargo test --release` either exercised a stale +/// debug binary or, with no debug build around, failed to spawn at all. +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_distree") +} + +fn run(args: &[&str], stdin: Option<&str>) -> (i32, String, String) { + let mut cmd = Command::new(bin()); + cmd.args(args); + if stdin.is_some() { + cmd.stdin(Stdio::piped()); + } + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = cmd.spawn().expect("failed to spawn distree binary"); + if let Some(input) = stdin { + child + .stdin + .take() + .unwrap() + .write_all(input.as_bytes()) + .unwrap(); + } + let out = child.wait_with_output().unwrap(); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +#[test] +fn test_binary_basic_patristic() { + // Write a temp tree file + let dir = tempfile::tempdir().expect("tempdir"); + let tree = dir.path().join("tree.nwk"); + std::fs::write(&tree, "((A:1.0,B:2.0):0.5,C:3.0);").unwrap(); + + let (code, stdout, _stderr) = run(&[tree.to_str().unwrap()], None); + assert_eq!(code, 0, "exit code should be 0"); + + // Header row: first field is empty, then A B C (sorted) + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!(lines.len(), 4, "header + 3 rows"); + assert!(lines[0].starts_with('\t'), "header starts with tab"); + assert!(lines[1].starts_with("A\t"), "first data row starts with A"); +} + +#[test] +fn test_binary_stdin() { + let newick = "(A:1.0,B:3.0);"; + let (code, stdout, _) = run(&["-"], Some(newick)); + assert_eq!(code, 0); + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!(lines.len(), 3); // header + 2 rows +} + +#[test] +fn test_binary_topology_flag() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1,B:2):3,C:4);").unwrap(); + + let (code, stdout, _) = run(&["--topology", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + // Topology values should be integers (no decimal point) + for line in stdout.lines().skip(1) { + for cell in line.split('\t').skip(1) { + assert!( + !cell.contains('.'), + "Topology output should be integers, got: {}", + cell + ); + } + } +} + +#[test] +fn test_binary_midpoint_does_not_inflate_topology() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1,B:1):1,(C:1,D:1):1);").unwrap(); + + let (code, plain, _) = run(&["--topology", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let (code, rooted, stderr) = run( + &["--topology", "--midpoint", tree.to_str().unwrap()], + None, + ); + assert_eq!(code, 0); + assert_eq!( + plain, rooted, + "rooting must not change the number of edges between two leaves" + ); + assert!(stderr.contains("--midpoint"), "stderr: {}", stderr); + // A-C crosses the root: 2 edges up + 2 edges down + assert!(plain.contains("\t4\t"), "unexpected matrix:\n{}", plain); +} + +#[test] +fn test_binary_midpoint_preserves_patristic() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(((A:0.5,B:0.3):0.4,C:0.9):0.1,D:1.2);").unwrap(); + + let (code, plain, _) = run(&[tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let (code, rooted, _) = run(&["--midpoint", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + assert_eq!( + plain, rooted, + "patristic distances do not depend on the root" + ); +} + +#[test] +fn test_binary_lower_triangle() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1,B:2):3,C:4);").unwrap(); + + let (code, stdout, _) = run(&["--lower", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let lines: Vec<&str> = stdout.lines().collect(); + // PHYLIP lower triangle: header line (taxa count) + 3 leaf rows + assert_eq!(lines.len(), 4, "should have 1 header + 3 rows, got: {:?}", lines); + // First line: taxa count + assert_eq!(lines[0].trim(), "3", "first line should be taxa count"); + // Row 0 (A): label only, 0 distance columns + assert_eq!(lines[1].trim(), "A", "first data row is just label A"); + // Row 1 (B): label + 1 distance + let cols_b: Vec<&str> = lines[2].split('\t').collect(); + assert_eq!(cols_b.len(), 2, "row B: label + 1 distance"); + assert_eq!(cols_b[0], "B"); + // Row 2 (C): label + 2 distances + let cols_c: Vec<&str> = lines[3].split('\t').collect(); + assert_eq!(cols_c.len(), 3, "row C: label + 2 distances"); + assert_eq!(cols_c[0], "C"); +} + +#[test] +fn test_binary_duplicate_leaf_error() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(A:1.0,A:2.0);").unwrap(); + + let (code, _stdout, stderr) = run(&[tree.to_str().unwrap()], None); + assert_ne!(code, 0, "should exit with error on duplicate leaves"); + assert!(stderr.contains("Duplicate"), "stderr should mention 'Duplicate': {}", stderr); +} + +#[test] +fn test_binary_empty_input_error() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("empty.nwk"); + std::fs::write(&tree, "").unwrap(); + + let (code, _stdout, stderr) = run(&[tree.to_str().unwrap()], None); + assert_ne!(code, 0); + assert!(stderr.contains("Empty") || stderr.contains("empty"), "stderr: {}", stderr); +} + +#[test] +fn test_binary_rejects_whitespace_label_in_lower_mode() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(('Taxon A':1.0,'Taxon B':2.0):0.5,C:3.0);").unwrap(); + + // PHYLIP readers split the name off at the first whitespace + let (code, _stdout, stderr) = run(&["--lower", tree.to_str().unwrap()], None); + assert_ne!(code, 0, "a spaced label should be rejected in --lower mode"); + assert!(stderr.contains("whitespace"), "stderr: {}", stderr); + + // The square TSV is tab-delimited and has no such problem + let (code, stdout, _) = run(&[tree.to_str().unwrap()], None); + assert_eq!(code, 0, "the same tree must still work without --lower"); + assert!(stdout.contains("Taxon A\t"), "stdout: {}", stdout); +} + +#[test] +fn test_binary_warns_lower_drops_lmm_diagonal() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1.95,B:3.25):0.35,(C:0.80,D:1.20):0.50);").unwrap(); + + let (code, _stdout, stderr) = run(&["--lmm", "--lower", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + assert!(stderr.contains("diagonal"), "stderr: {}", stderr); + + // Patristic loses nothing, so it must stay quiet + let (code, _stdout, stderr) = run(&["--lower", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + assert!(!stderr.contains("diagonal"), "stderr: {}", stderr); +} + +fn gzip(bytes: &[u8]) -> Vec { + use flate2::write::GzEncoder; + use flate2::Compression; + let mut enc = GzEncoder::new(Vec::new(), Compression::default()); + enc.write_all(bytes).unwrap(); + enc.finish().unwrap() +} + +#[test] +fn test_binary_reads_gzip_input() { + let dir = tempfile::tempdir().unwrap(); + let plain = dir.path().join("t.nwk"); + let packed = dir.path().join("t.nwk.gz"); + let newick = "((A:1.0,B:2.0):0.5,C:3.0);"; + std::fs::write(&plain, newick).unwrap(); + std::fs::write(&packed, gzip(newick.as_bytes())).unwrap(); + + let (code, from_plain, _) = run(&[plain.to_str().unwrap()], None); + assert_eq!(code, 0); + let (code, from_gz, _) = run(&[packed.to_str().unwrap()], None); + assert_eq!(code, 0); + assert_eq!(from_plain, from_gz, "gzip input must give the same matrix"); +} + +#[test] +fn test_binary_reads_gzip_from_stdin() { + // Detection is by magic bytes, so it works with no filename to go on + let packed = gzip(b"(A:1.0,B:3.0);"); + let mut cmd = Command::new(bin()); + cmd.arg("-").stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = cmd.spawn().unwrap(); + child.stdin.take().unwrap().write_all(&packed).unwrap(); + let out = child.wait_with_output().unwrap(); + + assert_eq!(out.status.code(), Some(0)); + let stdout = String::from_utf8_lossy(&out.stdout); + assert_eq!(stdout.lines().count(), 3, "header + 2 rows: {}", stdout); +} + +#[test] +fn test_binary_reports_corrupt_gzip() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk.gz"); + std::fs::write(&tree, [0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00]).unwrap(); + + let (code, _stdout, stderr) = run(&[tree.to_str().unwrap()], None); + assert_ne!(code, 0); + assert!(stderr.contains("gzip"), "stderr should name gzip: {}", stderr); +} + +#[test] +fn test_binary_taxa_subset() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1.0,B:2.0):0.5,(C:3.0,D:4.0):0.5);").unwrap(); + let taxa = dir.path().join("keep.txt"); + // Out of order, with a blank line, a comment and a duplicate + std::fs::write(&taxa, "D\n\n# keep these\nA\nA\n").unwrap(); + + let (code, subset, _) = run(&["--taxa", taxa.to_str().unwrap(), tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let lines: Vec<&str> = subset.lines().collect(); + assert_eq!(lines.len(), 3, "header + 2 rows: {:?}", lines); + assert_eq!(lines[0], "\tA\tD"); + + // The distance must be the one from the full tree: a subset filters the + // output, it does not prune the tree + let (_, full, _) = run(&[tree.to_str().unwrap()], None); + let full_ad = full + .lines() + .find(|l| l.starts_with("A\t")) + .unwrap() + .split('\t') + .nth(4) + .unwrap() + .to_string(); + let subset_ad = lines[1].split('\t').nth(2).unwrap(); + assert_eq!(subset_ad, full_ad, "A-D differs between subset and full matrix"); +} + +#[test] +fn test_binary_taxa_rejects_unknown_labels() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(A:1.0,B:2.0);").unwrap(); + let taxa = dir.path().join("keep.txt"); + std::fs::write(&taxa, "A\nNOT_IN_TREE\n").unwrap(); + + let (code, _stdout, stderr) = run(&["--taxa", taxa.to_str().unwrap(), tree.to_str().unwrap()], None); + assert_ne!(code, 0, "an unknown label should be an error, not a smaller matrix"); + assert!(stderr.contains("NOT_IN_TREE"), "stderr should name it: {}", stderr); +} + +/// Minimal .npy reader: check the header and return the f64 payload. +fn read_npy(path: &std::path::Path) -> (String, Vec) { + let bytes = std::fs::read(path).expect("npy file"); + assert_eq!(&bytes[..6], b"\x93NUMPY", "npy magic"); + assert_eq!(&bytes[6..8], &[1, 0], "npy version 1.0"); + let header_len = u16::from_le_bytes([bytes[8], bytes[9]]) as usize; + let header = String::from_utf8(bytes[10..10 + header_len].to_vec()).unwrap(); + assert!(header.contains("'descr': ' u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, n: usize) -> usize { + if n == 0 { + return 0; + } + (self.next_u64() % n as u64) as usize + } +} + +/// The bytes that mean something in Newick, plus a few that do not, so the +/// mutations land on structure rather than on label text. +const ALPHABET: &[u8] = b"(),:;[]'\"0123456789.eE+- \t\nABxy"; + +fn seed_trees() -> Vec { + vec![ + "(A:1.0,B:2.0);".to_string(), + "((A:1.0,B:2.0):0.5,C:3.0);".to_string(), + "(A:1,B:2,C:3);".to_string(), + "[&R] ((A:0.1[&&NHX:S=human],B:0.2):0.3,C:0.4);".to_string(), + "('Taxon A':1.0,\"Taxon B\":2.0)'my clade':0.5;".to_string(), + "(A:1.5e-3,B:2.0E+1,(C:0.0,D:-0.5):1e10);".to_string(), + "((((A:1):1):1):1);".to_string(), + "(it''s:1,B:2);".to_string(), + "(A,B,(C,D));".to_string(), + ";".to_string(), + ] +} + +/// Apply one random mutation to `bytes`. +fn mutate(rng: &mut Rng, bytes: &mut Vec) { + if bytes.is_empty() { + bytes.push(ALPHABET[rng.below(ALPHABET.len())]); + return; + } + match rng.below(6) { + // Truncate: an interrupted write or a partial download + 0 => { + let at = rng.below(bytes.len()); + bytes.truncate(at); + } + // Flip one byte to another meaningful one + 1 => { + let at = rng.below(bytes.len()); + bytes[at] = ALPHABET[rng.below(ALPHABET.len())]; + } + // Insert + 2 => { + let at = rng.below(bytes.len() + 1); + bytes.insert(at, ALPHABET[rng.below(ALPHABET.len())]); + } + // Delete + 3 => { + let at = rng.below(bytes.len()); + bytes.remove(at); + } + // Duplicate a run + 4 => { + let start = rng.below(bytes.len()); + let end = (start + 1 + rng.below(bytes.len() - start)).min(bytes.len()); + let chunk = bytes[start..end].to_vec(); + let at = rng.below(bytes.len() + 1); + for (k, b) in chunk.into_iter().enumerate() { + if bytes.len() < 4096 { + bytes.insert(at + k, b); + } + } + } + // Swap two bytes + _ => { + let a = rng.below(bytes.len()); + let b = rng.below(bytes.len()); + bytes.swap(a, b); + } + } +} + +/// Everything the binary does after a successful parse. +fn exercise_pipeline(text: &str) { + let Ok(raw) = parse_newick(text) else { + return; + }; + + for do_midpoint in [false, true] { + let mut nodes: Vec = Vec::new(); + let mut root = flatten_raw(&raw, None, &mut nodes); + + if do_midpoint { + // Midpoint rooting assumes non-negative, finite lengths; the binary + // warns about the rest rather than refusing. + if nodes.iter().any(|n| n.length < 0.0 || !n.length.is_finite()) { + continue; + } + root = midpoint_root(root, &mut nodes); + } + + let lca = build_lca_structure(root, &nodes); + + // The binary refuses a tree whose depths are too large to subtract, + // because d_i + d_j overflows and inf - inf is NaN. Hold the pipeline + // to the same precondition rather than asserting past it. + let deepest = lca.depth_len.iter().copied().fold(0.0_f64, f64::max); + if !(deepest * 2.0).is_finite() { + continue; + } + + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + for &i in leaves.iter().take(16) { + for &j in leaves.iter().take(16) { + for mode in [DistMode::Patristic, DistMode::Topology, DistMode::Lmm] { + let d = compute_distance(i, j, mode, &lca); + if i == j && mode != DistMode::Lmm { + assert_eq!(d, 0.0, "a leaf must be exactly 0.0 from itself"); + } + } + } + } + } +} + +#[test] +fn test_mutated_trees_never_panic() { + let seeds = seed_trees(); + + for (s, seed_tree) in seeds.iter().enumerate() { + let mut rng = Rng(0x243F_6A88_85A3_08D3 ^ (s as u64) << 32); + + for round in 0..4_000 { + let mut bytes = seed_tree.as_bytes().to_vec(); + // One to four mutations: one finds the shallow cases, four gets far + // enough from a valid tree to be interesting. + for _ in 0..=rng.below(4) { + mutate(&mut rng, &mut bytes); + } + // Every byte in ALPHABET is ASCII, so the result is still UTF-8 + let text = match std::str::from_utf8(&bytes) { + Ok(t) => t, + Err(_) => continue, + }; + + // A panic here fails the test with the seed and round that caused it + let result = std::panic::catch_unwind(|| { + let _ = parse_newick(text); + exercise_pipeline(text); + }); + assert!( + result.is_ok(), + "panicked on seed {} round {} with input {:?}", + s, + round, + text + ); + } + } +} + +#[test] +fn test_random_bytes_never_panic() { + let mut rng = Rng(0x13198A2E_03707344); + + for round in 0..4_000 { + let len = rng.below(64); + let bytes: Vec = (0..len).map(|_| ALPHABET[rng.below(ALPHABET.len())]).collect(); + let text = std::str::from_utf8(&bytes).expect("ALPHABET is ASCII"); + + let result = std::panic::catch_unwind(|| { + let _ = parse_newick(text); + exercise_pipeline(text); + }); + assert!(result.is_ok(), "panicked on round {} with input {:?}", round, text); + } +} + +#[test] +fn test_pathological_shapes_never_panic() { + // Shapes that are valid enough to reach the pipeline but are not what any + // of it was written with in mind. + let cases: Vec = vec![ + // A ladder, which makes the LCA depth equal to the node count + format!("{}A:1{};", "(".repeat(2_000), ")".repeat(2_000)), + // One node with thousands of children + format!("({});", (0..2_000).map(|i| format!("L{}:0.1", i)).collect::>().join(",")), + // Every branch zero, so the diameter is zero and midpoint has nothing to find + "((A:0,B:0):0,(C:0,D:0):0);".to_string(), + // Enormous and tiny lengths in the same tree + "((A:1e308,B:1e-308):1e300,C:0.0);".to_string(), + // A single leaf, and a single leaf with no length + "(A:1.0);".to_string(), + "A;".to_string(), + // Internal nodes carrying the labels instead of the tips + "((:1,:2)inner:3,:4)root;".to_string(), + // Deeply nested comments + format!("(A:1[{}],B:2);", "[".repeat(500) + &"]".repeat(500)), + ]; + + for (i, case) in cases.iter().enumerate() { + let result = std::panic::catch_unwind(|| exercise_pipeline(case)); + assert!(result.is_ok(), "panicked on pathological case {}", i); + } +}