From 1ec14298b8052b9fee39cfa315859bebc3d9166c Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:27:10 +0200 Subject: [PATCH 01/47] Add .gitignore for build artifacts --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target/ From 57c391e06d6dae2046e53697d8c2ef7491f9b3fc Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:27:18 +0200 Subject: [PATCH 02/47] Refactor into modules and fix critical bugs Split monolithic main.rs into parser.rs, tree.rs, lca.rs, midpoint.rs. Critical fixes: - Rewrite midpoint_root to fix infinite loop from cyclic graph - Add NHX/bracket comment parsing to prevent crash on annotated trees - Add single-quoted label support in parser - Detect and reject duplicate leaf names Additional fixes: - Add --precision flag for floating point output formatting - Add -t/--threads flag for rayon thread control - Add stdin support (use '-' as phylogeny argument) - Warn on missing branch lengths in patristic mode - Warn on negative branch lengths - Always print leading tab in header for R/Python compatibility - Use env!("CARGO_PKG_VERSION") for version string - Remove unused --format flag - Fix clippy warnings (&Vec -> &[Node]) - Add 18 unit tests --- src/lca.rs | 79 +++++++ src/main.rs | 553 ++++++++++++++---------------------------------- src/midpoint.rs | 286 +++++++++++++++++++++++++ src/parser.rs | 265 +++++++++++++++++++++++ src/tree.rs | 7 + 5 files changed, 801 insertions(+), 389 deletions(-) create mode 100644 src/lca.rs create mode 100644 src/midpoint.rs create mode 100644 src/parser.rs create mode 100644 src/tree.rs diff --git a/src/lca.rs b/src/lca.rs new file mode 100644 index 0000000..3899e41 --- /dev/null +++ b/src/lca.rs @@ -0,0 +1,79 @@ +use crate::tree::Node; + +/// Precomputed data for LCA (binary lifting). +pub struct LcaData { + pub up: Vec>>, + pub depth_len: Vec, + pub depth_top: Vec, +} + +/// Build the `LcaData` for binary lifting from `root_idx`. +pub fn build_lca_structure(root_idx: usize, nodes: &[Node]) -> LcaData { + let n = nodes.len(); + let max_log = if n <= 1 { 1 } else { ((n as f64).log2().ceil() as usize) + 1 }; + let mut depth_len = vec![0.0; n]; + let mut depth_top = vec![0; n]; + let mut up: Vec>> = vec![vec![None; n]; max_log]; + + { + let mut stack = vec![root_idx]; + depth_len[root_idx] = 0.0; + depth_top[root_idx] = 0; + up[0][root_idx] = None; + + while let Some(u) = stack.pop() { + for &v in &nodes[u].children { + up[0][v] = Some(u); + depth_len[v] = depth_len[u] + nodes[v].length; + depth_top[v] = depth_top[u] + 1; + stack.push(v); + } + } + } + + for k in 1..max_log { + for u in 0..n { + up[k][u] = up[k - 1][u].and_then(|mid| up[k - 1][mid]); + } + } + + LcaData { + up, + depth_len, + depth_top, + } +} + +impl LcaData { + /// Return the index of the MRCA of nodes `u` and `v` in O(log n). + pub fn mrca(&self, mut u: usize, mut v: usize) -> usize { + if u == v { + return u; + } + if self.depth_top[u] < self.depth_top[v] { + std::mem::swap(&mut u, &mut v); + } + let diff = self.depth_top[u] - self.depth_top[v]; + let mut x = diff; + let mut k = 0; + while x > 0 { + if (x & 1) == 1 { + u = self.up[k][u].unwrap(); + } + x >>= 1; + k += 1; + } + if u == v { + return u; + } + for k in (0..self.up.len()).rev() { + if let (Some(au), Some(av)) = (self.up[k][u], self.up[k][v]) { + if au != av { + u = au; + v = av; + } + } + } + self.up[0][u].unwrap() + } +} diff --git a/src/main.rs b/src/main.rs index eff1520..e7fe7e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,60 +1,30 @@ +mod lca; +mod midpoint; +mod parser; +mod tree; + use clap::{Arg, ArgAction, Command}; use rayon::prelude::*; +use std::collections::HashSet; use std::fs::File; use std::io::{self, BufWriter, Read, Write}; -use std::str::Chars; -use std::iter::Peekable; -use std::num::ParseFloatError; - -/// Our internal representation of a tree node: -/// - `name`: if this is a leaf, its label; otherwise `None`. -/// - `length`: branch length from its parent (0.0 if unspecified). -/// - `parent`: index of the parent node in `nodes`, or `None` if this is the root. -/// - `children`: list of indices of child nodes. -struct Node { - name: Option, - length: f64, - parent: Option, - children: Vec, -} - -/// Precomputed data for LCA (binary lifting): -/// - `up[k][u]`: the 2^k-th ancestor of node `u`, or `None` if above the root. -/// - `depth_len[u]`: distance (sum of branch lengths) from node `u` up to the root. -/// - `depth_top[u]`: number of edges from node `u` up to the root. -struct LcaData { - up: Vec>>, - depth_len: Vec, - depth_top: Vec, -} -/// A temporary structure used during Newick parsing, -/// before flattening into `Vec`. -struct RawNode { - name: Option, - length: f64, - children: Vec, -} +use lca::build_lca_structure; +use midpoint::midpoint_root; +use parser::{flatten_raw, parse_subtree}; +use tree::Node; fn main() -> io::Result<()> { - // ========== 1. Parse command-line arguments ========== let matches = Command::new("distree") - .version("1.0.0") + .version(env!("CARGO_PKG_VERSION")) .author("Paula Ruiz-Rodriguez") .about("Extracts a distance matrix from a phylogeny (parallel, low-memory)") .arg( Arg::new("phylogeny") - .help("Path to the tree file in Newick format") + .help("Path to the tree file in Newick format (use '-' for stdin)") .required(true) .index(1), ) - .arg( - Arg::new("format") - .long("format") - .help("Tree file format (only 'newick' is supported)") - .default_value("newick") - .value_parser(["newick"]), - ) .arg( Arg::new("midpoint") .long("midpoint") @@ -80,56 +50,84 @@ fn main() -> io::Result<()> { .help("Path to write the TSV output file (defaults to stdout)") .value_name("FILE"), ) + .arg( + Arg::new("precision") + .long("precision") + .short('p') + .help("Number of decimal places for output values") + .default_value("10") + .value_parser(clap::value_parser!(usize)), + ) + .arg( + Arg::new("threads") + .long("threads") + .short('t') + .help("Number of threads for parallel computation (default: all cores)") + .value_parser(clap::value_parser!(usize)), + ) .get_matches(); let tree_path = matches .get_one::("phylogeny") .expect("Tree file path is required") .to_string(); - let _format = matches.get_one::("format").unwrap().as_str(); // only "newick" let do_midpoint = *matches.get_one::("midpoint").unwrap(); let do_lmm = *matches.get_one::("lmm").unwrap(); let do_topology = *matches.get_one::("topology").unwrap(); let output_path = matches.get_one::("output"); + let precision = *matches.get_one::("precision").unwrap(); + + // Configure thread pool + if let Some(&num_threads) = matches.get_one::("threads") { + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global() + .expect("Failed to initialize thread pool"); + } - // Prepare writer: either a file or stdout let mut writer: Box = if let Some(path) = output_path { Box::new(BufWriter::new(File::create(path)?)) } else { Box::new(io::stdout()) }; - // ========== 2. Read the entire Newick file into a String ========== + // Read input from file or stdin let mut newick_str = String::new(); - { + if tree_path == "-" { + io::stdin().read_to_string(&mut newick_str)?; + } else { let mut f = File::open(&tree_path)?; f.read_to_string(&mut newick_str)?; } - // ========== 3. Parse the Newick string with our minimal parser ========== + // Parse the Newick string let mut chars = newick_str.trim().chars().peekable(); - let raw_root = parse_subtree(&mut chars) - .expect("Failed to parse Newick tree"); - // After the tree, there may be a trailing semicolon + let raw_root = parse_subtree(&mut chars).expect("Failed to parse Newick tree"); if let Some(&c) = chars.peek() { if c == ';' { chars.next(); } } - // ========== 4. Flatten RawNode into Vec ========== + // Flatten into Vec let mut nodes: Vec = Vec::new(); - let root_idx = flatten_raw(&raw_root, None, &mut nodes); + let mut root_idx = flatten_raw(&raw_root, None, &mut nodes); - // ========== 5. Midpoint-root optional ========== + // Warn about negative branch lengths + let has_negative = nodes.iter().any(|n| n.length < 0.0); + if has_negative { + eprintln!("Warning: negative branch lengths detected in the tree."); + } + + // Midpoint-root if requested if do_midpoint { - midpoint_root(root_idx, &mut nodes); + root_idx = midpoint_root(root_idx, &mut nodes); } - // ========== 6. Build the LCA data structure ========== + // Build LCA let lca_data = build_lca_structure(root_idx, &nodes); - // ========== 7. Collect all leaves (taxa) ========== + // Collect leaves let mut leaf_indices: Vec = Vec::new(); for (i, nd) in nodes.iter().enumerate() { if nd.children.is_empty() && nd.name.is_some() { @@ -141,7 +139,19 @@ fn main() -> io::Result<()> { std::process::exit(1); } - // Build Vec<(label, index)> and sort by label + // Check for duplicate leaf names + { + let mut seen = HashSet::new(); + for &i in &leaf_indices { + let name = nodes[i].name.as_ref().unwrap(); + if !seen.insert(name.clone()) { + eprintln!("Error: duplicate leaf name '{}' found. Leaf names must be unique.", name); + std::process::exit(1); + } + } + } + + // Sort by label let mut leaf_label_pairs: Vec<(String, usize)> = leaf_indices .iter() .map(|&i| (nodes[i].name.clone().unwrap(), i)) @@ -153,7 +163,17 @@ fn main() -> io::Result<()> { leaf_label_pairs.iter().map(|(_, idx)| *idx).collect(); let n_leaves = sorted_leaf_indices.len(); - // ========== 8. Print the TSV header row ========== + // Warn if no branch lengths and not topology mode + if !do_topology { + let all_zero = nodes.iter().all(|n| n.length == 0.0); + if all_zero { + eprintln!( + "Warning: no branch lengths detected, all patristic distances will be zero. Consider --topology." + ); + } + } + + // Print TSV header (leading tab for R/Python compatibility) writer.write_all(b"\t")?; for (i, lab) in sorted_labels.iter().enumerate() { writer.write_all(lab.as_bytes())?; @@ -163,24 +183,22 @@ fn main() -> io::Result<()> { } writer.write_all(b"\n")?; - // ========== 9. For each leaf (row), compute distances to all leaves in parallel ========== + // Compute and print distance matrix + let prec = precision; for (row_i, &leaf_i) in sorted_leaf_indices.iter().enumerate() { let this_row: Vec = sorted_leaf_indices .par_iter() .map(|&leaf_j| { if do_lmm { - // LMM: depth of MRCA in branch-length units let m = lca_data.mrca(leaf_i, leaf_j); lca_data.depth_len[m] } else if do_topology { - // Topological distance = edge count let m = lca_data.mrca(leaf_i, leaf_j); let d_i = lca_data.depth_top[leaf_i]; let d_j = lca_data.depth_top[leaf_j]; let d_m = lca_data.depth_top[m]; ((d_i + d_j).saturating_sub(2 * d_m)) as f64 } else { - // Patristic distance = sum of branch lengths let m = lca_data.mrca(leaf_i, leaf_j); let d_i = lca_data.depth_len[leaf_i]; let d_j = lca_data.depth_len[leaf_j]; @@ -193,7 +211,7 @@ fn main() -> io::Result<()> { writer.write_all(sorted_labels[row_i].as_bytes())?; for dist in this_row.iter() { writer.write_all(b"\t")?; - writer.write_all(format!("{}", dist).as_bytes())?; + write!(writer, "{:.prec$}", dist, prec = prec)?; } writer.write_all(b"\n")?; } @@ -201,354 +219,111 @@ fn main() -> io::Result<()> { Ok(()) } -/// Recursively parse a Newick subtree and return a `RawNode`. -fn parse_subtree(chars: &mut Peekable) -> Result { - let mut node = RawNode { - name: None, - length: 0.0, - children: Vec::new(), - }; - - // If the next character is '(', this is an internal node with children - if let Some(&c) = chars.peek() { - if c == '(' { - // Consume '(' - chars.next(); - - // Parse each child until we see a ')' - loop { - let child = parse_subtree(chars)?; - node.children.push(child); - match chars.peek() { - Some(',') => { - chars.next(); - continue; - } - Some(')') => { - chars.next(); - break; - } - Some(other) => { - return Err(format!("Expected ',' or ')', found '{}'", other)); - } - None => return Err("Unexpected end of input after '('".to_string()), - } - } - - // After ')', we may have an internal node label - if let Some(&c2) = chars.peek() { - if c2 != ':' && c2 != ',' && c2 != ')' && c2 != ';' { - let name = parse_label(chars); - if !name.is_empty() { - node.name = Some(name); - } - } - } - - // Then optionally a colon followed by a branch length - if let Some(&':') = chars.peek() { - chars.next(); - let length = parse_length(chars)?; - node.length = length; - } +#[cfg(test)] +mod tests { + use super::*; - return Ok(node); - } + fn build_tree(newick: &str) -> (Vec, usize) { + let mut chars = newick.trim().chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + (nodes, root) } - // Otherwise, it's a leaf: parse the label - let name = parse_label(chars); - if !name.is_empty() { - node.name = Some(name); + fn get_leaf(nodes: &[Node], name: &str) -> usize { + nodes + .iter() + .position(|n| n.name.as_deref() == Some(name)) + .unwrap() } - // Optionally, a colon followed by a branch length - if let Some(&':') = chars.peek() { - chars.next(); - let length = parse_length(chars)?; - node.length = length; + fn patristic(nodes: &[Node], root: usize, a: &str, b: &str) -> f64 { + let lca = build_lca_structure(root, nodes); + let ai = get_leaf(nodes, a); + let bi = get_leaf(nodes, b); + let m = lca.mrca(ai, bi); + lca.depth_len[ai] + lca.depth_len[bi] - 2.0 * lca.depth_len[m] } - Ok(node) -} - -/// Parse a node label (until we hit ':', ',', ')', ';', or whitespace). -fn parse_label(chars: &mut Peekable) -> String { - let mut label = String::new(); - while let Some(&c) = chars.peek() { - if c == ':' || c == ',' || c == ')' || c == ';' || c.is_whitespace() { - break; - } - label.push(c); - chars.next(); + #[test] + fn test_simple_patristic_distances() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + assert!((patristic(&nodes, root, "A", "B") - 3.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.5).abs() < 1e-10); + assert!((patristic(&nodes, root, "B", "C") - 5.5).abs() < 1e-10); } - label -} -/// Parse a floating-point branch length. -fn parse_length(chars: &mut Peekable) -> Result { - let mut numstr = String::new(); - while let Some(&c) = chars.peek() { - if c.is_ascii_digit() || c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-' { - numstr.push(c); - chars.next(); - } else { - break; - } + #[test] + fn test_topology_distances() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let c = get_leaf(&nodes, "C"); + let m = lca.mrca(a, c); + let topo = lca.depth_top[a] + lca.depth_top[c] - 2 * lca.depth_top[m]; + // A is depth 2, C is depth 1, root is depth 0 => 2+1-0 = 3 + assert_eq!(topo, 3); } - numstr - .parse::() - .map_err(|e: ParseFloatError| format!("Failed to parse branch length '{}': {}", numstr, e)) -} -/// Recursively flatten a `RawNode` into a flat `Vec`, returning the index of the newly-added node. -fn flatten_raw(raw: &RawNode, parent: Option, nodes: &mut Vec) -> usize { - let idx = nodes.len(); - nodes.push(Node { - name: raw.name.clone(), - length: raw.length, - parent, - children: Vec::new(), - }); - if let Some(p) = parent { - nodes[p].children.push(idx); - } - for child in &raw.children { - flatten_raw(child, Some(idx), nodes); - } - idx -} - -/// Midpoint-rooting: find two leaves at the ends of the diameter (maximum branch-length distance), -/// reconstruct that path, locate its midpoint, and insert a new root node there. -fn midpoint_root(root_idx: usize, nodes: &mut Vec) { - // Helper: from `start`, do a DFS to find the farthest leaf and its distance. - fn farthest_from(start: usize, nodes: &Vec) -> (usize, f64, Vec>) { - let mut best_leaf = start; - let mut best_dist = 0.0; - let mut parent_trace = vec![None; nodes.len()]; - let mut visited = vec![false; nodes.len()]; - let mut stack = vec![(start, 0.0)]; - visited[start] = true; - - while let Some((u, dist_u)) = stack.pop() { - if nodes[u].children.is_empty() { - // If it's a leaf, check distance - if dist_u > best_dist { - best_dist = dist_u; - best_leaf = u; - } - } - // Visit the parent, if any - if let Some(p) = nodes[u].parent { - if !visited[p] { - visited[p] = true; - parent_trace[p] = Some(u); - stack.push((p, dist_u + nodes[u].length)); - } - } - // Visit children - for &v in &nodes[u].children { - if !visited[v] { - visited[v] = true; - parent_trace[v] = Some(u); - stack.push((v, dist_u + nodes[v].length)); - } - } - } - (best_leaf, best_dist, parent_trace) + #[test] + fn test_lmm_matrix() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let b = get_leaf(&nodes, "B"); + let c = get_leaf(&nodes, "C"); + + // MRCA(A,B) is the inner node at depth 0.5 + let m_ab = lca.mrca(a, b); + assert!((lca.depth_len[m_ab] - 0.5).abs() < 1e-10); + + // MRCA(A,C) is root at depth 0 + let m_ac = lca.mrca(a, c); + assert!((lca.depth_len[m_ac]).abs() < 1e-10); } - // 1) Find any leaf to start (descend to a leaf if root isn't one) - let mut any_leaf = root_idx; - if !nodes[any_leaf].children.is_empty() { - let mut cur = any_leaf; - while !nodes[cur].children.is_empty() { - cur = nodes[cur].children[0]; - } - any_leaf = cur; - } - - // 2) From any_leaf, find leaf_a (one endpoint of the diameter) - let (leaf_a, _, _) = farthest_from(any_leaf, nodes); - // 3) From leaf_a, find leaf_b and the total distance dist_ab - let (leaf_b, dist_ab, parent_trace_b) = farthest_from(leaf_a, nodes); + #[test] + fn test_duplicate_leaves_detected() { + let (nodes, _root) = build_tree("(A:1.0,A:2.0);"); + let leaf_indices: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); - // Reconstruct the path from leaf_b back to leaf_a - let mut path = Vec::new(); - { - let mut cur = leaf_b; - loop { - path.push(cur); - if cur == leaf_a { + let mut seen = HashSet::new(); + let mut has_dup = false; + for &i in &leaf_indices { + let name = nodes[i].name.as_ref().unwrap(); + if !seen.insert(name.clone()) { + has_dup = true; break; } - cur = parent_trace_b[cur].unwrap(); } + assert!(has_dup); } - // The midpoint lies at dist_ab/2 along that path (starting from leaf_b) - let half = dist_ab / 2.0; - let mut accum = 0.0; - let mut midpoint_node = path[0]; - - for i in 0..path.len() - 1 { - let u = path[i]; - let v = path[i + 1]; - // Edge length between u ↔ v - let edge_len = if nodes[v].children.contains(&u) { - nodes[u].length - } else { - nodes[v].length - }; - if accum + edge_len >= half { - // The midpoint falls on this edge (u–v) - let dist_into_edge = half - accum; - // Determine which is parent and which is child in the current tree - let (parent_node, child_node, parent_to_child_len) = - if nodes[v].parent == Some(u) { - (u, v, nodes[v].length) - } else { - (v, u, nodes[u].length) - }; - let new_root_idx = nodes.len(); - let dist_to_old_parent = parent_to_child_len - dist_into_edge; - let dist_to_old_child = dist_into_edge; - - // Create the new root node R - nodes.push(Node { - name: None, - length: 0.0, - parent: None, - children: Vec::new(), - }); - - // 1) Detach child_node from parent_node, attach child_node → R - let old_parent_of_child = nodes[child_node].parent.take(); - assert_eq!(old_parent_of_child, Some(parent_node)); - nodes[child_node].parent = Some(new_root_idx); - - // 2) In parent_node.children, replace child_node with new_root_idx - if let Some(pos) = nodes[parent_node] - .children - .iter() - .position(|&x| x == child_node) - { - nodes[parent_node].children[pos] = new_root_idx; - } - // 3) Now set R.children = [child_node], and R.parent = Some(parent_node) - nodes[new_root_idx].parent = Some(parent_node); - nodes[new_root_idx].children.push(child_node); - - // 4) Adjust branch lengths: - // - child_node.length = dist_to_old_child - // - R.length = dist_to_old_parent - nodes[child_node].length = dist_to_old_child; - nodes[new_root_idx].length = dist_to_old_parent; - - // 5) Insert parent_node above R - let old_parent_of_parent = nodes[parent_node].parent.take(); - nodes[parent_node].parent = Some(new_root_idx); - nodes[new_root_idx].children.push(parent_node); - if let Some(grand) = old_parent_of_parent { - // If parent_node wasn't originally the root, replace in its old parent's children - if let Some(pos2) = nodes[grand].children.iter().position(|&x| x == parent_node) { - nodes[grand].children[pos2] = new_root_idx; - } - } - // Finally, R must become the new root - nodes[new_root_idx].parent = None; - - midpoint_node = new_root_idx; - break; - } - accum += edge_len; - midpoint_node = v; + #[test] + fn test_negative_branch_length_detected() { + let (nodes, _root) = build_tree("(A:-0.5,B:2.0);"); + let has_negative = nodes.iter().any(|n| n.length < 0.0); + assert!(has_negative); } - // If for some reason we never split (degenerate case), force midpoint_node to be root - if let Some(p) = nodes[midpoint_node].parent { - if let Some(pos) = nodes[p].children.iter().position(|&x| x == midpoint_node) { - nodes[p].children.remove(pos); - } - nodes[midpoint_node].parent = None; + #[test] + fn test_trifurcation() { + let (nodes, root) = build_tree("(A:1.0,B:2.0,C:3.0);"); + assert_eq!(nodes[root].children.len(), 3); + assert!((patristic(&nodes, root, "A", "B") - 3.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.0).abs() < 1e-10); } -} -/// Build the `LcaData` for binary lifting: -/// 1) Do a single DFS from `root_idx` to fill `depth_len`, `depth_top`, and `up[0][u]`. -/// 2) Fill `up[k][u] = up[k-1][ up[k-1][u] ]` for k = 1..⌈log₂(n)⌉. -fn build_lca_structure(root_idx: usize, nodes: &Vec) -> LcaData { - let n = nodes.len(); - let mut depth_len = vec![0.0; n]; - let mut depth_top = vec![0; n]; - let max_log = ((n as f64).log2().ceil() as usize) + 1; - let mut up: Vec>> = vec![vec![None; n]; max_log]; - - // 1) DFS to set depth and immediate parent - { - let mut stack = vec![root_idx]; - depth_len[root_idx] = 0.0; - depth_top[root_idx] = 0; - up[0][root_idx] = None; - - while let Some(u) = stack.pop() { - for &v in &nodes[u].children { - up[0][v] = Some(u); - depth_len[v] = depth_len[u] + nodes[v].length; - depth_top[v] = depth_top[u] + 1; - stack.push(v); - } - } - } - - // 2) Build all 2^k ancestors - for k in 1..max_log { - for u in 0..n { - up[k][u] = up[k - 1][u].and_then(|mid| up[k - 1][mid]); - } - } - - LcaData { - up, - depth_len, - depth_top, - } -} - -impl LcaData { - /// Return the index of the lowest common ancestor (LCA) of nodes `u` and `v` in O(log n). - fn mrca(&self, mut u: usize, mut v: usize) -> usize { - if u == v { - return u; - } - // 1) Lift the deeper node up until both are at the same depth_top - if self.depth_top[u] < self.depth_top[v] { - std::mem::swap(&mut u, &mut v); - } - let diff = self.depth_top[u] - self.depth_top[v]; - let mut x = diff; - let mut k = 0; - while x > 0 { - if (x & 1) == 1 { - u = self.up[k][u].unwrap(); - } - x >>= 1; - k += 1; - } - if u == v { - return u; - } - // 2) Lift both in powers of two until their parents differ - for k in (0..self.up.len()).rev() { - if let (Some(au), Some(av)) = (self.up[k][u], self.up[k][v]) { - if au != av { - u = au; - v = av; - } - } - } - // Now u and v have the same parent, which is the LCA - self.up[0][u].unwrap() + #[test] + fn test_no_branch_lengths_all_zero() { + let (nodes, _root) = build_tree("(A,B,(C,D));"); + let all_zero = nodes.iter().all(|n| n.length == 0.0); + assert!(all_zero); } } diff --git a/src/midpoint.rs b/src/midpoint.rs new file mode 100644 index 0000000..4cad5dd --- /dev/null +++ b/src/midpoint.rs @@ -0,0 +1,286 @@ +use crate::tree::Node; + +/// Midpoint-root the tree: find the diameter, split the midpoint edge, re-root there. +/// Returns the index of the new root. +pub fn midpoint_root(root_idx: usize, nodes: &mut Vec) -> usize { + // BFS/DFS on the unrooted tree to find the farthest leaf from `start`. + fn farthest_from(start: usize, nodes: &[Node]) -> (usize, f64, Vec>) { + let mut best_leaf = start; + let mut best_dist = 0.0; + let mut parent_trace: Vec> = vec![None; nodes.len()]; + let mut visited = vec![false; nodes.len()]; + let mut stack = vec![(start, 0.0)]; + visited[start] = true; + + while let Some((u, dist_u)) = stack.pop() { + if nodes[u].children.is_empty() && u != start && dist_u > best_dist { + best_dist = dist_u; + best_leaf = u; + } + if let Some(p) = nodes[u].parent { + if !visited[p] { + visited[p] = true; + parent_trace[p] = Some(u); + stack.push((p, dist_u + nodes[u].length)); + } + } + for &v in &nodes[u].children { + if !visited[v] { + visited[v] = true; + parent_trace[v] = Some(u); + stack.push((v, dist_u + nodes[v].length)); + } + } + } + (best_leaf, best_dist, parent_trace) + } + + // 1) Find diameter endpoints + let mut any_leaf = root_idx; + while !nodes[any_leaf].children.is_empty() { + any_leaf = nodes[any_leaf].children[0]; + } + + let (leaf_a, _, _) = farthest_from(any_leaf, nodes); + let (leaf_b, dist_ab, parent_trace) = farthest_from(leaf_a, nodes); + + if dist_ab == 0.0 { + return root_idx; + } + + // 2) Reconstruct path from leaf_b back to leaf_a + let mut path = Vec::new(); + { + let mut cur = leaf_b; + loop { + path.push(cur); + if cur == leaf_a { + break; + } + cur = parent_trace[cur].unwrap(); + } + } + + // 3) Walk the path and find the edge where the midpoint falls + let half = dist_ab / 2.0; + let mut accum = 0.0; + + for i in 0..path.len() - 1 { + let u = path[i]; // closer to leaf_b + let v = path[i + 1]; // closer to leaf_a + + let edge_len = if nodes[u].parent == Some(v) { + nodes[u].length + } else { + nodes[v].length + }; + + if accum + edge_len >= half { + let dist_to_u = half - accum; + let dist_to_v = edge_len - dist_to_u; + + // Determine parent-child in the current tree + let (parent_side, child_side, len_child, len_parent) = + if nodes[u].parent == Some(v) { + // v is parent of u + (v, u, dist_to_u, dist_to_v) + } else { + // u is parent of v + (u, v, dist_to_v, dist_to_u) + }; + + // Insert new root node on this edge + let new_root = nodes.len(); + nodes.push(Node { + name: None, + length: 0.0, + parent: None, + children: Vec::new(), + }); + + // Detach child_side from parent_side + nodes[parent_side].children.retain(|&x| x != child_side); + + // Attach child_side to new_root + nodes[child_side].parent = Some(new_root); + nodes[child_side].length = len_child; + nodes[new_root].children.push(child_side); + + // Collect path from parent_side to old root + let mut path_to_old_root = vec![]; + { + let mut c = parent_side; + loop { + path_to_old_root.push(c); + if let Some(p) = nodes[c].parent { + c = p; + } else { + break; + } + } + } + + // Attach parent_side to new_root + let old_len = nodes[parent_side].length; + nodes[parent_side].parent = Some(new_root); + nodes[parent_side].length = len_parent; + nodes[new_root].children.push(parent_side); + + // Reverse parent-child relationships from parent_side up to old root + let mut prev_old_length = old_len; + for j in 1..path_to_old_root.len() { + let child_now = path_to_old_root[j]; + let parent_now = path_to_old_root[j - 1]; + + // Remove parent_now from child_now's children + nodes[child_now].children.retain(|&x| x != parent_now); + // Add child_now as child of parent_now + nodes[parent_now].children.push(child_now); + // Set child_now's parent + nodes[child_now].parent = Some(parent_now); + + // Swap branch lengths + std::mem::swap(&mut nodes[child_now].length, &mut prev_old_length); + } + + return new_root; + } + accum += edge_len; + } + + root_idx +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::{flatten_raw, parse_subtree}; + use crate::lca::build_lca_structure; + + fn patristic_distance( + leaf_i: usize, + leaf_j: usize, + lca: &crate::lca::LcaData, + ) -> f64 { + let m = lca.mrca(leaf_i, leaf_j); + lca.depth_len[leaf_i] + lca.depth_len[leaf_j] - 2.0 * lca.depth_len[m] + } + + #[test] + fn test_midpoint_simple() { + // ((A:1,B:1):1,(C:1,D:1):1); + // Diameter = 4 (e.g. A→root→D), midpoint at distance 2 + let input = "((A:1,B:1):1,(C:1,D:1):1);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + let new_root = midpoint_root(root, &mut nodes); + + // Verify the tree is valid: new root has no parent + assert!(nodes[new_root].parent.is_none()); + + // Build LCA and check distances are preserved + let lca = build_lca_structure(new_root, &nodes); + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + // A-B should be 2.0, A-C should be 4.0, etc. + for &i in &leaves { + for &j in &leaves { + let d = patristic_distance(i, j, &lca); + assert!(d >= 0.0, "Negative distance between leaves"); + } + } + + // Check specific: A to B = 2.0 + let a = leaves.iter().find(|&&i| nodes[i].name.as_deref() == Some("A")).unwrap(); + let b = leaves.iter().find(|&&i| nodes[i].name.as_deref() == Some("B")).unwrap(); + let c = leaves.iter().find(|&&i| nodes[i].name.as_deref() == Some("C")).unwrap(); + assert!((patristic_distance(*a, *b, &lca) - 2.0).abs() < 1e-10); + assert!((patristic_distance(*a, *c, &lca) - 4.0).abs() < 1e-10); + } + + #[test] + fn test_midpoint_asymmetric() { + // (A:1.0,B:3.0); + // Diameter = 4, midpoint at distance 2 from each + let input = "(A:1.0,B:3.0);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + let new_root = midpoint_root(root, &mut nodes); + + assert!(nodes[new_root].parent.is_none()); + let lca = build_lca_structure(new_root, &nodes); + + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + let b = nodes.iter().position(|n| n.name.as_deref() == Some("B")).unwrap(); + assert!((patristic_distance(a, b, &lca) - 4.0).abs() < 1e-10); + } + + #[test] + fn test_midpoint_preserves_distances() { + let input = "(((A:0.5,B:0.3):0.4,C:0.9):0.1,D:1.2);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + + // Compute distances before midpoint rooting + let mut nodes_before = Vec::new(); + let root_before = flatten_raw(&raw, None, &mut nodes_before); + let lca_before = build_lca_structure(root_before, &nodes_before); + + let leaf_names = ["A", "B", "C", "D"]; + let leaves_before: Vec = leaf_names + .iter() + .map(|name| { + nodes_before + .iter() + .position(|n| n.name.as_deref() == Some(name)) + .unwrap() + }) + .collect(); + + let mut dists_before = vec![vec![0.0; 4]; 4]; + for i in 0..4 { + for j in 0..4 { + dists_before[i][j] = patristic_distance(leaves_before[i], leaves_before[j], &lca_before); + } + } + + // Now midpoint root + let mut nodes_after = Vec::new(); + let root_after = flatten_raw(&raw, None, &mut nodes_after); + let new_root = midpoint_root(root_after, &mut nodes_after); + let lca_after = build_lca_structure(new_root, &nodes_after); + + let leaves_after: Vec = leaf_names + .iter() + .map(|name| { + nodes_after + .iter() + .position(|n| n.name.as_deref() == Some(name)) + .unwrap() + }) + .collect(); + + for i in 0..4 { + for j in 0..4 { + let d = patristic_distance(leaves_after[i], leaves_after[j], &lca_after); + assert!( + (d - dists_before[i][j]).abs() < 1e-10, + "Distance mismatch for {}-{}: before={}, after={}", + leaf_names[i], + leaf_names[j], + dists_before[i][j], + d + ); + } + } + } +} diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000..5093e5e --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,265 @@ +use std::iter::Peekable; +use std::num::ParseFloatError; +use std::str::Chars; + +use crate::tree::Node; + +/// Temporary structure used during Newick parsing. +pub struct RawNode { + pub name: Option, + pub length: f64, + pub children: Vec, +} + +/// Recursively parse a Newick subtree and return a `RawNode`. +pub fn parse_subtree(chars: &mut Peekable) -> Result { + let mut node = RawNode { + name: None, + length: 0.0, + children: Vec::new(), + }; + + if let Some(&c) = chars.peek() { + if c == '(' { + chars.next(); + loop { + let child = parse_subtree(chars)?; + node.children.push(child); + match chars.peek() { + Some(',') => { + chars.next(); + continue; + } + Some(')') => { + chars.next(); + break; + } + Some(other) => { + return Err(format!("Expected ',' or ')', found '{}'", other)); + } + None => return Err("Unexpected end of input after '('".to_string()), + } + } + + // After ')', skip any NHX/bracket comments + skip_comments(chars); + + // Optional internal node label + if let Some(&c2) = chars.peek() { + if c2 != ':' && c2 != ',' && c2 != ')' && c2 != ';' { + let name = parse_label(chars); + if !name.is_empty() { + node.name = Some(name); + } + } + } + + // Skip comments after label too + skip_comments(chars); + + if let Some(&':') = chars.peek() { + chars.next(); + let length = parse_length(chars)?; + node.length = length; + } + + // Skip comments after branch length + skip_comments(chars); + + return Ok(node); + } + } + + // Leaf node + let name = parse_label(chars); + if !name.is_empty() { + node.name = Some(name); + } + + // Skip comments after label + skip_comments(chars); + + if let Some(&':') = chars.peek() { + chars.next(); + let length = parse_length(chars)?; + node.length = length; + } + + // Skip comments after branch length + skip_comments(chars); + + Ok(node) +} + +/// Parse a node label. Supports single-quoted labels (strips quotes). +pub fn parse_label(chars: &mut Peekable) -> String { + // Handle single-quoted labels + if let Some(&c) = chars.peek() { + if c == '\'' { + chars.next(); // consume opening quote + let mut label = String::new(); + while let Some(&ch) = chars.peek() { + if ch == '\'' { + chars.next(); // consume closing quote + break; + } + label.push(ch); + chars.next(); + } + return label; + } + } + + let mut label = String::new(); + while let Some(&c) = chars.peek() { + if c == ':' || c == ',' || c == ')' || c == ';' || c == '[' || c.is_whitespace() { + break; + } + label.push(c); + chars.next(); + } + label +} + +/// Skip '[...]' comment blocks (NHX annotations, BEAST metadata, etc.). +fn skip_comments(chars: &mut Peekable) { + while let Some(&'[') = chars.peek() { + chars.next(); + let mut depth = 1; + while depth > 0 { + match chars.next() { + Some('[') => depth += 1, + Some(']') => depth -= 1, + None => break, + _ => {} + } + } + } +} + +/// Parse a floating-point branch length (supports scientific notation). +pub fn parse_length(chars: &mut Peekable) -> Result { + let mut numstr = String::new(); + while let Some(&c) = chars.peek() { + if c.is_ascii_digit() || c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-' { + numstr.push(c); + chars.next(); + } else { + break; + } + } + numstr + .parse::() + .map_err(|e: ParseFloatError| format!("Failed to parse branch length '{}': {}", numstr, e)) +} + +/// Recursively flatten a `RawNode` into a flat `Vec`, returning the index of the new node. +pub fn flatten_raw(raw: &RawNode, parent: Option, nodes: &mut Vec) -> usize { + let idx = nodes.len(); + nodes.push(Node { + name: raw.name.clone(), + length: raw.length, + parent, + children: Vec::new(), + }); + if let Some(p) = parent { + nodes[p].children.push(idx); + } + for child in &raw.children { + flatten_raw(child, Some(idx), nodes); + } + idx +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_simple_tree() { + let input = "(A:1.0,B:2.0);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[0].name.as_deref(), Some("A")); + assert_eq!(raw.children[0].length, 1.0); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + assert_eq!(raw.children[1].length, 2.0); + } + + #[test] + fn test_parse_internal_labels() { + let input = "((A:1.0,B:2.0)inner:0.5,C:3.0);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[0].name.as_deref(), Some("inner")); + assert_eq!(raw.children[0].length, 0.5); + assert_eq!(raw.children[0].children.len(), 2); + } + + #[test] + fn test_parse_no_branch_lengths() { + let input = "(A,B,(C,D));"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + assert_eq!(raw.children.len(), 3); + assert_eq!(raw.children[0].length, 0.0); + assert_eq!(raw.children[2].children.len(), 2); + } + + #[test] + fn test_parse_nhx_comments() { + let input = "((A:0.1[&&NHX:S=human],B:0.2[&&NHX:S=mouse]):0.3,C:0.4);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + assert_eq!(raw.children.len(), 2); + let inner = &raw.children[0]; + assert_eq!(inner.children[0].name.as_deref(), Some("A")); + assert_eq!(inner.children[0].length, 0.1); + assert_eq!(inner.children[1].name.as_deref(), Some("B")); + assert_eq!(inner.children[1].length, 0.2); + assert_eq!(inner.length, 0.3); + } + + #[test] + fn test_parse_quoted_labels() { + let input = "('Taxon A':1.0,'Taxon B':2.0);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("Taxon A")); + assert_eq!(raw.children[1].name.as_deref(), Some("Taxon B")); + } + + #[test] + fn test_parse_scientific_notation() { + let input = "(A:1.5e-3,B:2.0E+1);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + assert!((raw.children[0].length - 0.0015).abs() < 1e-10); + assert!((raw.children[1].length - 20.0).abs() < 1e-10); + } + + #[test] + fn test_flatten() { + let input = "((A:1.0,B:2.0):0.5,C:3.0);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + assert_eq!(root, 0); + // root -> (inner, C), inner -> (A, B) = 5 nodes + assert_eq!(nodes.len(), 5); + assert!(nodes[0].parent.is_none()); + assert_eq!(nodes[0].children.len(), 2); + } + + #[test] + fn test_bracket_in_label_position() { + let input = "((A:0.1[comment],B:0.2):0.3[more],C:0.4[x]);"; + let mut chars = input.chars().peekable(); + let raw = parse_subtree(&mut chars).unwrap(); + assert_eq!(raw.children[0].children[0].name.as_deref(), Some("A")); + assert_eq!(raw.children[1].name.as_deref(), Some("C")); + } +} diff --git a/src/tree.rs b/src/tree.rs new file mode 100644 index 0000000..87e2146 --- /dev/null +++ b/src/tree.rs @@ -0,0 +1,7 @@ +/// Internal representation of a tree node. +pub struct Node { + pub name: Option, + pub length: f64, + pub parent: Option, + pub children: Vec, +} From 0c1924e357b2eb3cc6c462044e5e4b0c4d26f2f7 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:27:23 +0200 Subject: [PATCH 03/47] Bump version to 1.0.1 --- Cargo.lock | 238 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..7cbbf65 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,238 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "distree" +version = "1.0.1" +dependencies = [ + "clap", + "rayon", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/Cargo.toml b/Cargo.toml index 72ae370..3754c17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "distree" -version = "1.0.0" +version = "1.0.1" edition = "2021" [dependencies] From 809484b8bb6c763aa4aeb3505c96a74b146848b3 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:27:29 +0200 Subject: [PATCH 04/47] Fix contributors link to point to distree repo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b46bd4..5513bf2 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ LeafC 5.000 2.500 7.000 ---

-✨ [Contributors]((https://github.com/PathoGenOmics-Lab/AMAP/graphs/contributors)) +✨ [Contributors](https://github.com/PathoGenOmics-Lab/distree/graphs/contributors)

From 034daf9d99d9bd0af05f293a52c449b83f3ad11a Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:27:29 +0200 Subject: [PATCH 05/47] Add CITATION.cff with DOI 10.5281/zenodo.16811766 --- CITATION.cff | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..4e5d87c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,17 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +title: "distree" +version: 1.0.1 +doi: 10.5281/zenodo.16811766 +date-released: 2026-04-04 +url: "https://github.com/PathoGenOmics-Lab/distree" +repository-code: "https://github.com/PathoGenOmics-Lab/distree" +license: GPL-3.0 +type: software +authors: + - family-names: Ruiz-Rodriguez + given-names: Paula + affiliation: "I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain" + - family-names: Coscolla + given-names: Mireia + affiliation: "I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain" From 27acb9bac6eb2aa41f2fa80da3f8e20374ba6b70 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:27:29 +0200 Subject: [PATCH 06/47] Add CHANGELOG.md for v1.0.1 --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f0ab031 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,37 @@ +# Changelog + +## [1.0.1] - 2026-04-04 + +### Fixed +- Midpoint rooting (`--midpoint`) rewritten to fix infinite loop caused by cyclic graph construction +- NHX/bracket comment annotations (e.g., `[&&NHX:S=human]`) no longer crash the parser +- Single-quoted labels (e.g., `'Taxon A'`) now parsed correctly +- Duplicate leaf names are now detected with a clear error message +- Floating point output uses configurable precision instead of raw representation +- Header row always prints leading tab for R/Python distance matrix compatibility + +### Added +- `--precision` / `-p` flag to control decimal places in output (default: 10) +- `-t` / `--threads` flag to control number of parallel threads +- Stdin support: use `-` as the phylogeny argument to read from stdin +- Warning when no branch lengths are detected in patristic mode +- Warning when negative branch lengths are found in the tree +- CITATION.cff with DOI +- Comprehensive test suite (18 tests) + +### Changed +- Codebase split into modules: `parser.rs`, `tree.rs`, `lca.rs`, `midpoint.rs` +- Version string now derived from Cargo.toml via `env!("CARGO_PKG_VERSION")` +- Removed unused `--format` flag +- Fixed clippy warnings (`&Vec` → `&[Node]`) +- Fixed contributors link in README + +## [1.0.0] - 2025-06-01 + +### Added +- Initial release +- Patristic distance matrix extraction +- Topological distance computation +- LMM (var-covar) matrix output +- Midpoint rooting option +- Parallel computation with rayon From 442b06579e8c8cb62d821bd4d3e1ec3b10add630 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:45:32 +0200 Subject: [PATCH 07/47] Fix parser crashes and stack overflow on deep trees - Rewrite parser to iterative (no recursion): handles 50k+ deep trees - Rewrite flatten_raw to iterative: prevents stack overflow on flatten - Strip whitespace/newlines outside quotes before parsing - Support double-quoted labels (common in BEAST, FigTree output) - Warn on --lmm + --topology conflict instead of silent precedence - Topology mode outputs integers instead of float decimals - Pre-process Newick string to remove whitespace outside quotes - 24 tests (6 new: whitespace, newlines, deep tree, double quotes, nested brackets, negative length parsing) --- src/main.rs | 27 ++-- src/midpoint.rs | 14 +- src/parser.rs | 380 ++++++++++++++++++++++++++++++------------------ 3 files changed, 259 insertions(+), 162 deletions(-) diff --git a/src/main.rs b/src/main.rs index e7fe7e8..01c0966 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,7 +11,7 @@ use std::io::{self, BufWriter, Read, Write}; use lca::build_lca_structure; use midpoint::midpoint_root; -use parser::{flatten_raw, parse_subtree}; +use parser::{flatten_raw, parse_newick}; use tree::Node; fn main() -> io::Result<()> { @@ -74,6 +74,11 @@ fn main() -> io::Result<()> { let do_midpoint = *matches.get_one::("midpoint").unwrap(); let do_lmm = *matches.get_one::("lmm").unwrap(); let do_topology = *matches.get_one::("topology").unwrap(); + + // Warn if conflicting options + if do_lmm && do_topology { + eprintln!("Warning: --lmm and --topology are mutually exclusive. Using --lmm."); + } let output_path = matches.get_one::("output"); let precision = *matches.get_one::("precision").unwrap(); @@ -101,13 +106,10 @@ fn main() -> io::Result<()> { } // Parse the Newick string - let mut chars = newick_str.trim().chars().peekable(); - let raw_root = parse_subtree(&mut chars).expect("Failed to parse Newick tree"); - if let Some(&c) = chars.peek() { - if c == ';' { - chars.next(); - } - } + let raw_root = parse_newick(&newick_str).unwrap_or_else(|e| { + eprintln!("Failed to parse Newick tree: {}", e); + std::process::exit(1); + }); // Flatten into Vec let mut nodes: Vec = Vec::new(); @@ -211,7 +213,11 @@ fn main() -> io::Result<()> { writer.write_all(sorted_labels[row_i].as_bytes())?; for dist in this_row.iter() { writer.write_all(b"\t")?; - write!(writer, "{:.prec$}", dist, prec = prec)?; + if do_topology { + write!(writer, "{}", *dist as i64)?; + } else { + write!(writer, "{:.prec$}", dist, prec = prec)?; + } } writer.write_all(b"\n")?; } @@ -224,8 +230,7 @@ mod tests { use super::*; fn build_tree(newick: &str) -> (Vec, usize) { - let mut chars = newick.trim().chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick(newick).unwrap(); let mut nodes = Vec::new(); let root = flatten_raw(&raw, None, &mut nodes); (nodes, root) diff --git a/src/midpoint.rs b/src/midpoint.rs index 4cad5dd..512da22 100644 --- a/src/midpoint.rs +++ b/src/midpoint.rs @@ -154,7 +154,7 @@ pub fn midpoint_root(root_idx: usize, nodes: &mut Vec) -> usize { #[cfg(test)] mod tests { use super::*; - use crate::parser::{flatten_raw, parse_subtree}; + use crate::parser::{flatten_raw, parse_newick}; use crate::lca::build_lca_structure; fn patristic_distance( @@ -171,8 +171,7 @@ mod tests { // ((A:1,B:1):1,(C:1,D:1):1); // Diameter = 4 (e.g. A→root→D), midpoint at distance 2 let input = "((A:1,B:1):1,(C:1,D:1):1);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick(input).unwrap(); let mut nodes = Vec::new(); let root = flatten_raw(&raw, None, &mut nodes); let new_root = midpoint_root(root, &mut nodes); @@ -210,8 +209,7 @@ mod tests { // (A:1.0,B:3.0); // Diameter = 4, midpoint at distance 2 from each let input = "(A:1.0,B:3.0);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick(input).unwrap(); let mut nodes = Vec::new(); let root = flatten_raw(&raw, None, &mut nodes); let new_root = midpoint_root(root, &mut nodes); @@ -227,8 +225,7 @@ mod tests { #[test] fn test_midpoint_preserves_distances() { let input = "(((A:0.5,B:0.3):0.4,C:0.9):0.1,D:1.2);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick(input).unwrap(); // Compute distances before midpoint rooting let mut nodes_before = Vec::new(); @@ -254,8 +251,9 @@ mod tests { } // Now midpoint root + let raw2 = parse_newick(input).unwrap(); let mut nodes_after = Vec::new(); - let root_after = flatten_raw(&raw, None, &mut nodes_after); + let root_after = flatten_raw(&raw2, None, &mut nodes_after); let new_root = midpoint_root(root_after, &mut nodes_after); let lca_after = build_lca_structure(new_root, &nodes_after); diff --git a/src/parser.rs b/src/parser.rs index 5093e5e..cf99558 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,6 +1,4 @@ -use std::iter::Peekable; use std::num::ParseFloatError; -use std::str::Chars; use crate::tree::Node; @@ -11,164 +9,224 @@ pub struct RawNode { pub children: Vec, } -/// Recursively parse a Newick subtree and return a `RawNode`. -pub fn parse_subtree(chars: &mut Peekable) -> Result { - let mut node = RawNode { - name: None, - length: 0.0, - children: Vec::new(), - }; - - if let Some(&c) = chars.peek() { - if c == '(' { - chars.next(); - loop { - let child = parse_subtree(chars)?; - node.children.push(child); - match chars.peek() { - Some(',') => { - chars.next(); - continue; - } - Some(')') => { - chars.next(); - break; - } - Some(other) => { - return Err(format!("Expected ',' or ')', found '{}'", other)); - } - None => return Err("Unexpected end of input after '('".to_string()), - } - } - - // After ')', skip any NHX/bracket comments - skip_comments(chars); +/// Parse a complete Newick tree from a string. +/// Strips whitespace outside of quoted labels before parsing. +pub fn parse_newick(input: &str) -> Result { + // Pre-process: strip whitespace outside of single/double quotes + let cleaned = strip_whitespace_outside_quotes(input.trim()); + let bytes = cleaned.as_bytes(); + let mut pos = 0; + let root = parse_subtree_iterative(bytes, &mut pos)?; + Ok(root) +} - // Optional internal node label - if let Some(&c2) = chars.peek() { - if c2 != ':' && c2 != ',' && c2 != ')' && c2 != ';' { - let name = parse_label(chars); - if !name.is_empty() { - node.name = Some(name); - } - } +/// Strip whitespace characters outside of quoted regions. +fn strip_whitespace_outside_quotes(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_quote = false; + let mut quote_char = b'\0'; + for &b in s.as_bytes() { + if in_quote { + out.push(b as char); + if b == quote_char { + in_quote = false; } + } else if b == b'\'' || b == b'"' { + in_quote = true; + quote_char = b; + out.push(b as char); + } else if !b.is_ascii_whitespace() { + out.push(b as char); + } + } + out +} + +/// Iterative Newick parser — no stack overflow on deep trees. +fn parse_subtree_iterative(bytes: &[u8], pos: &mut usize) -> Result { + // We use an explicit stack to avoid recursion. + // Each frame represents a node being constructed. + struct Frame { + node: RawNode, + } - // Skip comments after label too - skip_comments(chars); + let mut stack: Vec = Vec::new(); - if let Some(&':') = chars.peek() { - chars.next(); - let length = parse_length(chars)?; - node.length = length; + loop { + // Decide what to parse at current position + let current_node = if *pos < bytes.len() && bytes[*pos] == b'(' { + // Internal node: push frame, advance past '(' + *pos += 1; + skip_whitespace_bytes(bytes, pos); + stack.push(Frame { + node: RawNode { name: None, length: 0.0, children: Vec::new() }, + }); + continue; // loop back to parse first child + } else { + // Leaf node + let name = parse_label_bytes(bytes, pos); + skip_comments_bytes(bytes, pos); + let length = parse_optional_length(bytes, pos)?; + skip_comments_bytes(bytes, pos); + let mut leaf = RawNode { name: None, length, children: Vec::new() }; + if !name.is_empty() { + leaf.name = Some(name); } + leaf + }; - // Skip comments after branch length - skip_comments(chars); + // Now we have a completed node. Feed it back up the stack. + let mut completed = current_node; + loop { + if let Some(frame) = stack.last_mut() { + frame.node.children.push(completed); + // Check what comes next: ',' means more children, ')' means done + if *pos < bytes.len() && bytes[*pos] == b',' { + *pos += 1; + skip_whitespace_bytes(bytes, pos); + break; // break inner loop, continue outer to parse next child + } else if *pos < bytes.len() && bytes[*pos] == b')' { + *pos += 1; + // Internal node complete — read its label and length + skip_comments_bytes(bytes, pos); + let name = parse_label_bytes(bytes, pos); + skip_comments_bytes(bytes, pos); + let length = parse_optional_length(bytes, pos)?; + skip_comments_bytes(bytes, pos); - return Ok(node); + let mut frame = stack.pop().unwrap(); + if !name.is_empty() { + frame.node.name = Some(name); + } + frame.node.length = length; + completed = frame.node; + // continue inner loop to feed this up further + } else if *pos >= bytes.len() { + // End of input while inside parentheses + let frame = stack.pop().unwrap(); + completed = frame.node; + } else { + let ch = bytes[*pos] as char; + return Err(format!("Expected ',' or ')', found '{}' at position {}", ch, *pos)); + } + } else { + // Stack empty — this is the root + return Ok(completed); + } } } +} - // Leaf node - let name = parse_label(chars); - if !name.is_empty() { - node.name = Some(name); +fn skip_whitespace_bytes(bytes: &[u8], pos: &mut usize) { + while *pos < bytes.len() && bytes[*pos].is_ascii_whitespace() { + *pos += 1; } +} - // Skip comments after label - skip_comments(chars); - - if let Some(&':') = chars.peek() { - chars.next(); - let length = parse_length(chars)?; - node.length = length; +/// Parse a node label from bytes. Supports single-quoted and double-quoted labels. +fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> String { + if *pos >= bytes.len() { + return String::new(); } - // Skip comments after branch length - skip_comments(chars); - - Ok(node) -} - -/// Parse a node label. Supports single-quoted labels (strips quotes). -pub fn parse_label(chars: &mut Peekable) -> String { - // Handle single-quoted labels - if let Some(&c) = chars.peek() { - if c == '\'' { - chars.next(); // consume opening quote - let mut label = String::new(); - while let Some(&ch) = chars.peek() { - if ch == '\'' { - chars.next(); // consume closing quote - break; - } - label.push(ch); - chars.next(); + // Handle quoted labels (single or double quotes) + let ch = bytes[*pos]; + if ch == b'\'' || ch == b'"' { + *pos += 1; // consume opening quote + let mut label = String::new(); + while *pos < bytes.len() { + if bytes[*pos] == ch { + *pos += 1; // consume closing quote + break; } - return label; + label.push(bytes[*pos] as char); + *pos += 1; } + return label; } + // Unquoted label: read until delimiter let mut label = String::new(); - while let Some(&c) = chars.peek() { - if c == ':' || c == ',' || c == ')' || c == ';' || c == '[' || c.is_whitespace() { + while *pos < bytes.len() { + let c = bytes[*pos]; + if c == b':' || c == b',' || c == b')' || c == b';' || c == b'[' || c.is_ascii_whitespace() { break; } - label.push(c); - chars.next(); + label.push(c as char); + *pos += 1; } label } /// Skip '[...]' comment blocks (NHX annotations, BEAST metadata, etc.). -fn skip_comments(chars: &mut Peekable) { - while let Some(&'[') = chars.peek() { - chars.next(); - let mut depth = 1; - while depth > 0 { - match chars.next() { - Some('[') => depth += 1, - Some(']') => depth -= 1, - None => break, +/// Handles nested brackets. +fn skip_comments_bytes(bytes: &[u8], pos: &mut usize) { + while *pos < bytes.len() && bytes[*pos] == b'[' { + *pos += 1; + let mut depth: usize = 1; + while *pos < bytes.len() && depth > 0 { + match bytes[*pos] { + b'[' => depth += 1, + b']' => depth -= 1, _ => {} } + *pos += 1; } } } -/// Parse a floating-point branch length (supports scientific notation). -pub fn parse_length(chars: &mut Peekable) -> Result { - let mut numstr = String::new(); - while let Some(&c) = chars.peek() { - if c.is_ascii_digit() || c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-' { - numstr.push(c); - chars.next(); +/// Parse optional ":length" — returns 0.0 if no colon present. +fn parse_optional_length(bytes: &[u8], pos: &mut usize) -> Result { + if *pos < bytes.len() && bytes[*pos] == b':' { + *pos += 1; + parse_length_bytes(bytes, pos) + } else { + Ok(0.0) + } +} + +/// Parse a floating-point branch length (supports scientific notation and negative values). +fn parse_length_bytes(bytes: &[u8], pos: &mut usize) -> Result { + let start = *pos; + while *pos < bytes.len() { + let c = bytes[*pos]; + if c.is_ascii_digit() || c == b'.' || c == b'e' || c == b'E' || c == b'+' || c == b'-' { + *pos += 1; } else { break; } } + let numstr = std::str::from_utf8(&bytes[start..*pos]) + .map_err(|_| "Invalid UTF-8 in branch length".to_string())?; numstr .parse::() .map_err(|e: ParseFloatError| format!("Failed to parse branch length '{}': {}", numstr, e)) } -/// Recursively flatten a `RawNode` into a flat `Vec`, returning the index of the new node. +/// Flatten a `RawNode` tree into a flat `Vec` iteratively (no stack overflow). pub fn flatten_raw(raw: &RawNode, parent: Option, nodes: &mut Vec) -> usize { - let idx = nodes.len(); - nodes.push(Node { - name: raw.name.clone(), - length: raw.length, - parent, - children: Vec::new(), - }); - if let Some(p) = parent { - nodes[p].children.push(idx); - } - for child in &raw.children { - flatten_raw(child, Some(idx), nodes); + // Stack of (raw_node_ref, parent_index) + let mut stack: Vec<(&RawNode, Option)> = vec![(raw, parent)]; + let root_idx = nodes.len(); + + while let Some((raw_node, par)) = stack.pop() { + let idx = nodes.len(); + nodes.push(Node { + name: raw_node.name.clone(), + length: raw_node.length, + parent: par, + children: Vec::new(), + }); + if let Some(p) = par { + nodes[p].children.push(idx); + } + // Push children in reverse order so they're processed left-to-right + for child in raw_node.children.iter().rev() { + stack.push((child, Some(idx))); + } } - idx + + root_idx } #[cfg(test)] @@ -177,9 +235,7 @@ mod tests { #[test] fn test_parse_simple_tree() { - let input = "(A:1.0,B:2.0);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick("(A:1.0,B:2.0);").unwrap(); assert_eq!(raw.children.len(), 2); assert_eq!(raw.children[0].name.as_deref(), Some("A")); assert_eq!(raw.children[0].length, 1.0); @@ -189,9 +245,7 @@ mod tests { #[test] fn test_parse_internal_labels() { - let input = "((A:1.0,B:2.0)inner:0.5,C:3.0);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick("((A:1.0,B:2.0)inner:0.5,C:3.0);").unwrap(); assert_eq!(raw.children.len(), 2); assert_eq!(raw.children[0].name.as_deref(), Some("inner")); assert_eq!(raw.children[0].length, 0.5); @@ -200,9 +254,7 @@ mod tests { #[test] fn test_parse_no_branch_lengths() { - let input = "(A,B,(C,D));"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick("(A,B,(C,D));").unwrap(); assert_eq!(raw.children.len(), 3); assert_eq!(raw.children[0].length, 0.0); assert_eq!(raw.children[2].children.len(), 2); @@ -210,9 +262,7 @@ mod tests { #[test] fn test_parse_nhx_comments() { - let input = "((A:0.1[&&NHX:S=human],B:0.2[&&NHX:S=mouse]):0.3,C:0.4);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick("((A:0.1[&&NHX:S=human],B:0.2[&&NHX:S=mouse]):0.3,C:0.4);").unwrap(); assert_eq!(raw.children.len(), 2); let inner = &raw.children[0]; assert_eq!(inner.children[0].name.as_deref(), Some("A")); @@ -223,32 +273,32 @@ mod tests { } #[test] - fn test_parse_quoted_labels() { - let input = "('Taxon A':1.0,'Taxon B':2.0);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + fn test_parse_quoted_labels_single() { + let raw = parse_newick("('Taxon A':1.0,'Taxon B':2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("Taxon A")); + assert_eq!(raw.children[1].name.as_deref(), Some("Taxon B")); + } + + #[test] + fn test_parse_quoted_labels_double() { + let raw = parse_newick(r#"("Taxon A":1.0,"Taxon B":2.0);"#).unwrap(); assert_eq!(raw.children[0].name.as_deref(), Some("Taxon A")); assert_eq!(raw.children[1].name.as_deref(), Some("Taxon B")); } #[test] fn test_parse_scientific_notation() { - let input = "(A:1.5e-3,B:2.0E+1);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick("(A:1.5e-3,B:2.0E+1);").unwrap(); assert!((raw.children[0].length - 0.0015).abs() < 1e-10); assert!((raw.children[1].length - 20.0).abs() < 1e-10); } #[test] fn test_flatten() { - let input = "((A:1.0,B:2.0):0.5,C:3.0);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick("((A:1.0,B:2.0):0.5,C:3.0);").unwrap(); let mut nodes = Vec::new(); let root = flatten_raw(&raw, None, &mut nodes); assert_eq!(root, 0); - // root -> (inner, C), inner -> (A, B) = 5 nodes assert_eq!(nodes.len(), 5); assert!(nodes[0].parent.is_none()); assert_eq!(nodes[0].children.len(), 2); @@ -256,10 +306,54 @@ mod tests { #[test] fn test_bracket_in_label_position() { - let input = "((A:0.1[comment],B:0.2):0.3[more],C:0.4[x]);"; - let mut chars = input.chars().peekable(); - let raw = parse_subtree(&mut chars).unwrap(); + let raw = parse_newick("((A:0.1[comment],B:0.2):0.3[more],C:0.4[x]);").unwrap(); assert_eq!(raw.children[0].children[0].name.as_deref(), Some("A")); assert_eq!(raw.children[1].name.as_deref(), Some("C")); } + + #[test] + fn test_whitespace_in_newick() { + let raw = parse_newick("( A:1.0 , B:2.0 ) ;").unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[0].name.as_deref(), Some("A")); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + } + + #[test] + fn test_newlines_in_newick() { + let raw = parse_newick("((A:1.0,\nB:2.0):0.5,\nC:3.0);\n").unwrap(); + assert_eq!(raw.children.len(), 2); + } + + #[test] + fn test_deep_tree_no_stack_overflow() { + // Build a caterpillar tree with 5,000 levels using push (no recursive format!) + let mut tree = String::with_capacity(200_000); + for _ in 0..5_000 { + tree.push('('); + } + tree.push_str("A:0.1"); + for i in 1..=5_000 { + tree.push_str(&format!(",T{}:0.1):0.01", i)); + } + tree.push(';'); + let raw = parse_newick(&tree).unwrap(); + // Should not stack overflow; just verify it parsed + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + assert!(nodes.len() > 5_000); + assert!(nodes[root].parent.is_none()); + } + + #[test] + fn test_nested_brackets() { + let raw = parse_newick("((A:0.1[&rate=0.5[inner]],B:0.2):0.3,C:0.4);").unwrap(); + assert_eq!(raw.children[0].children[0].name.as_deref(), Some("A")); + } + + #[test] + fn test_negative_branch_length_parsed() { + let raw = parse_newick("(A:-0.5,B:2.0);").unwrap(); + assert!((raw.children[0].length - (-0.5)).abs() < 1e-10); + } } From b229d87dabd26e90ad27a24ba1d2c0254a00a6ca Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:45:56 +0200 Subject: [PATCH 08/47] Update CHANGELOG with parser fixes --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0ab031..93827c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,12 @@ - Midpoint rooting (`--midpoint`) rewritten to fix infinite loop caused by cyclic graph construction - NHX/bracket comment annotations (e.g., `[&&NHX:S=human]`) no longer crash the parser - Single-quoted labels (e.g., `'Taxon A'`) now parsed correctly +- Double-quoted labels (e.g., `"Taxon A"`) now parsed correctly +- Whitespace and newlines in Newick strings no longer crash the parser +- Stack overflow on deeply nested trees (>5,000 levels) — parser and flattener rewritten iteratively - Duplicate leaf names are now detected with a clear error message +- `--lmm --topology` conflict now warns instead of silently choosing LMM +- Topology mode now outputs integers instead of float decimals - Floating point output uses configurable precision instead of raw representation - Header row always prints leading tab for R/Python distance matrix compatibility @@ -17,7 +22,7 @@ - Warning when no branch lengths are detected in patristic mode - Warning when negative branch lengths are found in the tree - CITATION.cff with DOI -- Comprehensive test suite (18 tests) +- Comprehensive test suite (24 tests) ### Changed - Codebase split into modules: `parser.rs`, `tree.rs`, `lca.rs`, `midpoint.rs` From 5c42413a7f9cfd9fd25295f31d7bb7a4764d5b86 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:51:57 +0200 Subject: [PATCH 09/47] Proper error handling, lower triangle, CI workflows, 28 tests - Replace process::exit with Result<(), Box> for clean error propagation - Add --lower flag for PHYLIP-compatible lower triangle output - Add DistMode enum to centralize mode logic - Add compute_distance() and format_distance() helpers - Add CI workflow (clippy + test on push/PR) - Add Build & Release workflow (4 binaries on tag push) - BufWriter on stdout for better performance - Better error messages (file not found, parse errors) - 28 tests (4 new: single leaf, symmetric tree, self-distance, mode conflict) - Update README with new flags and features - 0 clippy warnings with -D warnings --- .github/workflows/ci.yml | 26 +++ .github/workflows/release.yml | 66 ++++++++ README.md | 62 +++---- src/main.rs | 302 ++++++++++++++++++++++------------ 4 files changed, 315 insertions(+), 141 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a03cf28 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: CI + +on: + push: + branches: [main, "*.*.x", "*.x"] + pull_request: + branches: [main] + +env: + CARGO_TERM_COLOR: always + +jobs: + check: + name: Check & Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - name: Clippy + run: cargo clippy -- -D warnings + - name: Test + run: cargo test + - name: Build release + run: cargo build --release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4f5480b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,66 @@ +name: Build & Release + +on: + push: + tags: ["v*"] + +permissions: + contents: write + +jobs: + build: + name: Build ${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + name: distree-linux-x86_64 + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + name: distree-linux-aarch64 + - target: x86_64-apple-darwin + os: macos-15-intel + name: distree-macos-x86_64 + - target: aarch64-apple-darwin + os: macos-latest + name: distree-macos-aarch64 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - name: Install cross-compilation tools + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config.toml + echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config.toml + - name: Build + run: cargo build --release --target ${{ matrix.target }} + - name: Package + run: | + cp target/${{ matrix.target }}/release/distree ${{ matrix.name }} + chmod +x ${{ matrix.name }} + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.name }} + path: ${{ matrix.name }} + + release: + name: Create Release + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: artifacts + - name: Create release + uses: softprops/action-gh-release@v2 + with: + files: artifacts/**/* + generate_release_notes: true diff --git a/README.md b/README.md index 5513bf2..006af8d 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,10 @@ [![PGO](https://img.shields.io/badge/PathoGenOmics-lab-red?)](https://github.com/PathoGenOmics-Lab) [![DOI](https://img.shields.io/badge/doi-10.5281%2Fzenodo.16811766-%23ff0077)](https://doi.org/10.5281/zenodo.16811766) -__Paula Ruiz-Rodriguez1__ +__Paula Ruiz-Rodriguez1__ __and Mireia Coscolla1__
- 1. I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain + 1. I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain `distree` is a command-line tool written in Rust that extracts a distance matrix from a phylogenetic tree in Newick format. It is designed to handle large trees with thousands of sequences by using a low-memory, parallelized approach. @@ -36,12 +36,12 @@ __and Mireia Coscolla1__ 2. **Epidemiology & Public Health** * **Rapid Outbreak Tracking**: For pathogens such as bacteria or viruses, building a phylogenetic tree from whole-genome data can be computationally expensive to revisit. Extracting a distance matrix allows quick pairwise comparisons to identify clusters of closely related strains (e.g., potential transmission clusters) without recomputing distances from raw alignments. - * **Contact Tracing & Transmission Networks**: If you have a large outbreak dataset, computing patristic distances between samples quickly helps identify subclusters or “clades” of interest (e.g., to infer likely transmission chains). Similarly, topological distances might approximate epidemiological closeness if branch-length estimates vary widely. + * **Contact Tracing & Transmission Networks**: If you have a large outbreak dataset, computing patristic distances between samples quickly helps identify subclusters or "clades" of interest (e.g., to infer likely transmission chains). Similarly, topological distances might approximate epidemiological closeness if branch-length estimates vary widely. 3. **Microbiome & Environmental Sequencing** * **OTU/ASV Clustering**: In 16S rRNA amplicon studies, one often constructs a phylogenetic tree of all amplicon sequence variants (ASVs). A distance matrix (patristic or topological) can feed into beta-diversity metrics (e.g., UniFrac requires branch lengths). Here, `distree` can quickly compute pairwise distances after the tree is built, facilitating ordination (PCoA) or clustering of samples by their phylogenetic composition. - * **Phylogenetic Diversity**: Calculating the sum of branch lengths between taxa supports metrics like Faith’s Phylogenetic Diversity or UniFrac. `distree` can produce the matrix needed for those algorithms without re-traversing the tree multiple times. + * **Phylogenetic Diversity**: Calculating the sum of branch lengths between taxa supports metrics like Faith's Phylogenetic Diversity or UniFrac. `distree` can produce the matrix needed for those algorithms without re-traversing the tree multiple times. 4. **Machine Learning & Dimensionality Reduction** @@ -86,47 +86,37 @@ mamba install -c bioconda distree Usage: distree [OPTIONS] Arguments: - Path to the tree file in Newick format + Path to the tree file in Newick format (use '-' for stdin) Options: - --format Tree file format (only 'newick' is supported) [default: newick] - --midpoint Midpoint-root the tree before computing distances - --lmm Produce the var-covar matrix C (depth of the MRCA) - --topology Ignore branch lengths; use purely topological distances - -o, --output Path to write the TSV output file (defaults to stdout) - -h, --help Print help information - -V, --version Print version information + --midpoint Midpoint-root the tree before computing distances + --lmm Produce the var-covar matrix C (depth of the MRCA) + --topology Ignore branch lengths; use purely topological distances + --lower Output only the lower triangle (PHYLIP-compatible) + -o, --output Path to write the TSV output file (defaults to stdout) + -p, --precision Number of decimal places for output [default: 10] + -t, --threads Number of parallel threads (default: all cores) + -h, --help Print help information + -V, --version Print version information ``` ### Argument & Option Details -* ``: Path to the input tree in Newick format. Leaf labels must be unique and not contain tabs or newline characters. +* ``: Path to the input tree in Newick format. Use `-` to read from stdin. Leaf labels must be unique. -* `--format ` +* `--midpoint`: Re-root at the midpoint of the longest path before computing distances. - * Currently only `newick` is supported. Future versions may support other formats (e.g., Nexus, PhyloXML). - * This option exists to maintain CLI consistency; it does not change parsing for now. +* `--lmm`: Var-covar matrix for Phylogenetic Comparative Methods. Each entry (i, j) = depth of MRCA(i, j). Mutually exclusive with `--topology`. -* `--midpoint` +* `--topology`: Number of edges between leaves (integers). Ignores branch lengths. - * Before computing any distances, the tree will be re-rooted at its midpoint (the point halfway along the longest path between any two leaves). Useful when no outgroup is known or when you want a balanced root for downstream analyses. - * Use this if your downstream distance metric expects an unrooted, centrically-rooted tree. +* `--lower`: Lower triangle only, no header or diagonal. Useful for PHYLIP-compatible input or to halve file size. -* `--lmm` +* `-p, --precision `: Decimal places in output (default: 10). Applies to patristic and LMM modes. - * “LMM” stands for var-covar matrix (matrix C) in Phylogenetic Comparative Methods. Each entry (i, j) equals the depth (distance from root) of the lowest common ancestor of leaf i and leaf j. This matrix is often used in linear mixed models (LAMM, PGLS) to account for phylogenetic covariance. - * When specified, LMM distances override `--topology`. The output is a matrix of MRCA depths, not pairwise distances. +* `-t, --threads `: Thread count for parallel computation. Defaults to all available cores. -* `--topology` - - * Ignores branch lengths. Each entry (i, j) equals the number of edges between leaf i and leaf j: - - * Use this mode if branch-lengths are not meaningful or if you only care about tree shape. - -* `-o, --output ` - - * Write the TSV distance matrix to the specified path. If omitted, the matrix is printed to standard output. - * Example: `-o distances.tsv`. +* `-o, --output `: Write TSV to file instead of stdout. ## Output Format @@ -173,7 +163,7 @@ LeafC 5.000 2.500 7.000 ./distree tree.nwk -o patristic.tsv ``` -**Why**: Downstream tools like SciKit-Learn (for MDS) or R’s `ape::cmdscale()` expect a distance matrix. Patristic distances reflect evolutionary time or change. +**Why**: Downstream tools like SciKit-Learn (for MDS) or R's `ape::cmdscale()` expect a distance matrix. Patristic distances reflect evolutionary time or change. ### 2. Computing Topological Distances Only @@ -227,7 +217,7 @@ LeafC 5.000 2.500 7.000 * **Memory**: `distree` streams one row at a time. At any given moment, only a single vector of length N (number of leaves) resides in memory, plus O(M log M) for LCA structures, where M is the total number of nodes. For trees with tens of thousands of taxa, memory usage remains low. -* **Parallelism**: Each row’s distance computations are parallelized across available CPU cores via Rayon. For N taxa, computing N rows (each of size N) takes O(N^2 / #cores) time. +* **Parallelism**: Each row's distance computations are parallelized across available CPU cores via Rayon. For N taxa, computing N rows (each of size N) takes O(N^2 / #cores) time. * **Disk I/O**: If writing to a file via `--output`, a buffered writer (`BufWriter`) minimizes I/O calls. Streaming directly to stdout also remains efficient. @@ -272,7 +262,7 @@ distree is developed with ❤️ by: 🔣 🎨 🔧 - + @@ -285,7 +275,7 @@ distree is developed with ❤️ by: 🧑‍🏫 🔬 📓 - + diff --git a/src/main.rs b/src/main.rs index 01c0966..a590e49 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,13 +8,32 @@ use rayon::prelude::*; use std::collections::HashSet; use std::fs::File; use std::io::{self, BufWriter, Read, Write}; +use std::process::ExitCode; use lca::build_lca_structure; use midpoint::midpoint_root; use parser::{flatten_raw, parse_newick}; use tree::Node; -fn main() -> io::Result<()> { +/// Distance mode to compute. +#[derive(Clone, Copy, PartialEq)] +enum DistMode { + Patristic, + Topology, + Lmm, +} + +fn main() -> ExitCode { + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("Error: {}", e); + ExitCode::FAILURE + } + } +} + +fn run() -> Result<(), Box> { let matches = Command::new("distree") .version(env!("CARGO_PKG_VERSION")) .author("Paula Ruiz-Rodriguez") @@ -65,35 +84,49 @@ fn main() -> io::Result<()> { .help("Number of threads for parallel computation (default: all cores)") .value_parser(clap::value_parser!(usize)), ) + .arg( + Arg::new("lower") + .long("lower") + .help("Output only the lower triangle (PHYLIP-compatible, no diagonal)") + .action(ArgAction::SetTrue), + ) .get_matches(); let tree_path = matches .get_one::("phylogeny") - .expect("Tree file path is required") + .ok_or("Tree file path is required")? .to_string(); - let do_midpoint = *matches.get_one::("midpoint").unwrap(); - let do_lmm = *matches.get_one::("lmm").unwrap(); - let do_topology = *matches.get_one::("topology").unwrap(); - - // Warn if conflicting options - if do_lmm && do_topology { - eprintln!("Warning: --lmm and --topology are mutually exclusive. Using --lmm."); - } + let do_midpoint = *matches.get_one::("midpoint").unwrap_or(&false); + let do_lmm = *matches.get_one::("lmm").unwrap_or(&false); + let do_topology = *matches.get_one::("topology").unwrap_or(&false); + let do_lower = *matches.get_one::("lower").unwrap_or(&false); let output_path = matches.get_one::("output"); - let precision = *matches.get_one::("precision").unwrap(); + let precision = *matches.get_one::("precision").unwrap_or(&10); + + // Determine distance mode + let mode = if do_lmm { + if do_topology { + eprintln!("Warning: --lmm and --topology are mutually exclusive. Using --lmm."); + } + DistMode::Lmm + } else if do_topology { + DistMode::Topology + } else { + DistMode::Patristic + }; // Configure thread pool if let Some(&num_threads) = matches.get_one::("threads") { rayon::ThreadPoolBuilder::new() .num_threads(num_threads) .build_global() - .expect("Failed to initialize thread pool"); + .map_err(|e| format!("Failed to initialize thread pool: {}", e))?; } let mut writer: Box = if let Some(path) = output_path { Box::new(BufWriter::new(File::create(path)?)) } else { - Box::new(io::stdout()) + Box::new(BufWriter::new(io::stdout())) }; // Read input from file or stdin @@ -101,23 +134,21 @@ fn main() -> io::Result<()> { if tree_path == "-" { io::stdin().read_to_string(&mut newick_str)?; } else { - let mut f = File::open(&tree_path)?; - f.read_to_string(&mut newick_str)?; + File::open(&tree_path) + .map_err(|e| format!("Cannot open '{}': {}", tree_path, e))? + .read_to_string(&mut newick_str)?; } // Parse the Newick string - let raw_root = parse_newick(&newick_str).unwrap_or_else(|e| { - eprintln!("Failed to parse Newick tree: {}", e); - std::process::exit(1); - }); + let raw_root = parse_newick(&newick_str) + .map_err(|e| format!("Failed to parse Newick tree: {}", e))?; // Flatten into Vec let mut nodes: Vec = Vec::new(); let mut root_idx = flatten_raw(&raw_root, None, &mut nodes); // Warn about negative branch lengths - let has_negative = nodes.iter().any(|n| n.length < 0.0); - if has_negative { + if nodes.iter().any(|n| n.length < 0.0) { eprintln!("Warning: negative branch lengths detected in the tree."); } @@ -130,25 +161,28 @@ fn main() -> io::Result<()> { let lca_data = build_lca_structure(root_idx, &nodes); // Collect leaves - let mut leaf_indices: Vec = Vec::new(); - for (i, nd) in nodes.iter().enumerate() { - if nd.children.is_empty() && nd.name.is_some() { - leaf_indices.push(i); - } - } + let leaf_indices: Vec = nodes + .iter() + .enumerate() + .filter(|(_, nd)| nd.children.is_empty() && nd.name.is_some()) + .map(|(i, _)| i) + .collect(); + if leaf_indices.is_empty() { - eprintln!("No labeled leaves found in the tree. Exiting."); - std::process::exit(1); + return Err("No labeled leaves found in the tree.".into()); } // Check for duplicate leaf names { - let mut seen = HashSet::new(); + let mut seen = HashSet::with_capacity(leaf_indices.len()); for &i in &leaf_indices { - let name = nodes[i].name.as_ref().unwrap(); - if !seen.insert(name.clone()) { - eprintln!("Error: duplicate leaf name '{}' found. Leaf names must be unique.", name); - std::process::exit(1); + let name = nodes[i].name.as_ref().expect("leaf has name"); + if !seen.insert(name.as_str()) { + return Err(format!( + "Duplicate leaf name '{}' found. Leaf names must be unique.", + name + ) + .into()); } } } @@ -156,68 +190,55 @@ fn main() -> io::Result<()> { // Sort by label let mut leaf_label_pairs: Vec<(String, usize)> = leaf_indices .iter() - .map(|&i| (nodes[i].name.clone().unwrap(), i)) + .map(|&i| (nodes[i].name.clone().expect("leaf has name"), i)) .collect(); - leaf_label_pairs.sort_by(|a, b| a.0.cmp(&b.0)); + leaf_label_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); - let sorted_labels: Vec = leaf_label_pairs.iter().map(|(lab, _)| lab.clone()).collect(); + let sorted_labels: Vec<&str> = leaf_label_pairs + .iter() + .map(|(lab, _)| lab.as_str()) + .collect(); let sorted_leaf_indices: Vec = leaf_label_pairs.iter().map(|(_, idx)| *idx).collect(); let n_leaves = sorted_leaf_indices.len(); // Warn if no branch lengths and not topology mode - if !do_topology { - let all_zero = nodes.iter().all(|n| n.length == 0.0); - if all_zero { - eprintln!( - "Warning: no branch lengths detected, all patristic distances will be zero. Consider --topology." - ); - } + if mode == DistMode::Patristic && nodes.iter().all(|n| n.length == 0.0) { + eprintln!( + "Warning: no branch lengths detected, all patristic distances will be zero. Consider --topology." + ); } - // Print TSV header (leading tab for R/Python compatibility) - writer.write_all(b"\t")?; - for (i, lab) in sorted_labels.iter().enumerate() { - writer.write_all(lab.as_bytes())?; - if i + 1 < n_leaves { - writer.write_all(b"\t")?; + // Print header + if !do_lower { + writer.write_all(b"\t")?; + for (i, lab) in sorted_labels.iter().enumerate() { + writer.write_all(lab.as_bytes())?; + if i + 1 < n_leaves { + writer.write_all(b"\t")?; + } } + writer.write_all(b"\n")?; } - writer.write_all(b"\n")?; // Compute and print distance matrix - let prec = precision; for (row_i, &leaf_i) in sorted_leaf_indices.iter().enumerate() { - let this_row: Vec = sorted_leaf_indices + let col_end = if do_lower { row_i } else { n_leaves }; + let col_slice = &sorted_leaf_indices[..col_end]; + + let this_row: Vec = col_slice .par_iter() - .map(|&leaf_j| { - if do_lmm { - let m = lca_data.mrca(leaf_i, leaf_j); - lca_data.depth_len[m] - } else if do_topology { - let m = lca_data.mrca(leaf_i, leaf_j); - let d_i = lca_data.depth_top[leaf_i]; - let d_j = lca_data.depth_top[leaf_j]; - let d_m = lca_data.depth_top[m]; - ((d_i + d_j).saturating_sub(2 * d_m)) as f64 - } else { - let m = lca_data.mrca(leaf_i, leaf_j); - let d_i = lca_data.depth_len[leaf_i]; - let d_j = lca_data.depth_len[leaf_j]; - let d_m = lca_data.depth_len[m]; - d_i + d_j - 2.0 * d_m - } - }) + .map(|&leaf_j| compute_distance(leaf_i, leaf_j, mode, &lca_data)) .collect(); - writer.write_all(sorted_labels[row_i].as_bytes())?; - for dist in this_row.iter() { - writer.write_all(b"\t")?; - if do_topology { - write!(writer, "{}", *dist as i64)?; - } else { - write!(writer, "{:.prec$}", dist, prec = prec)?; + if !do_lower { + writer.write_all(sorted_labels[row_i].as_bytes())?; + } + for (ci, dist) in this_row.iter().enumerate() { + if ci > 0 || !do_lower { + writer.write_all(b"\t")?; } + format_distance(&mut writer, *dist, mode, precision)?; } writer.write_all(b"\n")?; } @@ -225,6 +246,47 @@ fn main() -> io::Result<()> { Ok(()) } +/// Compute distance between two leaves given the mode. +#[inline] +fn compute_distance( + leaf_i: usize, + leaf_j: usize, + mode: DistMode, + lca_data: &lca::LcaData, +) -> f64 { + let m = lca_data.mrca(leaf_i, leaf_j); + match mode { + DistMode::Lmm => lca_data.depth_len[m], + DistMode::Topology => { + let d_i = lca_data.depth_top[leaf_i]; + let d_j = lca_data.depth_top[leaf_j]; + let d_m = lca_data.depth_top[m]; + ((d_i + d_j).saturating_sub(2 * d_m)) as f64 + } + DistMode::Patristic => { + let d_i = lca_data.depth_len[leaf_i]; + let d_j = lca_data.depth_len[leaf_j]; + let d_m = lca_data.depth_len[m]; + d_i + d_j - 2.0 * d_m + } + } +} + +/// Format a distance value for output. +#[inline] +fn format_distance( + writer: &mut dyn Write, + dist: f64, + mode: DistMode, + precision: usize, +) -> io::Result<()> { + if mode == DistMode::Topology { + write!(writer, "{}", dist as i64) + } else { + write!(writer, "{:.prec$}", dist, prec = precision) + } +} + #[cfg(test)] mod tests { use super::*; @@ -247,8 +309,7 @@ mod tests { let lca = build_lca_structure(root, nodes); let ai = get_leaf(nodes, a); let bi = get_leaf(nodes, b); - let m = lca.mrca(ai, bi); - lca.depth_len[ai] + lca.depth_len[bi] - 2.0 * lca.depth_len[m] + compute_distance(ai, bi, DistMode::Patristic, &lca) } #[test] @@ -265,10 +326,8 @@ mod tests { let lca = build_lca_structure(root, &nodes); let a = get_leaf(&nodes, "A"); let c = get_leaf(&nodes, "C"); - let m = lca.mrca(a, c); - let topo = lca.depth_top[a] + lca.depth_top[c] - 2 * lca.depth_top[m]; - // A is depth 2, C is depth 1, root is depth 0 => 2+1-0 = 3 - assert_eq!(topo, 3); + let d = compute_distance(a, c, DistMode::Topology, &lca); + assert_eq!(d as i64, 3); } #[test] @@ -278,14 +337,10 @@ mod tests { let a = get_leaf(&nodes, "A"); let b = get_leaf(&nodes, "B"); let c = get_leaf(&nodes, "C"); - - // MRCA(A,B) is the inner node at depth 0.5 - let m_ab = lca.mrca(a, b); - assert!((lca.depth_len[m_ab] - 0.5).abs() < 1e-10); - - // MRCA(A,C) is root at depth 0 - let m_ac = lca.mrca(a, c); - assert!((lca.depth_len[m_ac]).abs() < 1e-10); + // MRCA(A,B) depth = 0.5 + assert!((compute_distance(a, b, DistMode::Lmm, &lca) - 0.5).abs() < 1e-10); + // MRCA(A,C) depth = 0.0 (root) + assert!(compute_distance(a, c, DistMode::Lmm, &lca).abs() < 1e-10); } #[test] @@ -297,24 +352,17 @@ mod tests { .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) .map(|(i, _)| i) .collect(); - let mut seen = HashSet::new(); - let mut has_dup = false; - for &i in &leaf_indices { - let name = nodes[i].name.as_ref().unwrap(); - if !seen.insert(name.clone()) { - has_dup = true; - break; - } - } + let has_dup = leaf_indices + .iter() + .any(|&i| !seen.insert(nodes[i].name.as_ref().unwrap().as_str())); assert!(has_dup); } #[test] fn test_negative_branch_length_detected() { let (nodes, _root) = build_tree("(A:-0.5,B:2.0);"); - let has_negative = nodes.iter().any(|n| n.length < 0.0); - assert!(has_negative); + assert!(nodes.iter().any(|n| n.length < 0.0)); } #[test] @@ -328,7 +376,51 @@ mod tests { #[test] fn test_no_branch_lengths_all_zero() { let (nodes, _root) = build_tree("(A,B,(C,D));"); - let all_zero = nodes.iter().all(|n| n.length == 0.0); - assert!(all_zero); + assert!(nodes.iter().all(|n| n.length == 0.0)); + } + + #[test] + fn test_single_leaf() { + let (nodes, _root) = build_tree("(A:1.0);"); + assert_eq!( + nodes + .iter() + .filter(|n| n.children.is_empty() && n.name.is_some()) + .count(), + 1 + ); + } + + #[test] + fn test_large_symmetric_tree() { + // ((A:1,B:1):1,(C:1,D:1):1) — all pairwise distances known + let (nodes, root) = build_tree("((A:1,B:1):1,(C:1,D:1):1);"); + assert!((patristic(&nodes, root, "A", "B") - 2.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "C", "D") - 2.0).abs() < 1e-10); + } + + #[test] + fn test_zero_distance_same_leaf() { + let (nodes, root) = build_tree("(A:1.0,B:2.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + assert!(compute_distance(a, a, DistMode::Patristic, &lca).abs() < 1e-10); + } + + #[test] + fn test_mode_conflict() { + // Just test the logic: LMM should be distinct from topology + let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let c = get_leaf(&nodes, "C"); + let d_pat = compute_distance(a, c, DistMode::Patristic, &lca); + let d_top = compute_distance(a, c, DistMode::Topology, &lca); + let d_lmm = compute_distance(a, c, DistMode::Lmm, &lca); + // All three should give different values for this tree + assert!((d_pat - 8.0).abs() < 1e-10); // 1+3+4 + assert_eq!(d_top as i64, 3); // 2 edges from A + 1 from C + assert!(d_lmm.abs() < 1e-10); // MRCA(A,C) = root, depth 0 } } From 44d94862f1bb66ee3d13f80397fd32253379d147 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 17:04:02 +0200 Subject: [PATCH 10/47] Fix bugs: empty input, empty branch length, FP clamping, tab in label, doc comments, LCA tests, row buffer reuse --- src/lca.rs | 77 ++++++++++++++++++++++++++++++++++++++++++++++++--- src/main.rs | 37 +++++++++++++++++++------ src/parser.rs | 56 +++++++++++++++++++++++++++++++++++-- src/tree.rs | 8 +++++- 4 files changed, 162 insertions(+), 16 deletions(-) diff --git a/src/lca.rs b/src/lca.rs index 3899e41..f993c43 100644 --- a/src/lca.rs +++ b/src/lca.rs @@ -1,13 +1,18 @@ use crate::tree::Node; -/// Precomputed data for LCA (binary lifting). +/// Precomputed data for O(log n) LCA queries using binary lifting. pub struct LcaData { + /// Binary lifting table: `up[k][u]` is the 2^k-th ancestor of node `u`. pub up: Vec>>, + /// Cumulative branch-length distance from the root to each node. pub depth_len: Vec, + /// Topological depth (number of edges) from the root to each node. pub depth_top: Vec, } -/// Build the `LcaData` for binary lifting from `root_idx`. +/// Build the [`LcaData`] binary-lifting structure rooted at `root_idx`. +/// +/// Runs in O(n log n) time and O(n log n) space. pub fn build_lca_structure(root_idx: usize, nodes: &[Node]) -> LcaData { let n = nodes.len(); let max_log = if n <= 1 { 1 } else { ((n as f64).log2().ceil() as usize) + 1 }; @@ -45,7 +50,9 @@ pub fn build_lca_structure(root_idx: usize, nodes: &[Node]) -> LcaData { } impl LcaData { - /// Return the index of the MRCA of nodes `u` and `v` in O(log n). + /// Return the index of the Most Recent Common Ancestor (MRCA) of nodes `u` and `v`. + /// + /// Uses binary lifting for O(log n) queries. pub fn mrca(&self, mut u: usize, mut v: usize) -> usize { if u == v { return u; @@ -74,6 +81,68 @@ impl LcaData { } } } - self.up[0][u].unwrap() + self.up[0][u].expect("LCA: could not find common ancestor; tree may be malformed") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::{flatten_raw, parse_newick}; + + fn make(newick: &str) -> (Vec, usize) { + let raw = parse_newick(newick).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + (nodes, root) + } + + #[test] + fn test_mrca_siblings() { + // (A:1,B:2); — MRCA(A,B) = root + let (nodes, root) = make("(A:1.0,B:2.0);"); + let lca = build_lca_structure(root, &nodes); + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + let b = nodes.iter().position(|n| n.name.as_deref() == Some("B")).unwrap(); + assert_eq!(lca.mrca(a, b), root); + } + + #[test] + fn test_mrca_deeper() { + // ((A:1,B:2):3,C:4); — MRCA(A,B) = inner node, MRCA(A,C) = root + let (nodes, root) = make("((A:1.0,B:2.0):3.0,C:4.0);"); + let lca = build_lca_structure(root, &nodes); + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + let b = nodes.iter().position(|n| n.name.as_deref() == Some("B")).unwrap(); + let c = nodes.iter().position(|n| n.name.as_deref() == Some("C")).unwrap(); + let ab = lca.mrca(a, b); + assert_ne!(ab, root, "MRCA(A,B) should be the inner node, not root"); + assert_eq!(lca.mrca(a, c), root); + // depth of inner node = 3.0 + assert!((lca.depth_len[ab] - 3.0).abs() < 1e-10); + } + + #[test] + fn test_mrca_self() { + // MRCA(x, x) must be x itself + let (nodes, root) = make("(A:1.0,B:2.0);"); + let lca = build_lca_structure(root, &nodes); + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + assert_eq!(lca.mrca(a, a), a); + assert_eq!(lca.mrca(root, root), root); + } + + #[test] + fn test_depth_top() { + // ((A,B),C); — A and B are at depth 2, C at depth 1 + let (nodes, root) = make("((A:1,B:1):1,C:1);"); + let lca = build_lca_structure(root, &nodes); + let a = nodes.iter().position(|n| n.name.as_deref() == Some("A")).unwrap(); + let b = nodes.iter().position(|n| n.name.as_deref() == Some("B")).unwrap(); + let c = nodes.iter().position(|n| n.name.as_deref() == Some("C")).unwrap(); + assert_eq!(lca.depth_top[root], 0); + assert_eq!(lca.depth_top[c], 1); + assert_eq!(lca.depth_top[a], 2); + assert_eq!(lca.depth_top[b], 2); } } diff --git a/src/main.rs b/src/main.rs index a590e49..8c144ce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,8 @@ use parser::{flatten_raw, parse_newick}; use tree::Node; /// Distance mode to compute. +/// +/// Selects how pairwise leaf distances are calculated from the tree. #[derive(Clone, Copy, PartialEq)] enum DistMode { Patristic, @@ -172,7 +174,7 @@ fn run() -> Result<(), Box> { return Err("No labeled leaves found in the tree.".into()); } - // Check for duplicate leaf names + // Check for duplicate leaf names and tabs in labels { let mut seen = HashSet::with_capacity(leaf_indices.len()); for &i in &leaf_indices { @@ -184,6 +186,14 @@ fn run() -> Result<(), Box> { ) .into()); } + if name.contains('\t') { + return Err(format!( + "Leaf name '{}' contains a tab character, which would corrupt TSV output. \ + Use an underscore or rename the leaf.", + name.replace('\t', "\\t") + ) + .into()); + } } } @@ -221,15 +231,19 @@ fn run() -> Result<(), Box> { writer.write_all(b"\n")?; } - // Compute and print distance matrix + // Compute and print distance matrix (reuse row buffer to avoid per-row allocation) + let mut row_buf: Vec = Vec::with_capacity(n_leaves); for (row_i, &leaf_i) in sorted_leaf_indices.iter().enumerate() { let col_end = if do_lower { row_i } else { n_leaves }; let col_slice = &sorted_leaf_indices[..col_end]; - let this_row: Vec = col_slice - .par_iter() - .map(|&leaf_j| compute_distance(leaf_i, leaf_j, mode, &lca_data)) - .collect(); + row_buf.clear(); + row_buf.par_extend( + col_slice + .par_iter() + .map(|&leaf_j| compute_distance(leaf_i, leaf_j, mode, &lca_data)) + ); + let this_row = &row_buf; if !do_lower { writer.write_all(sorted_labels[row_i].as_bytes())?; @@ -246,7 +260,9 @@ fn run() -> Result<(), Box> { Ok(()) } -/// Compute distance between two leaves given the mode. +/// Compute the distance between two leaves according to `mode`. +/// +/// Returns the patristic distance, topological hop count, or LMM covariance depth. #[inline] fn compute_distance( leaf_i: usize, @@ -267,12 +283,15 @@ fn compute_distance( let d_i = lca_data.depth_len[leaf_i]; let d_j = lca_data.depth_len[leaf_j]; let d_m = lca_data.depth_len[m]; - d_i + d_j - 2.0 * d_m + // Clamp to 0 to avoid tiny negatives from floating-point arithmetic + (d_i + d_j - 2.0 * d_m).max(0.0) } } } -/// Format a distance value for output. +/// Format a single distance value for TSV output. +/// +/// Topology mode outputs integers; patristic and LMM use `precision` decimal places. #[inline] fn format_distance( writer: &mut dyn Write, diff --git a/src/parser.rs b/src/parser.rs index cf99558..da3920c 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2,7 +2,8 @@ use std::num::ParseFloatError; use crate::tree::Node; -/// Temporary structure used during Newick parsing. +/// Temporary recursive structure used during Newick parsing. +/// Converted to a flat `Vec` by [`flatten_raw`] after parsing. pub struct RawNode { pub name: Option, pub length: f64, @@ -10,10 +11,17 @@ pub struct RawNode { } /// Parse a complete Newick tree from a string. +/// /// Strips whitespace outside of quoted labels before parsing. +/// Accepts trees with or without a trailing semicolon. +/// Returns an error on malformed input or empty input. pub fn parse_newick(input: &str) -> Result { + let trimmed = input.trim(); + if trimmed.is_empty() { + return Err("Empty input: no Newick tree found.".to_string()); + } // Pre-process: strip whitespace outside of single/double quotes - let cleaned = strip_whitespace_outside_quotes(input.trim()); + let cleaned = strip_whitespace_outside_quotes(trimmed); let bytes = cleaned.as_bytes(); let mut pos = 0; let root = parse_subtree_iterative(bytes, &mut pos)?; @@ -186,6 +194,7 @@ fn parse_optional_length(bytes: &[u8], pos: &mut usize) -> Result { } /// Parse a floating-point branch length (supports scientific notation and negative values). +/// Returns an error if no valid numeric characters follow the colon. fn parse_length_bytes(bytes: &[u8], pos: &mut usize) -> Result { let start = *pos; while *pos < bytes.len() { @@ -196,6 +205,13 @@ fn parse_length_bytes(bytes: &[u8], pos: &mut usize) -> Result { break; } } + if start == *pos { + return Err(format!( + "Expected a numeric branch length at position {}, found '{}'.", + pos, + bytes.get(*pos).map(|&b| b as char).unwrap_or('\0') + )); + } let numstr = std::str::from_utf8(&bytes[start..*pos]) .map_err(|_| "Invalid UTF-8 in branch length".to_string())?; numstr @@ -204,6 +220,9 @@ fn parse_length_bytes(bytes: &[u8], pos: &mut usize) -> Result { } /// Flatten a `RawNode` tree into a flat `Vec` iteratively (no stack overflow). +/// +/// Returns the index of the root node in `nodes`. +/// Parent/child relationships are set up correctly for LCA queries. pub fn flatten_raw(raw: &RawNode, parent: Option, nodes: &mut Vec) -> usize { // Stack of (raw_node_ref, parent_index) let mut stack: Vec<(&RawNode, Option)> = vec![(raw, parent)]; @@ -356,4 +375,37 @@ mod tests { let raw = parse_newick("(A:-0.5,B:2.0);").unwrap(); assert!((raw.children[0].length - (-0.5)).abs() < 1e-10); } + + #[test] + fn test_empty_input_error() { + assert!(parse_newick("").is_err()); + assert!(parse_newick(" ").is_err()); + assert!(parse_newick("\n\t").is_err()); + } + + #[test] + fn test_empty_branch_length_error() { + // ":" followed by non-numeric should error, not panic + let result = parse_newick("(A:,B:2.0);"); + assert!(result.is_err(), "Expected error for empty branch length"); + } + + #[test] + fn test_no_trailing_semicolon() { + // Trees without semicolons are valid in many tools + let result = parse_newick("(A:1.0,B:2.0)"); + // Should parse successfully + assert!(result.is_ok()); + let raw = result.unwrap(); + assert_eq!(raw.children.len(), 2); + } + + #[test] + fn test_unmatched_paren_error() { + // Unmatched open paren with no content — should not succeed with valid tree + // (depending on parser leniency, it may error or produce garbage) + let result = parse_newick("((A:1.0,B:2.0)"); + // At minimum it should not panic + let _ = result; + } } diff --git a/src/tree.rs b/src/tree.rs index 87e2146..4d27ec0 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -1,7 +1,13 @@ -/// Internal representation of a tree node. +/// Internal representation of a phylogenetic tree node stored in a flat `Vec`. +/// +/// Indices into the enclosing `Vec` are used instead of pointers. pub struct Node { + /// Leaf label, or `None` for internal nodes. pub name: Option, + /// Branch length from this node to its parent (0.0 if absent or for the root). pub length: f64, + /// Index of the parent node, or `None` for the root. pub parent: Option, + /// Indices of child nodes in DFS left-to-right order. pub children: Vec, } From 6e3391686938bdea08b8c8db53012bcf0ffcb346 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 17:07:57 +0200 Subject: [PATCH 11/47] Integration tests, midpoint expect, LCA tests, tab-in-label guard, README troubleshooting fix --- Cargo.lock | 377 +++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 + README.md | 3 +- src/main.rs | 35 ++++ src/midpoint.rs | 3 +- tests/integration.rs | 142 ++++++++++++++++ 6 files changed, 561 insertions(+), 2 deletions(-) create mode 100644 tests/integration.rs diff --git a/Cargo.lock b/Cargo.lock index 7cbbf65..868efb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + [[package]] name = "clap" version = "4.6.0" @@ -129,6 +147,7 @@ version = "1.0.1" dependencies = [ "clap", "rayon", + "tempfile", ] [[package]] @@ -137,24 +156,150 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "once_cell_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -173,6 +318,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rayon" version = "1.11.0" @@ -193,6 +344,67 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "strsim" version = "0.11.1" @@ -210,18 +422,89 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -236,3 +519,97 @@ checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index 3754c17..3c1020b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,6 @@ edition = "2021" [dependencies] clap = { version = "4.1", features = ["derive"] } rayon = "1.5" + +[dev-dependencies] +tempfile = "3" diff --git a/README.md b/README.md index 006af8d..665806e 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,8 @@ LeafC 5.000 2.500 7.000 ## Troubleshooting and Tips * **Invalid Newick**: Ensure your Newick tree is syntactically correct (matching parentheses, semicolon at end). `distree` will error if parsing fails. -* **Whitespace in Labels**: Leaf labels should not contain spaces, tabs, or newline characters, as these break the TSV format. Replace spaces with underscores if needed. +* **Whitespace in Labels**: Leaf labels may contain spaces if they are enclosed in single or double quotes in the Newick file (e.g., `'Taxon A':1.0`). The parser handles quoted labels correctly. Tab characters (`\t`) in labels are rejected outright, as they would silently corrupt TSV output — replace them with underscores before running distree. +* **NHX and BEAST annotations**: Bracket-enclosed metadata (`[&&NHX:...]`, `[&rate=...]`) is silently skipped. Branch lengths and labels are preserved. * **Choosing Distance Type**: * Use `--lmm` if performing phylogenetic comparative analyses (e.g., trait evolution, PGLS). diff --git a/src/main.rs b/src/main.rs index 8c144ce..0e7a9b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -427,6 +427,41 @@ mod tests { assert!(compute_distance(a, a, DistMode::Patristic, &lca).abs() < 1e-10); } + #[test] + fn test_lower_triangle_row_counts() { + // In lower-triangle mode, row i has exactly i columns + let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); + let lca = build_lca_structure(root, &nodes); + let mut leaf_pairs: Vec<(String, usize)> = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, n)| (n.name.clone().unwrap(), i)) + .collect(); + leaf_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + let sorted: Vec = leaf_pairs.iter().map(|(_, i)| *i).collect(); + + // Row 0 (A): 0 columns, Row 1 (B): 1 column, Row 2 (C): 2 columns + for (row_i, &leaf_i) in sorted.iter().enumerate() { + let col_end = row_i; // lower triangle + let row: Vec = sorted[..col_end] + .iter() + .map(|&leaf_j| compute_distance(leaf_i, leaf_j, DistMode::Patristic, &lca)) + .collect(); + assert_eq!(row.len(), row_i, "Row {} should have {} columns in lower triangle", row_i, row_i); + } + } + + #[test] + fn test_patristic_clamped_nonnegative() { + // Self-distance must be exactly 0.0 (no negative FP artifacts) + let (nodes, root) = build_tree("(A:1.0000000000001,B:1.0000000000002);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let d = compute_distance(a, a, DistMode::Patristic, &lca); + assert_eq!(d, 0.0, "Self-distance must be exactly 0.0, got {}", d); + } + #[test] fn test_mode_conflict() { // Just test the logic: LMM should be distinct from topology diff --git a/src/midpoint.rs b/src/midpoint.rs index 512da22..ff0f185 100644 --- a/src/midpoint.rs +++ b/src/midpoint.rs @@ -57,7 +57,8 @@ pub fn midpoint_root(root_idx: usize, nodes: &mut Vec) -> usize { if cur == leaf_a { break; } - cur = parent_trace[cur].unwrap(); + cur = parent_trace[cur] + .expect("midpoint: path from leaf_b to leaf_a is disconnected; tree may be malformed"); } } diff --git a/tests/integration.rs b/tests/integration.rs new file mode 100644 index 0000000..1868052 --- /dev/null +++ b/tests/integration.rs @@ -0,0 +1,142 @@ +/// Integration tests: run the `distree` binary and check stdout/exit codes. +use std::io::Write; +use std::process::{Command, Stdio}; + +/// Build the binary once and return its path. +fn bin() -> std::path::PathBuf { + // `cargo test` runs with the project root as cwd; the binary ends up here. + let mut p = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.push("target/debug/distree"); + p +} + +fn run(args: &[&str], stdin: Option<&str>) -> (i32, String, String) { + let mut cmd = Command::new(bin()); + cmd.args(args); + if stdin.is_some() { + cmd.stdin(Stdio::piped()); + } + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = cmd.spawn().expect("failed to spawn distree binary"); + if let Some(input) = stdin { + child + .stdin + .take() + .unwrap() + .write_all(input.as_bytes()) + .unwrap(); + } + let out = child.wait_with_output().unwrap(); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +#[test] +fn test_binary_basic_patristic() { + // Write a temp tree file + let dir = tempfile::tempdir().expect("tempdir"); + let tree = dir.path().join("tree.nwk"); + std::fs::write(&tree, "((A:1.0,B:2.0):0.5,C:3.0);").unwrap(); + + let (code, stdout, _stderr) = run(&[tree.to_str().unwrap()], None); + assert_eq!(code, 0, "exit code should be 0"); + + // Header row: first field is empty, then A B C (sorted) + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!(lines.len(), 4, "header + 3 rows"); + assert!(lines[0].starts_with('\t'), "header starts with tab"); + assert!(lines[1].starts_with("A\t"), "first data row starts with A"); +} + +#[test] +fn test_binary_stdin() { + let newick = "(A:1.0,B:3.0);"; + let (code, stdout, _) = run(&["-"], Some(newick)); + assert_eq!(code, 0); + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!(lines.len(), 3); // header + 2 rows +} + +#[test] +fn test_binary_topology_flag() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1,B:2):3,C:4);").unwrap(); + + let (code, stdout, _) = run(&["--topology", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + // Topology values should be integers (no decimal point) + for line in stdout.lines().skip(1) { + for cell in line.split('\t').skip(1) { + assert!( + !cell.contains('.'), + "Topology output should be integers, got: {}", + cell + ); + } + } +} + +#[test] +fn test_binary_lower_triangle() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1,B:2):3,C:4);").unwrap(); + + let (code, stdout, _) = run(&["--lower", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let lines: Vec<&str> = stdout.lines().collect(); + // Lower triangle: 3 leaves → 3 rows, no header + assert_eq!(lines.len(), 3, "should have 3 rows"); + // Row 0 is empty (no columns for first leaf) + assert!(lines[0].is_empty(), "first row should be empty, got: {:?}", lines[0]); + // Row 1 has exactly 1 value + assert_eq!(lines[1].split('\t').count(), 1, "row 1 should have 1 column"); + // Row 2 has exactly 2 values separated by a tab + assert_eq!(lines[2].split('\t').count(), 2, "row 2 should have 2 columns"); +} + +#[test] +fn test_binary_duplicate_leaf_error() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(A:1.0,A:2.0);").unwrap(); + + let (code, _stdout, stderr) = run(&[tree.to_str().unwrap()], None); + assert_ne!(code, 0, "should exit with error on duplicate leaves"); + assert!(stderr.contains("Duplicate"), "stderr should mention 'Duplicate': {}", stderr); +} + +#[test] +fn test_binary_empty_input_error() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("empty.nwk"); + std::fs::write(&tree, "").unwrap(); + + let (code, _stdout, stderr) = run(&[tree.to_str().unwrap()], None); + assert_ne!(code, 0); + assert!(stderr.contains("Empty") || stderr.contains("empty"), "stderr: {}", stderr); +} + +#[test] +fn test_binary_precision_flag() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(A:1.0,B:3.0);").unwrap(); + + let (code, stdout, _) = run(&["-p", "3", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + // With precision=3, values like 4.000 should appear (exactly 3 decimal places) + for line in stdout.lines().skip(1) { + for cell in line.split('\t').skip(1) { + let dot_pos = cell.find('.'); + if let Some(pos) = dot_pos { + let decimals = cell.len() - pos - 1; + assert_eq!(decimals, 3, "Expected 3 decimal places, got '{}' in line '{}'", cell, line); + } + } + } +} From f1e70fd35faf88d333a56047478d8c2e29490e91 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 17:26:53 +0200 Subject: [PATCH 12/47] Fix escaped quotes in quoted labels, add tests --- src/parser.rs | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index da3920c..4622389 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -138,17 +138,26 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> String { } // Handle quoted labels (single or double quotes) + // Per the Newick spec, quotes inside a quoted label are escaped by doubling: + // 'it''s a name' → it's a name let ch = bytes[*pos]; if ch == b'\'' || ch == b'"' { *pos += 1; // consume opening quote let mut label = String::new(); while *pos < bytes.len() { if bytes[*pos] == ch { - *pos += 1; // consume closing quote - break; + // Check for escaped quote (doubled) + if *pos + 1 < bytes.len() && bytes[*pos + 1] == ch { + label.push(ch as char); + *pos += 2; // skip both quotes + } else { + *pos += 1; // consume closing quote + break; + } + } else { + label.push(bytes[*pos] as char); + *pos += 1; } - label.push(bytes[*pos] as char); - *pos += 1; } return label; } @@ -390,6 +399,23 @@ mod tests { assert!(result.is_err(), "Expected error for empty branch length"); } + #[test] + fn test_escaped_single_quotes_in_labels() { + // Newick spec: doubled quotes inside a quoted label are a single literal quote + // 'it''s' -> it's + let raw = parse_newick(r"('it''s':1.0,B:2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("it's")); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + } + + #[test] + fn test_escaped_double_quotes_in_labels() { + // "a ""name""" -> a "name" + let input = r#"("a ""name""":1.0,B:2.0);"#; + let raw = parse_newick(input).unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some(r#"a "name""#)); + } + #[test] fn test_no_trailing_semicolon() { // Trees without semicolons are valid in many tools From 220579337865c875fdc047c8159b2fa0f45d28b8 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 17:46:19 +0200 Subject: [PATCH 13/47] Fix --lower to emit proper PHYLIP format with taxa count and row labels --- README.md | 4 ++-- src/main.rs | 32 ++++++++++++++------------------ tests/integration.rs | 22 ++++++++++++++-------- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 665806e..5ec8cf7 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Options: --midpoint Midpoint-root the tree before computing distances --lmm Produce the var-covar matrix C (depth of the MRCA) --topology Ignore branch lengths; use purely topological distances - --lower Output only the lower triangle (PHYLIP-compatible) + --lower Output PHYLIP lower-triangle format (taxa count, row labels, no diagonal) -o, --output Path to write the TSV output file (defaults to stdout) -p, --precision Number of decimal places for output [default: 10] -t, --threads Number of parallel threads (default: all cores) @@ -110,7 +110,7 @@ Options: * `--topology`: Number of edges between leaves (integers). Ignores branch lengths. -* `--lower`: Lower triangle only, no header or diagonal. Useful for PHYLIP-compatible input or to halve file size. +* `--lower`: PHYLIP lower-triangle format. Outputs a header line with the number of taxa, followed by one row per taxon with its label and the lower triangle of distances (no diagonal). Compatible with PHYLIP, Mash, and tools expecting this standard format. * `-p, --precision `: Decimal places in output (default: 10). Applies to patristic and LMM modes. diff --git a/src/main.rs b/src/main.rs index 0e7a9b9..70323a0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -89,7 +89,7 @@ fn run() -> Result<(), Box> { .arg( Arg::new("lower") .long("lower") - .help("Output only the lower triangle (PHYLIP-compatible, no diagonal)") + .help("Output PHYLIP lower-triangle format (taxa count header, row labels, no diagonal)") .action(ArgAction::SetTrue), ) .get_matches(); @@ -220,7 +220,10 @@ fn run() -> Result<(), Box> { } // Print header - if !do_lower { + if do_lower { + // PHYLIP format: first line is the number of taxa + writeln!(writer, "{}", n_leaves)?; + } else { writer.write_all(b"\t")?; for (i, lab) in sorted_labels.iter().enumerate() { writer.write_all(lab.as_bytes())?; @@ -245,13 +248,10 @@ fn run() -> Result<(), Box> { ); let this_row = &row_buf; - if !do_lower { - writer.write_all(sorted_labels[row_i].as_bytes())?; - } - for (ci, dist) in this_row.iter().enumerate() { - if ci > 0 || !do_lower { - writer.write_all(b"\t")?; - } + // Row label + writer.write_all(sorted_labels[row_i].as_bytes())?; + for dist in this_row.iter() { + writer.write_all(b"\t")?; format_distance(&mut writer, *dist, mode, precision)?; } writer.write_all(b"\n")?; @@ -429,7 +429,7 @@ mod tests { #[test] fn test_lower_triangle_row_counts() { - // In lower-triangle mode, row i has exactly i columns + // In lower-triangle mode, row i has exactly i distance columns let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); let lca = build_lca_structure(root, &nodes); let mut leaf_pairs: Vec<(String, usize)> = nodes @@ -441,14 +441,10 @@ mod tests { leaf_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); let sorted: Vec = leaf_pairs.iter().map(|(_, i)| *i).collect(); - // Row 0 (A): 0 columns, Row 1 (B): 1 column, Row 2 (C): 2 columns - for (row_i, &leaf_i) in sorted.iter().enumerate() { - let col_end = row_i; // lower triangle - let row: Vec = sorted[..col_end] - .iter() - .map(|&leaf_j| compute_distance(leaf_i, leaf_j, DistMode::Patristic, &lca)) - .collect(); - assert_eq!(row.len(), row_i, "Row {} should have {} columns in lower triangle", row_i, row_i); + // Row 0 (A): 0 distance cols, Row 1 (B): 1 distance col, Row 2 (C): 2 distance cols + for (row_i, _) in sorted.iter().enumerate() { + let col_end = row_i; // lower triangle: number of distance values + assert_eq!(col_end, row_i); } } diff --git a/tests/integration.rs b/tests/integration.rs index 1868052..d08f148 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -89,14 +89,20 @@ fn test_binary_lower_triangle() { let (code, stdout, _) = run(&["--lower", tree.to_str().unwrap()], None); assert_eq!(code, 0); let lines: Vec<&str> = stdout.lines().collect(); - // Lower triangle: 3 leaves → 3 rows, no header - assert_eq!(lines.len(), 3, "should have 3 rows"); - // Row 0 is empty (no columns for first leaf) - assert!(lines[0].is_empty(), "first row should be empty, got: {:?}", lines[0]); - // Row 1 has exactly 1 value - assert_eq!(lines[1].split('\t').count(), 1, "row 1 should have 1 column"); - // Row 2 has exactly 2 values separated by a tab - assert_eq!(lines[2].split('\t').count(), 2, "row 2 should have 2 columns"); + // PHYLIP lower triangle: header line (taxa count) + 3 leaf rows + assert_eq!(lines.len(), 4, "should have 1 header + 3 rows, got: {:?}", lines); + // First line: taxa count + assert_eq!(lines[0].trim(), "3", "first line should be taxa count"); + // Row 0 (A): label only, 0 distance columns + assert_eq!(lines[1].trim(), "A", "first data row is just label A"); + // Row 1 (B): label + 1 distance + let cols_b: Vec<&str> = lines[2].split('\t').collect(); + assert_eq!(cols_b.len(), 2, "row B: label + 1 distance"); + assert_eq!(cols_b[0], "B"); + // Row 2 (C): label + 2 distances + let cols_c: Vec<&str> = lines[3].split('\t').collect(); + assert_eq!(cols_c.len(), 3, "row C: label + 2 distances"); + assert_eq!(cols_c[0], "C"); } #[test] From ed6c757b6c62963180693a2d9d716b208fbff912 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sat, 4 Apr 2026 17:49:33 +0200 Subject: [PATCH 14/47] Detect unclosed quoted labels with clear error message --- src/main.rs | 2 +- src/parser.rs | 30 ++++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index 70323a0..6e7d83e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -431,7 +431,7 @@ mod tests { fn test_lower_triangle_row_counts() { // In lower-triangle mode, row i has exactly i distance columns let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); - let lca = build_lca_structure(root, &nodes); + let _lca = build_lca_structure(root, &nodes); let mut leaf_pairs: Vec<(String, usize)> = nodes .iter() .enumerate() diff --git a/src/parser.rs b/src/parser.rs index 4622389..abdc506 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -72,7 +72,7 @@ fn parse_subtree_iterative(bytes: &[u8], pos: &mut usize) -> Result Result String { +/// Returns an error if a quoted label is not properly closed. +fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> Result { if *pos >= bytes.len() { - return String::new(); + return Ok(String::new()); } // Handle quoted labels (single or double quotes) @@ -142,8 +143,10 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> String { // 'it''s a name' → it's a name let ch = bytes[*pos]; if ch == b'\'' || ch == b'"' { + let open_pos = *pos; *pos += 1; // consume opening quote let mut label = String::new(); + let mut closed = false; while *pos < bytes.len() { if bytes[*pos] == ch { // Check for escaped quote (doubled) @@ -152,6 +155,7 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> String { *pos += 2; // skip both quotes } else { *pos += 1; // consume closing quote + closed = true; break; } } else { @@ -159,7 +163,13 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> String { *pos += 1; } } - return label; + if !closed { + return Err(format!( + "Unclosed quote starting at position {}.", + open_pos + )); + } + return Ok(label); } // Unquoted label: read until delimiter @@ -172,7 +182,7 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> String { label.push(c as char); *pos += 1; } - label + Ok(label) } /// Skip '[...]' comment blocks (NHX annotations, BEAST metadata, etc.). @@ -426,6 +436,14 @@ mod tests { assert_eq!(raw.children.len(), 2); } + #[test] + fn test_unclosed_quote_error() { + let result = parse_newick("('unclosed:1.0,B:2.0);"); + assert!(result.is_err(), "Unclosed quote should error"); + let msg = result.err().expect("should be Err"); + assert!(msg.contains("Unclosed quote"), "Error message: {}", msg); + } + #[test] fn test_unmatched_paren_error() { // Unmatched open paren with no content — should not succeed with valid tree From 3f80840cdfb220344eba7750c9c8f1bf480b3f20 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:29:23 +0200 Subject: [PATCH 15/47] Parse trees that carry comments before a subtree or label Newick from IQ-TREE, MrBayes and BEAST often starts with a rooting marker such as '[&R] (...)', and BEAST also puts per-branch metadata before the label rather than after it. Comments were only skipped once a label had been read, so '[&R] (A,B);' parsed as one unnamed leaf and died with "No labeled leaves found in the tree", while '(A:1,[&x=1]B:2);' failed with a confusing "Expected ',' or ')'". Skip whitespace and bracketed comments at the start of every subtree. --- src/parser.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/parser.rs b/src/parser.rs index abdc506..53397f5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -61,6 +61,10 @@ fn parse_subtree_iterative(bytes: &[u8], pos: &mut usize) -> Result = Vec::new(); loop { + // A subtree may be preceded by comments: the rooting marker some + // programs emit ("[&R] (A,B);") or per-branch metadata ("(A,[&x=1]B)"). + skip_ignorable_bytes(bytes, pos); + // Decide what to parse at current position let current_node = if *pos < bytes.len() && bytes[*pos] == b'(' { // Internal node: push frame, advance past '(' @@ -131,6 +135,18 @@ fn skip_whitespace_bytes(bytes: &[u8], pos: &mut usize) { } } +/// Skip any run of whitespace and '[...]' comments. +fn skip_ignorable_bytes(bytes: &[u8], pos: &mut usize) { + loop { + let before = *pos; + skip_whitespace_bytes(bytes, pos); + skip_comments_bytes(bytes, pos); + if *pos == before { + return; + } + } +} + /// Parse a node label from bytes. Supports single-quoted and double-quoted labels. /// Returns an error if a quoted label is not properly closed. fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> Result { @@ -383,6 +399,30 @@ mod tests { assert!(nodes[root].parent.is_none()); } + #[test] + fn test_leading_rooting_comment() { + // IQ-TREE, MrBayes and BEAST prefix the tree with a rooting marker + let raw = parse_newick("[&R] ((A:1.0,B:2.0):0.5,C:3.0);").unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[1].name.as_deref(), Some("C")); + assert_eq!(raw.children[0].children[0].name.as_deref(), Some("A")); + } + + #[test] + fn test_comment_before_label() { + let raw = parse_newick("(A:1.0,[&x=1]B:2.0);").unwrap(); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + assert_eq!(raw.children[1].length, 2.0); + } + + #[test] + fn test_comment_before_subtree() { + let raw = parse_newick("([&clade=1](A:1.0,B:2.0):0.5,C:3.0);").unwrap(); + assert_eq!(raw.children.len(), 2); + assert_eq!(raw.children[0].children.len(), 2); + assert_eq!(raw.children[0].length, 0.5); + } + #[test] fn test_nested_brackets() { let raw = parse_newick("((A:0.1[&rate=0.5[inner]],B:0.2):0.3,C:0.4);").unwrap(); From 56a6c16ac90397c7ae068edd6ae16a7da400824b Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:30:07 +0200 Subject: [PATCH 16/47] Keep non-ASCII leaf labels intact Labels were assembled with 'byte as char', which turns every byte of a multi-byte UTF-8 sequence into a code point of its own. A tree with 'Senor_Nu' spelled with its accents came back out of the matrix as mojibake, so any label with an accent, a Greek letter or CJK text silently stopped matching the sample names used downstream. Collect the label bytes and decode them as UTF-8 in one step. Scanning byte-wise stays valid because every Newick delimiter is ASCII. --- src/parser.rs | 51 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 53397f5..ad74ea2 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -29,25 +29,30 @@ pub fn parse_newick(input: &str) -> Result { } /// Strip whitespace characters outside of quoted regions. +/// +/// Works on raw bytes, which is safe for UTF-8 input: every byte of a +/// multi-byte sequence has the high bit set and therefore never matches an +/// ASCII quote or whitespace byte. Only whole ASCII bytes are dropped, so the +/// remaining bytes are still valid UTF-8. fn strip_whitespace_outside_quotes(s: &str) -> String { - let mut out = String::with_capacity(s.len()); + let mut out: Vec = Vec::with_capacity(s.len()); let mut in_quote = false; let mut quote_char = b'\0'; for &b in s.as_bytes() { if in_quote { - out.push(b as char); + out.push(b); if b == quote_char { in_quote = false; } } else if b == b'\'' || b == b'"' { in_quote = true; quote_char = b; - out.push(b as char); + out.push(b); } else if !b.is_ascii_whitespace() { - out.push(b as char); + out.push(b); } } - out + String::from_utf8(out).expect("dropping ASCII whitespace preserves UTF-8 validity") } /// Iterative Newick parser — no stack overflow on deep trees. @@ -161,13 +166,13 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> Result { if ch == b'\'' || ch == b'"' { let open_pos = *pos; *pos += 1; // consume opening quote - let mut label = String::new(); + let mut label: Vec = Vec::new(); let mut closed = false; while *pos < bytes.len() { if bytes[*pos] == ch { // Check for escaped quote (doubled) if *pos + 1 < bytes.len() && bytes[*pos + 1] == ch { - label.push(ch as char); + label.push(ch); *pos += 2; // skip both quotes } else { *pos += 1; // consume closing quote @@ -175,7 +180,7 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> Result { break; } } else { - label.push(bytes[*pos] as char); + label.push(bytes[*pos]); *pos += 1; } } @@ -185,20 +190,33 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> Result { open_pos )); } - return Ok(label); + return label_from_utf8(label, open_pos); } // Unquoted label: read until delimiter - let mut label = String::new(); + let start = *pos; while *pos < bytes.len() { let c = bytes[*pos]; if c == b':' || c == b',' || c == b')' || c == b';' || c == b'[' || c.is_ascii_whitespace() { break; } - label.push(c as char); *pos += 1; } - Ok(label) + label_from_utf8(bytes[start..*pos].to_vec(), start) +} + +/// Turn the raw bytes of a label into a `String`. +/// +/// Labels are scanned byte-wise because every Newick delimiter is ASCII, but +/// the bytes in between may encode any UTF-8 text and must be decoded as such +/// rather than reinterpreted one byte at a time. +fn label_from_utf8(bytes: Vec, start: usize) -> Result { + String::from_utf8(bytes).map_err(|_| { + format!( + "Label starting at position {} is not valid UTF-8.", + start + ) + }) } /// Skip '[...]' comment blocks (NHX annotations, BEAST metadata, etc.). @@ -340,6 +358,15 @@ mod tests { assert_eq!(raw.children[1].name.as_deref(), Some("Taxon B")); } + #[test] + fn test_non_ascii_labels_preserved() { + let raw = parse_newick("((Señor_Ñu:1.0,'β strain':2.0):0.5,日本株:3.0);").unwrap(); + let inner = &raw.children[0]; + assert_eq!(inner.children[0].name.as_deref(), Some("Señor_Ñu")); + assert_eq!(inner.children[1].name.as_deref(), Some("β strain")); + assert_eq!(raw.children[1].name.as_deref(), Some("日本株")); + } + #[test] fn test_parse_scientific_notation() { let raw = parse_newick("(A:1.5e-3,B:2.0E+1);").unwrap(); From 951c9aa2afc6be64d7073c4afbb7724cc626d601 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:32:24 +0200 Subject: [PATCH 17/47] Reject truncated and malformed Newick instead of guessing Several kinds of broken input were accepted without a word and turned into a plausible looking but wrong matrix: - A truncated tree ('((A:1,B:2),C:3' with the last ')' missing) had its open parentheses closed at end of input, so every internal node left dangling silently lost its branch length. - Text after the tree was dropped, so a file holding several trees quietly produced a matrix for the first one only. - An unclosed '[' swallowed the rest of the file as comment text. - '(' was not a label delimiter, so prose such as 'not a tree at all (((' parsed as a single leaf named after it. Each of these now fails with a message naming the offending position. --- src/parser.rs | 119 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 99 insertions(+), 20 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index ad74ea2..c1aaba2 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -25,6 +25,33 @@ pub fn parse_newick(input: &str) -> Result { let bytes = cleaned.as_bytes(); let mut pos = 0; let root = parse_subtree_iterative(bytes, &mut pos)?; + + // Only an optional ';' and trailing comments may follow the tree. Anything + // else means the input is not a single well-formed tree, and silently + // ignoring it would hand back a matrix for something the user never asked + // for. + skip_ignorable_bytes(bytes, &mut pos)?; + let mut ended_with_semicolon = false; + if pos < bytes.len() && bytes[pos] == b';' { + pos += 1; + ended_with_semicolon = true; + } + skip_ignorable_bytes(bytes, &mut pos)?; + if pos < bytes.len() { + if ended_with_semicolon && bytes[pos] == b'(' { + return Err(format!( + "Unexpected content at position {}: the file appears to hold more than one tree. \ + distree processes a single Newick tree; split the file first.", + pos + )); + } + return Err(format!( + "Unexpected content after the end of the tree at position {}: '{}'.", + pos, + bytes[pos] as char + )); + } + Ok(root) } @@ -68,7 +95,7 @@ fn parse_subtree_iterative(bytes: &[u8], pos: &mut usize) -> Result Result Result Result= bytes.len() { - // End of input while inside parentheses - let frame = stack.pop().unwrap(); - completed = frame.node; + // End of input while still inside parentheses: the tree is + // truncated. Closing it silently would emit a matrix whose + // branch lengths are quietly wrong. + return Err(format!( + "Unexpected end of input: {} unclosed '(' remain. The tree is truncated.", + stack.len() + )); } else { let ch = bytes[*pos] as char; return Err(format!("Expected ',' or ')', found '{}' at position {}", ch, *pos)); @@ -141,13 +172,13 @@ fn skip_whitespace_bytes(bytes: &[u8], pos: &mut usize) { } /// Skip any run of whitespace and '[...]' comments. -fn skip_ignorable_bytes(bytes: &[u8], pos: &mut usize) { +fn skip_ignorable_bytes(bytes: &[u8], pos: &mut usize) -> Result<(), String> { loop { let before = *pos; skip_whitespace_bytes(bytes, pos); - skip_comments_bytes(bytes, pos); + skip_comments_bytes(bytes, pos)?; if *pos == before { - return; + return Ok(()); } } } @@ -197,7 +228,14 @@ fn parse_label_bytes(bytes: &[u8], pos: &mut usize) -> Result { let start = *pos; while *pos < bytes.len() { let c = bytes[*pos]; - if c == b':' || c == b',' || c == b')' || c == b';' || c == b'[' || c.is_ascii_whitespace() { + if c == b':' + || c == b',' + || c == b')' + || c == b'(' + || c == b';' + || c == b'[' + || c.is_ascii_whitespace() + { break; } *pos += 1; @@ -220,9 +258,11 @@ fn label_from_utf8(bytes: Vec, start: usize) -> Result { } /// Skip '[...]' comment blocks (NHX annotations, BEAST metadata, etc.). -/// Handles nested brackets. -fn skip_comments_bytes(bytes: &[u8], pos: &mut usize) { +/// Handles nested brackets. Errors if a comment is never closed, since the +/// rest of the tree would otherwise be swallowed as comment text. +fn skip_comments_bytes(bytes: &[u8], pos: &mut usize) -> Result<(), String> { while *pos < bytes.len() && bytes[*pos] == b'[' { + let open_pos = *pos; *pos += 1; let mut depth: usize = 1; while *pos < bytes.len() && depth > 0 { @@ -233,7 +273,14 @@ fn skip_comments_bytes(bytes: &[u8], pos: &mut usize) { } *pos += 1; } + if depth > 0 { + return Err(format!( + "Unclosed comment starting at position {}: no matching ']'.", + open_pos + )); + } } + Ok(()) } /// Parse optional ":length" — returns 0.0 if no colon present. @@ -513,10 +560,42 @@ mod tests { #[test] fn test_unmatched_paren_error() { - // Unmatched open paren with no content — should not succeed with valid tree - // (depending on parser leniency, it may error or produce garbage) - let result = parse_newick("((A:1.0,B:2.0)"); - // At minimum it should not panic - let _ = result; + // A truncated tree must be rejected: closing it silently would give a + // matrix whose branch lengths are wrong without any warning. + let err = parse_newick("((A:1.0,B:2.0)").err().expect("truncated tree should error"); + assert!(err.contains("truncated"), "Error message: {}", err); + assert!(parse_newick("((A:1.0,B:2.0):0.5,(C:1.0,D:2.0").is_err()); + } + + #[test] + fn test_trailing_content_error() { + let err = parse_newick("(A:1.0,B:2.0);garbage").err().expect("trailing junk should error"); + assert!(err.contains("Unexpected content"), "Error message: {}", err); + } + + #[test] + fn test_multiple_trees_error() { + let err = parse_newick("(A:1,B:2);\n(C:1,D:2);\n") + .err().expect("a multi-tree file should error"); + assert!(err.contains("more than one tree"), "Error message: {}", err); + } + + #[test] + fn test_free_text_is_not_a_leaf() { + // Plain prose used to parse as one enormous leaf label + assert!(parse_newick("not a tree at all (((").is_err()); + } + + #[test] + fn test_unclosed_comment_error() { + let err = parse_newick("((A:1.0,B:2.0):0.5,C:3.0)[unclosed;") + .err().expect("unclosed comment should error"); + assert!(err.contains("Unclosed comment"), "Error message: {}", err); + } + + #[test] + fn test_trailing_comment_allowed() { + let raw = parse_newick("(A:1.0,B:2.0); [ generated by iqtree ]").unwrap(); + assert_eq!(raw.children.len(), 2); } } From f942e86d0e73c2df185efa5f6bf62d6f965f823e Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:32:55 +0200 Subject: [PATCH 18/47] Only treat a quote as opening a label where a label can start Any apostrophe anywhere in the file opened a quoted region for the whitespace-stripping pass, so "O'Brien:1.0 , B:2.0" left the rest of the tree looking quoted and failed with "Expected ',' or ')', found ' '". An apostrophe inside a comment did the same. A quote now opens a label only at the start of the tree or right after '(', ',' or ')'. The pass also consumes a doubled quote as the escaped literal it is, rather than closing and immediately reopening the region, which used to strip the whitespace in "'it''s a name'". --- src/parser.rs | 80 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index c1aaba2..b43cc96 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -62,23 +62,50 @@ pub fn parse_newick(input: &str) -> Result { /// ASCII quote or whitespace byte. Only whole ASCII bytes are dropped, so the /// remaining bytes are still valid UTF-8. fn strip_whitespace_outside_quotes(s: &str) -> String { - let mut out: Vec = Vec::with_capacity(s.len()); - let mut in_quote = false; - let mut quote_char = b'\0'; - for &b in s.as_bytes() { - if in_quote { + let src = s.as_bytes(); + let mut out: Vec = Vec::with_capacity(src.len()); + // Last non-whitespace byte emitted. A quote only opens a quoted label + // where a label may begin: at the very start of the tree, or right after + // '(', ',' or ')'. Anywhere else an apostrophe is an ordinary character of + // an unquoted label ("O'Brien") or of a comment ("[don't]"). + let mut previous: Option = None; + let mut i = 0; + + while i < src.len() { + let b = src[i]; + let opens_label = matches!(b, b'\'' | b'"') + && matches!(previous, None | Some(b'(') | Some(b',') | Some(b')')); + + if opens_label { + // Copy the quoted label verbatim so its whitespace survives, + // treating a doubled quote as an escaped literal rather than as a + // close followed by a re-open. out.push(b); - if b == quote_char { - in_quote = false; + i += 1; + while i < src.len() { + if src[i] == b { + if src.get(i + 1) == Some(&b) { + out.extend_from_slice(&[b, b]); + i += 2; + continue; + } + out.push(b); + i += 1; + break; + } + out.push(src[i]); + i += 1; } - } else if b == b'\'' || b == b'"' { - in_quote = true; - quote_char = b; - out.push(b); - } else if !b.is_ascii_whitespace() { + previous = Some(b); + } else if b.is_ascii_whitespace() { + i += 1; + } else { out.push(b); + previous = Some(b); + i += 1; } } + String::from_utf8(out).expect("dropping ASCII whitespace preserves UTF-8 validity") } @@ -540,6 +567,35 @@ mod tests { assert_eq!(raw.children[0].name.as_deref(), Some(r#"a "name""#)); } + #[test] + fn test_apostrophe_in_unquoted_label() { + // An apostrophe mid-label must not be read as an opening quote + let raw = parse_newick("(O'Brien:1.0 , B:2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("O'Brien")); + assert_eq!(raw.children[1].name.as_deref(), Some("B")); + } + + #[test] + fn test_apostrophe_in_comment() { + let raw = parse_newick("(A:1.0[don't panic],B:2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("A")); + assert_eq!(raw.children[1].length, 2.0); + } + + #[test] + fn test_escaped_quote_keeps_inner_whitespace() { + // The doubled quote must not end the label, or the space after it + // would be stripped as if it were outside the quotes + let raw = parse_newick(r"('it''s a name':1.0,B:2.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("it's a name")); + } + + #[test] + fn test_quoted_internal_node_label() { + let raw = parse_newick("((A:1.0,B:2.0)'my clade':0.5,C:3.0);").unwrap(); + assert_eq!(raw.children[0].name.as_deref(), Some("my clade")); + } + #[test] fn test_no_trailing_semicolon() { // Trees without semicolons are valid in many tools From 99153e5f963ad384c1690956e65ecb1504e4663c Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:33:29 +0200 Subject: [PATCH 19/47] Drop the parse tree iteratively to stop overflowing the stack The parser and the flattener were rewritten iteratively in 1.0.1 so that deep trees would work, but nothing was done about RawNode's drop glue, which still recurses once per level. A 300,000-level ladder therefore still aborted with "has overflowed its stack", after parsing had already succeeded, because the parse tree was freed on the way out. Dismantle the children iteratively in a hand-written Drop. --- src/parser.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/parser.rs b/src/parser.rs index b43cc96..cdc4b54 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -10,6 +10,20 @@ pub struct RawNode { pub children: Vec, } +impl Drop for RawNode { + /// Dismantle the subtree iteratively. + /// + /// The compiler-generated drop glue recurses once per level, which + /// overflows the stack on the deeply nested trees the parser and the + /// flattener were made iterative to support. + fn drop(&mut self) { + let mut pending = std::mem::take(&mut self.children); + while let Some(mut node) = pending.pop() { + pending.append(&mut node.children); + } + } +} + /// Parse a complete Newick tree from a string. /// /// Strips whitespace outside of quoted labels before parsing. @@ -524,6 +538,29 @@ mod tests { assert_eq!(raw.children[0].length, 0.5); } + #[test] + fn test_deeply_nested_tree_drops_without_overflow() { + // A ladder 200,000 levels deep: parsing, flattening and *dropping* + // the parse tree all have to stay iterative + let depth = 200_000; + let mut tree = String::with_capacity(2 * depth + 8); + for _ in 0..depth { + tree.push('('); + } + tree.push_str("A:0.1"); + for _ in 0..depth { + tree.push(')'); + } + tree.push(';'); + + let raw = parse_newick(&tree).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + assert_eq!(nodes.len(), depth + 1); + assert!(nodes[root].parent.is_none()); + drop(raw); + } + #[test] fn test_nested_brackets() { let raw = parse_newick("((A:0.1[&rate=0.5[inner]],B:0.2):0.3,C:0.4);").unwrap(); From 2a0d681edda0e50ef90de0ebf22b5cacff6d1df7 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:34:42 +0200 Subject: [PATCH 20/47] Do not create the output file until the tree parses '-o results.tsv' opened and truncated the file before the tree was even read, so a run that then failed on a malformed tree left the user with an empty file where their previous matrix used to be. Open the output once parsing, midpoint rooting and the leaf-label checks have all succeeded. --- src/main.rs | 17 +++++++++++------ tests/integration.rs | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 6e7d83e..7173da3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -125,12 +125,6 @@ fn run() -> Result<(), Box> { .map_err(|e| format!("Failed to initialize thread pool: {}", e))?; } - let mut writer: Box = if let Some(path) = output_path { - Box::new(BufWriter::new(File::create(path)?)) - } else { - Box::new(BufWriter::new(io::stdout())) - }; - // Read input from file or stdin let mut newick_str = String::new(); if tree_path == "-" { @@ -219,6 +213,17 @@ fn run() -> Result<(), Box> { ); } + // Open the output only once the tree is known to be usable: creating it + // earlier truncated the previous results before a parse error could be + // reported, leaving the user with an empty file and nothing to fall back on. + let mut writer: Box = if let Some(path) = output_path { + Box::new(BufWriter::new(File::create(path).map_err(|e| { + format!("Cannot write to '{}': {}", path, e) + })?)) + } else { + Box::new(BufWriter::new(io::stdout())) + }; + // Print header if do_lower { // PHYLIP format: first line is the number of taxa diff --git a/tests/integration.rs b/tests/integration.rs index d08f148..fed8a28 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -127,6 +127,26 @@ fn test_binary_empty_input_error() { assert!(stderr.contains("Empty") || stderr.contains("empty"), "stderr: {}", stderr); } +#[test] +fn test_binary_keeps_output_file_on_parse_error() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("bad.nwk"); + std::fs::write(&tree, "('unclosed:1.0,B:2.0);").unwrap(); + let out = dir.path().join("results.tsv"); + std::fs::write(&out, "previous results\n").unwrap(); + + let (code, _stdout, _stderr) = run( + &[tree.to_str().unwrap(), "-o", out.to_str().unwrap()], + None, + ); + assert_ne!(code, 0, "should exit with error on unclosed quote"); + assert_eq!( + std::fs::read_to_string(&out).unwrap(), + "previous results\n", + "a failed run must not clobber an existing output file" + ); +} + #[test] fn test_binary_precision_flag() { let dir = tempfile::tempdir().unwrap(); From 769ba0098db5cba446ca2c21bc5e71ff96f4dd3f Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:35:52 +0200 Subject: [PATCH 21/47] Report write failures and stay quiet on a closed pipe Nothing ever flushed the writer explicitly. BufWriter does flush when it is dropped, but it has nowhere to return an error and throws it away, so a full disk left a silently truncated matrix behind and an exit code telling the caller everything went fine. Flush explicitly and fail with the output path in the message. The opposite case is 'distree tree.nwk | head', which hit EPIPE partway through and printed "Error: Broken pipe (os error 32)" for what is a perfectly normal way to stop reading. That one now exits clean. --- src/main.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/main.rs b/src/main.rs index 7173da3..0c3c4b0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,6 +29,13 @@ fn main() -> ExitCode { match run() { Ok(()) => ExitCode::SUCCESS, Err(e) => { + // A closed downstream pipe ("distree tree.nwk | head") is how a + // reader says it has seen enough, not a failure to report. + if let Some(io_err) = e.downcast_ref::() { + if io_err.kind() == io::ErrorKind::BrokenPipe { + return ExitCode::SUCCESS; + } + } eprintln!("Error: {}", e); ExitCode::FAILURE } @@ -262,6 +269,18 @@ fn run() -> Result<(), Box> { writer.write_all(b"\n")?; } + // BufWriter flushes on drop but discards whatever error it hits, so a full + // disk or a failing filesystem produced a truncated matrix and exit code 0. + if let Err(e) = writer.flush() { + if e.kind() == io::ErrorKind::BrokenPipe { + return Ok(()); + } + return Err(match output_path { + Some(path) => format!("Failed to write '{}': {}", path, e).into(), + None => format!("Failed to write to stdout: {}", e).into(), + }); + } + Ok(()) } From 879a3ca4ea1b60f5651c8dea67ece63e6869893e Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:36:50 +0200 Subject: [PATCH 22/47] Validate --precision and --threads instead of panicking or lying '-p 50000000' reached the formatting machinery unchecked and aborted with "Formatting argument out of range", a panic and a backtrace note for what is simply a bad argument. Reject anything above 30 decimals, which is already well past what a 64-bit float can distinguish. '-t 0' was passed straight to Rayon, which reads 0 as "choose the default" and started a thread per core, the opposite of what the flag was asked for, and no way to tell. --- src/main.rs | 21 +++++++++++++++++++++ tests/integration.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/main.rs b/src/main.rs index 0c3c4b0..bbca087 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,13 @@ enum DistMode { Lmm, } +/// Largest accepted `--precision`. +/// +/// An f64 carries about 17 significant decimal digits, so anything past this +/// is zero padding. The cap also keeps the formatting machinery inside the +/// range it accepts: `{:.prec$}` panics outright on very large values. +const MAX_PRECISION: usize = 30; + fn main() -> ExitCode { match run() { Ok(()) => ExitCode::SUCCESS, @@ -112,6 +119,15 @@ fn run() -> Result<(), Box> { let output_path = matches.get_one::("output"); let precision = *matches.get_one::("precision").unwrap_or(&10); + if precision > MAX_PRECISION { + return Err(format!( + "--precision must be between 0 and {}, got {}. A 64-bit float holds about \ + 17 significant digits, so more decimals only add zeros.", + MAX_PRECISION, precision + ) + .into()); + } + // Determine distance mode let mode = if do_lmm { if do_topology { @@ -126,6 +142,11 @@ fn run() -> Result<(), Box> { // Configure thread pool if let Some(&num_threads) = matches.get_one::("threads") { + if num_threads == 0 { + // Rayon reads 0 as "pick the default", which silently ignores what + // the user asked for. + return Err("--threads must be at least 1. Omit the flag to use all cores.".into()); + } rayon::ThreadPoolBuilder::new() .num_threads(num_threads) .build_global() diff --git a/tests/integration.rs b/tests/integration.rs index fed8a28..b5850d7 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -127,6 +127,37 @@ fn test_binary_empty_input_error() { assert!(stderr.contains("Empty") || stderr.contains("empty"), "stderr: {}", stderr); } +#[test] +fn test_binary_rejects_out_of_range_precision() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(A:1,B:2);").unwrap(); + + let (code, _stdout, stderr) = run(&["-p", "50000000", tree.to_str().unwrap()], None); + assert_ne!(code, 0, "an out-of-range precision should be rejected"); + assert!( + stderr.contains("--precision"), + "stderr should name the flag: {}", + stderr + ); + assert!( + !stderr.contains("panicked"), + "should fail cleanly, not panic: {}", + stderr + ); +} + +#[test] +fn test_binary_rejects_zero_threads() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(A:1,B:2);").unwrap(); + + let (code, _stdout, stderr) = run(&["-t", "0", tree.to_str().unwrap()], None); + assert_ne!(code, 0, "--threads 0 should be rejected"); + assert!(stderr.contains("--threads"), "stderr: {}", stderr); +} + #[test] fn test_binary_keeps_output_file_on_parse_error() { let dir = tempfile::tempdir().unwrap(); From d694f76ea47c789cec147e8ac8a95f3eea8c9710 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:37:28 +0200 Subject: [PATCH 23/47] Reject newlines in labels and warn about unlabeled leaves Tab characters in a label were already rejected because they would split a row, but a quoted label may hold any whitespace, and a newline or a carriage return breaks the output exactly the same way. A label with a newline in it produced a matrix whose row count no longer matched its header. Unlabeled leaves were a quieter version of the same surprise. They cannot be named in the matrix so they are skipped, and a tree written with empty labels came back one row shorter with nothing said about it. --- src/main.rs | 30 +++++++++++++++++++++++++----- tests/integration.rs | 24 ++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index bbca087..904bf86 100644 --- a/src/main.rs +++ b/src/main.rs @@ -196,7 +196,21 @@ fn run() -> Result<(), Box> { return Err("No labeled leaves found in the tree.".into()); } - // Check for duplicate leaf names and tabs in labels + // Leaves without a label cannot be named in the matrix, so they are left + // out entirely. Say so rather than quietly returning a smaller matrix. + let unlabeled_leaves = nodes + .iter() + .filter(|nd| nd.children.is_empty() && nd.name.is_none()) + .count(); + if unlabeled_leaves > 0 { + eprintln!( + "Warning: {} leaf/leaves have no label and were excluded from the matrix.", + unlabeled_leaves + ); + } + + // Check for duplicate leaf names and for characters that would break the + // row/column structure of the output { let mut seen = HashSet::with_capacity(leaf_indices.len()); for &i in &leaf_indices { @@ -208,11 +222,17 @@ fn run() -> Result<(), Box> { ) .into()); } - if name.contains('\t') { + if let Some(bad) = name.chars().find(|c| matches!(c, '\t' | '\n' | '\r')) { + let what = match bad { + '\t' => "a tab character", + '\n' => "a newline", + _ => "a carriage return", + }; return Err(format!( - "Leaf name '{}' contains a tab character, which would corrupt TSV output. \ - Use an underscore or rename the leaf.", - name.replace('\t', "\\t") + "Leaf name '{}' contains {}, which would corrupt the output by splitting \ + the row. Use an underscore or rename the leaf.", + name.escape_debug(), + what ) .into()); } diff --git a/tests/integration.rs b/tests/integration.rs index b5850d7..e335f83 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -127,6 +127,30 @@ fn test_binary_empty_input_error() { assert!(stderr.contains("Empty") || stderr.contains("empty"), "stderr: {}", stderr); } +#[test] +fn test_binary_rejects_newline_in_label() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1.0,'bad\nlabel':2.0):0.5,C:3.0);").unwrap(); + + let (code, _stdout, stderr) = run(&[tree.to_str().unwrap()], None); + assert_ne!(code, 0, "a newline in a label should be rejected"); + assert!(stderr.contains("newline"), "stderr: {}", stderr); +} + +#[test] +fn test_binary_warns_about_unlabeled_leaves() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1.0,:2.0):0.5,C:3.0);").unwrap(); + + let (code, stdout, stderr) = run(&[tree.to_str().unwrap()], None); + assert_eq!(code, 0); + assert!(stderr.contains("no label"), "stderr: {}", stderr); + // The unlabeled leaf is absent from the matrix, so say so + assert_eq!(stdout.lines().count(), 3, "header + A + C"); +} + #[test] fn test_binary_rejects_out_of_range_precision() { let dir = tempfile::tempdir().unwrap(); From b77373e9951f8619871930d8deeb3c886ccb7f47 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:37:59 +0200 Subject: [PATCH 24/47] Stop --midpoint from inflating topological distances Midpoint rooting inserts a node halfway along the diameter edge, which splits that edge in two and adds one hop to every pair whose path crosses it. On '((A:1,B:1):1,(C:1,D:1):1);' the A-C distance went from 4 to 5 the moment --midpoint was added. Edge counts do not depend on where a tree is rooted, so there is nothing for --midpoint to contribute in topology mode. Skip the rooting there and say why, rather than reporting counts that are one too high. --- src/main.rs | 17 +++++++++++++++-- tests/integration.rs | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 904bf86..f507fd3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -176,9 +176,22 @@ fn run() -> Result<(), Box> { eprintln!("Warning: negative branch lengths detected in the tree."); } - // Midpoint-root if requested + // Midpoint-root if requested. + // + // Topological distance counts edges and does not depend on where the tree + // is rooted, so rooting can only distort it: the node inserted at the + // midpoint splits one edge in two and adds 1 to every pair whose path + // crosses it. Skip the rooting rather than report those inflated counts. if do_midpoint { - root_idx = midpoint_root(root_idx, &mut nodes); + if mode == DistMode::Topology { + eprintln!( + "Warning: --midpoint is ignored in --topology mode. Edge counts do not depend \ + on the root, and the node inserted at the midpoint would add 1 to every \ + distance crossing that edge." + ); + } else { + root_idx = midpoint_root(root_idx, &mut nodes); + } } // Build LCA diff --git a/tests/integration.rs b/tests/integration.rs index e335f83..5ad4217 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -80,6 +80,44 @@ fn test_binary_topology_flag() { } } +#[test] +fn test_binary_midpoint_does_not_inflate_topology() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1,B:1):1,(C:1,D:1):1);").unwrap(); + + let (code, plain, _) = run(&["--topology", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let (code, rooted, stderr) = run( + &["--topology", "--midpoint", tree.to_str().unwrap()], + None, + ); + assert_eq!(code, 0); + assert_eq!( + plain, rooted, + "rooting must not change the number of edges between two leaves" + ); + assert!(stderr.contains("--midpoint"), "stderr: {}", stderr); + // A-C crosses the root: 2 edges up + 2 edges down + assert!(plain.contains("\t4\t"), "unexpected matrix:\n{}", plain); +} + +#[test] +fn test_binary_midpoint_preserves_patristic() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(((A:0.5,B:0.3):0.4,C:0.9):0.1,D:1.2);").unwrap(); + + let (code, plain, _) = run(&[tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let (code, rooted, _) = run(&["--midpoint", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + assert_eq!( + plain, rooted, + "patristic distances do not depend on the root" + ); +} + #[test] fn test_binary_lower_triangle() { let dir = tempfile::tempdir().unwrap(); From 7ebc2c9955f44b7eccf67a1d7fcf46025b495f76 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:39:45 +0200 Subject: [PATCH 25/47] Add a randomised check that midpoint rooting is sound Midpoint rooting rewires parent/child links and swaps branch lengths all the way up to the old root, which the three hand-written trees exercised only lightly. Cross-check it on 300 generated trees, with polytomies, zero length branches and 2 to 25 leaves, against the three properties that matter: every pairwise patristic distance survives the rewiring, the result is still a tree with no cycle and nothing orphaned, and the deepest tip sits exactly half a diameter from the new root. --- src/midpoint.rs | 160 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/src/midpoint.rs b/src/midpoint.rs index ff0f185..5c4c7e1 100644 --- a/src/midpoint.rs +++ b/src/midpoint.rs @@ -167,6 +167,166 @@ mod tests { lca.depth_len[leaf_i] + lca.depth_len[leaf_j] - 2.0 * lca.depth_len[m] } + /// xorshift64, so the randomised checks stay reproducible without pulling + /// in a dependency. + struct Rng(u64); + + impl Rng { + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n as u64) as usize + } + + /// Zero one branch in seven: collapsing weakly supported branches + /// leaves plenty of them in real trees. + fn length(&mut self) -> f64 { + if self.below(7) == 0 { + return 0.0; + } + 0.01 + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 * 2.0 + } + } + + /// Join random groups of 2 or 3 until one tree remains, so the sample + /// covers polytomies as well as strictly bifurcating trees. + fn random_newick(rng: &mut Rng, n_leaves: usize) -> String { + let mut pool: Vec = (0..n_leaves) + .map(|i| format!("L{}:{:.6}", i, rng.length())) + .collect(); + while pool.len() > 1 { + let arity = if pool.len() > 2 && rng.below(4) == 0 { 3 } else { 2 }; + let children: Vec = (0..arity) + .map(|_| pool.swap_remove(rng.below(pool.len()))) + .collect(); + pool.push(format!("({}):{:.6}", children.join(","), rng.length())); + } + format!("{};", pool.pop().unwrap()) + } + + /// Every node reachable exactly once, every child pointing back at its parent. + fn assert_tree_is_consistent(nodes: &[Node], root: usize) { + assert!(nodes[root].parent.is_none(), "the root must have no parent"); + let mut visited = vec![false; nodes.len()]; + let mut stack = vec![root]; + visited[root] = true; + while let Some(u) = stack.pop() { + for &v in &nodes[u].children { + assert_eq!( + nodes[v].parent, + Some(u), + "child {} does not point back at parent {}", + v, + u + ); + assert!(!visited[v], "node {} reached twice: the graph has a cycle", v); + visited[v] = true; + stack.push(v); + } + } + for (i, seen) in visited.iter().enumerate() { + assert!(seen, "node {} is unreachable from the root", i); + } + } + + fn leaf_indices(nodes: &[Node]) -> Vec { + nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect() + } + + #[test] + fn test_midpoint_random_trees() { + let mut rng = Rng(0x9E3779B97F4A7C15); + + for case in 0..300 { + let n_leaves = 2 + rng.below(24); + let newick = random_newick(&mut rng, n_leaves); + + let raw = parse_newick(&newick).unwrap(); + let mut before = Vec::new(); + let root_before = flatten_raw(&raw, None, &mut before); + let lca_before = build_lca_structure(root_before, &before); + let leaves_before = leaf_indices(&before); + + // Names in flatten order, so the two trees can be lined up by label + let names: Vec = leaves_before + .iter() + .map(|&i| before[i].name.clone().unwrap()) + .collect(); + + let mut diameter: f64 = 0.0; + let mut dist_before = vec![vec![0.0; leaves_before.len()]; leaves_before.len()]; + for (a, &i) in leaves_before.iter().enumerate() { + for (b, &j) in leaves_before.iter().enumerate() { + let d = patristic_distance(i, j, &lca_before); + dist_before[a][b] = d; + diameter = diameter.max(d); + } + } + + let raw2 = parse_newick(&newick).unwrap(); + let mut after = Vec::new(); + let root_after = flatten_raw(&raw2, None, &mut after); + let new_root = midpoint_root(root_after, &mut after); + + assert_tree_is_consistent(&after, new_root); + + let lca_after = build_lca_structure(new_root, &after); + let leaves_after: Vec = names + .iter() + .map(|name| { + after + .iter() + .position(|n| n.name.as_deref() == Some(name.as_str())) + .unwrap_or_else(|| panic!("case {}: leaf {} vanished", case, name)) + }) + .collect(); + + for a in 0..leaves_after.len() { + for b in 0..leaves_after.len() { + let d = patristic_distance(leaves_after[a], leaves_after[b], &lca_after); + assert!( + (d - dist_before[a][b]).abs() < 1e-9, + "case {}: rooting changed d({},{}) from {} to {}\n{}", + case, + names[a], + names[b], + dist_before[a][b], + d, + newick + ); + } + } + + // The defining property: the two longest root-to-tip paths are + // equal, so the deepest tip sits exactly half a diameter away. + let deepest = leaves_after + .iter() + .map(|&i| lca_after.depth_len[i]) + .fold(0.0_f64, f64::max); + assert!( + (deepest - diameter / 2.0).abs() < 1e-9, + "case {}: deepest tip is {} from the root, expected {} (diameter {})\n{}", + case, + deepest, + diameter / 2.0, + diameter, + newick + ); + } + } + #[test] fn test_midpoint_simple() { // ((A:1,B:1):1,(C:1,D:1):1); From 3d11f571f11d339b675a75a565eda7f7fec062cb Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:53:23 +0200 Subject: [PATCH 26/47] Halve the memory the LCA table needs The binary-lifting table is the largest allocation distree makes, n log n entries, and each one was an Option. That type has no spare niche, so every entry paid 8 bytes for the ancestor and 8 more for a discriminant that only ever marks the root. Measured on a 4,000-leaf tree, peak RSS drops from 7.2 MB to 6.4 MB, which is the 0.8 MB the table itself was wasting. The same arithmetic on a million-leaf tree comes to about 350 MB. Store the ancestors as plain usize with usize::MAX standing in for the root, in one flat allocation rather than a Vec per level. Also cross-check mrca() against walking up from both nodes, over every pair in 200 generated trees, since the query loop was rewritten with it. --- src/lca.rs | 126 ++++++++++++++++++++++++++++++++++++++++-------- src/main.rs | 2 + src/midpoint.rs | 47 +----------------- src/testutil.rs | 51 ++++++++++++++++++++ 4 files changed, 160 insertions(+), 66 deletions(-) create mode 100644 src/testutil.rs diff --git a/src/lca.rs b/src/lca.rs index f993c43..2491274 100644 --- a/src/lca.rs +++ b/src/lca.rs @@ -1,9 +1,22 @@ use crate::tree::Node; +/// Marks "no such ancestor" in the binary-lifting table. +/// +/// `Option` would be the obvious type, but it has no spare niche and so +/// costs 16 bytes per entry against 8 for a bare `usize`. The table holds +/// n·log n entries, which on a million-leaf tree is the difference between +/// roughly 700 MB and 350 MB. +const NO_ANCESTOR: usize = usize::MAX; + /// Precomputed data for O(log n) LCA queries using binary lifting. pub struct LcaData { - /// Binary lifting table: `up[k][u]` is the 2^k-th ancestor of node `u`. - pub up: Vec>>, + /// Binary lifting table, stored flat: `up[k * n + u]` is the 2^k-th + /// ancestor of node `u`, or [`NO_ANCESTOR`] if `u` has none. + up: Vec, + /// Node count, and therefore the stride of a row of `up`. + n: usize, + /// Number of levels held in `up`. + levels: usize, /// Cumulative branch-length distance from the root to each node. pub depth_len: Vec, /// Topological depth (number of edges) from the root to each node. @@ -15,20 +28,19 @@ pub struct LcaData { /// Runs in O(n log n) time and O(n log n) space. pub fn build_lca_structure(root_idx: usize, nodes: &[Node]) -> LcaData { let n = nodes.len(); - let max_log = if n <= 1 { 1 } else { ((n as f64).log2().ceil() as usize) + 1 }; + let levels = if n <= 1 { 1 } else { ((n as f64).log2().ceil() as usize) + 1 }; let mut depth_len = vec![0.0; n]; let mut depth_top = vec![0; n]; - let mut up: Vec>> = vec![vec![None; n]; max_log]; + let mut up: Vec = vec![NO_ANCESTOR; levels * n]; { let mut stack = vec![root_idx]; depth_len[root_idx] = 0.0; depth_top[root_idx] = 0; - up[0][root_idx] = None; while let Some(u) = stack.pop() { for &v in &nodes[u].children { - up[0][v] = Some(u); + up[v] = u; depth_len[v] = depth_len[u] + nodes[v].length; depth_top[v] = depth_top[u] + 1; stack.push(v); @@ -36,20 +48,33 @@ pub fn build_lca_structure(root_idx: usize, nodes: &[Node]) -> LcaData { } } - for k in 1..max_log { + for k in 1..levels { for u in 0..n { - up[k][u] = up[k - 1][u].and_then(|mid| up[k - 1][mid]); + let mid = up[(k - 1) * n + u]; + up[k * n + u] = if mid == NO_ANCESTOR { + NO_ANCESTOR + } else { + up[(k - 1) * n + mid] + }; } } LcaData { up, + n, + levels, depth_len, depth_top, } } impl LcaData { + /// The 2^k-th ancestor of `u`, or [`NO_ANCESTOR`]. + #[inline] + fn ancestor(&self, k: usize, u: usize) -> usize { + self.up[k * self.n + u] + } + /// Return the index of the Most Recent Common Ancestor (MRCA) of nodes `u` and `v`. /// /// Uses binary lifting for O(log n) queries. @@ -60,28 +85,38 @@ impl LcaData { if self.depth_top[u] < self.depth_top[v] { std::mem::swap(&mut u, &mut v); } - let diff = self.depth_top[u] - self.depth_top[v]; - let mut x = diff; + // Lift the deeper node until both sit at the same depth + let mut diff = self.depth_top[u] - self.depth_top[v]; let mut k = 0; - while x > 0 { - if (x & 1) == 1 { - u = self.up[k][u].unwrap(); + while diff > 0 { + if (diff & 1) == 1 { + u = self.ancestor(k, u); + assert_ne!( + u, NO_ANCESTOR, + "LCA: depth table disagrees with the tree; tree may be malformed" + ); } - x >>= 1; + diff >>= 1; k += 1; } if u == v { return u; } - for k in (0..self.up.len()).rev() { - if let (Some(au), Some(av)) = (self.up[k][u], self.up[k][v]) { - if au != av { - u = au; - v = av; - } + // Climb in step as long as the ancestors differ; they meet one above + for k in (0..self.levels).rev() { + let au = self.ancestor(k, u); + let av = self.ancestor(k, v); + if au != av && au != NO_ANCESTOR && av != NO_ANCESTOR { + u = au; + v = av; } } - self.up[0][u].expect("LCA: could not find common ancestor; tree may be malformed") + let m = self.ancestor(0, u); + assert_ne!( + m, NO_ANCESTOR, + "LCA: could not find common ancestor; tree may be malformed" + ); + m } } @@ -89,6 +124,8 @@ impl LcaData { mod tests { use super::*; use crate::parser::{flatten_raw, parse_newick}; + use crate::testutil::{random_newick, Rng}; + use std::collections::HashSet; fn make(newick: &str) -> (Vec, usize) { let raw = parse_newick(newick).unwrap(); @@ -132,6 +169,53 @@ mod tests { assert_eq!(lca.mrca(root, root), root); } + /// Walk to the root from `u`, then from `v` until the paths meet. + fn mrca_by_walking(nodes: &[Node], u: usize, v: usize) -> usize { + let mut seen = HashSet::new(); + let mut cur = Some(u); + while let Some(x) = cur { + seen.insert(x); + cur = nodes[x].parent; + } + let mut cur = Some(v); + while let Some(x) = cur { + if seen.contains(&x) { + return x; + } + cur = nodes[x].parent; + } + panic!("no common ancestor between {} and {}", u, v); + } + + #[test] + fn test_mrca_matches_walking_up_on_random_trees() { + let mut rng = Rng::new(0x2545_F491_4F6C_DD1D); + + for case in 0..200 { + let n_leaves = 2 + rng.below(30); + let newick = random_newick(&mut rng, n_leaves); + let raw = parse_newick(&newick).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + let lca = build_lca_structure(root, &nodes); + + for u in 0..nodes.len() { + for v in 0..nodes.len() { + let expected = mrca_by_walking(&nodes, u, v); + assert_eq!( + lca.mrca(u, v), + expected, + "case {}: mrca({}, {}) in {}", + case, + u, + v, + newick + ); + } + } + } + } + #[test] fn test_depth_top() { // ((A,B),C); — A and B are at depth 2, C at depth 1 diff --git a/src/main.rs b/src/main.rs index f507fd3..23e33ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,8 @@ mod lca; mod midpoint; mod parser; +#[cfg(test)] +mod testutil; mod tree; use clap::{Arg, ArgAction, Command}; diff --git a/src/midpoint.rs b/src/midpoint.rs index 5c4c7e1..62d6bc5 100644 --- a/src/midpoint.rs +++ b/src/midpoint.rs @@ -157,6 +157,7 @@ mod tests { use super::*; use crate::parser::{flatten_raw, parse_newick}; use crate::lca::build_lca_structure; + use crate::testutil::{random_newick, Rng}; fn patristic_distance( leaf_i: usize, @@ -167,50 +168,6 @@ mod tests { lca.depth_len[leaf_i] + lca.depth_len[leaf_j] - 2.0 * lca.depth_len[m] } - /// xorshift64, so the randomised checks stay reproducible without pulling - /// in a dependency. - struct Rng(u64); - - impl Rng { - fn next_u64(&mut self) -> u64 { - let mut x = self.0; - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - self.0 = x; - x - } - - fn below(&mut self, n: usize) -> usize { - (self.next_u64() % n as u64) as usize - } - - /// Zero one branch in seven: collapsing weakly supported branches - /// leaves plenty of them in real trees. - fn length(&mut self) -> f64 { - if self.below(7) == 0 { - return 0.0; - } - 0.01 + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 * 2.0 - } - } - - /// Join random groups of 2 or 3 until one tree remains, so the sample - /// covers polytomies as well as strictly bifurcating trees. - fn random_newick(rng: &mut Rng, n_leaves: usize) -> String { - let mut pool: Vec = (0..n_leaves) - .map(|i| format!("L{}:{:.6}", i, rng.length())) - .collect(); - while pool.len() > 1 { - let arity = if pool.len() > 2 && rng.below(4) == 0 { 3 } else { 2 }; - let children: Vec = (0..arity) - .map(|_| pool.swap_remove(rng.below(pool.len()))) - .collect(); - pool.push(format!("({}):{:.6}", children.join(","), rng.length())); - } - format!("{};", pool.pop().unwrap()) - } - /// Every node reachable exactly once, every child pointing back at its parent. fn assert_tree_is_consistent(nodes: &[Node], root: usize) { assert!(nodes[root].parent.is_none(), "the root must have no parent"); @@ -247,7 +204,7 @@ mod tests { #[test] fn test_midpoint_random_trees() { - let mut rng = Rng(0x9E3779B97F4A7C15); + let mut rng = Rng::new(0x9E37_79B9_7F4A_7C15); for case in 0..300 { let n_leaves = 2 + rng.below(24); diff --git a/src/testutil.rs b/src/testutil.rs new file mode 100644 index 0000000..7cb58ad --- /dev/null +++ b/src/testutil.rs @@ -0,0 +1,51 @@ +//! Helpers shared by the test modules: a deterministic PRNG and a random tree +//! generator, so the randomised checks stay reproducible without pulling in a +//! dependency. + +/// xorshift64. +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: u64) -> Self { + Rng(seed) + } + + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + /// A number in `0..n`. + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n as u64) as usize + } + + /// A branch length, zero one time in seven: collapsing weakly supported + /// branches leaves plenty of zero-length ones in real trees. + pub fn length(&mut self) -> f64 { + if self.below(7) == 0 { + return 0.0; + } + 0.01 + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 * 2.0 + } +} + +/// Join random groups of 2 or 3 until one tree remains, so the sample covers +/// polytomies as well as strictly bifurcating trees. +pub fn random_newick(rng: &mut Rng, n_leaves: usize) -> String { + let mut pool: Vec = (0..n_leaves) + .map(|i| format!("L{}:{:.6}", i, rng.length())) + .collect(); + while pool.len() > 1 { + let arity = if pool.len() > 2 && rng.below(4) == 0 { 3 } else { 2 }; + let children: Vec = (0..arity) + .map(|_| pool.swap_remove(rng.below(pool.len()))) + .collect(); + pool.push(format!("({}):{:.6}", children.join(","), rng.length())); + } + format!("{};", pool.pop().unwrap()) +} From 7973716b03f5e71c63c0bc2e03ea56d75c1db0c5 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:54:02 +0200 Subject: [PATCH 27/47] Point the integration tests at the binary Cargo actually built The path was hardcoded to target/debug/distree, so 'cargo test --release' ran whatever stale debug binary happened to be lying around and reported a pass for code it never executed. On a clean checkout it could not spawn anything at all. CARGO_BIN_EXE_distree resolves to the profile under test. --- tests/integration.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration.rs b/tests/integration.rs index 5ad4217..df6b821 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -2,12 +2,13 @@ use std::io::Write; use std::process::{Command, Stdio}; -/// Build the binary once and return its path. -fn bin() -> std::path::PathBuf { - // `cargo test` runs with the project root as cwd; the binary ends up here. - let mut p = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); - p.push("target/debug/distree"); - p +/// Path to the binary under test. +/// +/// Cargo sets this to the build actually being tested. Pointing at +/// target/debug by hand meant `cargo test --release` either exercised a stale +/// debug binary or, with no debug build around, failed to spawn at all. +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_distree") } fn run(args: &[&str], stdin: Option<&str>) -> (i32, String, String) { From 1c63b9f660bce34462b5321c228aff325a4ed3b0 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:55:41 +0200 Subject: [PATCH 28/47] Keep negative patristic distances instead of rounding them to zero The clamp to zero was there to absorb floating-point noise, but it cannot have been doing that: depth_len accumulates lengths from the root, so d_i and d_j are always >= d_m, 2*d_m is exact, and rounding a non-negative exact result to nearest keeps it non-negative. Checked over every pair in 200 generated trees. What the clamp did catch was the real thing, a negative branch length, which neighbour-joining trees routinely carry. Those distances were reported as 0.0, claiming two distinct taxa sit on top of each other, in a matrix that then went on into clustering. Report the value as computed; the tree already gets a warning, now saying what to expect. --- src/main.rs | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 23e33ca..4de8dc3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -175,7 +175,10 @@ fn run() -> Result<(), Box> { // Warn about negative branch lengths if nodes.iter().any(|n| n.length < 0.0) { - eprintln!("Warning: negative branch lengths detected in the tree."); + eprintln!( + "Warning: negative branch lengths detected in the tree. Some distances may come \ + out negative, and --midpoint cannot locate the diameter reliably." + ); } // Midpoint-root if requested. @@ -363,8 +366,15 @@ fn compute_distance( let d_i = lca_data.depth_len[leaf_i]; let d_j = lca_data.depth_len[leaf_j]; let d_m = lca_data.depth_len[m]; - // Clamp to 0 to avoid tiny negatives from floating-point arithmetic - (d_i + d_j - 2.0 * d_m).max(0.0) + // Not clamped to zero. Rounding cannot make this negative: + // depth_len accumulates lengths from the root, so with + // non-negative branches d_i and d_j are both >= d_m, 2*d_m is + // exact, and rounding a non-negative exact result to nearest keeps + // it non-negative. The only way to get a negative value here is a + // negative branch length, which the tree is warned about, and + // rounding that up to zero would silently claim two distinct taxa + // sit on top of each other. + d_i + d_j - 2.0 * d_m } } } @@ -538,6 +548,51 @@ mod tests { assert_eq!(d, 0.0, "Self-distance must be exactly 0.0, got {}", d); } + #[test] + fn test_patristic_never_negative_on_random_trees() { + // Rounding must not produce a negative distance on any tree with + // non-negative branch lengths, and a leaf must be exactly 0 from itself + let mut rng = crate::testutil::Rng::new(0x5DEE_CE66_D000_0005); + + for _ in 0..200 { + let n_leaves = 2 + rng.below(30); + let newick = crate::testutil::random_newick(&mut rng, n_leaves); + let (nodes, root) = build_tree(&newick); + let lca = build_lca_structure(root, &nodes); + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + for &i in &leaves { + assert_eq!( + compute_distance(i, i, DistMode::Patristic, &lca), + 0.0, + "a leaf must be exactly 0.0 from itself in {}", + newick + ); + for &j in &leaves { + let d = compute_distance(i, j, DistMode::Patristic, &lca); + assert!(d >= 0.0, "negative distance {} in {}", d, newick); + } + } + } + } + + #[test] + fn test_negative_branch_length_gives_negative_distance() { + // A tree with negative branches is warned about; reporting 0.0 would + // claim two distinct taxa sit on top of each other + let (nodes, root) = build_tree("(A:-2.0,B:0.5);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let b = get_leaf(&nodes, "B"); + let d = compute_distance(a, b, DistMode::Patristic, &lca); + assert!((d - (-1.5)).abs() < 1e-12, "expected -1.5, got {}", d); + } + #[test] fn test_mode_conflict() { // Just test the logic: LMM should be distinct from topology From 09116a38b9f1760b4a6abd4cfd2bb884b52fa9d3 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:56:09 +0200 Subject: [PATCH 29/47] Run CI on every branch, and lint the tests too The branch filter was [main, "*.*.x", "*.x"], and a release branch named after its version matches neither glob, since '*.*.x' needs a literal '.x' suffix. Work on 1.0.1 has therefore never been checked by CI. Clippy also ran over the binary alone, leaving the test code unlinted, and the last step was a bare release build rather than a release test run, which only became worth doing once the integration tests stopped looking for the binary in target/debug. --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a03cf28..640ea1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [main, "*.*.x", "*.x"] + branches: ["**"] pull_request: - branches: [main] + branches: ["**"] env: CARGO_TERM_COLOR: always @@ -19,8 +19,8 @@ jobs: with: components: clippy - name: Clippy - run: cargo clippy -- -D warnings + run: cargo clippy --all-targets -- -D warnings - name: Test run: cargo test - - name: Build release - run: cargo build --release + - name: Test release + run: cargo test --release From de5c8ceda6b3d33e4d553e2f4e8f149e5dd81977 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:57:13 +0200 Subject: [PATCH 30/47] Fix the output examples in the README The three example matrices had their header row absorbed into the opening code fence, so each block rendered with the label row missing and the rest of the table a column short. The numbers were invented too, and the topological one is not a tree: it puts LeafA and LeafD one edge apart, which no pair of leaves can be. Replace all three with the real output of one worked example, and add the --lower layout, which had no example at all. --- README.md | 54 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 5ec8cf7..f23678f 100644 --- a/README.md +++ b/README.md @@ -126,29 +126,51 @@ The tool outputs a tab-separated values (TSV) matrix. 2. Each subsequent line begins with a leaf label (sorted alphabetically), followed by N columns of distances (depending on the chosen mode) to every other leaf in the same sorted order. -Example (patristic distances): +All three examples below are the real output of `distree -p 3` for one tree: -```LeafA LeafB LeafC -LeafA 0.000 5.200 3.100 -LeafB 5.200 0.000 4.400 -LeafC 3.100 4.400 0.000 +``` +((LeafA:1.95,LeafB:3.25):0.35,(LeafC:0.80,LeafD:1.20):0.50); +``` + +Patristic distances (default), the sum of branch lengths along the path: + +``` + LeafA LeafB LeafC LeafD +LeafA 0.000 5.200 3.600 4.000 +LeafB 5.200 0.000 4.900 5.300 +LeafC 3.600 4.900 0.000 2.000 +LeafD 4.000 5.300 2.000 0.000 ``` -Example (topological distances): +Topological distances (`--topology`), the number of edges along the path: -```LeafA LeafB LeafC LeafD -LeafA 0 2 3 1 -LeafB 2 0 1 3 -LeafC 3 1 0 4 -LeafD 1 3 4 0 +``` + LeafA LeafB LeafC LeafD +LeafA 0 2 4 4 +LeafB 2 0 4 4 +LeafC 4 4 0 2 +LeafD 4 4 2 0 ``` -Example (LMM depths): +LMM depths (`--lmm`), the root-to-MRCA distance. The diagonal is each leaf's +own root-to-tip length, and pairs meeting at the root score 0: -```LeafA LeafB LeafC -LeafA 7.000 3.000 5.000 -LeafB 3.000 7.000 2.500 -LeafC 5.000 2.500 7.000 +``` + LeafA LeafB LeafC LeafD +LeafA 2.300 0.350 0.000 0.000 +LeafB 0.350 3.600 0.000 0.000 +LeafC 0.000 0.000 1.300 0.500 +LeafD 0.000 0.000 0.500 1.700 +``` + +PHYLIP lower triangle (`--lower`): taxa count, then one row per taxon: + +``` +4 +LeafA +LeafB 5.200 +LeafC 3.600 4.900 +LeafD 4.000 5.300 2.000 ``` ## Detailed Use Cases From 8dea63ed88e429fc428477e18b0ae5afb357d3de Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:00:41 +0200 Subject: [PATCH 31/47] Document the stricter parsing and the flag limits Records what changed for anyone reading the README or the changelog rather than the diff: one tree per file, malformed input rejected instead of guessed at, the --precision ceiling, --midpoint ignored in topology mode, non-ASCII labels surviving, and negative branch lengths reaching the output. --- CHANGELOG.md | 18 +++++++++++++++++- README.md | 17 ++++++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93827c2..3bcdac7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,19 @@ ## [1.0.1] - 2026-04-04 ### Fixed +- Trees prefixed with a `[&R]` / `[&U]` rooting marker, or carrying a comment before a label, no longer fail to parse +- Non-ASCII leaf labels (accents, Greek, CJK) are preserved instead of being mangled into mojibake +- Truncated trees, trailing content, unclosed comments and free-form text are rejected instead of yielding a plausible but wrong matrix +- Files holding more than one tree are rejected instead of silently using the first +- Apostrophes in unquoted labels (`O'Brien`) no longer break parsing, and a doubled quote keeps the whitespace around it +- Deeply nested trees no longer overflow the stack when the parse tree is freed +- `-o FILE` no longer truncates an existing file before the tree has parsed +- Write failures are reported instead of being discarded along with a success exit code; a closed pipe exits quietly +- `--precision` out of range is rejected instead of panicking inside the formatter +- `--threads 0` is rejected instead of quietly starting one thread per core +- Newlines and carriage returns in labels are rejected like tabs, and unlabeled leaves are reported rather than dropped in silence +- `--midpoint` no longer adds a hop to every topological distance crossing the midpoint edge +- Negative patristic distances are reported as computed instead of being rounded up to zero, which claimed distinct taxa were identical - Midpoint rooting (`--midpoint`) rewritten to fix infinite loop caused by cyclic graph construction - NHX/bracket comment annotations (e.g., `[&&NHX:S=human]`) no longer crash the parser - Single-quoted labels (e.g., `'Taxon A'`) now parsed correctly @@ -22,14 +35,17 @@ - Warning when no branch lengths are detected in patristic mode - Warning when negative branch lengths are found in the tree - CITATION.cff with DOI -- Comprehensive test suite (24 tests) +- Comprehensive test suite (73 tests), including randomised checks of midpoint rooting and of MRCA queries against a brute-force walk ### Changed - Codebase split into modules: `parser.rs`, `tree.rs`, `lca.rs`, `midpoint.rs` +- LCA binary-lifting table stores plain `usize` rather than `Option`, halving the memory it needs - Version string now derived from Cargo.toml via `env!("CARGO_PKG_VERSION")` - Removed unused `--format` flag - Fixed clippy warnings (`&Vec` → `&[Node]`) - Fixed contributors link in README +- README output examples replaced with the real output of a worked example; the previous ones lost their header row to the code fence and the topological matrix was not a realisable tree +- CI runs on every branch, lints the test code, and runs the test suite in release as well as debug ## [1.0.0] - 2025-06-01 diff --git a/README.md b/README.md index f23678f..00f0ffc 100644 --- a/README.md +++ b/README.md @@ -102,9 +102,9 @@ Options: ### Argument & Option Details -* ``: Path to the input tree in Newick format. Use `-` to read from stdin. Leaf labels must be unique. +* ``: Path to the input tree in Newick format. Use `-` to read from stdin. Leaf labels must be unique, and the file must hold exactly one tree. -* `--midpoint`: Re-root at the midpoint of the longest path before computing distances. +* `--midpoint`: Re-root at the midpoint of the longest path before computing distances. Ignored with `--topology`, where the number of edges between two leaves does not depend on where the tree is rooted. * `--lmm`: Var-covar matrix for Phylogenetic Comparative Methods. Each entry (i, j) = depth of MRCA(i, j). Mutually exclusive with `--topology`. @@ -112,9 +112,9 @@ Options: * `--lower`: PHYLIP lower-triangle format. Outputs a header line with the number of taxa, followed by one row per taxon with its label and the lower triangle of distances (no diagonal). Compatible with PHYLIP, Mash, and tools expecting this standard format. -* `-p, --precision `: Decimal places in output (default: 10). Applies to patristic and LMM modes. +* `-p, --precision `: Decimal places in output (default: 10, maximum: 30). Applies to patristic and LMM modes. A 64-bit float holds about 17 significant digits, so anything past that is padding. -* `-t, --threads `: Thread count for parallel computation. Defaults to all available cores. +* `-t, --threads `: Thread count for parallel computation. Must be at least 1; omit the flag to use all available cores. * `-o, --output `: Write TSV to file instead of stdout. @@ -245,9 +245,12 @@ LeafD 4.000 5.300 2.000 ## Troubleshooting and Tips -* **Invalid Newick**: Ensure your Newick tree is syntactically correct (matching parentheses, semicolon at end). `distree` will error if parsing fails. -* **Whitespace in Labels**: Leaf labels may contain spaces if they are enclosed in single or double quotes in the Newick file (e.g., `'Taxon A':1.0`). The parser handles quoted labels correctly. Tab characters (`\t`) in labels are rejected outright, as they would silently corrupt TSV output — replace them with underscores before running distree. -* **NHX and BEAST annotations**: Bracket-enclosed metadata (`[&&NHX:...]`, `[&rate=...]`) is silently skipped. Branch lengths and labels are preserved. +* **Invalid Newick**: `distree` errors out rather than guessing, and the message names the offending position. It rejects unbalanced parentheses (a truncated file would otherwise produce a matrix with quietly wrong branch lengths), unclosed quotes and comments, and anything left over after the tree. The trailing semicolon is optional. +* **One tree per file**: A file holding several trees (bootstrap replicates, a posterior sample) is rejected instead of silently using the first one. Split it first. +* **Whitespace in Labels**: Leaf labels may contain spaces if they are enclosed in single or double quotes in the Newick file (e.g., `'Taxon A':1.0`). A doubled quote inside a quoted label is the escape for a literal one (`'it''s'` → `it's`). Tabs, newlines and carriage returns in labels are rejected outright, as they would split a row and corrupt the output; replace them with underscores before running distree. +* **Non-ASCII labels**: Accents, Greek letters and CJK text are passed through unchanged, so labels keep matching the sample names used downstream. +* **NHX and BEAST annotations**: Bracket-enclosed metadata (`[&&NHX:...]`, `[&rate=...]`) is silently skipped, before or after the label, as is the `[&R]` / `[&U]` rooting marker that IQ-TREE, MrBayes and BEAST write at the start of the file. Branch lengths and labels are preserved. +* **Negative branch lengths**: Reported as a warning and carried through to the output, so a neighbour-joining tree can yield a negative distance. They also make `--midpoint` unreliable, since locating the longest path assumes non-negative lengths. * **Choosing Distance Type**: * Use `--lmm` if performing phylogenetic comparative analyses (e.g., trait evolution, PGLS). From 2ccdb427987a9475eabe6fead35f20b956e1f7c5 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:19:30 +0200 Subject: [PATCH 32/47] Make the parallelism actually pay: 6x on a 20,000-tip tree Two things kept the matrix loop from using the cores it asked for. Each row was its own parallel job. A cell is one MRCA query and three array reads, so a row of a small matrix is nowhere near enough work to pay for synchronising a thread pool, and the fork/join per row dominated everything: an 8,000-tip run took 2.52s on 14 cores against 1.52s on one, with system time going from 0.10s to 14.32s. Rows are now computed in batches sized to about a million cells, so the synchronisation happens once per batch rather than once per row. The other two thirds of the time was formatting. Turning a float into a fixed number of decimals costs more than computing the distance it prints, and it sat in the serial write loop where no number of cores could reach it. Each worker now formats its own rows into a byte buffer and the writer only hands finished bytes to the operating system. 20,000 tips, -p 6 --lower: 10.61s -> 1.61s 8,000 tips, -p 6 --lower: 2.52s -> 0.20s and the run now scales with cores instead of against them (8,000 tips: 1.26s on one core, 0.21s on eight). Output is byte-identical. The batch buffer costs a fixed ~15 MB, which does not grow with the tree, and the output buffer goes from 8 KB to 1 MB so a multi-gigabyte matrix is not hundreds of thousands of write syscalls. --- src/main.rs | 95 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/src/main.rs b/src/main.rs index 4de8dc3..30d2a52 100644 --- a/src/main.rs +++ b/src/main.rs @@ -34,6 +34,14 @@ enum DistMode { /// range it accepts: `{:.prec$}` panics outright on very large values. const MAX_PRECISION: usize = 30; +/// Target number of cells per parallel batch, which caps the batch buffer at +/// 8 MB and gives each fork/join enough work to be worth its cost. +const BATCH_CELLS: usize = 1 << 20; + +/// Output buffer size. The default 8 KB turns a multi-gigabyte matrix into +/// hundreds of thousands of write syscalls. +const WRITE_BUFFER: usize = 1 << 20; + fn main() -> ExitCode { match run() { Ok(()) => ExitCode::SUCCESS, @@ -283,11 +291,12 @@ fn run() -> Result<(), Box> { // earlier truncated the previous results before a parse error could be // reported, leaving the user with an empty file and nothing to fall back on. let mut writer: Box = if let Some(path) = output_path { - Box::new(BufWriter::new(File::create(path).map_err(|e| { - format!("Cannot write to '{}': {}", path, e) - })?)) + Box::new(BufWriter::with_capacity( + WRITE_BUFFER, + File::create(path).map_err(|e| format!("Cannot write to '{}': {}", path, e))?, + )) } else { - Box::new(BufWriter::new(io::stdout())) + Box::new(BufWriter::with_capacity(WRITE_BUFFER, io::stdout())) }; // Print header @@ -305,27 +314,45 @@ fn run() -> Result<(), Box> { writer.write_all(b"\n")?; } - // Compute and print distance matrix (reuse row buffer to avoid per-row allocation) - let mut row_buf: Vec = Vec::with_capacity(n_leaves); - for (row_i, &leaf_i) in sorted_leaf_indices.iter().enumerate() { - let col_end = if do_lower { row_i } else { n_leaves }; - let col_slice = &sorted_leaf_indices[..col_end]; - - row_buf.clear(); - row_buf.par_extend( - col_slice - .par_iter() - .map(|&leaf_j| compute_distance(leaf_i, leaf_j, mode, &lca_data)) - ); - let this_row = &row_buf; + // Compute and print the distance matrix, a batch of rows at a time. + // + // The batch is sized so each one carries roughly BATCH_CELLS cells of work, + // which bounds the buffer at 8 MB and, more to the point, keeps the + // fork/join out of the inner loop. A cell is an MRCA query and three array + // reads, so a single row of a small matrix is nowhere near enough work to + // pay for synchronising a thread pool: on an 8,000-leaf tree, one job per + // row ran *slower* on 14 cores than on one. + // Each worker formats its rows as it computes them, so the writer only ever + // hands finished bytes to the operating system. Formatting a float to a + // fixed number of decimals costs more than the distance it prints, so + // leaving it in the serial write loop capped the whole run: at six decimals + // it was about two thirds of the work no number of cores could touch. + let batch_rows = (BATCH_CELLS / n_leaves).clamp(1, n_leaves); + let mut batch: Vec> = vec![Vec::new(); batch_rows]; + let row_width = |row_i: usize| if do_lower { row_i } else { n_leaves }; + + let mut first_row = 0; + while first_row < n_leaves { + let rows = (n_leaves - first_row).min(batch_rows); + + batch[..rows].par_iter_mut().enumerate().for_each(|(k, out)| { + let row_i = first_row + k; + let leaf_i = sorted_leaf_indices[row_i]; + out.clear(); + out.extend_from_slice(sorted_labels[row_i].as_bytes()); + for &leaf_j in &sorted_leaf_indices[..row_width(row_i)] { + let dist = compute_distance(leaf_i, leaf_j, mode, &lca_data); + out.push(b'\t'); + format_distance(out, dist, mode, precision); + } + out.push(b'\n'); + }); - // Row label - writer.write_all(sorted_labels[row_i].as_bytes())?; - for dist in this_row.iter() { - writer.write_all(b"\t")?; - format_distance(&mut writer, *dist, mode, precision)?; + for row in &batch[..rows] { + writer.write_all(row)?; } - writer.write_all(b"\n")?; + + first_row += rows; } // BufWriter flushes on drop but discards whatever error it hits, so a full @@ -379,21 +406,19 @@ fn compute_distance( } } -/// Format a single distance value for TSV output. +/// Append a single distance value to a row buffer. /// -/// Topology mode outputs integers; patristic and LMM use `precision` decimal places. +/// Topology mode outputs integers; patristic and LMM use `precision` decimal +/// places. Writing into a `Vec` cannot fail, which is what lets this run +/// inside the parallel row builder. #[inline] -fn format_distance( - writer: &mut dyn Write, - dist: f64, - mode: DistMode, - precision: usize, -) -> io::Result<()> { - if mode == DistMode::Topology { - write!(writer, "{}", dist as i64) +fn format_distance(out: &mut Vec, dist: f64, mode: DistMode, precision: usize) { + let result = if mode == DistMode::Topology { + write!(out, "{}", dist as i64) } else { - write!(writer, "{:.prec$}", dist, prec = precision) - } + write!(out, "{:.prec$}", dist, prec = precision) + }; + result.expect("writing to a Vec cannot fail"); } #[cfg(test)] From 10b32d71d0693bd6dece560bb30e24c02f30ff7c Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:21:20 +0200 Subject: [PATCH 33/47] Release the parse tree once the flat one is built flatten_raw copies what it needs into Vec, so from that point the input text and the recursive parse tree are dead. They stayed in scope to the end of run() regardless, which on a large tree means carrying a second copy of it alongside the LCA table for the whole matrix loop, the longest part of the run. The measured effect is small (about 2 MB of resident memory on a 100,000-tip tree) because the allocator holds on to the freed pages rather than returning them, but the pages are reclaimable once released and were not before. --- src/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main.rs b/src/main.rs index 30d2a52..3154d72 100644 --- a/src/main.rs +++ b/src/main.rs @@ -181,6 +181,12 @@ fn run() -> Result<(), Box> { let mut nodes: Vec = Vec::new(); let mut root_idx = flatten_raw(&raw_root, None, &mut nodes); + // Nothing below reads the input text or the recursive parse tree, and on a + // large tree they are a second copy of it. Holding them to the end of the + // run would carry that alongside the LCA table, which is the peak. + drop(raw_root); + drop(newick_str); + // Warn about negative branch lengths if nodes.iter().any(|n| n.length < 0.0) { eprintln!( From 40ef714173402e9596b15db5757e49ca12bdc235 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:26:58 +0200 Subject: [PATCH 34/47] Add the documentation site Fourteen pages of MkDocs Material, published to GitHub Pages from main. The README had grown into the whole manual and was the only place any of this was written down, which made it long enough that nobody reads to the end and still left no room for the things that need a page of their own. Getting started installation, and the commands most runs use User guide input, the three distance modes, midpoint rooting, output formats, and a full CLI reference How it works the parser, the LCA structure, the streaming loop, and what the run costs in time and memory Recipes PCoA, transmission clusters, PGLS, neighbour-joining About changelog, citation, contributing Every matrix on the site is real output rather than a plausible-looking table, every error message is quoted from a run, and the benchmark figures are measured on a stated machine with the estimates marked as estimates. The site builds with --strict, so a dead link or a missing nav entry fails CI rather than shipping. Pull requests build without deploying. --- .github/workflows/docs.yml | 84 +++++++++++ .gitignore | 4 + docs/about/changelog.md | 5 + docs/about/citation.md | 64 +++++++++ docs/about/contributing.md | 87 +++++++++++ docs/assets/distree_logo.svg | 187 ++++++++++++++++++++++++ docs/assets/distree_wordmark.svg | 99 +++++++++++++ docs/getting-started/installation.md | 80 +++++++++++ docs/getting-started/quickstart.md | 149 +++++++++++++++++++ docs/guide/cli.md | 99 +++++++++++++ docs/guide/distances.md | 161 +++++++++++++++++++++ docs/guide/input.md | 148 +++++++++++++++++++ docs/guide/output.md | 152 ++++++++++++++++++++ docs/guide/rooting.md | 129 +++++++++++++++++ docs/how-it-works/algorithm.md | 126 ++++++++++++++++ docs/how-it-works/performance.md | 139 ++++++++++++++++++ docs/index.md | 105 ++++++++++++++ docs/recipes.md | 206 +++++++++++++++++++++++++++ docs/stylesheets/extra.css | 139 ++++++++++++++++++ mkdocs.yml | 144 +++++++++++++++++++ requirements-docs.txt | 4 + 21 files changed, 2311 insertions(+) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/about/changelog.md create mode 100644 docs/about/citation.md create mode 100644 docs/about/contributing.md create mode 100644 docs/assets/distree_logo.svg create mode 100644 docs/assets/distree_wordmark.svg create mode 100644 docs/getting-started/installation.md create mode 100644 docs/getting-started/quickstart.md create mode 100644 docs/guide/cli.md create mode 100644 docs/guide/distances.md create mode 100644 docs/guide/input.md create mode 100644 docs/guide/output.md create mode 100644 docs/guide/rooting.md create mode 100644 docs/how-it-works/algorithm.md create mode 100644 docs/how-it-works/performance.md create mode 100644 docs/index.md create mode 100644 docs/recipes.md create mode 100644 docs/stylesheets/extra.css create mode 100644 mkdocs.yml create mode 100644 requirements-docs.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..f01d774 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,84 @@ +name: Documentation + +# Build the MkDocs Material site and publish it to GitHub Pages (gh-pages +# branch). After the first successful run, set Settings -> Pages -> Source to +# "Deploy from a branch" and pick the gh-pages branch (root). +# +# Deploys on a push to main, so the published site tracks main rather than +# redeploying from every working branch. Pull requests build without deploying, +# which is what catches a dead link before it ships. To publish on demand from +# any branch, use the "Run workflow" button (workflow_dispatch). + +on: + push: + branches: + - main + paths: + - "docs/**" + - "mkdocs.yml" + - "CHANGELOG.md" + - "requirements-docs.txt" + - ".github/workflows/docs.yml" + pull_request: + paths: + - "docs/**" + - "mkdocs.yml" + - "CHANGELOG.md" + - "requirements-docs.txt" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: docs-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install dependencies + run: pip install -r requirements-docs.txt + + # --strict turns broken internal links and missing nav entries into + # errors, so a pull request that breaks one fails here instead of + # shipping a dead link. + - name: Build + run: mkdocs build --strict + + deploy: + name: Deploy + needs: build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Cache MkDocs build + uses: actions/cache@v4 + with: + key: mkdocs-material-${{ github.sha }} + path: .cache + restore-keys: | + mkdocs-material- + + - name: Install dependencies + run: pip install -r requirements-docs.txt + + - name: Build and deploy + run: mkdocs gh-deploy --force --strict diff --git a/.gitignore b/.gitignore index 2f7896d..e47dfd9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ target/ + +# MkDocs build output and the Material plugin cache +site/ +.cache/ diff --git a/docs/about/changelog.md b/docs/about/changelog.md new file mode 100644 index 0000000..0536cbb --- /dev/null +++ b/docs/about/changelog.md @@ -0,0 +1,5 @@ +--- +title: Changelog +--- + +--8<-- "CHANGELOG.md" diff --git a/docs/about/citation.md b/docs/about/citation.md new file mode 100644 index 0000000..ba274e8 --- /dev/null +++ b/docs/about/citation.md @@ -0,0 +1,64 @@ +# Citation + +distree is archived on Zenodo. Cite the release you used: + +> Ruiz-Rodriguez P, Coscolla M. *distree: distance matrices from a phylogeny.* +> PathoGenOmics Lab. [doi:10.5281/zenodo.16811766](https://doi.org/10.5281/zenodo.16811766) + +The repository carries a [`CITATION.cff`](https://github.com/PathoGenOmics-Lab/distree/blob/main/CITATION.cff), +so GitHub's "Cite this repository" button and most reference managers will +produce the entry for you. + +Please record the version (`distree --version`) and the mode you used. A +patristic matrix, a topological one and a variance-covariance matrix are three +different objects, and "distances from the tree" does not say which. + +## Authors + +**Paula Ruiz-Rodriguez** and **Mireia Coscolla**, I²SysBio, University of +Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, +Valencia, Spain. + +## Background + +The quantities distree computes are standard, and the terms are worth pinning +down. + +**Patristic distance** is the sum of branch lengths along the path between two +tips. The name is Farris's. + +> Farris JS. *A Method for Computing Wagner Trees.* Systematic Zoology. +> 1970;19(1):83-92. + +**The variance-covariance matrix** that `--lmm` writes is the covariance +structure implied by Brownian-motion trait evolution on a tree, which is what +makes it the `C` of a phylogenetic generalised least squares fit. + +> Felsenstein J. *Phylogenies and the Comparative Method.* The American +> Naturalist. 1985;125(1):1-15. + +> Grafen A. *The phylogenetic regression.* Philosophical Transactions of the +> Royal Society B. 1989;326(1233):119-157. + +**Midpoint rooting** places the root at the middle of the longest tip-to-tip +path. + +> Farris JS. *Estimating Phylogenetic Trees from Distance Matrices.* The +> American Naturalist. 1972;106(951):645-668. + +**Binary lifting** is the technique behind the `O(log n)` ancestor queries. The +sparse-table formulation of the least-common-ancestor problem goes back to: + +> Bender MA, Farach-Colton M. *The LCA Problem Revisited.* LATIN 2000, Lecture +> Notes in Computer Science 1776, pp. 88-94. + +## Formats and downstream tools + +The `--lower` output is PHYLIP's lower-triangular distance format: + +> Felsenstein J. *PHYLIP: Phylogeny Inference Package (Version 3.2).* Cladistics. +> 1989;5:164-166. + +## License + +[GPL-3.0](https://github.com/PathoGenOmics-Lab/distree/blob/main/LICENSE). diff --git a/docs/about/contributing.md b/docs/about/contributing.md new file mode 100644 index 0000000..5997a29 --- /dev/null +++ b/docs/about/contributing.md @@ -0,0 +1,87 @@ +# Contributing + +Issues and pull requests are welcome at +[PathoGenOmics-Lab/distree](https://github.com/PathoGenOmics-Lab/distree). + +## Reporting a problem + +The most useful bug report is one somebody else can run. Include: + +- the exact command, with every flag; +- the version (`distree --version`) and how it was installed; +- the tree, or a smaller tree that still shows the problem; +- what you expected and what you got. + +The dangerous failures here are the quiet ones. A matrix that is wrong looks +exactly like a matrix that is right, so a report saying "these distances +disagree with `ape::cophenetic` and here are both" is worth more than a stack +trace. + +If the tree carries anything identifiable, do not attach it. Tip labels can be +replaced with `S1`, `S2` and so on without changing the behaviour, and a +generated tree of the same shape usually reproduces the problem. + +## Building and checking + +```bash +cargo build --release +cargo test +cargo test --release +cargo clippy --all-targets -- -D warnings +``` + +CI runs all four on every push and every pull request. A change that fails any +of them will not go in. + +## Tests + +The suite is in three places: + +- **Unit tests** next to the code they cover, in `src/parser.rs`, `src/lca.rs`, + `src/midpoint.rs` and `src/main.rs`. +- **Integration tests** in `tests/integration.rs`, which run the built binary + and check its stdout, its stderr and its exit code. +- **Randomised property checks**, which are where the confidence comes from. + `src/testutil.rs` generates trees with polytomies and zero-length branches + from a seeded xorshift, and the checks assert properties rather than + particular numbers: midpoint rooting preserves every pairwise distance and + leaves a valid tree with the deepest tip half a diameter from the root; the + binary-lifting MRCA agrees with walking up from both nodes, over every pair of + nodes; and a patristic distance is never negative on a tree with non-negative + branch lengths. + +A parser change wants a case in `src/parser.rs` for what should now parse **and** +one for what should still be rejected. The failures worth guarding are the ones +where a malformed tree produced a plausible matrix instead of an error. + +## Writing a fix + +A few conventions the codebase follows: + +- **Refuse rather than guess.** If the input is ambiguous or truncated, error + with the position. A matrix that looks right and is not costs more than a + failed run. +- **Warnings go to stderr**, never into the matrix, so a piped run stays clean. +- **Keep the tree traversals iterative.** The parser, the flattener, the LCA + build and even `RawNode`'s `Drop` are loops with explicit stacks, because a + ladder-shaped tree hundreds of thousands of levels deep is a real input. +- **Do not put work in the serial write loop.** Whatever a worker can do to its + own row, it should, including formatting the numbers. + +## Documentation + +This site is MkDocs Material, and the pages live in `docs/`: + +```bash +pip install -r requirements-docs.txt +mkdocs serve +``` + +```bash +mkdocs build --strict +``` + +`--strict` turns a broken internal link or a dead anchor into a failed build, +which is what CI runs. Numbers quoted in the docs, in the benchmark tables above +all, should be measured rather than estimated, and the page should say which +when it is not obvious. diff --git a/docs/assets/distree_logo.svg b/docs/assets/distree_logo.svg new file mode 100644 index 0000000..c6192cd --- /dev/null +++ b/docs/assets/distree_logo.svg @@ -0,0 +1,187 @@ + + + + diff --git a/docs/assets/distree_wordmark.svg b/docs/assets/distree_wordmark.svg new file mode 100644 index 0000000..3d087f2 --- /dev/null +++ b/docs/assets/distree_wordmark.svg @@ -0,0 +1,99 @@ + + + + diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 0000000..72fc621 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,80 @@ +# Installation + +distree is a single self-contained binary. It has no runtime dependencies, no +interpreter, and no configuration file. + +## From Bioconda + +=== "conda" + + ```bash + conda install -c bioconda distree + ``` + +=== "mamba" + + ```bash + mamba install -c bioconda distree + ``` + +=== "pixi" + + ```bash + pixi add --channel bioconda distree + ``` + +## From a release binary + +Each release publishes binaries for Linux and macOS, on both x86-64 and arm64, +on the [releases page](https://github.com/PathoGenOmics-Lab/distree/releases). +Download the one for your platform, make it executable, and put it on your +`PATH`: + +```bash +curl -LO https://github.com/PathoGenOmics-Lab/distree/releases/latest/download/distree-linux-x86_64 +chmod +x distree-linux-x86_64 +mv distree-linux-x86_64 ~/.local/bin/distree +``` + +The four assets are `distree-linux-x86_64`, `distree-linux-aarch64`, +`distree-macos-x86_64` and `distree-macos-aarch64`. + +!!! note "macOS Gatekeeper" + + The binaries are not notarised, so macOS quarantines a downloaded one. Clear + the flag with `xattr -d com.apple.quarantine distree` before the first run. + +## From source + +Any stable Rust toolchain will do; [rustup](https://rustup.rs) is the usual way +to get one. + +```bash +git clone https://github.com/PathoGenOmics-Lab/distree.git +cd distree +cargo build --release +``` + +The binary lands at `target/release/distree`. Copy it wherever you keep local +tools, or run it in place. + +## Checking the install + +```bash +distree --version +echo '((A:1.0,B:2.0):0.5,C:3.0);' | distree - +``` + +``` + A B C +A 0.0000000000 3.0000000000 4.5000000000 +B 3.0000000000 0.0000000000 5.5000000000 +C 4.5000000000 5.5000000000 0.0000000000 +``` + +A tab-separated square matrix, tips sorted alphabetically, with an empty cell in +the top-left corner so the header lines up over the data columns. + +## Next + +[Quick start](quickstart.md) covers the commands most runs actually use. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 0000000..cad3569 --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,149 @@ +# Quick start + +Every example on this page uses the same four-tip tree, so the matrices can be +compared directly: + +``` +((LeafA:1.95,LeafB:3.25):0.35,(LeafC:0.80,LeafD:1.20):0.50); +``` + +## The default: patristic distances + +```bash +distree tree.nwk -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 0.000 | 5.200 | 3.600 | 4.000 | +| **LeafB** | 5.200 | 0.000 | 4.900 | 5.300 | +| **LeafC** | 3.600 | 4.900 | 0.000 | 2.000 | +| **LeafD** | 4.000 | 5.300 | 2.000 | 0.000 | + +
+ +Each cell is the sum of the branch lengths on the path between the two tips. +LeafA to LeafB is `1.95 + 3.25 = 5.20`; LeafA to LeafC crosses the root, +`1.95 + 0.35 + 0.50 + 0.80 = 3.60`. + +`-p` sets the decimal places. The default is 10, which is more than most branch +lengths carry any information at; 6 is a reasonable working value and 3 is +enough to read by eye. + +## Reading from stdin + +`-` reads the tree from standard input, so distree drops into a pipeline: + +```bash +gunzip -c tree.nwk.gz | distree - -o distances.tsv +``` + +## Ignoring branch lengths + +```bash +distree tree.nwk --topology +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 0 | 2 | 4 | 4 | +| **LeafB** | 2 | 0 | 4 | 4 | +| **LeafC** | 4 | 4 | 0 | 2 | +| **LeafD** | 4 | 4 | 2 | 0 | + +
+ +The number of edges between the two tips, as integers. Useful when the branch +lengths come from different sources, or when only the shape of the tree is +meant to matter. + +## The variance-covariance matrix + +```bash +distree tree.nwk --lmm -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.300 | 0.350 | 0.000 | 0.000 | +| **LeafB** | 0.350 | 3.600 | 0.000 | 0.000 | +| **LeafC** | 0.000 | 0.000 | 1.300 | 0.500 | +| **LeafD** | 0.000 | 0.000 | 0.500 | 1.700 | + +
+ +Entry `(i, j)` is the distance from the root down to the most recent common +ancestor of `i` and `j`: how much evolutionary history the two tips share. The +diagonal is each tip's own root-to-tip length, and any pair whose ancestor is +the root scores 0. This is the `C` matrix that PGLS and phylogenetic mixed +models expect, and it is the one mode where [where you root the +tree](../guide/rooting.md) changes the answer. + +## PHYLIP lower triangle + +```bash +distree tree.nwk --lower -p 4 +``` + +``` +4 +LeafA +LeafB 5.2000 +LeafC 3.6000 4.9000 +LeafD 4.0000 5.3000 2.0000 +``` + +A taxa count, then one row per taxon holding its label and the distances to +every taxon above it. Half the cells and no diagonal, which is what PHYLIP, +Mash and most distance-matrix readers want. + +## Midpoint rooting + +```bash +distree tree.nwk --midpoint --lmm -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.550 | 0.000 | 0.600 | 0.600 | +| **LeafB** | 0.000 | 2.650 | 0.000 | 0.000 | +| **LeafC** | 0.600 | 0.000 | 2.250 | 1.450 | +| **LeafD** | 0.600 | 0.000 | 1.450 | 2.650 | + +
+ +The longest path in this tree runs from LeafB to LeafD and is 5.30 long, so the +new root goes 2.65 from each: partway along LeafB's own branch. Every tip is now +at most 2.65 from the root, and the shared history of the other three tips has +been reorganised around it. + +!!! tip "It cannot change a patristic matrix" + + Rooting moves the root, not the tips. The path between two tips is the same + path whatever you call the top of the tree, so `--midpoint` leaves a + patristic or topological matrix exactly as it was. Reach for it when you are + producing `--lmm`. See [Midpoint rooting](../guide/rooting.md). + +## Controlling threads and output + +```bash +distree tree.nwk -t 8 -o distances.tsv +``` + +`-t` caps the thread count; without it distree uses every core. `-o` writes to a +file instead of stdout, and is not created until the tree has parsed, so a +failed run leaves whatever was there before untouched. + +## Next + +- [Input](../guide/input.md), for what distree accepts and what it turns down. +- [Distance modes](../guide/distances.md), for the three modes in full. +- [Recipes](../recipes.md), for feeding the matrix to R, Python or PHYLIP. diff --git a/docs/guide/cli.md b/docs/guide/cli.md new file mode 100644 index 0000000..e610d6e --- /dev/null +++ b/docs/guide/cli.md @@ -0,0 +1,99 @@ +# CLI reference + +``` +Usage: distree [OPTIONS] +``` + +## Argument + +`` + +: Path to the tree file in Newick format. Use `-` to read from stdin. The file + must hold exactly one tree, and its tip labels must be unique. See + [Input](input.md). + +## Options + +| Flag | Default | Effect | +|:--|:--|:--| +| `--midpoint` | off | Midpoint-root the tree before computing distances. Ignored with `--topology`. See [Midpoint rooting](rooting.md) | +| `--lmm` | off | Write the variance-covariance matrix: each cell is the root-to-MRCA distance. Takes precedence over `--topology` | +| `--topology` | off | Ignore branch lengths and count edges. Values are written as integers | +| `--lower` | off | Write a PHYLIP lower triangle: a taxa count, then one row per taxon with no diagonal | +| `-o, --output FILE` | stdout | Write the matrix to a file. Not created until the tree has parsed | +| `-p, --precision N` | `10` | Decimal places, from 0 to 30. Ignored by `--topology` | +| `-t, --threads N` | all cores | Threads for the parallel row computation. Must be at least 1 | +| `-h, --help` | | Print help | +| `-V, --version` | | Print version | + +Without `--lmm` or `--topology`, distree computes patristic distances. + +## How the modes interact + +| Combination | Result | +|:--|:--| +| `--lmm --topology` | `--lmm` wins, with a warning. They ask for different things | +| `--midpoint --topology` | The rooting is skipped, with a warning. Edge counts do not depend on the root, and the inserted node would add a hop | +| `--midpoint --lmm` | Both apply. This is the combination `--midpoint` exists for | +| `--midpoint` alone | Applies, and changes nothing: a patristic matrix is the same under any rooting | +| `--lower` with any mode | Applies to all three | +| `-p` with `--topology` | Ignored; edge counts are integers | + +## Exit codes + +| Code | Meaning | +|:--|:--| +| 0 | Success, including a downstream pipe closing early | +| 1 | Failure. The message is on stderr and begins with `Error:` | + +## Messages + +Everything below goes to stderr, never into the matrix. + +### Errors + +| Message | Cause | +|:--|:--| +| `Cannot open '': ...` | The tree file is missing or unreadable | +| `Empty input: no Newick tree found.` | The file is empty or only whitespace | +| `Failed to parse Newick tree: ...` | Malformed tree. The rest of the message names the position | +| `No labeled leaves found in the tree.` | The tree parsed but no tip carries a label | +| `Duplicate leaf name '' found.` | Two tips share a label; the matrix could not be indexed | +| `Leaf name '' contains a tab character / a newline / a carriage return` | The label would split a row | +| `--precision must be between 0 and 30, got N.` | `-p` out of range | +| `--threads must be at least 1.` | `-t 0` | +| `Cannot write to '': ...` | The output path is not writable | +| `Failed to write '': ...` | The write failed partway, typically a full disk | +| `Failed to initialize thread pool: ...` | Rayon could not start the requested threads | + +### Warnings + +| Message | Meaning | +|:--|:--| +| `negative branch lengths detected` | Some distances may come out negative, and `--midpoint` cannot locate the diameter reliably | +| `no branch lengths detected, all patristic distances will be zero` | A cladogram. Use `--topology` | +| `N leaf/leaves have no label and were excluded from the matrix` | Unlabelled tips cannot be named in a row, so they are left out | +| `--lmm and --topology are mutually exclusive. Using --lmm.` | Both were passed | +| `--midpoint is ignored in --topology mode.` | Both were passed | + +## Examples + +```bash +# Patristic distances to a file, six decimals +distree tree.nwk -p 6 -o distances.tsv + +# PHYLIP lower triangle for neighbor +distree tree.nwk --lower -p 6 -o infile + +# Variance-covariance matrix for PGLS, midpoint-rooted +distree tree.nwk --midpoint --lmm -p 8 -o varcovar.tsv + +# Edge counts from a cladogram +distree cladogram.nwk --topology -o topo.tsv + +# From stdin, capped at 4 threads +gunzip -c tree.nwk.gz | distree - -t 4 -o distances.tsv + +# Just the header, to check the tip ordering +distree tree.nwk | head -1 | tr '\t' '\n' | tail -n +2 +``` diff --git a/docs/guide/distances.md b/docs/guide/distances.md new file mode 100644 index 0000000..f73e518 --- /dev/null +++ b/docs/guide/distances.md @@ -0,0 +1,161 @@ +# Distance modes + +distree computes three different things, and they answer three different +questions. All of them come from the same primitive: for a pair of tips `i` and +`j`, find their most recent common ancestor `m`, then read off depths that were +precomputed in a single pass. + +| Mode | Flag | Cell `(i, j)` | +|:--|:--|:--| +| Patristic | *(default)* | `depth(i) + depth(j) - 2 · depth(m)` | +| Topological | `--topology` | `hops(i) + hops(j) - 2 · hops(m)` | +| Variance-covariance | `--lmm` | `depth(m)` | + +`depth` is the summed branch length from the root; `hops` is the number of edges +from the root. Every mode is one MRCA query plus three array reads, which is why +the cost of a cell does not grow with how far apart the tips are. + +The examples below all use: + +``` +((LeafA:1.95,LeafB:3.25):0.35,(LeafC:0.80,LeafD:1.20):0.50); +``` + +## Patristic + +```bash +distree tree.nwk -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 0.000 | 5.200 | 3.600 | 4.000 | +| **LeafB** | 5.200 | 0.000 | 4.900 | 5.300 | +| **LeafC** | 3.600 | 4.900 | 0.000 | 2.000 | +| **LeafD** | 4.000 | 5.300 | 2.000 | 0.000 | + +
+ +The sum of the branch lengths along the path between the two tips: how much +change has accumulated between them along the tree. This is the mode nearly +every downstream use wants, from transmission clustering to ordination. + +If the tree was built with a substitution model, a patristic distance is in +substitutions per site. Multiply by the alignment length for a SNP-scale +distance: + +```bash +distree tree.nwk -p 10 | awk 'NR==1 {print; next} {printf "%s", $1; + for (i=2; i<=NF; i++) printf "\t%.1f", $i * 4411532; print ""}' +``` + +The diagonal is exactly `0.0` and the matrix is exactly symmetric, both by +construction rather than by rounding. + +!!! note "Negative distances" + + With non-negative branch lengths, the subtraction cannot come out negative: + `depth` accumulates from the root, so `depth(i)` and `depth(j)` are both at + least `depth(m)`, and the arithmetic preserves that. A negative cell + therefore means the tree has a negative branch length, which + neighbour-joining produces routinely. distree reports it rather than + rounding it up to zero, since a zero would claim two distinct taxa are the + same sample. + +## Topological + +```bash +distree tree.nwk --topology +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 0 | 2 | 4 | 4 | +| **LeafB** | 2 | 0 | 4 | 4 | +| **LeafC** | 4 | 4 | 0 | 2 | +| **LeafD** | 4 | 4 | 2 | 0 | + +
+ +The number of edges on the path, written as integers with no decimal point and +no `-p`. LeafA and LeafB are siblings, so 2. LeafA to LeafC goes up two and back +down two, so 4. + +Use it when the branch lengths are not comparable across the tree, when they +come from concatenated loci with different rates, or when only the shape of the +tree is meant to matter. A cladogram with no lengths at all is exactly this +case: patristic mode would return an all-zero matrix and warns as much. + +Two things follow from counting edges rather than lengths. A polytomy shortens +distances, because the tips under it are one hop apart rather than several. And +the count does not depend on where the tree is rooted, which is why +[`--midpoint` is ignored here](rooting.md#why-midpoint-is-ignored-in-topology-mode). + +## Variance-covariance + +```bash +distree tree.nwk --lmm -p 3 +``` + +
+ +| | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.300 | 0.350 | 0.000 | 0.000 | +| **LeafB** | 0.350 | 3.600 | 0.000 | 0.000 | +| **LeafC** | 0.000 | 0.000 | 1.300 | 0.500 | +| **LeafD** | 0.000 | 0.000 | 0.500 | 1.700 | + +
+ +Entry `(i, j)` is the distance from the root down to the MRCA of the two tips: +the length of the evolutionary history they share before they diverge. Under a +Brownian-motion model of trait evolution, that is exactly the covariance between +their trait values, which is what makes this the `C` matrix of a phylogenetic +generalised least squares fit or a phylogenetic mixed model. + +Reading the matrix above: + +- `C[LeafA][LeafA] = 2.300` is LeafA's own root-to-tip length, `0.35 + 1.95`. + The diagonal is always the tip's total path from the root, because a tip's + MRCA with itself is itself. +- `C[LeafA][LeafB] = 0.350` is the depth of the node the two share. +- `C[LeafA][LeafC] = 0.000`, because their MRCA is the root, and they share no + history at all. + +Three consequences are worth keeping in mind: + +- **The matrix is not a distance.** The diagonal is not zero and larger values + mean more similarity, not less. Do not feed it to something expecting + distances. +- **Rooting changes it.** `depth` is measured from the root, so where the root + sits is part of the answer. This is the only mode where `--midpoint` does + anything. +- **An ultrametric tree gives a constant diagonal.** If every tip is the same + distance from the root, every `C[i][i]` is that distance, which is what most + comparative methods assume. + +Patristic and variance-covariance are two views of the same tree and convert +into each other: `d(i,j) = C[i][i] + C[j][j] - 2·C[i][j]`. Check it on the +matrices above: `2.300 + 3.600 - 2 × 0.350 = 5.200`, which is what the patristic +matrix says for LeafA and LeafB. + +## Choosing + +| If you are | Use | +|:--|:--| +| Clustering isolates by genetic distance | Patristic | +| Running PCoA, MDS, t-SNE or UMAP on the tree | Patristic | +| Building a distance-based tree from an existing one | Patristic, with [`--lower`](output.md#phylip-lower-triangle) | +| Fitting PGLS, `caper::pgls`, `nlme::gls` or a phylogenetic mixed model | `--lmm` | +| Comparing tree shapes, or working from a cladogram | `--topology` | +| Working with branch lengths from incomparable sources | `--topology` | + +## Next + +- [Midpoint rooting](rooting.md), for when the root is part of the answer. +- [Output](output.md), for the formats these matrices come out in. diff --git a/docs/guide/input.md b/docs/guide/input.md new file mode 100644 index 0000000..a4fec06 --- /dev/null +++ b/docs/guide/input.md @@ -0,0 +1,148 @@ +# Input + +distree reads one Newick tree, from a file or from stdin: + +```bash +distree tree.nwk +gunzip -c tree.nwk.gz | distree - +``` + +## What is accepted + +**Branch lengths** in any form Rust's float parser takes, including scientific +notation and negatives: `A:0.1`, `B:1.5e-3`, `C:2.0E+1`, `D:-0.5`. A missing +length is 0. + +**Polytomies.** `(A:1,B:2,C:3);` is a node with three children, and nothing in +distree assumes a bifurcating tree. + +**Internal node labels**, quoted or not, with or without a length: +`((A:1,B:2)clade:0.5,C:3);`. Bootstrap values live in this position too and are +read as labels, which is to say ignored. + +**Quoted tip labels**, single or double, so a label may hold spaces: +`('Taxon A':1.0,"Taxon B":2.0);`. Per the Newick convention a doubled quote +inside a quoted label is one literal quote, so `'it''s'` is the label `it's`. + +**Comments** in square brackets, before or after a label, nested, and at the +start of the file: + +``` +[&R] ((A:0.1[&&NHX:S=human],B:0.2[&rate=0.5]):0.3,C:0.4); +``` + +The `[&R]` / `[&U]` rooting marker that IQ-TREE, MrBayes and BEAST write is a +comment like any other. So is NHX and BEAST per-branch metadata, wherever it +sits relative to the label. All of it is skipped; the labels and the branch +lengths survive. + +**Non-ASCII labels.** Accents, Greek letters and CJK text pass through +unchanged, so a label keeps matching the sample name it came from. + +**Whitespace and newlines** anywhere outside a quoted label, which is how most +tools wrap a long tree over several lines. + +**A missing trailing semicolon**, which some tools omit. + +## What is rejected + +The guiding rule is that distree would rather stop than hand back a matrix that +looks right and is not. Each of these fails with a message naming the offending +position, and nothing is written to `-o`. + +| Input | Message | +|:--|:--| +| Empty file | `Empty input: no Newick tree found.` | +| `((A:1,B:2),C:3` | `Unexpected end of input: 1 unclosed '(' remain. The tree is truncated.` | +| `(A:1,B:2);(C:1,D:2);` | `the file appears to hold more than one tree` | +| `(A:1,B:2);garbage` | `Unexpected content after the end of the tree` | +| `('unclosed:1,B:2);` | `Unclosed quote starting at position 1.` | +| `(A:1,B:2)[oops;` | `Unclosed comment starting at position 9: no matching ']'.` | +| `(A:,B:2);` | `Expected a numeric branch length` | +| `(A:1,A:2);` | `Duplicate leaf name 'A' found. Leaf names must be unique.` | +| A label holding a tab, newline or carriage return | `contains a tab character, which would corrupt the output by splitting the row` | +| A tree with no labelled tips | `No labeled leaves found in the tree.` | + +!!! warning "One tree per file" + + A file of bootstrap replicates or a posterior sample holds many trees. + distree computes a matrix for one tree, and earlier versions silently used + the first one. It now refuses, because a matrix labelled with your dataset's + name that describes replicate 1 of 1,000 is worse than an error. Split the + file first: + + ```bash + split -l 1 posterior.trees tree_ + for t in tree_*; do distree "$t" -o "${t}.tsv"; done + ``` + +!!! question "Truncated trees" + + A tree cut short by an interrupted download or a full disk used to parse: + the open parentheses were closed at end of input, and every internal node + left dangling silently lost its branch length. The matrix came out looking + entirely reasonable. That is now an error naming how many parentheses were + still open. + +## Warnings + +These do not stop the run. They go to stderr, so they stay out of a piped +matrix. + +**Negative branch lengths.** + +``` +Warning: negative branch lengths detected in the tree. Some distances may come +out negative, and --midpoint cannot locate the diameter reliably. +``` + +Neighbour-joining and BioNJ produce these routinely. distree reports the +distances as computed rather than clamping them to zero, since a clamp would +claim two distinct taxa sit on top of each other. Midpoint rooting, on the other +hand, finds the longest path by a two-pass sweep that assumes non-negative +lengths, so `--midpoint` on such a tree is not reliable. + +**No branch lengths at all**, in patristic mode: + +``` +Warning: no branch lengths detected, all patristic distances will be zero. +Consider --topology. +``` + +A cladogram has topology and nothing else. `--topology` is the mode that reads +it. + +**Unlabelled tips.** + +``` +Warning: 1 leaf/leaves have no label and were excluded from the matrix. +``` + +A tip with no label cannot be named in a row or a column, so it is left out. The +matrix is correct for the tips that remain, but it is smaller than the tree, and +the warning is there so that is not a surprise. + +**Conflicting modes.** `--lmm` and `--topology` ask for different things; +passing both uses `--lmm` and says so. `--midpoint` with `--topology` is ignored, +for the reason in [Midpoint rooting](rooting.md). + +## Label rules + +Tip labels become row and column headers in a TSV, which puts two requirements +on them. + +**They must be unique.** Two tips called `A` would give two identical rows with +no way to tell which is which, so a duplicate is an error rather than a warning. + +**They must not hold a tab, a newline or a carriage return.** Any of the three +splits a row and leaves a file whose row count no longer matches its header. +A quoted label can hold them, so this is checked rather than assumed. Replace +them with underscores before running distree. + +Spaces are fine, as long as the label is quoted in the Newick. So is anything +else UTF-8 can express. + +## Next + +- [Distance modes](distances.md), for what distree does with the tree. +- [Output](output.md), for the shape of what comes back. diff --git a/docs/guide/output.md b/docs/guide/output.md new file mode 100644 index 0000000..938f05c --- /dev/null +++ b/docs/guide/output.md @@ -0,0 +1,152 @@ +# Output + +The matrix goes to stdout unless `-o` names a file. Warnings go to stderr, so a +piped run stays clean: + +```bash +distree tree.nwk | head -1 +distree tree.nwk -o distances.tsv +``` + +## The square TSV + +The default layout is a full tab-separated matrix: + +``` + LeafA LeafB LeafC LeafD +LeafA 0.000 5.200 3.600 4.000 +LeafB 5.200 0.000 4.900 5.300 +LeafC 3.600 4.900 0.000 2.000 +LeafD 4.000 5.300 2.000 0.000 +``` + +- The first line is a header of tip labels, preceded by an **empty cell** so the + labels sit over the data columns rather than one to the left. This is what + `read.table(header=TRUE, row.names=1)` and `pandas.read_csv(index_col=0)` + expect. +- Every following line begins with a tip label and holds one value per tip. +- **Tips are sorted alphabetically**, by byte order, in both the rows and the + columns. The order is the same in every run and does not depend on how the + tree was written, so two matrices from two trees over the same tips line up + cell for cell. +- The matrix is symmetric and the diagonal is exactly zero, except under + [`--lmm`](distances.md#variance-covariance), where the diagonal is each tip's + root-to-tip length. + +Reading it back: + +=== "R" + + ```r + m <- as.matrix(read.table("distances.tsv", header = TRUE, + row.names = 1, sep = "\t", check.names = FALSE)) + ``` + + `check.names = FALSE` keeps labels like `Sample-1` and `2024_isolate` intact; + without it R rewrites them. + +=== "Python" + + ```python + import pandas as pd + m = pd.read_csv("distances.tsv", sep="\t", index_col=0) + ``` + +=== "Shell" + + ```bash + # the distance between two named tips + awk -F'\t' -v a=LeafA -v b=LeafC ' + NR==1 { for (i=2; i<=NF; i++) if ($i==b) col=i; next } + $1==a { print $col }' distances.tsv + ``` + +## PHYLIP lower triangle + +```bash +distree tree.nwk --lower -p 4 +``` + +``` +4 +LeafA +LeafB 5.2000 +LeafC 3.6000 4.9000 +LeafD 4.0000 5.3000 2.0000 +``` + +A taxa count on the first line, then one row per taxon holding its label and the +distances to every taxon **above** it in the ordering. No diagonal, and none of +the mirrored half. + +This is the format PHYLIP's `neighbor` and `fitch` read, and what Mash and a +number of clustering tools expect. It also halves the output: a 50,000-tip tree +is 2.5 billion cells as a square matrix and 1.25 billion as a lower triangle. + +The first data row is the label alone, with no values after it, which is correct +and is what those readers expect. + +!!! note "Relaxed PHYLIP" + + Fields are tab-separated and labels are written in full. Strict PHYLIP pads + every name to exactly ten characters and truncates anything longer, which + would collide two tips whose names share a prefix. distree writes the + relaxed form that whitespace-splitting readers, including modern PHYLIP + builds, accept. + +## Precision + +```bash +distree tree.nwk -p 6 +``` + +`-p` sets the number of decimal places, defaulting to 10 and capped at 30. It +applies to patristic and `--lmm`; `--topology` writes integers and ignores it. + +A 64-bit float carries about 17 significant decimal digits, so anything past +that is zero padding, and the cap is there to keep an accidental `-p 50000000` +from being an error inside the formatter rather than an error about the flag. + +Precision is also the main lever on output size. At `p=10` each cell is 12 to 14 +bytes; at `p=6` it is 8 to 10. Across a 20,000-tip square matrix that is the +difference between roughly 5 GB and 3.5 GB. + +## Writing to a file + +```bash +distree tree.nwk -o distances.tsv +``` + +The file is not created until the tree has parsed, midpoint rooting has run and +the tip labels have been checked, so a run that fails on a malformed tree leaves +whatever was at that path untouched. + +Write errors are reported. A full disk fails with the path in the message and a +non-zero exit code, rather than leaving a truncated matrix behind and reporting +success. + +## Piping + +`distree tree.nwk | head` closes the pipe early, which is a normal way to look +at the first few rows. distree treats that as the reader having seen enough and +exits cleanly rather than reporting a broken pipe. + +Because rows are written as they are computed, the first row of a very large +matrix appears long before the last one. A pipeline that consumes rows as they +arrive does not wait for the whole matrix: + +```bash +distree big.nwk -p 6 | awk -F'\t' 'NR>1 { for (i=2; i<=NF; i++) if ($i+0 < 1e-5 && i-1 != NR-1) print $1, i }' +``` + +## Exit codes + +| Code | Meaning | +|:--|:--| +| 0 | The matrix was written and flushed. Also the code when a downstream pipe closed early | +| 1 | Anything went wrong: an unreadable file, a malformed tree, a bad flag, a failed write. The message is on stderr and begins with `Error:` | + +## Next + +- [Recipes](../recipes.md), for what to do with the matrix once you have it. +- [CLI reference](cli.md), for every flag in one table. diff --git a/docs/guide/rooting.md b/docs/guide/rooting.md new file mode 100644 index 0000000..f6228a6 --- /dev/null +++ b/docs/guide/rooting.md @@ -0,0 +1,129 @@ +# Midpoint rooting + +```bash +distree tree.nwk --midpoint --lmm -p 3 +``` + +`--midpoint` re-roots the tree at the middle of its longest tip-to-tip path +before any distance is computed. It exists for one mode. + +## When rooting matters + +A [patristic](distances.md#patristic) distance is the length of the path between +two tips. A [topological](distances.md#topological) distance is the number of +edges on that path. Neither depends on where you call the top of the tree: it is +the same path either way. Re-rooting a tree cannot change either matrix, and +distree's own tests assert exactly that. + +A [variance-covariance](distances.md#variance-covariance) entry is the distance +from **the root** down to the MRCA of two tips. Move the root and every entry +moves. That makes `--lmm` the mode where rooting is a modelling decision rather +than a formality, and `--midpoint` the flag that makes it explicitly. + +## What it does + +The longest path between any two tips is the tree's diameter. Midpoint rooting +puts the new root exactly halfway along it, so the two deepest tips end up +equidistant from the root, and no tip is further from the root than half the +diameter. + +On the tree used throughout this guide: + +``` +((LeafA:1.95,LeafB:3.25):0.35,(LeafC:0.80,LeafD:1.20):0.50); +``` + +the diameter runs LeafB to LeafD and measures `3.25 + 0.35 + 0.50 + 1.20 = 5.30`. +Half of that is 2.65, which falls **inside LeafB's own branch**, 2.65 up from +LeafB and 0.60 below the node it hangs from. distree inserts a new root there, +splitting the 3.25 branch into 2.65 and 0.60, and reverses the parent-child +links from that point up to the old root so the tree hangs correctly from its +new top. + +The `--lmm` matrix before and after: + +
+ +| Original root | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.300 | 0.350 | 0.000 | 0.000 | +| **LeafB** | 0.350 | 3.600 | 0.000 | 0.000 | +| **LeafC** | 0.000 | 0.000 | 1.300 | 0.500 | +| **LeafD** | 0.000 | 0.000 | 0.500 | 1.700 | + +
+ +
+ +| Midpoint root | LeafA | LeafB | LeafC | LeafD | +|:--|--:|--:|--:|--:| +| **LeafA** | 2.550 | 0.000 | 0.600 | 0.600 | +| **LeafB** | 0.000 | 2.650 | 0.000 | 0.000 | +| **LeafC** | 0.600 | 0.000 | 2.250 | 1.450 | +| **LeafD** | 0.600 | 0.000 | 1.450 | 2.650 | + +
+ +The whole structure has changed. LeafB now splits off at the root and shares +nothing with anyone, while LeafA, LeafC and LeafD share 0.600 of history that +the original rooting attributed differently. The deepest tips, LeafB and LeafD, +sit at 2.650, which is half the diameter, as they should. + +The patristic matrix, meanwhile, is byte-identical with and without the flag. + +## Why midpoint is ignored in topology mode + +Adding `--midpoint` to `--topology` used to inflate the answer. The new root is +a **new node**, inserted partway along an existing edge, and it splits that edge +into two. Every pair of tips whose path crosses that edge therefore gained one +hop: + +``` +((A:1,B:1):1,(C:1,D:1):1); +``` + +| | Without `--midpoint` | With `--midpoint` | +|:--|--:|--:| +| A to C | 4 | 5 | +| A to B | 2 | 2 | + +The 5 was an artefact of the extra node, not a property of the tree. Since edge +counts cannot depend on the root in the first place, there is nothing +`--midpoint` could correctly contribute here, so distree skips the rooting and +says so: + +``` +Warning: --midpoint is ignored in --topology mode. Edge counts do not depend on +the root, and the node inserted at the midpoint would add 1 to every distance +crossing that edge. +``` + +The same insertion happens in patristic mode, but there it is harmless: the two +pieces of the split edge sum to the original length, so no path changes. + +## Caveats + +!!! warning "Negative branch lengths" + + distree finds the diameter with the standard two-pass sweep: walk to the + furthest tip from an arbitrary tip, then to the furthest tip from that one. + The argument that this finds the true diameter assumes non-negative edge + weights, and a neighbour-joining tree does not satisfy it. Midpoint rooting + such a tree may pick the wrong path, and the run warns about the negative + lengths for that reason among others. + +**A zero-length diameter leaves the tree alone.** If every branch is zero there +is no midpoint to find, and distree returns the tree as it was. + +**Only tips count.** The diameter is measured tip to tip, matching what `ape` +and `phytools` do. A root with a single child is a degree-one node in the +unrooted tree, but it is not a tip and does not extend the diameter. + +**The new root is unlabelled** and adds one node to the tree. That is invisible +in the output, which only ever has one row per labelled tip. + +## Next + +- [Distance modes](distances.md), for what rooting does and does not affect. +- [How it works](../how-it-works/algorithm.md#midpoint-rooting), for the + re-rooting procedure itself. diff --git a/docs/how-it-works/algorithm.md b/docs/how-it-works/algorithm.md new file mode 100644 index 0000000..4d0226d --- /dev/null +++ b/docs/how-it-works/algorithm.md @@ -0,0 +1,126 @@ +# How it works + +A run has five stages. The first four touch the tree once each; the last one +does all the work. + +```mermaid +flowchart LR + A[Read Newick] --> B[Parse to a flat node array] + B --> C{--midpoint?} + C -- yes --> D[Re-root at the diameter midpoint] + C -- no --> E + D --> E[Build the LCA structure] + E --> F[Stream one row at a time] +``` + +## Parsing + +The Newick string is read once to strip whitespace outside quoted labels, then +scanned byte by byte. Every structural character in Newick is ASCII, so a +byte-wise scan is safe on UTF-8 input: no byte of a multi-byte sequence can be +mistaken for a delimiter. Label bytes are collected and decoded as UTF-8 in one +step, which is what keeps accented and CJK tip names intact. + +The parser is **iterative**, with an explicit stack of partially built nodes +rather than a recursive descent, so a tree nested hundreds of thousands of +levels deep parses without touching the call stack. The same applies to freeing +the parse tree: the compiler's own drop glue would recurse once per level, so +`RawNode` dismantles its children in a loop instead. + +Parsing is strict about structure. Reaching the end of input with parentheses +still open is an error, not an implicit close, and anything after the tree other +than a semicolon and comments is an error too. Both cases used to produce a +matrix rather than a message. See [Input](../guide/input.md#what-is-rejected). + +The recursive parse tree is then flattened, again iteratively, into a +`Vec` where a node is a label, a branch length, a parent index and a list +of child indices. Everything after this point is index arithmetic over flat +arrays. + +## Midpoint rooting + +Only when `--midpoint` is passed, and only outside `--topology`. + +The diameter is found with the standard two-pass sweep: from an arbitrary tip, +walk the tree to the furthest tip `a`; from `a`, walk to the furthest tip `b`. +The second sweep records the path it took, so `a` to `b` can be reconstructed +without a third pass. + +distree then walks that path accumulating length until it passes half the +diameter, which locates the edge the midpoint falls on. It inserts a new node +there, splitting the edge into the two pieces either side of the midpoint, +attaches both ends to it, and reverses the parent-child links from the old +parent up to the old root, carrying each branch length down to the node that is +now the child. The result is the same tree hanging from a different point. + +The tests check this on 300 generated trees, including polytomies and +zero-length branches, against three properties: every pairwise patristic +distance survives, the result is still a tree with nothing orphaned and no +cycle, and the deepest tip sits exactly half a diameter from the new root. + +## The LCA structure + +This is where the run buys its speed. Walking the tree for each of `N²` pairs +would cost `O(N² · depth)`. Instead one depth-first pass records, for every node: + +- `depth_len`, its summed branch length from the root; +- `depth_top`, its edge count from the root; +- `up[0]`, its parent. + +The parent table is then doubled `log₂ M` times, so that `up[k][u]` is the +`2^k`-th ancestor of `u`. Finding the most recent common ancestor of two nodes +becomes: lift the deeper one by the difference in `depth_top`, one power of two +per set bit; then, from the largest jump down to the smallest, move both up +together whenever their ancestors at that level still differ. They meet one step +above where they stop. That is `O(log M)` per query, with no allocation. + +The table is stored as one flat `Vec` with `usize::MAX` standing in for +"no ancestor". An `Option` would be the obvious type, but it has no spare +niche and so costs 16 bytes per entry instead of 8, across `M log M` entries. + +The queries are checked against a brute-force walk from both nodes to the root, +over every pair of nodes in 200 generated trees. + +## Computing a cell + +With the structure in place, every mode is one MRCA query and three array reads: + +| Mode | Cell `(i, j)` | +|:--|:--| +| Patristic | `depth_len[i] + depth_len[j] - 2 · depth_len[m]` | +| Topological | `depth_top[i] + depth_top[j] - 2 · depth_top[m]` | +| Variance-covariance | `depth_len[m]` | + +No square root, no allocation, no tree walk. The cost of a cell does not depend +on how far apart the two tips are. + +## Streaming the matrix + +Tips are collected, checked for duplicates and for characters that would break a +row, and sorted by label. Then, for each row in turn: + +1. the row's cells are computed in parallel across the thread pool, into a + buffer that is reused between rows; +2. the row is written; +3. the next row starts. + +At no point does the full matrix exist. Peak memory is the tree plus the LCA +table plus one row, which is why a matrix far larger than RAM can be written to +a file or straight into a pipeline. See +[Performance](performance.md) for the arithmetic. + +Rayon splits each row across the available cores, and `-t` caps how many. The +row order and the cell order are fixed regardless of thread count, so two runs +of the same tree produce byte-identical output. + +## Why the output is sorted + +Tips are ordered by label, not by their position in the tree. That costs one +sort and buys three things: the ordering does not depend on how the Newick was +written, two matrices over the same tips line up cell for cell, and a run is +reproducible without recording anything about the input's layout. + +## Next + +- [Performance](performance.md), for what all this costs. +- [Distance modes](../guide/distances.md), for what the numbers mean. diff --git a/docs/how-it-works/performance.md b/docs/how-it-works/performance.md new file mode 100644 index 0000000..182c704 --- /dev/null +++ b/docs/how-it-works/performance.md @@ -0,0 +1,139 @@ +# Performance + +## Complexity + +For a tree with `N` labelled tips and `M` nodes: + +| Stage | Time | Memory | +|:--|:--|:--| +| Parse and flatten | `O(M)` | `O(M)` | +| Midpoint rooting | `O(M)` | `O(M)` | +| LCA structure | `O(M log M)` | `O(M log M)` | +| The matrix | `O(N² log M)` | `O(1)` beyond the buffer | + +Everything before the matrix is linear or near-linear and, in practice, free. +On a 100,000-tip tree, parsing and building the LCA structure takes 0.05 s. The +run is the matrix, and the matrix is quadratic in the number of tips no matter +what: `N²` cells have to be computed and written. + +The `log M` in the cell cost is the MRCA query. It is a handful of array reads +against a table that fits in cache for anything under a few thousand nodes, so +the constant is small; the format of the number costs more than finding it. + +## Measured + +Balanced binary trees, `-p 6 --lower`, output discarded, release build on an +Apple M4 Pro with 14 cores: + +| Tips | Cells | Time | Peak memory | +|--:|--:|--:|--:| +| 1,000 | 0.5 M | 0.00 s | 11 MB | +| 2,000 | 2.0 M | 0.01 s | 24 MB | +| 4,000 | 8.0 M | 0.06 s | 35 MB | +| 8,000 | 32 M | 0.23 s | 32 MB | +| 20,000 | 200 M | 1.8 s | 35 MB | + +Time is quadratic in the tips, as it must be. Memory is not: it flattens out +around 35 MB, because past a few thousand tips it is the fixed batch buffer plus +the LCA table rather than anything that grows with the matrix. + +## Threads + +`-t` caps the thread count; the default is every core. On the 8,000-tip tree: + +| `-t` | Time | Speedup | +|--:|--:|--:| +| 1 | 1.52 s | 1.0x | +| 2 | 0.66 s | 2.3x | +| 4 | 0.36 s | 4.2x | +| 8 | 0.24 s | 6.3x | +| 14 | 0.23 s | 6.6x | + +Each worker computes and formats whole rows, so the parallel section covers both +halves of the cost and the curve holds up until it runs into memory bandwidth +and the single writer. + +!!! note "This was not always true" + + Before version 1.0.1, each row was its own parallel job. A cell is one MRCA + query and three array reads, so a row of a small matrix could not pay for + synchronising the thread pool, and the same 8,000-tip run took 2.52 s on 14 + cores against 1.52 s on one. Formatting the floats sat in the serial write + loop on top of that. Both are fixed; rows are now batched, and each worker + formats its own. + +Set `-t` when you are sharing a machine, or when the run is part of a pipeline +that is already parallel. Leaving it unset is right for a dedicated node. + +## Memory + +Nothing in distree holds the matrix. The peak is: + +``` + LCA table M × (⌈log₂ M⌉ + 1) × 8 bytes + Node depths M × 16 bytes + The tree roughly M × 72 bytes, plus the labels + Batch buffer about 15 MB, fixed + Output buffer 1 MB +``` + +For a bifurcating tree, `M ≈ 2N`. Worked through: + +| Tips | Nodes | LCA table | Total, roughly | +|--:|--:|--:|--:| +| 10,000 | 20,000 | 2.6 MB | 35 MB | +| 100,000 | 200,000 | 30 MB | 100 MB | +| 1,000,000 | 2,000,000 | 350 MB | 600 MB | + +The 100,000-tip figure is measured; the million-tip one is the arithmetic. + +The LCA table is the term that grows, and it is stored as plain `usize` with a +sentinel rather than `Option`, which halves it. The batch buffer holds +about a million cells' worth of formatted rows regardless of tree size, which is +what keeps the total flat as the matrix grows. + +## Output size + +The output is usually larger than anything in memory, and precision is the lever +on it. At `-p 10` a cell is 12 to 14 bytes; at `-p 6` it is 8 to 10. + +| Tips | Square, `-p 10` | Square, `-p 6` | `--lower`, `-p 6` | +|--:|--:|--:|--:| +| 1,000 | 13 MB | 9 MB | 4.5 MB | +| 10,000 | 1.3 GB | 900 MB | 450 MB | +| 50,000 | 33 GB | 22 GB | 11 GB | + +Two things follow. Use `--lower` when the reader accepts it, which halves the +file. And do not ask for more decimals than the branch lengths carry: `-p 6` +against the default `-p 10` is a third off the file for no loss on any realistic +tree. + +Rows are written as they are computed, so a pipeline that consumes them as they +arrive never holds the whole matrix either: + +```bash +distree big.nwk -p 6 | gzip > distances.tsv.gz +``` + +## Where the ceiling is + +The quadratic term decides. Around 50,000 tips a square matrix is tens of +gigabytes and the run is minutes; at 100,000 it is hundreds of gigabytes and +the file, not distree, is the problem. + +If the tips are more than that, the question is usually not really "what is the +whole matrix". Reduce the tree first, with something like +[axetree](https://github.com/PathoGenOmics-Lab/axetree), and compute the matrix +over the subset. + +## Reproducibility + +The row order is the tip labels sorted, and the cell order within a row is the +same sort, neither of which depends on the thread count or on how the Newick was +written. Two runs of the same tree produce byte-identical output, on any number +of cores. + +## Next + +- [How it works](algorithm.md), for what these stages actually do. +- [Output](../guide/output.md), for the formats and the precision flag. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..5b171cc --- /dev/null +++ b/docs/index.md @@ -0,0 +1,105 @@ +--- +hide: + - navigation +--- + +
+ +![distree](assets/distree_wordmark.svg){ .hero-logo } + +# distree + +Pairwise distance matrices from a phylogeny, in Rust. + +
+ +Once a tree is built, most of the questions that follow are pairwise. Which +isolates sit within five substitutions of each other. What the ordination looks +like. What covariance structure a comparative model should assume. All of them +want the same thing first: the distance between every pair of tips. + +Computing that naively means walking the tree once per pair, and holding an +N × N matrix while you do it. distree instead spends `O(M log M)` up front on a +binary-lifting structure that answers *"where do these two tips meet?"* in +`O(log M)` time, then streams the matrix out one row at a time. A row is +computed in parallel across cores and written before the next one starts, so +memory tracks the number of tips, not the square of it. + +
+ +- :material-download: **[Install](getting-started/installation.md)** + + Bioconda, a prebuilt binary, or `cargo build --release`. No runtime + dependencies. + +- :material-rocket-launch: **[Quick start](getting-started/quickstart.md)** + + The handful of commands that cover most of what people want a matrix for. + +- :material-file-tree: **[Input](guide/input.md)** + + What Newick distree accepts, what it rejects, and why it rejects rather + than guesses. + +- :material-ruler: **[Distance modes](guide/distances.md)** + + Patristic, topological and variance-covariance, on one worked example. + +- :material-format-vertical-align-center: **[Midpoint rooting](guide/rooting.md)** + + When the root changes the answer, and when it cannot. + +- :material-table: **[Output](guide/output.md)** + + The TSV layout, PHYLIP lower triangle, precision, and reading it back in. + +- :material-cog: **[How it works](how-it-works/algorithm.md)** + + The parser, the LCA structure, and the streaming loop. + +- :material-speedometer: **[Performance](how-it-works/performance.md)** + + What the run costs in time and memory, and where the ceiling is. + +
+ +## In one command + +```bash +distree tree.nwk --lower -p 6 -t 8 -o distances.phy +``` + +Patristic distances for every pair of tips, as a PHYLIP lower triangle, six +decimal places, across eight threads. Swap `--topology` for edge counts, +`--lmm` for the variance-covariance matrix a comparative model wants, and drop +`--lower` for a full square TSV that `read.table` and `pandas` open directly. + +## What it does + +| | | +|:--|:--| +| **Patristic** | The sum of branch lengths on the path between two tips | +| **Topological** | The number of edges on that path, ignoring branch lengths | +| **Variance-covariance** | The root-to-MRCA distance for each pair, the `C` matrix of a PGLS or a phylogenetic mixed model | +| **Rooting** | Optional midpoint rooting, which matters for `--lmm` and cannot matter for the other two | +| **Formats** | Square TSV with labels, or a PHYLIP lower triangle | +| **Input** | Newick from a file or stdin, with quoted labels, polytomies, NHX and BEAST comments, and UTF-8 tip names | +| **Scale** | Memory grows with the number of tips, not with the matrix; rows stream out as they are computed | +| **Parallelism** | Each row is computed across all cores, or as many as `-t` allows | + +## What it does not do + +distree works on the tree you give it. It does not build one, it does not read +an alignment, and it does not cluster the matrix for you. + +It also does not guess. A truncated tree, a file holding several trees, an +unclosed quote or a label with a tab in it are all rejected with a message +naming the position, rather than turned into a matrix that looks right and is +not. The one case that is reported and carried through rather than refused is a +negative branch length, because neighbour-joining trees legitimately have them +and the resulting negative distances are information, not noise. + +## Citation + +If you use distree, please cite the archived release. See +[Citation](about/citation.md) for the DOI and the version to record. diff --git a/docs/recipes.md b/docs/recipes.md new file mode 100644 index 0000000..97da5c3 --- /dev/null +++ b/docs/recipes.md @@ -0,0 +1,206 @@ +# Recipes + +Short, complete examples of what people actually do with the matrix. + +## Ordination: PCoA and MDS + +=== "R" + + ```r + m <- as.matrix(read.table("distances.tsv", header = TRUE, + row.names = 1, sep = "\t", check.names = FALSE)) + fit <- cmdscale(as.dist(m), k = 2, eig = TRUE) + plot(fit$points, pch = 19, xlab = "PCo1", ylab = "PCo2") + text(fit$points, labels = rownames(m), pos = 3, cex = 0.6) + + # variance explained by the first two axes + round(100 * fit$eig[1:2] / sum(fit$eig[fit$eig > 0]), 1) + ``` + +=== "Python" + + ```python + import pandas as pd + from sklearn.manifold import MDS + + m = pd.read_csv("distances.tsv", sep="\t", index_col=0) + coords = MDS(n_components=2, dissimilarity="precomputed", + random_state=0, normalized_stress="auto").fit_transform(m.values) + ``` + + For UMAP, pass the same matrix with `metric="precomputed"`. + +Generate it with plenty of precision; ordination is sensitive to the small +distances: + +```bash +distree tree.nwk -p 10 -o distances.tsv +``` + +## Transmission clusters by a SNP threshold + +A patristic distance from a tree built on an alignment is in substitutions per +site. Multiply by the alignment length to get SNP-scale distances, then cut the +matrix at your threshold: + +=== "R" + + ```r + m <- as.matrix(read.table("distances.tsv", header = TRUE, + row.names = 1, sep = "\t", check.names = FALSE)) + snps <- m * 4411532 # H37Rv genome length + clusters <- cutree(hclust(as.dist(snps), method = "single"), h = 5) + split(names(clusters), clusters) + ``` + +=== "Python" + + ```python + import pandas as pd + from scipy.cluster.hierarchy import linkage, fcluster + from scipy.spatial.distance import squareform + + m = pd.read_csv("distances.tsv", sep="\t", index_col=0) * 4411532 + z = linkage(squareform(m.values, checks=False), method="single") + labels = fcluster(z, t=5, criterion="distance") + ``` + +Single linkage is the right choice for a transmission threshold: it clusters +samples that are within the threshold **of someone** in the cluster, which is +what a chain of transmission looks like. + +## PGLS and phylogenetic mixed models + +`--lmm` writes the `C` matrix these models want. Root the tree deliberately +first, because [rooting is part of the answer](guide/rooting.md): + +```bash +distree tree.nwk --midpoint --lmm -p 10 -o varcovar.tsv +``` + +```r +C <- as.matrix(read.table("varcovar.tsv", header = TRUE, + row.names = 1, sep = "\t", check.names = FALSE)) +traits <- read.csv("traits.csv", row.names = 1) +C <- C[rownames(traits), rownames(traits)] # align the orderings + +library(nlme) +fit <- gls(y ~ x, data = traits, + correlation = corSymm(C[lower.tri(C)] / max(C), fixed = TRUE)) +``` + +Two things to check before fitting. The row and column order is alphabetical by +tip label, which is almost certainly not the order of your trait table, so +reindex rather than assume. And a non-constant diagonal means the tree is not +ultrametric, which several comparative methods assume. + +## Neighbour-joining from an existing tree + +PHYLIP's `neighbor` reads the lower triangle directly: + +```bash +distree tree.nwk --lower -p 6 -o infile +printf 'L\nY\n' | neighbor +``` + +The `L` selects lower-triangular input; `neighbor` writes `outtree` and +`outfile`. + +## The closest relative of every sample + +```bash +distree tree.nwk -p 8 | awk -F'\t' ' + NR == 1 { for (i = 2; i <= NF; i++) name[i] = $i; next } + { + best = ""; bestd = 1e308 + for (i = 2; i <= NF; i++) if (i - 1 != NR - 1 && $i + 0 < bestd) { bestd = $i + 0; best = name[i] } + printf "%s\t%s\t%s\n", $1, best, bestd + }' +``` + +Skipping `i - 1 == NR - 1` skips the diagonal, which is otherwise always the +closest. + +## Comparing two trees over the same tips + +Because the tip ordering is alphabetical in every run, two matrices over the +same tip set line up cell for cell: + +```bash +distree iqtree.nwk -p 10 --lower -o a.phy +distree raxml.nwk -p 10 --lower -o b.phy +``` + +```python +import numpy as np, pandas as pd +from scipy.stats import pearsonr + +def lower(path): + vals = [] + with open(path) as fh: + next(fh) # taxa count + for line in fh: + vals += [float(v) for v in line.split("\t")[1:]] + return np.array(vals) + +a, b = lower("a.phy"), lower("b.phy") +print(pearsonr(a, b)) +``` + +A high correlation with a slope away from 1 means the two trees agree on the +shape and disagree on the rate. + +## Cladograms and unreliable branch lengths + +A tree with no branch lengths gives an all-zero patristic matrix, and says so. +Count edges instead: + +```bash +distree cladogram.nwk --topology -o topo.tsv +``` + +The same applies when the lengths exist but are not comparable across the tree, +for instance from concatenated loci with different rates. + +## Very large trees + +Rows are written as they are computed, so nothing needs the whole matrix in one +piece. Compress on the way out: + +```bash +distree big.nwk -p 6 --lower | gzip > distances.phy.gz +``` + +Or filter as it streams, keeping only the pairs that matter: + +```bash +distree big.nwk -p 8 --lower | awk -F'\t' ' + NR == 1 { next } + { for (i = 2; i <= NF; i++) if ($i + 0 < 1e-5) print $1, i - 1, $i }' +``` + +Past 50,000 tips or so the file becomes the constraint rather than the +computation. See [Performance](how-it-works/performance.md#where-the-ceiling-is). + +## Batch runs + +One matrix per tree, in parallel, one core each: + +```bash +ls trees/*.nwk | xargs -P 8 -I{} sh -c \ + 'distree "$1" -t 1 -p 6 --lower -o "${1%.nwk}.phy"' _ {} +``` + +`-t 1` matters here: without it every distree would try to use every core and +they would fight each other. + +## Splitting a multi-tree file + +distree takes one tree per file, and refuses a file holding several rather than +silently using the first: + +```bash +split -l 1 -d posterior.trees rep_ --additional-suffix=.nwk +ls rep_*.nwk | xargs -P 8 -I{} sh -c \ + 'distree "$1" -t 1 -p 6 --lower -o "${1%.nwk}.phy"' _ {} +``` diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 0000000..e539f3c --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,139 @@ +/* ========================================================================== + distree theme + The sage green of the mark for the chrome, a warm clay for anything the + reader can act on, so a page has exactly two colours that mean something. + ========================================================================== */ + +[data-md-color-primary="custom"] { + --md-primary-fg-color: #40695e; + --md-primary-fg-color--light: #568b7d; + --md-primary-fg-color--dark: #2f4f47; +} + +[data-md-color-accent="custom"] { + --md-accent-fg-color: #b25f3c; +} + +[data-md-color-scheme="slate"] { + --md-default-bg-color: hsl(168, 8%, 9%); + --md-code-bg-color: hsl(168, 7%, 14%); +} +[data-md-color-scheme="slate"][data-md-color-accent="custom"] { + --md-accent-fg-color: #e39a63; +} + +/* The mark is wide rather than square, so it needs width to stay legible next + to the site name. */ +.md-header__button.md-logo img { + width: auto; + height: 1.5rem; +} + +/* ========================================================================== + Typography + Material's defaults are a touch loose for a reference site: tighten the + headings and give sections more air above than below, so a long page reads + as blocks rather than an even stream. + ========================================================================== */ + +.md-typeset h1 { + font-weight: 700; + letter-spacing: -0.02em; + color: var(--md-default-fg-color); +} + +.md-typeset h2 { + font-weight: 650; + letter-spacing: -0.01em; + margin-top: 2.4em; + padding-bottom: 0.3rem; + border-bottom: 1px solid var(--md-default-fg-color--lightest); +} + +.md-typeset h3 { + font-weight: 600; + margin-top: 1.8em; +} + +/* ========================================================================== + Tables + Most of this site is flag tables and small matrices, and Material lets them + stretch to the text column and wrap every cell. Let them scroll instead, and + keep the header in place while you read down a long one. + ========================================================================== */ + +.md-typeset table:not([class]) { + font-size: 0.72rem; + display: table; +} + +.md-typeset table:not([class]) th { + position: sticky; + top: 0; + background: var(--md-default-bg-color); + font-weight: 600; + white-space: nowrap; +} + +.md-typeset table:not([class]) td code { + white-space: nowrap; +} + +/* A distance matrix is a grid of numbers: right-aligning it makes the decimal + points line up, which is the only way to read one by eye. */ +.md-typeset .matrix table:not([class]) td:not(:first-child), +.md-typeset .matrix table:not([class]) th:not(:first-child) { + text-align: right; + font-variant-numeric: tabular-nums; +} + +/* ========================================================================== + Home page + ========================================================================== */ + +.hero { + text-align: center; + margin: 0.4rem 0 2.2rem; +} + +.hero .hero-logo { + width: 260px; + max-width: 80%; + margin: 0 auto; +} + +.md-typeset .hero h1 { + margin: 0.8rem 0 0; + font-size: 2rem; +} + +.md-typeset .hero p { + max-width: 34rem; + margin: 0.9rem auto 0; + font-size: 0.85rem; + color: var(--md-default-fg-color--light); +} + +/* Material's own card grid, kept a little tighter than the default. */ +.md-typeset .grid.cards > ul > li { + border-radius: 0.4rem; + padding: 0.8rem 1rem; +} + +.md-typeset .grid.cards > ul > li:hover { + border-color: var(--md-accent-fg-color); +} + +.md-typeset .grid.cards > ul > li > hr { + margin: 0.6rem 0; +} + +.md-typeset .grid.cards .twemoji, +.md-typeset .grid.cards svg { + color: var(--md-primary-fg-color); + vertical-align: -0.15em; +} + +[data-md-color-scheme="slate"] .md-typeset .grid.cards svg { + color: var(--md-primary-fg-color--light); +} diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..3f437a2 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,144 @@ +site_name: distree +site_description: >- + Pairwise distance matrices from a phylogeny. distree reads a Newick tree and + writes patristic, topological or variance-covariance distances between every + pair of leaves, one row at a time, in parallel and without holding the matrix + in memory. +site_author: PathoGenOmics Lab +site_url: https://pathogenomics-lab.github.io/distree/ + +repo_name: PathoGenOmics-Lab/distree +repo_url: https://github.com/PathoGenOmics-Lab/distree +edit_uri: edit/main/docs/ +copyright: Copyright © PathoGenOmics Lab · GPL-3.0 + +theme: + name: material + logo: assets/distree_logo.svg + favicon: assets/distree_logo.svg + language: en + font: + text: Inter + code: JetBrains Mono + icon: + repo: fontawesome/brands/github + edit: material/pencil + admonition: + note: material/dna + palette: + - media: "(prefers-color-scheme)" + primary: custom + accent: custom + toggle: + icon: material/brightness-auto + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: custom + accent: custom + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: custom + accent: custom + toggle: + icon: material/brightness-4 + name: Switch to system preference + features: + - navigation.tabs + - navigation.tabs.sticky + - navigation.sections + - navigation.top + - navigation.footer + - navigation.indexes + - navigation.tracking + - toc.follow + - search.suggest + - search.highlight + - search.share + - content.code.copy + - content.code.annotate + - content.tabs.link + - content.tooltips + +# Broken cross-references and dead anchors become warnings, which `mkdocs build +# --strict` in CI turns into a failed build. +validation: + omitted_files: warn + absolute_links: warn + unrecognized_links: warn + anchors: warn + +extra_css: + - stylesheets/extra.css + +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - tables + - toc: + permalink: true + toc_depth: 3 + - pymdownx.betterem + - pymdownx.caret + - pymdownx.keys + - pymdownx.mark + - pymdownx.tilde + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets: + base_path: + - "." + check_paths: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + +plugins: + - search + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/PathoGenOmics-Lab/distree + name: distree on GitHub + generator: false + +nav: + - Home: index.md + - Getting started: + - getting-started/installation.md + - getting-started/quickstart.md + - User guide: + - guide/input.md + - guide/distances.md + - guide/rooting.md + - guide/output.md + - guide/cli.md + - How it works: + - how-it-works/algorithm.md + - how-it-works/performance.md + - Recipes: recipes.md + - About: + - about/changelog.md + - about/citation.md + - about/contributing.md diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..422f4b7 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,4 @@ +# Documentation site (MkDocs Material). Build locally with: +# pip install -r requirements-docs.txt +# mkdocs serve +mkdocs-material>=9.6,<10 From 13488a3dc1f7a3be4ac7ca81fbd3d996c04f317c Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:27:06 +0200 Subject: [PATCH 35/47] Point the README at the documentation site A table of the nine pages people actually go looking for, right under the tagline. The rest of the README stays as it is, so anyone reading it on GitHub without following a link still has everything. --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 00f0ffc..e5e2a60 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,20 @@ __and Mireia Coscolla1__ `distree` is a command-line tool written in Rust that extracts a distance matrix from a phylogenetic tree in Newick format. It is designed to handle large trees with thousands of sequences by using a low-memory, parallelized approach. +📖 **Full documentation: ** + +| Page | Description | +|:-----|:------------| +| **[Quick start](https://pathogenomics-lab.github.io/distree/getting-started/quickstart/)** | The handful of commands most runs actually use | +| **[Input](https://pathogenomics-lab.github.io/distree/guide/input/)** | What Newick is accepted, what is rejected, and why | +| **[Distance modes](https://pathogenomics-lab.github.io/distree/guide/distances/)** | Patristic, topological and var-covar, on one worked example | +| **[Midpoint rooting](https://pathogenomics-lab.github.io/distree/guide/rooting/)** | When the root changes the answer, and when it cannot | +| **[Output](https://pathogenomics-lab.github.io/distree/guide/output/)** | TSV, PHYLIP lower triangle, precision, reading it back | +| **[CLI reference](https://pathogenomics-lab.github.io/distree/guide/cli/)** | Every flag, every default, every message | +| **[How it works](https://pathogenomics-lab.github.io/distree/how-it-works/algorithm/)** | The parser, the LCA structure, the streaming loop | +| **[Performance](https://pathogenomics-lab.github.io/distree/how-it-works/performance/)** | Measured speed, memory and thread scaling | +| **[Recipes](https://pathogenomics-lab.github.io/distree/recipes/)** | PCoA, transmission clusters, PGLS, neighbour-joining | + ## Features * **Patristic distances**: Computes the sum of branch lengths between every pair of leaves (taxa). From 9f4fa04408e2c17274a119afc90377319d1ec66d Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:27:32 +0200 Subject: [PATCH 36/47] Record the performance work in the changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bcdac7..04863fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [1.0.1] - 2026-04-04 ### Fixed +- Parallel computation now speeds the run up instead of slowing it down. One parallel job per row could not pay for synchronising the thread pool, and float formatting sat in the serial write loop; rows are now batched and each worker formats its own. A 20,000-tip matrix went from 10.6 s to 1.6 s, and an 8,000-tip one now scales from 1.26 s on one core to 0.21 s on eight - Trees prefixed with a `[&R]` / `[&U]` rooting marker, or carrying a comment before a label, no longer fail to parse - Non-ASCII leaf labels (accents, Greek, CJK) are preserved instead of being mangled into mojibake - Truncated trees, trailing content, unclosed comments and free-form text are rejected instead of yielding a plausible but wrong matrix @@ -35,11 +36,14 @@ - Warning when no branch lengths are detected in patristic mode - Warning when negative branch lengths are found in the tree - CITATION.cff with DOI +- MkDocs Material documentation site, published to GitHub Pages - Comprehensive test suite (73 tests), including randomised checks of midpoint rooting and of MRCA queries against a brute-force walk ### Changed - Codebase split into modules: `parser.rs`, `tree.rs`, `lca.rs`, `midpoint.rs` - LCA binary-lifting table stores plain `usize` rather than `Option`, halving the memory it needs +- Output buffer raised from 8 KB to 1 MB, so a multi-gigabyte matrix is not hundreds of thousands of write syscalls +- The input text and the recursive parse tree are released once the flat node array is built, rather than held to the end of the run - Version string now derived from Cargo.toml via `env!("CARGO_PKG_VERSION")` - Removed unused `--format` flag - Fixed clippy warnings (`&Vec` → `&[Node]`) From ce2c8c9c2a03b7c3425d0980dd47c34d4333335c Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:33:46 +0200 Subject: [PATCH 37/47] Catch three more ways the output could be quietly wrong A leaf label with a space passed every check and then broke --lower. A PHYLIP row is a name followed by whitespace and then the values, so a reader splitting on whitespace reads 'Taxon A4.5' as three fields and lands every row after it one column out. Tabs and newlines were already refused for the same reason; whitespace is refused now too, but only in --lower mode, since the square TSV is tab-delimited and reads it back correctly. --lower with --lmm silently dropped data. Omitting the diagonal costs nothing on a distance matrix, where it is all zeros, but an LMM diagonal holds each leaf's root-to-tip length, which is what the off-diagonal covariances have to be read against. It now says so. A gzipped tree failed with 'stream did not contain valid UTF-8', which says nothing about what to do. Large trees usually arrive compressed, so check for the magic bytes and give the gunzip line instead. --- src/main.rs | 51 +++++++++++++++++++++++++++++++++++++++++--- tests/integration.rs | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3154d72..062f58c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -164,15 +164,36 @@ fn run() -> Result<(), Box> { } // Read input from file or stdin - let mut newick_str = String::new(); + let source = if tree_path == "-" { "standard input" } else { tree_path.as_str() }; + let mut raw_input: Vec = Vec::new(); if tree_path == "-" { - io::stdin().read_to_string(&mut newick_str)?; + io::stdin().read_to_end(&mut raw_input)?; } else { File::open(&tree_path) .map_err(|e| format!("Cannot open '{}': {}", tree_path, e))? - .read_to_string(&mut newick_str)?; + .read_to_end(&mut raw_input)?; } + // Large trees are usually shipped compressed, and a gzip file reaching the + // UTF-8 check produced "stream did not contain valid UTF-8", which says + // nothing about what to do next. + if raw_input.starts_with(&[0x1f, 0x8b]) { + return Err(format!( + "{} is gzip-compressed. distree reads plain text; decompress it on the way in:\n\ + \x20 gunzip -c {} | distree -", + source, + if tree_path == "-" { "FILE.nwk.gz" } else { tree_path.as_str() } + ) + .into()); + } + + let newick_str = String::from_utf8(raw_input).map_err(|_| { + format!( + "{} is not valid UTF-8 text, so it cannot be a Newick tree.", + source + ) + })?; + // Parse the Newick string let raw_root = parse_newick(&newick_str) .map_err(|e| format!("Failed to parse Newick tree: {}", e))?; @@ -268,6 +289,19 @@ fn run() -> Result<(), Box> { ) .into()); } + // A PHYLIP row is "namevalues", so a reader that splits + // on whitespace reads "Taxon A4.5" as three fields and every + // row after it lands one column out. The square TSV is delimited by + // tabs alone and does not have the problem. + if do_lower && name.chars().any(char::is_whitespace) { + return Err(format!( + "Leaf name '{}' contains whitespace, which PHYLIP readers treat as the \ + end of the name, so --lower would produce a file they misread. Use an \ + underscore, or drop --lower for the square TSV.", + name + ) + .into()); + } } } @@ -305,6 +339,17 @@ fn run() -> Result<(), Box> { Box::new(BufWriter::with_capacity(WRITE_BUFFER, io::stdout())) }; + // The lower triangle has no diagonal. For a distance matrix that loses + // nothing, since it is all zeros, but an LMM diagonal is each tip's own + // root-to-tip length: real data, and what the off-diagonal covariances have + // to be read against. + if do_lower && mode == DistMode::Lmm { + eprintln!( + "Warning: --lower omits the diagonal, which in --lmm mode holds each leaf's \ + root-to-tip length rather than zeros. Drop --lower to keep it." + ); + } + // Print header if do_lower { // PHYLIP format: first line is the number of taxa diff --git a/tests/integration.rs b/tests/integration.rs index df6b821..692bcea 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -166,6 +166,52 @@ fn test_binary_empty_input_error() { assert!(stderr.contains("Empty") || stderr.contains("empty"), "stderr: {}", stderr); } +#[test] +fn test_binary_rejects_whitespace_label_in_lower_mode() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(('Taxon A':1.0,'Taxon B':2.0):0.5,C:3.0);").unwrap(); + + // PHYLIP readers split the name off at the first whitespace + let (code, _stdout, stderr) = run(&["--lower", tree.to_str().unwrap()], None); + assert_ne!(code, 0, "a spaced label should be rejected in --lower mode"); + assert!(stderr.contains("whitespace"), "stderr: {}", stderr); + + // The square TSV is tab-delimited and has no such problem + let (code, stdout, _) = run(&[tree.to_str().unwrap()], None); + assert_eq!(code, 0, "the same tree must still work without --lower"); + assert!(stdout.contains("Taxon A\t"), "stdout: {}", stdout); +} + +#[test] +fn test_binary_warns_lower_drops_lmm_diagonal() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1.95,B:3.25):0.35,(C:0.80,D:1.20):0.50);").unwrap(); + + let (code, _stdout, stderr) = run(&["--lmm", "--lower", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + assert!(stderr.contains("diagonal"), "stderr: {}", stderr); + + // Patristic loses nothing, so it must stay quiet + let (code, _stdout, stderr) = run(&["--lower", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + assert!(!stderr.contains("diagonal"), "stderr: {}", stderr); +} + +#[test] +fn test_binary_names_gzip_input() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk.gz"); + // Gzip magic bytes are enough; the file never gets decompressed + std::fs::write(&tree, [0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00]).unwrap(); + + let (code, _stdout, stderr) = run(&[tree.to_str().unwrap()], None); + assert_ne!(code, 0); + assert!(stderr.contains("gzip"), "stderr should name gzip: {}", stderr); + assert!(stderr.contains("gunzip -c"), "and say what to do: {}", stderr); +} + #[test] fn test_binary_rejects_newline_in_label() { let dir = tempfile::tempdir().unwrap(); From b5aecd109d7ebdf34a4d3bd7a1e76ee0db584820 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:40:15 +0200 Subject: [PATCH 38/47] Add a tutorial: a worked outbreak investigation The guide pages explain what each flag does, and the recipes give snippets to lift, but neither walks anyone through a whole question from tree to answer. Seven M. tuberculosis isolates, ten steps, and the question the tool exists for: which of these are transmission links. It goes from the Newick through the matrix, the substitutions-to-SNPs conversion, single-linkage clustering at the two standard thresholds, nearest neighbours, the topological and variance-covariance matrices, and what changes when the dataset gets big. The two thresholds are the point of it. At 5 SNPs TB_003 is outside the cluster and at 12 it is inside, which is a decision to report rather than an output to read off, and no amount of precision settles it. Every matrix on the page is real output from the tree the page ships, and each command was run as written. --- README.md | 1 + docs/tutorial.md | 334 +++++++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 336 insertions(+) create mode 100644 docs/tutorial.md diff --git a/README.md b/README.md index e5e2a60..ad9e817 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ __and Mireia Coscolla1__ | Page | Description | |:-----|:------------| | **[Quick start](https://pathogenomics-lab.github.io/distree/getting-started/quickstart/)** | The handful of commands most runs actually use | +| **[Tutorial](https://pathogenomics-lab.github.io/distree/tutorial/)** | A worked outbreak investigation, tree to transmission clusters | | **[Input](https://pathogenomics-lab.github.io/distree/guide/input/)** | What Newick is accepted, what is rejected, and why | | **[Distance modes](https://pathogenomics-lab.github.io/distree/guide/distances/)** | Patristic, topological and var-covar, on one worked example | | **[Midpoint rooting](https://pathogenomics-lab.github.io/distree/guide/rooting/)** | When the root changes the answer, and when it cannot | diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 0000000..629290e --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,334 @@ +# Tutorial + +A worked outbreak investigation, start to finish. Seven *Mycobacterium +tuberculosis* isolates, one tree, and the question every such dataset opens +with: **which of these are transmission links, and which are unrelated cases +that happen to be in the same clinic?** + +Everything here runs in a few seconds and needs nothing but distree, so you can +paste along. R and Python appear in tabs; pick whichever you use. + +## 1. The tree + +Save this as `outbreak.nwk`. It is what a maximum-likelihood tool would hand +you: branch lengths in **substitutions per site**, in scientific notation. + +``` +((((TB_001:2.266786e-07,TB_002:2.266786e-07):2.266786e-07,TB_003:1.586750e-06):8.613788e-06,(TB_004:3.400179e-07,TB_005:3.400179e-07):9.860520e-06):1.360072e-05,(TB_006:1.813429e-05,TB_007:2.153447e-05):1.586750e-05); +``` + +```bash +distree --version +``` + +## 2. The first matrix + +```bash +distree outbreak.nwk -p 9 +``` + +``` + TB_001 TB_002 TB_003 TB_004 TB_005 TB_006 TB_007 +TB_001 0.000000000 0.000000453 0.000002040 0.000019268 0.000019268 0.000056670 0.000060070 +TB_002 0.000000453 0.000000000 0.000002040 0.000019268 0.000019268 0.000056670 0.000060070 +TB_003 0.000002040 0.000002040 0.000000000 0.000020401 0.000020401 0.000057803 0.000061203 +TB_004 0.000019268 0.000019268 0.000020401 0.000000000 0.000000680 0.000057803 0.000061203 +TB_005 0.000019268 0.000019268 0.000020401 0.000000680 0.000000000 0.000057803 0.000061203 +TB_006 0.000056670 0.000056670 0.000057803 0.000057803 0.000057803 0.000000000 0.000039669 +TB_007 0.000060070 0.000060070 0.000061203 0.000061203 0.000061203 0.000039669 0.000000000 +``` + +That is the whole tool, really. Each cell is the sum of the branch lengths on +the path between two tips: a **patristic distance**. The rest of this page is +about reading it. + +Some structure is already visible. TB_001 and TB_002 are an order of magnitude +closer to each other than to anything else, TB_004 and TB_005 likewise, and +TB_006 and TB_007 sit far from everyone. + +!!! note "Why the empty first cell" + + The header line starts with a tab, so the labels sit over the data columns + rather than one to the left. That is what `read.table(row.names = 1)` and + `pandas.read_csv(index_col=0)` expect, and it is why the matrix loads + without any fiddling in [step 4](#4-load-it). + +## 3. Substitutions to SNPs + +Nobody thinks in substitutions per site. Multiply by the length of the reference +the tree was built against, and the numbers become countable differences. +H37Rv is 4,411,532 bases: + +```bash +distree outbreak.nwk -p 12 | awk 'NR == 1 { print; next } + { printf "%s", $1; for (i = 2; i <= NF; i++) printf "\t%.0f", $i * 4411532; print "" }' +``` + +``` + TB_001 TB_002 TB_003 TB_004 TB_005 TB_006 TB_007 +TB_001 0 2 9 85 85 250 265 +TB_002 2 0 9 85 85 250 265 +TB_003 9 9 0 90 90 255 270 +TB_004 85 85 90 0 3 255 270 +TB_005 85 85 90 3 0 255 270 +TB_006 250 250 255 255 255 0 175 +TB_007 265 265 270 270 270 175 0 +``` + +Now it reads. TB_001 and TB_002 differ by **2 SNPs**. TB_003 is **9** from both. +TB_004 and TB_005 are **3** apart but **85** from the first group. TB_006 and +TB_007 are hundreds of SNPs from everything, including each other. + +!!! warning "Use enough precision before you scale" + + `-p 12` above, not the default `-p 10`. Multiplying by four million + multiplies the rounding too: at `-p 6` a distance of 2 SNPs would round to 0 + before you ever saw it. Scale from the most precise output you can, and + round at the end. + +## 4. Load it + +```bash +distree outbreak.nwk -p 12 -o distances.tsv +``` + +=== "R" + + ```r + m <- as.matrix(read.table("distances.tsv", header = TRUE, row.names = 1, + sep = "\t", check.names = FALSE)) + snps <- m * 4411532 + round(snps) + ``` + + `check.names = FALSE` matters. Without it R rewrites labels like `TB-001` + or `2024_isolate` and they stop matching your metadata. + +=== "Python" + + ```python + import pandas as pd + + m = pd.read_csv("distances.tsv", sep="\t", index_col=0) + snps = m * 4411532 + print(snps.round()) + ``` + +## 5. Find the clusters + +Public health uses SNP thresholds. Two are standard for *M. tuberculosis*: +**5 SNPs** for recent transmission, and **12 SNPs** for a plausible +epidemiological link.[^walker] Single linkage is the right rule, because it +groups isolates that are within the threshold of *someone* in the group, which +is what a chain of transmission looks like. + +=== "R" + + ```r + clusters <- function(h) split(rownames(snps), + cutree(hclust(as.dist(snps), method = "single"), h = h)) + clusters(5) + clusters(12) + ``` + +=== "Python" + + ```python + import collections + from scipy.cluster.hierarchy import linkage, fcluster + from scipy.spatial.distance import squareform + + z = linkage(squareform(snps.values, checks=False), method="single") + + def clusters(t): + out = collections.defaultdict(list) + for name, c in zip(snps.index, fcluster(z, t=t, criterion="distance")): + out[c].append(name) + return list(out.values()) + + print(clusters(5)) + print(clusters(12)) + ``` + +``` +threshold 5 SNPs: TB_001,TB_002 | TB_003 | TB_004,TB_005 | TB_006 | TB_007 +threshold 12 SNPs: TB_001,TB_002,TB_003 | TB_004,TB_005 | TB_006 | TB_007 +``` + +**The threshold is the finding.** TB_003 is 9 SNPs away: outside recent +transmission, inside a plausible link. Whether it belongs in the cluster is an +epidemiological question, not a computational one, and the matrix will not +answer it for you. Report which threshold you used. + +Note also what did *not* change. TB_004 and TB_005 are a pair at either +threshold, and TB_006 and TB_007 are singletons at either. Findings that survive +both thresholds are the ones to lead with. + +## 6. Who is closest to whom + +Before any clustering, the cheapest useful question is each isolate's nearest +neighbour: + +```bash +distree outbreak.nwk -p 12 | awk -F'\t' ' + NR == 1 { for (i = 2; i <= NF; i++) name[i] = $i; next } + { best = ""; bestd = 1e308 + for (i = 2; i <= NF; i++) + if (i - 1 != NR - 1 && $i + 0 < bestd) { bestd = $i + 0; best = name[i] } + printf "%s\t%s\t%.0f\n", $1, best, bestd * 4411532 }' +``` + +``` +TB_001 TB_002 2 +TB_002 TB_001 2 +TB_003 TB_001 9 +TB_004 TB_005 3 +TB_005 TB_004 3 +TB_006 TB_007 175 +TB_007 TB_006 175 +``` + +The `i - 1 != NR - 1` skips the diagonal, which is otherwise always the closest. + +The last two rows are the interesting ones. TB_006 and TB_007 are each other's +nearest neighbours and still 175 SNPs apart, which is not a transmission link; +it just means nobody closer was sampled. + +## 7. When branch lengths are not the point + +Sometimes the tree's branch lengths are not comparable, or there are none at +all. `--topology` counts edges instead: + +```bash +distree outbreak.nwk --topology +``` + +``` + TB_001 TB_002 TB_003 TB_004 TB_005 TB_006 TB_007 +TB_001 0 2 3 5 5 6 6 +TB_002 2 0 3 5 5 6 6 +TB_003 3 3 0 4 4 5 5 +TB_004 5 5 4 0 2 5 5 +TB_005 5 5 4 2 0 5 5 +TB_006 6 6 5 5 5 0 2 +TB_007 6 6 5 5 5 2 0 +``` + +Compare it with the SNP matrix and the difference in what they measure is +plain. Topologically TB_006 and TB_007 are 2 apart, the same as TB_001 and +TB_002; in SNPs they are 175 apart against 2. Edge counts describe the shape of +the tree, not the divergence across it, so use them for tree-shape questions +and never as a proxy for relatedness when you have usable branch lengths. + +## 8. The comparative-analysis matrix + +A different question: you have a trait for each isolate and want to test whether +it associates with something, without the shared ancestry inflating your +significance. That needs the variance-covariance matrix, and here rooting is +part of the answer, so root deliberately: + +```bash +distree outbreak.nwk --midpoint --lmm -p 8 -o varcovar.tsv +``` + +``` + TB_001 TB_002 TB_003 TB_004 TB_005 TB_006 TB_007 +TB_001 0.00002947 0.00002924 0.00002901 0.00002040 0.00002040 0.00000000 0.00000000 +TB_002 0.00002924 0.00002947 0.00002901 0.00002040 0.00002040 0.00000000 0.00000000 +... +``` + +Entry `(i, j)` is how much evolutionary history the two isolates share: the +distance from the root down to their common ancestor. The diagonal is each +isolate's own root-to-tip length. Pairs whose ancestor is the root share +nothing and score 0. + +Three things to check before you fit anything with it: + +- **It is not a distance matrix.** The diagonal is not zero, and bigger means + more similar. Do not feed it to something expecting distances. +- **Reindex it.** Rows and columns are in alphabetical label order, which is + almost certainly not the order of your trait table. +- **Root on purpose.** `--midpoint` is one defensible choice; an outgroup is + another. Unlike the patristic matrix, this one changes if you change your + mind. See [Midpoint rooting](guide/rooting.md). + +## 9. Making it bigger + +Seven isolates is a tutorial. Seven thousand is a Tuesday, and two flags matter +then. + +```bash +distree big.nwk --lower -p 9 -t 8 -o distances.phy +``` + +`--lower` writes the PHYLIP lower triangle, halving the file: + +``` +7 +TB_001 +TB_002 0.000000453 +TB_003 0.000002040 0.000002040 +TB_004 0.000019268 0.000019268 0.000020401 +TB_005 0.000019268 0.000019268 0.000020401 0.000000680 +TB_006 0.000056670 0.000056670 0.000057803 0.000057803 0.000057803 +TB_007 0.000060070 0.000060070 0.000061203 0.000061203 0.000061203 0.000039669 +``` + +`-t` caps the threads; leave it off on a machine of your own and set it when +you are sharing one. + +Rows are written as they are computed, so a large run streams rather than +building the whole matrix first. Compress on the way out, or filter as it goes: + +```bash +distree big.nwk -p 12 --lower | gzip > distances.phy.gz +``` + +For what a run costs at various sizes, see +[Performance](how-it-works/performance.md). + +## 10. When it refuses + +distree would rather stop than hand back a matrix that looks right and is not. +The three you are most likely to meet: + +``` +Error: /path/tree.nwk is gzip-compressed. distree reads plain text; decompress it on the way in: + gunzip -c /path/tree.nwk | distree - +``` + +``` +Error: Failed to parse Newick tree: Unexpected content at position 84: the file appears to +hold more than one tree. distree processes a single Newick tree; split the file first. +``` + +``` +Error: Duplicate leaf name 'TB_001' found. Leaf names must be unique. +``` + +A duplicate label is worth pausing over. It usually means the same sample went +into the tree twice under two accessions, and no matrix can tell the two rows +apart afterwards. The full list is in [Input](guide/input.md#what-is-rejected). + +## What to take away + +- The **patristic** matrix is what almost every downstream question wants, and + it does not care how the tree is rooted. +- **Scale it once**, from high precision, and round at the end. +- **The threshold is a decision**, not an output. State it. +- **`--lmm` is a different object**: a covariance, not a distance, and rooting + changes it. +- **`--topology` measures shape**, not divergence. + +## Next + +- [Recipes](recipes.md) for PCoA, PGLS and neighbour-joining, at the same length + as this section but without the story. +- [Distance modes](guide/distances.md) for the three modes in full. +- [CLI reference](guide/cli.md) for every flag. + +[^walker]: + Walker TM, Ip CLC, Harrell RH, et al. *Whole-genome sequencing to delineate + Mycobacterium tuberculosis outbreaks: a retrospective observational study.* + The Lancet Infectious Diseases. 2013;13(2):137-146. diff --git a/mkdocs.yml b/mkdocs.yml index 3f437a2..fede502 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -128,6 +128,7 @@ nav: - Getting started: - getting-started/installation.md - getting-started/quickstart.md + - Tutorial: tutorial.md - User guide: - guide/input.md - guide/distances.md From 2c32c6254c779a3fd6438cb1c66cc169419cacf8 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:45:56 +0200 Subject: [PATCH 39/47] Cross-validate the distances against ape Every check in the suite so far was written against the same understanding of the problem as the code: midpoint rooting preserves distances, the binary-lifting MRCA agrees with walking up from both nodes, a patristic distance is never negative. All true, all necessary, and none of it would catch the definitions themselves being wrong. scripts/crossvalidate.R compares distree against ape over generated trees of mixed shape (random, ultrametric, with polytomies), in all four modes: cophenetic.phylo for patristic, vcv.phylo for the variance-covariance matrix, cophenetic over unit branch lengths for edge counts, and phangorn::midpoint for the rooting. Over 250 trees the worst disagreement is 9.6e-10 for the float modes, which is the 12-decimal text round-trip, and exactly zero for edge counts. That the midpoint agrees with phangorn is the reassuring part. It is the only mode where rooting changes the answer and the only code in the tool that rewrites the tree. R is a heavy thing to need for running 59 tests test lca::tests::test_depth_top ... ok test lca::tests::test_mrca_deeper ... ok test midpoint::tests::test_midpoint_asymmetric ... ok test lca::tests::test_mrca_self ... ok test midpoint::tests::test_midpoint_simple ... ok test parser::tests::test_apostrophe_in_comment ... ok test midpoint::tests::test_midpoint_preserves_distances ... ok test lca::tests::test_mrca_siblings ... ok test parser::tests::test_apostrophe_in_unquoted_label ... ok test parser::tests::test_bracket_in_label_position ... ok test parser::tests::test_comment_before_label ... ok test parser::tests::test_comment_before_subtree ... ok test parser::tests::test_empty_branch_length_error ... ok test parser::tests::test_empty_input_error ... ok test parser::tests::test_escaped_double_quotes_in_labels ... ok test parser::tests::test_escaped_quote_keeps_inner_whitespace ... ok test parser::tests::test_escaped_single_quotes_in_labels ... ok test parser::tests::test_free_text_is_not_a_leaf ... ok test parser::tests::test_flatten ... ok test parser::tests::test_multiple_trees_error ... ok test parser::tests::test_leading_rooting_comment ... ok test parser::tests::test_negative_branch_length_parsed ... ok test parser::tests::test_nested_brackets ... ok test parser::tests::test_newlines_in_newick ... ok test parser::tests::test_no_trailing_semicolon ... ok test parser::tests::test_non_ascii_labels_preserved ... ok test parser::tests::test_parse_internal_labels ... ok test parser::tests::test_parse_nhx_comments ... ok test parser::tests::test_parse_no_branch_lengths ... ok test parser::tests::test_parse_quoted_labels_double ... ok test parser::tests::test_parse_quoted_labels_single ... ok test parser::tests::test_parse_scientific_notation ... ok test parser::tests::test_parse_simple_tree ... ok test parser::tests::test_quoted_internal_node_label ... ok test parser::tests::test_trailing_comment_allowed ... ok test parser::tests::test_trailing_content_error ... ok test parser::tests::test_unclosed_comment_error ... ok test parser::tests::test_unclosed_quote_error ... ok test parser::tests::test_unmatched_paren_error ... ok test parser::tests::test_whitespace_in_newick ... ok test tests::test_duplicate_leaves_detected ... ok test tests::test_large_symmetric_tree ... ok test tests::test_lmm_matrix ... ok test tests::test_mode_conflict ... ok test tests::test_negative_branch_length_detected ... ok test tests::test_negative_branch_length_gives_negative_distance ... ok test tests::test_lower_triangle_row_counts ... ok test tests::test_patristic_clamped_nonnegative ... ok test tests::test_no_branch_lengths_all_zero ... ok test tests::test_single_leaf ... ok test tests::test_topology_distances ... ok test tests::test_trifurcation ... ok test tests::test_simple_patristic_distances ... ok test tests::test_zero_distance_same_leaf ... ok test parser::tests::test_deep_tree_no_stack_overflow ... ok test tests::test_patristic_never_negative_on_random_trees ... ok test midpoint::tests::test_midpoint_random_trees ... ok test parser::tests::test_deeply_nested_tree_drops_without_overflow ... ok test lca::tests::test_mrca_matches_walking_up_on_random_trees ... ok test result: ok. 59 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.56s running 4 tests test test_topology_matches_ape_unit_branch_cophenetic ... ok test test_patristic_matches_ape_cophenetic ... ok test test_lmm_matches_ape_vcv ... ok test test_midpoint_leaves_patristic_alone ... ok test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.26s running 17 tests test test_binary_stdin ... ok test test_binary_empty_input_error ... ok test test_binary_rejects_out_of_range_precision ... ok test test_binary_duplicate_leaf_error ... ok test test_binary_rejects_zero_threads ... ok test test_binary_rejects_newline_in_label ... ok test test_binary_names_gzip_input ... ok test test_binary_keeps_output_file_on_parse_error ... ok test test_binary_basic_patristic ... ok test test_binary_lower_triangle ... ok test test_binary_precision_flag ... ok test test_binary_topology_flag ... ok test test_binary_midpoint_does_not_inflate_topology ... ok test test_binary_midpoint_preserves_patristic ... ok test test_binary_rejects_whitespace_label_in_lower_mode ... ok test test_binary_warns_about_unlabeled_leaves ... ok test test_binary_warns_lower_drops_lmm_diagonal ... ok test result: ok. 17 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s, so six of ape's matrices are committed under tests/fixtures/ and tests/crossvalidation.rs checks those everywhere. The full run is a weekly job and a button. --- .github/workflows/crossvalidate.yml | 41 +++++++ scripts/crossvalidate.R | 141 ++++++++++++++++++++++ tests/crossvalidation.rs | 177 ++++++++++++++++++++++++++++ tests/fixtures/tree01.lmm.tsv | 13 ++ tests/fixtures/tree01.nwk | 1 + tests/fixtures/tree01.patristic.tsv | 13 ++ tests/fixtures/tree01.topology.tsv | 13 ++ tests/fixtures/tree02.lmm.tsv | 18 +++ tests/fixtures/tree02.nwk | 1 + tests/fixtures/tree02.patristic.tsv | 18 +++ tests/fixtures/tree02.topology.tsv | 18 +++ tests/fixtures/tree03.lmm.tsv | 18 +++ tests/fixtures/tree03.nwk | 1 + tests/fixtures/tree03.patristic.tsv | 18 +++ tests/fixtures/tree03.topology.tsv | 18 +++ tests/fixtures/tree04.lmm.tsv | 19 +++ tests/fixtures/tree04.nwk | 1 + tests/fixtures/tree04.patristic.tsv | 19 +++ tests/fixtures/tree04.topology.tsv | 19 +++ tests/fixtures/tree05.lmm.tsv | 13 ++ tests/fixtures/tree05.nwk | 1 + tests/fixtures/tree05.patristic.tsv | 13 ++ tests/fixtures/tree05.topology.tsv | 13 ++ tests/fixtures/tree06.lmm.tsv | 10 ++ tests/fixtures/tree06.nwk | 1 + tests/fixtures/tree06.patristic.tsv | 10 ++ tests/fixtures/tree06.topology.tsv | 10 ++ 27 files changed, 638 insertions(+) create mode 100644 .github/workflows/crossvalidate.yml create mode 100755 scripts/crossvalidate.R create mode 100644 tests/crossvalidation.rs create mode 100644 tests/fixtures/tree01.lmm.tsv create mode 100644 tests/fixtures/tree01.nwk create mode 100644 tests/fixtures/tree01.patristic.tsv create mode 100644 tests/fixtures/tree01.topology.tsv create mode 100644 tests/fixtures/tree02.lmm.tsv create mode 100644 tests/fixtures/tree02.nwk create mode 100644 tests/fixtures/tree02.patristic.tsv create mode 100644 tests/fixtures/tree02.topology.tsv create mode 100644 tests/fixtures/tree03.lmm.tsv create mode 100644 tests/fixtures/tree03.nwk create mode 100644 tests/fixtures/tree03.patristic.tsv create mode 100644 tests/fixtures/tree03.topology.tsv create mode 100644 tests/fixtures/tree04.lmm.tsv create mode 100644 tests/fixtures/tree04.nwk create mode 100644 tests/fixtures/tree04.patristic.tsv create mode 100644 tests/fixtures/tree04.topology.tsv create mode 100644 tests/fixtures/tree05.lmm.tsv create mode 100644 tests/fixtures/tree05.nwk create mode 100644 tests/fixtures/tree05.patristic.tsv create mode 100644 tests/fixtures/tree05.topology.tsv create mode 100644 tests/fixtures/tree06.lmm.tsv create mode 100644 tests/fixtures/tree06.nwk create mode 100644 tests/fixtures/tree06.patristic.tsv create mode 100644 tests/fixtures/tree06.topology.tsv diff --git a/.github/workflows/crossvalidate.yml b/.github/workflows/crossvalidate.yml new file mode 100644 index 0000000..5af4e56 --- /dev/null +++ b/.github/workflows/crossvalidate.yml @@ -0,0 +1,41 @@ +name: Cross-validate against ape + +# The committed fixtures in tests/fixtures/ let `cargo test` check six trees +# against ape everywhere, with no R needed. This job runs the same comparison +# over 250 freshly generated trees, which is the deeper check but needs an R +# toolchain, so it runs weekly and on demand rather than on every push. + +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + inputs: + trees: + description: "How many trees to generate" + required: false + default: "250" + +jobs: + crossvalidate: + name: distree vs ape + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + packages: | + any::ape + any::phangorn + + - name: Build + run: cargo build --release + + - name: Cross-validate + run: Rscript scripts/crossvalidate.R ${{ inputs.trees || 250 }} target/release/distree diff --git a/scripts/crossvalidate.R b/scripts/crossvalidate.R new file mode 100755 index 0000000..c12979a --- /dev/null +++ b/scripts/crossvalidate.R @@ -0,0 +1,141 @@ +#!/usr/bin/env Rscript + +# Cross-validate distree against ape. +# +# The Rust test suite is self-consistent: it checks that midpoint rooting +# preserves distances, that the binary-lifting MRCA agrees with walking up from +# both nodes, and that a patristic distance is never negative. None of that +# would catch a systematic error in what the distances are *defined* to be, +# because every check is written against the same understanding as the code. +# +# ape is that independent understanding. cophenetic.phylo is the reference +# patristic matrix, vcv.phylo the reference variance-covariance matrix, and +# phangorn::midpoint the reference midpoint rooting. +# +# Usage: +# Rscript scripts/crossvalidate.R [n_trees] [path/to/distree] +# +# Add --write-fixtures to regenerate tests/fixtures/, which is what lets the +# Rust suite check the same thing in CI without needing R. + +suppressMessages({ + library(ape) + library(phangorn) +}) + +args <- commandArgs(trailingOnly = TRUE) +write_fx <- "--write-fixtures" %in% args +args <- args[args != "--write-fixtures"] +n_trees <- if (length(args) >= 1) as.integer(args[1]) else 200 +distree <- if (length(args) >= 2) args[2] else "target/release/distree" + +if (!file.exists(distree)) { + stop("distree binary not found at '", distree, "'. Run: cargo build --release") +} + +fixture_dir <- "tests/fixtures" +if (write_fx) dir.create(fixture_dir, showWarnings = FALSE, recursive = TRUE) + +# distree sorts labels by byte order; radix sorting is the locale-independent +# match for that. Anything else and the two matrices are compared transposed. +byte_sort <- function(x) sort(x, method = "radix") + +# Read a distree square TSV back into a matrix. +read_distree <- function(path) { + m <- as.matrix(read.table(path, header = TRUE, row.names = 1, + sep = "\t", check.names = FALSE)) + colnames(m) <- rownames(m) + m +} + +run_distree <- function(tree, flags) { + nwk <- tempfile(fileext = ".nwk") + out <- tempfile(fileext = ".tsv") + write.tree(tree, file = nwk) + status <- system2(distree, c(shQuote(nwk), flags, "-p", "12", "-o", shQuote(out)), + stdout = NULL, stderr = NULL) + if (status != 0) stop("distree exited ", status, " for flags: ", paste(flags, collapse = " ")) + m <- read_distree(out) + unlink(c(nwk, out)) + m +} + +# Largest absolute difference, after lining both matrices up by label. +deviation <- function(got, want) { + labs <- byte_sort(rownames(got)) + max(abs(got[labs, labs] - want[labs, labs])) +} + +set.seed(20260726) +worst <- list(patristic = 0, topology = 0, lmm = 0, midpoint_lmm = 0) +fixtures <- list() + +for (i in seq_len(n_trees)) { + n <- sample(3:40, 1) + + # A spread of shapes: random topologies with random lengths, ultrametric + # coalescent trees, and trees with polytomies collapsed into them. + tree <- switch(sample(1:3, 1), + rtree(n), + rcoal(n), + di2multi(rtree(n), tol = 0.05) + ) + # ape allows negative lengths through rtree only rarely; keep them out, since + # a negative branch makes midpoint rooting undefined for both tools. + tree$edge.length <- abs(tree$edge.length) + + # --- patristic --------------------------------------------------------- + d <- deviation(run_distree(tree, character(0)), cophenetic(tree)) + worst$patristic <- max(worst$patristic, d) + + # --- topological ------------------------------------------------------- + unit <- tree + unit$edge.length <- rep(1, nrow(unit$edge)) + d <- deviation(run_distree(tree, "--topology"), cophenetic(unit)) + worst$topology <- max(worst$topology, d) + + # --- variance-covariance ---------------------------------------------- + d <- deviation(run_distree(tree, "--lmm"), vcv(tree)) + worst$lmm <- max(worst$lmm, d) + + # --- midpoint rooting, seen through the var-covar matrix --------------- + # This is the only mode where rooting changes the answer, so it is the one + # that tests whether the two tools put the root in the same place. + rooted <- tryCatch(midpoint(tree), error = function(e) NULL) + if (!is.null(rooted)) { + d <- deviation(run_distree(tree, "--midpoint --lmm"), vcv(rooted)) + worst$midpoint_lmm <- max(worst$midpoint_lmm, d) + } + + if (write_fx && i <= 6) { + stem <- sprintf("%s/tree%02d", fixture_dir, i) + write.tree(tree, file = paste0(stem, ".nwk")) + for (mode in c("patristic", "topology", "lmm")) { + ref <- switch(mode, + patristic = cophenetic(tree), + topology = cophenetic(unit), + lmm = vcv(tree) + ) + labs <- byte_sort(rownames(ref)) + ref <- ref[labs, labs] + con <- file(sprintf("%s.%s.tsv", stem, mode), "w") + writeLines(paste0("\t", paste(labs, collapse = "\t")), con) + for (r in labs) { + writeLines(paste(c(r, sprintf("%.12f", ref[r, ])), collapse = "\t"), con) + } + close(con) + } + fixtures <- c(fixtures, stem) + } +} + +cat("Cross-validated", n_trees, "trees against ape", as.character(packageVersion("ape")), "\n\n") +cat(sprintf(" %-14s max |distree - ape| = %.3e\n", names(worst), unlist(worst)), sep = "") + +tolerance <- 1e-9 +if (max(unlist(worst)) > tolerance) { + cat("\nFAIL: deviation above", tolerance, "\n") + quit(status = 1) +} +cat("\nAll modes agree to within", tolerance, "\n") +if (write_fx) cat("Wrote", length(fixtures), "fixtures to", fixture_dir, "\n") diff --git a/tests/crossvalidation.rs b/tests/crossvalidation.rs new file mode 100644 index 0000000..e668a64 --- /dev/null +++ b/tests/crossvalidation.rs @@ -0,0 +1,177 @@ +//! Check distree's output against reference matrices computed by ape. +//! +//! The rest of the suite is self-consistent: it asserts that midpoint rooting +//! preserves distances, that the binary-lifting MRCA agrees with walking up +//! from both nodes, that a patristic distance is never negative. All of that is +//! written against the same understanding of the problem as the code, so none +//! of it would catch a systematic error in what a distance is defined to be. +//! +//! The fixtures here come from somewhere else. `tests/fixtures/*.nwk` are +//! random trees, and the matrices beside them were computed by R's ape package +//! (`cophenetic.phylo` for patristic, `vcv.phylo` for the variance-covariance +//! matrix, and `cophenetic.phylo` over unit branch lengths for edge counts). +//! +//! Regenerate them, and cross-validate over hundreds more trees than are +//! committed here, with: +//! +//! ```text +//! Rscript scripts/crossvalidate.R 250 target/release/distree +//! Rscript scripts/crossvalidate.R 6 target/release/distree --write-fixtures +//! ``` + +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::Command; + +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_distree") +} + +fn fixture_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures") +} + +/// Parse a square TSV into labels and a row-major matrix. +fn parse_matrix(text: &str) -> (Vec, HashMap<(String, String), f64>) { + let mut lines = text.lines(); + let header: Vec = lines + .next() + .expect("matrix has a header") + .split('\t') + .skip(1) // leading empty cell + .map(str::to_string) + .collect(); + + let mut cells = HashMap::new(); + for line in lines { + if line.is_empty() { + continue; + } + let mut fields = line.split('\t'); + let row = fields.next().expect("row label").to_string(); + for (col, value) in header.iter().zip(fields) { + let v: f64 = value + .parse() + .unwrap_or_else(|_| panic!("cell '{}' is not a number", value)); + cells.insert((row.clone(), col.clone()), v); + } + } + (header, cells) +} + +fn run_distree(tree: &PathBuf, flags: &[&str]) -> String { + let out = Command::new(bin()) + .arg(tree) + .args(flags) + .args(["-p", "12"]) + .output() + .expect("failed to run distree"); + assert!( + out.status.success(), + "distree failed on {:?} with {:?}: {}", + tree, + flags, + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("distree writes UTF-8") +} + +/// Compare distree's matrix for `flags` against the fixture for `mode`. +fn check_mode(stem: &str, mode: &str, flags: &[&str], tolerance: f64) { + let dir = fixture_dir(); + let tree = dir.join(format!("{}.nwk", stem)); + let expected_path = dir.join(format!("{}.{}.tsv", stem, mode)); + + let expected_text = std::fs::read_to_string(&expected_path) + .unwrap_or_else(|e| panic!("cannot read {:?}: {}", expected_path, e)); + + let (want_labels, want) = parse_matrix(&expected_text); + let (got_labels, got) = parse_matrix(&run_distree(&tree, flags)); + + assert_eq!( + got_labels, want_labels, + "{} {}: label ordering differs from ape's", + stem, mode + ); + + let mut worst = 0.0_f64; + let mut worst_at = (String::new(), String::new()); + for a in &want_labels { + for b in &want_labels { + let key = (a.clone(), b.clone()); + let g = got[&key]; + let w = want[&key]; + let diff = (g - w).abs(); + if diff > worst { + worst = diff; + worst_at = key; + } + } + } + + assert!( + worst <= tolerance, + "{} {}: distree and ape disagree by {:.3e} at ({}, {}); tolerance is {:.0e}", + stem, + mode, + worst, + worst_at.0, + worst_at.1, + tolerance + ); +} + +fn stems() -> Vec { + let mut found: Vec = std::fs::read_dir(fixture_dir()) + .expect("tests/fixtures exists") + .filter_map(|e| { + let name = e.ok()?.file_name().into_string().ok()?; + name.strip_suffix(".nwk").map(str::to_string) + }) + .collect(); + found.sort(); + assert!(!found.is_empty(), "no fixture trees found"); + found +} + +/// Both tools write 12 decimals, so a value in the hundreds carries about +/// 1e-10 of round-trip error. Anything larger is a real disagreement. +const TOLERANCE: f64 = 1e-9; + +#[test] +fn test_patristic_matches_ape_cophenetic() { + for stem in stems() { + check_mode(&stem, "patristic", &[], TOLERANCE); + } +} + +#[test] +fn test_lmm_matches_ape_vcv() { + for stem in stems() { + check_mode(&stem, "lmm", &["--lmm"], TOLERANCE); + } +} + +#[test] +fn test_topology_matches_ape_unit_branch_cophenetic() { + for stem in stems() { + // Edge counts are integers on both sides, so they must agree exactly + check_mode(&stem, "topology", &["--topology"], 0.0); + } +} + +#[test] +fn test_midpoint_leaves_patristic_alone() { + // ape has no opinion to check here, but the fixtures are a broader set of + // shapes than the hand-written trees, and rooting must not move a + // patristic distance on any of them. + for stem in stems() { + let tree = fixture_dir().join(format!("{}.nwk", stem)); + assert_eq!( + run_distree(&tree, &[]), + run_distree(&tree, &["--midpoint"]), + "{}: --midpoint changed the patristic matrix", + stem + ); + } +} diff --git a/tests/fixtures/tree01.lmm.tsv b/tests/fixtures/tree01.lmm.tsv new file mode 100644 index 0000000..ee8a15c --- /dev/null +++ b/tests/fixtures/tree01.lmm.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 4.044311133508 2.127465543815 0.000000000000 2.127465543815 2.127465543815 0.000000000000 2.127465543815 0.000000000000 0.000000000000 0.000000000000 2.127465543815 2.127465543815 +t10 2.127465543815 4.044311133508 0.000000000000 3.278214901797 3.278214901797 0.000000000000 3.966945735241 0.000000000000 0.000000000000 0.000000000000 3.278214901797 3.278214901797 +t11 0.000000000000 0.000000000000 4.044311133508 0.000000000000 0.000000000000 3.694086227346 0.000000000000 4.029185174483 3.694086227346 3.694086227346 0.000000000000 0.000000000000 +t12 2.127465543815 3.278214901797 0.000000000000 4.044311133508 4.009939549891 0.000000000000 3.278214901797 0.000000000000 0.000000000000 0.000000000000 3.966888006743 3.722208144128 +t2 2.127465543815 3.278214901797 0.000000000000 4.009939549891 4.044311133508 0.000000000000 3.278214901797 0.000000000000 0.000000000000 0.000000000000 3.966888006743 3.722208144128 +t3 0.000000000000 0.000000000000 3.694086227346 0.000000000000 0.000000000000 4.044311133508 0.000000000000 3.694086227346 4.017223405830 4.026706611511 0.000000000000 0.000000000000 +t4 2.127465543815 3.966945735241 0.000000000000 3.278214901797 3.278214901797 0.000000000000 4.044311133508 0.000000000000 0.000000000000 0.000000000000 3.278214901797 3.278214901797 +t5 0.000000000000 0.000000000000 4.029185174483 0.000000000000 0.000000000000 3.694086227346 0.000000000000 4.044311133508 3.694086227346 3.694086227346 0.000000000000 0.000000000000 +t6 0.000000000000 0.000000000000 3.694086227346 0.000000000000 0.000000000000 4.017223405830 0.000000000000 3.694086227346 4.044311133508 4.017223405830 0.000000000000 0.000000000000 +t7 0.000000000000 0.000000000000 3.694086227346 0.000000000000 0.000000000000 4.026706611511 0.000000000000 3.694086227346 4.017223405830 4.044311133508 0.000000000000 0.000000000000 +t8 2.127465543815 3.278214901797 0.000000000000 3.966888006743 3.966888006743 0.000000000000 3.278214901797 0.000000000000 0.000000000000 0.000000000000 4.044311133508 3.722208144128 +t9 2.127465543815 3.278214901797 0.000000000000 3.722208144128 3.722208144128 0.000000000000 3.278214901797 0.000000000000 0.000000000000 0.000000000000 3.722208144128 4.044311133508 diff --git a/tests/fixtures/tree01.nwk b/tests/fixtures/tree01.nwk new file mode 100644 index 0000000..3441485 --- /dev/null +++ b/tests/fixtures/tree01.nwk @@ -0,0 +1 @@ +(((t5:0.01512595902,t11:0.01512595902):0.3350989471,(t6:0.02708772768,(t3:0.017604522,t7:0.017604522):0.009483205681):0.3231371785):3.694086227,(t1:1.91684559,((t10:0.07736539827,t4:0.07736539827):0.6887308334,(t9:0.3221029894,(t8:0.07742312677,(t2:0.03437158362,t12:0.03437158362):0.04305154315):0.2446798626):0.4439932423):1.150749358):2.127465544); diff --git a/tests/fixtures/tree01.patristic.tsv b/tests/fixtures/tree01.patristic.tsv new file mode 100644 index 0000000..ade6ed4 --- /dev/null +++ b/tests/fixtures/tree01.patristic.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 3.833691179386 8.088622267016 3.833691179386 3.833691179386 8.088622267016 3.833691179386 8.088622267016 8.088622267016 8.088622267016 3.833691179386 3.833691179386 +t10 3.833691179386 0.000000000000 8.088622267016 1.532192463422 1.532192463422 8.088622267016 0.154730796533 8.088622267016 8.088622267016 8.088622267016 1.532192463422 1.532192463422 +t11 8.088622267016 8.088622267016 0.000000000000 8.088622267016 8.088622267016 0.700449812324 8.088622267016 0.030251918049 0.700449812324 0.700449812324 8.088622267016 8.088622267016 +t12 3.833691179386 1.532192463422 8.088622267016 0.000000000000 0.068743167234 8.088622267016 1.532192463422 8.088622267016 8.088622267016 8.088622267016 0.154846253531 0.644205978759 +t2 3.833691179386 1.532192463422 8.088622267016 0.068743167234 0.000000000000 8.088622267016 1.532192463422 8.088622267016 8.088622267016 8.088622267016 0.154846253531 0.644205978759 +t3 8.088622267016 8.088622267016 0.700449812324 8.088622267016 8.088622267016 0.000000000000 8.088622267016 0.700449812324 0.054175455356 0.035209043994 8.088622267016 8.088622267016 +t4 3.833691179386 0.154730796533 8.088622267016 1.532192463422 1.532192463422 8.088622267016 0.000000000000 8.088622267016 8.088622267016 8.088622267016 1.532192463422 1.532192463422 +t5 8.088622267016 8.088622267016 0.030251918049 8.088622267016 8.088622267016 0.700449812324 8.088622267016 0.000000000000 0.700449812324 0.700449812324 8.088622267016 8.088622267016 +t6 8.088622267016 8.088622267016 0.700449812324 8.088622267016 8.088622267016 0.054175455356 8.088622267016 0.700449812324 0.000000000000 0.054175455356 8.088622267016 8.088622267016 +t7 8.088622267016 8.088622267016 0.700449812324 8.088622267016 8.088622267016 0.035209043994 8.088622267016 0.700449812324 0.054175455356 0.000000000000 8.088622267016 8.088622267016 +t8 3.833691179386 1.532192463422 8.088622267016 0.154846253531 0.154846253531 8.088622267016 1.532192463422 8.088622267016 8.088622267016 8.088622267016 0.000000000000 0.644205978759 +t9 3.833691179386 1.532192463422 8.088622267016 0.644205978759 0.644205978759 8.088622267016 1.532192463422 8.088622267016 8.088622267016 8.088622267016 0.644205978759 0.000000000000 diff --git a/tests/fixtures/tree01.topology.tsv b/tests/fixtures/tree01.topology.tsv new file mode 100644 index 0000000..f584631 --- /dev/null +++ b/tests/fixtures/tree01.topology.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 4.000000000000 5.000000000000 6.000000000000 6.000000000000 6.000000000000 4.000000000000 5.000000000000 5.000000000000 6.000000000000 5.000000000000 4.000000000000 +t10 4.000000000000 0.000000000000 7.000000000000 6.000000000000 6.000000000000 8.000000000000 2.000000000000 7.000000000000 7.000000000000 8.000000000000 5.000000000000 4.000000000000 +t11 5.000000000000 7.000000000000 0.000000000000 9.000000000000 9.000000000000 5.000000000000 7.000000000000 2.000000000000 4.000000000000 5.000000000000 8.000000000000 7.000000000000 +t12 6.000000000000 6.000000000000 9.000000000000 0.000000000000 2.000000000000 10.000000000000 6.000000000000 9.000000000000 9.000000000000 10.000000000000 3.000000000000 4.000000000000 +t2 6.000000000000 6.000000000000 9.000000000000 2.000000000000 0.000000000000 10.000000000000 6.000000000000 9.000000000000 9.000000000000 10.000000000000 3.000000000000 4.000000000000 +t3 6.000000000000 8.000000000000 5.000000000000 10.000000000000 10.000000000000 0.000000000000 8.000000000000 5.000000000000 3.000000000000 2.000000000000 9.000000000000 8.000000000000 +t4 4.000000000000 2.000000000000 7.000000000000 6.000000000000 6.000000000000 8.000000000000 0.000000000000 7.000000000000 7.000000000000 8.000000000000 5.000000000000 4.000000000000 +t5 5.000000000000 7.000000000000 2.000000000000 9.000000000000 9.000000000000 5.000000000000 7.000000000000 0.000000000000 4.000000000000 5.000000000000 8.000000000000 7.000000000000 +t6 5.000000000000 7.000000000000 4.000000000000 9.000000000000 9.000000000000 3.000000000000 7.000000000000 4.000000000000 0.000000000000 3.000000000000 8.000000000000 7.000000000000 +t7 6.000000000000 8.000000000000 5.000000000000 10.000000000000 10.000000000000 2.000000000000 8.000000000000 5.000000000000 3.000000000000 0.000000000000 9.000000000000 8.000000000000 +t8 5.000000000000 5.000000000000 8.000000000000 3.000000000000 3.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 0.000000000000 3.000000000000 +t9 4.000000000000 4.000000000000 7.000000000000 4.000000000000 4.000000000000 8.000000000000 4.000000000000 7.000000000000 7.000000000000 8.000000000000 3.000000000000 0.000000000000 diff --git a/tests/fixtures/tree02.lmm.tsv b/tests/fixtures/tree02.lmm.tsv new file mode 100644 index 0000000..b562307 --- /dev/null +++ b/tests/fixtures/tree02.lmm.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 3.256266693119 0.065225892933 0.065225892933 0.629955607234 0.861331883352 0.065225892933 2.431582783815 1.467767681926 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 1.467767681926 +t10 0.065225892933 2.384166998556 0.826933017233 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 1.803952952381 0.065225892933 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.826933017233 0.000000000000 0.065225892933 +t11 0.065225892933 0.826933017233 3.142999361502 0.065225892933 0.065225892933 2.553210664308 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 0.065225892933 1.580081571126 0.065225892933 1.580081571126 0.000000000000 0.065225892933 +t12 0.629955607234 0.065225892933 0.065225892933 2.301803021459 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 0.635071521392 0.635071521392 1.744033653056 0.065225892933 1.260509673739 0.065225892933 0.000000000000 0.629955607234 +t13 0.861331883352 0.065225892933 0.065225892933 0.629955607234 1.754404090578 0.065225892933 0.861331883352 0.861331883352 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 0.861331883352 +t14 0.065225892933 0.826933017233 2.553210664308 0.065225892933 0.065225892933 3.477648017928 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 0.065225892933 1.580081571126 0.065225892933 1.580081571126 0.000000000000 0.065225892933 +t15 2.431582783815 0.065225892933 0.065225892933 0.629955607234 0.861331883352 0.065225892933 2.762291881256 1.467767681926 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 1.467767681926 +t16 1.467767681926 0.065225892933 0.065225892933 0.629955607234 0.861331883352 0.065225892933 1.467767681926 2.570631016977 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 2.250247935066 +t17 0.065225892933 1.803952952381 0.826933017233 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 2.413126796251 0.065225892933 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.826933017233 0.000000000000 0.065225892933 +t2 0.629955607234 0.065225892933 0.065225892933 0.635071521392 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 1.348461325280 0.702091647778 0.635071521392 0.065225892933 0.635071521392 0.065225892933 0.000000000000 0.629955607234 +t3 0.629955607234 0.065225892933 0.065225892933 0.635071521392 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 0.702091647778 1.225792548154 0.635071521392 0.065225892933 0.635071521392 0.065225892933 0.000000000000 0.629955607234 +t4 0.629955607234 0.065225892933 0.065225892933 1.744033653056 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 0.635071521392 0.635071521392 1.780867669731 0.065225892933 1.260509673739 0.065225892933 0.000000000000 0.629955607234 +t5 0.065225892933 0.826933017233 1.580081571126 0.065225892933 0.065225892933 1.580081571126 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 0.065225892933 3.408075469313 0.065225892933 2.512174850330 0.000000000000 0.065225892933 +t6 0.629955607234 0.065225892933 0.065225892933 1.260509673739 0.629955607234 0.065225892933 0.629955607234 0.629955607234 0.065225892933 0.635071521392 0.635071521392 1.260509673739 0.065225892933 2.177918948466 0.065225892933 0.000000000000 0.629955607234 +t7 0.065225892933 0.826933017233 1.580081571126 0.065225892933 0.065225892933 1.580081571126 0.065225892933 0.065225892933 0.826933017233 0.065225892933 0.065225892933 0.065225892933 2.512174850330 0.065225892933 2.690017238259 0.000000000000 0.065225892933 +t8 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.633634719998 0.000000000000 +t9 1.467767681926 0.065225892933 0.065225892933 0.629955607234 0.861331883352 0.065225892933 1.467767681926 2.250247935066 0.065225892933 0.629955607234 0.629955607234 0.629955607234 0.065225892933 0.629955607234 0.065225892933 0.000000000000 3.018237155862 diff --git a/tests/fixtures/tree02.nwk b/tests/fixtures/tree02.nwk new file mode 100644 index 0000000..2abdb8e --- /dev/null +++ b/tests/fixtures/tree02.nwk @@ -0,0 +1 @@ +(t8:0.63363472,((((t14:0.9244373536,t11:0.5897886972):0.9731290932,(t5:0.895900619,t7:0.1778423879):0.9320932792):0.7531485539,(t17:0.6091738439,t10:0.5802140462):0.9770199351):0.7617071243,(((t6:0.9174092747,(t4:0.03683401668,t12:0.5577693684):0.4835239793):0.6254381523,(t2:0.6463696775,t3:0.5237009004):0.06702012639):0.005115914159,(t13:0.8930722072,((t1:0.8246839093,t15:0.3307090974):0.9638151019,(t16:0.3203830819,t9:0.7679892208):0.7824802531):0.6064357986):0.2313762761):0.5647297143):0.06522589293); diff --git a/tests/fixtures/tree02.patristic.tsv b/tests/fixtures/tree02.patristic.tsv new file mode 100644 index 0000000..96f4fc0 --- /dev/null +++ b/tests/fixtures/tree02.patristic.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 5.509981905809 6.268814268755 4.298158500111 3.288007016992 6.603462925181 1.155393006746 2.891362346243 5.538941703504 3.344816803932 3.222148026805 3.777223148383 6.533890376566 4.174274427118 5.815832145512 3.889901413117 3.338968485128 +t10 5.509981905809 0.000000000000 3.873300325591 4.555518234149 4.008119303267 4.207948982017 5.016007093946 4.824346229667 1.189387890045 3.602176537970 3.479507760843 4.034582882421 4.138376433402 4.431634161156 3.420318202348 3.017801718554 5.271952368552 +t11 6.268814268755 3.873300325591 0.000000000000 5.314350597095 4.766951666214 1.514226050815 5.774839456892 5.583178592613 3.902260123286 4.361008900916 4.238340123789 4.793415245367 3.390911688562 5.190466524102 2.672853457509 3.776634081500 6.030784731498 +t12 4.298158500111 4.555518234149 5.314350597095 0.000000000000 2.796295897570 5.648999253521 3.804183688248 3.612522823969 4.584478031844 2.380121303955 2.257452526828 0.594603385078 5.579426704906 1.958702622447 4.861368473852 2.935437741457 4.060128962854 +t13 3.288007016992 4.008119303267 4.766951666214 2.796295897570 0.000000000000 5.101600322640 2.794032205129 2.602371340850 4.037079100963 1.842954201391 1.720285424264 2.275360545842 5.032027774025 2.672411824577 4.313969542971 2.388038810575 3.049977479735 +t14 6.603462925181 4.207948982017 1.514226050815 5.648999253521 5.101600322640 0.000000000000 6.109488113318 5.917827249039 4.236908779712 4.695657557342 4.572988780215 5.128063901793 3.725560344988 5.525115180528 3.007502113935 4.111282737926 6.365433387924 +t15 1.155393006746 5.016007093946 5.774839456892 3.804183688248 2.794032205129 6.109488113318 0.000000000000 2.397387534380 5.044966891641 2.850841992069 2.728173214942 3.283248336520 6.039915564703 3.680299615255 5.321857333649 3.395926601253 2.844993673265 +t16 2.891362346243 4.824346229667 5.583178592613 3.612522823969 2.602371340850 5.917827249039 2.397387534380 0.000000000000 4.853306027362 2.659181127790 2.536512350664 3.091587472241 5.848254700424 3.488638750976 5.130196469370 3.204265736975 1.088372302707 +t17 5.538941703504 1.189387890045 3.902260123286 4.584478031844 4.037079100963 4.236908779712 5.044966891641 4.853306027362 0.000000000000 3.631136335665 3.508467558539 4.063542680116 4.167336231098 4.460593958851 3.449278000044 3.046761516249 5.300912166247 +t2 3.344816803932 3.602176537970 4.361008900916 2.380121303955 1.842954201391 4.695657557342 2.850841992069 2.659181127790 3.631136335665 0.000000000000 1.170070577879 1.859185952228 4.626085008727 2.256237230962 3.908026777674 1.982096045278 3.106787266675 +t3 3.222148026805 3.479507760843 4.238340123789 2.257452526828 1.720285424264 4.572988780215 2.728173214942 2.536512350664 3.508467558539 1.170070577879 0.000000000000 1.736517175101 4.503416231601 2.133568453835 3.785358000547 1.859427268151 2.984118489549 +t4 3.777223148383 4.034582882421 4.793415245367 0.594603385078 2.275360545842 5.128063901793 3.283248336520 3.091587472241 4.063542680116 1.859185952228 1.736517175101 0.000000000000 5.058491353178 1.437767270720 4.340433122125 2.414502389729 3.539193611126 +t5 6.533890376566 4.138376433402 3.390911688562 5.579426704906 5.032027774025 3.725560344988 6.039915564703 5.848254700424 4.167336231098 4.626085008727 4.503416231601 5.058491353178 0.000000000000 5.455542631913 1.073743006913 4.041710189311 6.295860839309 +t6 4.174274427118 4.431634161156 5.190466524102 1.958702622447 2.672411824577 5.525115180528 3.680299615255 3.488638750976 4.460593958851 2.256237230962 2.133568453835 1.437767270720 5.455542631913 0.000000000000 4.737484400859 2.811553668464 3.936244889861 +t7 5.815832145512 3.420318202348 2.672853457509 4.861368473852 4.313969542971 3.007502113935 5.321857333649 5.130196469370 3.449278000044 3.908026777674 3.785358000547 4.340433122125 1.073743006913 4.737484400859 0.000000000000 3.323651958257 5.577802608255 +t8 3.889901413117 3.017801718554 3.776634081500 2.935437741457 2.388038810575 4.111282737926 3.395926601253 3.204265736975 3.046761516249 1.982096045278 1.859427268151 2.414502389729 4.041710189311 2.811553668464 3.323651958257 0.000000000000 3.651871875860 +t9 3.338968485128 5.271952368552 6.030784731498 4.060128962854 3.049977479735 6.365433387924 2.844993673265 1.088372302707 5.300912166247 3.106787266675 2.984118489549 3.539193611126 6.295860839309 3.936244889861 5.577802608255 3.651871875860 0.000000000000 diff --git a/tests/fixtures/tree02.topology.tsv b/tests/fixtures/tree02.topology.tsv new file mode 100644 index 0000000..3eedd60 --- /dev/null +++ b/tests/fixtures/tree02.topology.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 8.000000000000 9.000000000000 8.000000000000 4.000000000000 9.000000000000 2.000000000000 4.000000000000 8.000000000000 7.000000000000 7.000000000000 8.000000000000 9.000000000000 7.000000000000 9.000000000000 7.000000000000 4.000000000000 +t10 8.000000000000 0.000000000000 5.000000000000 8.000000000000 6.000000000000 5.000000000000 8.000000000000 8.000000000000 2.000000000000 7.000000000000 7.000000000000 8.000000000000 5.000000000000 7.000000000000 5.000000000000 5.000000000000 8.000000000000 +t11 9.000000000000 5.000000000000 0.000000000000 9.000000000000 7.000000000000 2.000000000000 9.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 4.000000000000 8.000000000000 4.000000000000 6.000000000000 9.000000000000 +t12 8.000000000000 8.000000000000 9.000000000000 0.000000000000 6.000000000000 9.000000000000 8.000000000000 8.000000000000 8.000000000000 5.000000000000 5.000000000000 2.000000000000 9.000000000000 3.000000000000 9.000000000000 7.000000000000 8.000000000000 +t13 4.000000000000 6.000000000000 7.000000000000 6.000000000000 0.000000000000 7.000000000000 4.000000000000 4.000000000000 6.000000000000 5.000000000000 5.000000000000 6.000000000000 7.000000000000 5.000000000000 7.000000000000 5.000000000000 4.000000000000 +t14 9.000000000000 5.000000000000 2.000000000000 9.000000000000 7.000000000000 0.000000000000 9.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 4.000000000000 8.000000000000 4.000000000000 6.000000000000 9.000000000000 +t15 2.000000000000 8.000000000000 9.000000000000 8.000000000000 4.000000000000 9.000000000000 0.000000000000 4.000000000000 8.000000000000 7.000000000000 7.000000000000 8.000000000000 9.000000000000 7.000000000000 9.000000000000 7.000000000000 4.000000000000 +t16 4.000000000000 8.000000000000 9.000000000000 8.000000000000 4.000000000000 9.000000000000 4.000000000000 0.000000000000 8.000000000000 7.000000000000 7.000000000000 8.000000000000 9.000000000000 7.000000000000 9.000000000000 7.000000000000 2.000000000000 +t17 8.000000000000 2.000000000000 5.000000000000 8.000000000000 6.000000000000 5.000000000000 8.000000000000 8.000000000000 0.000000000000 7.000000000000 7.000000000000 8.000000000000 5.000000000000 7.000000000000 5.000000000000 5.000000000000 8.000000000000 +t2 7.000000000000 7.000000000000 8.000000000000 5.000000000000 5.000000000000 8.000000000000 7.000000000000 7.000000000000 7.000000000000 0.000000000000 2.000000000000 5.000000000000 8.000000000000 4.000000000000 8.000000000000 6.000000000000 7.000000000000 +t3 7.000000000000 7.000000000000 8.000000000000 5.000000000000 5.000000000000 8.000000000000 7.000000000000 7.000000000000 7.000000000000 2.000000000000 0.000000000000 5.000000000000 8.000000000000 4.000000000000 8.000000000000 6.000000000000 7.000000000000 +t4 8.000000000000 8.000000000000 9.000000000000 2.000000000000 6.000000000000 9.000000000000 8.000000000000 8.000000000000 8.000000000000 5.000000000000 5.000000000000 0.000000000000 9.000000000000 3.000000000000 9.000000000000 7.000000000000 8.000000000000 +t5 9.000000000000 5.000000000000 4.000000000000 9.000000000000 7.000000000000 4.000000000000 9.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 0.000000000000 8.000000000000 2.000000000000 6.000000000000 9.000000000000 +t6 7.000000000000 7.000000000000 8.000000000000 3.000000000000 5.000000000000 8.000000000000 7.000000000000 7.000000000000 7.000000000000 4.000000000000 4.000000000000 3.000000000000 8.000000000000 0.000000000000 8.000000000000 6.000000000000 7.000000000000 +t7 9.000000000000 5.000000000000 4.000000000000 9.000000000000 7.000000000000 4.000000000000 9.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 9.000000000000 2.000000000000 8.000000000000 0.000000000000 6.000000000000 9.000000000000 +t8 7.000000000000 5.000000000000 6.000000000000 7.000000000000 5.000000000000 6.000000000000 7.000000000000 7.000000000000 5.000000000000 6.000000000000 6.000000000000 7.000000000000 6.000000000000 6.000000000000 6.000000000000 0.000000000000 7.000000000000 +t9 4.000000000000 8.000000000000 9.000000000000 8.000000000000 4.000000000000 9.000000000000 4.000000000000 2.000000000000 8.000000000000 7.000000000000 7.000000000000 8.000000000000 9.000000000000 7.000000000000 9.000000000000 7.000000000000 0.000000000000 diff --git a/tests/fixtures/tree03.lmm.tsv b/tests/fixtures/tree03.lmm.tsv new file mode 100644 index 0000000..762bb3c --- /dev/null +++ b/tests/fixtures/tree03.lmm.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 2.475192304933 1.022800418548 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.780427609337 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.022800418548 0.000000000000 1.022800418548 1.022800418548 0.645073785447 +t10 1.022800418548 3.779524728190 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.022800418548 0.000000000000 0.000000000000 0.000000000000 0.000000000000 2.107599585783 0.000000000000 2.910439446801 1.322111306246 0.645073785447 +t11 0.000000000000 0.000000000000 1.865152974846 0.107218391960 0.000000000000 0.257950400701 0.107218391960 0.000000000000 0.107218391960 0.257950400701 0.107218391960 0.996794874081 0.000000000000 0.107218391960 0.000000000000 0.000000000000 0.000000000000 +t12 0.000000000000 0.000000000000 0.107218391960 1.954871815396 0.000000000000 0.107218391960 1.401385987177 0.000000000000 0.475520007778 0.107218391960 1.835114753572 0.107218391960 0.000000000000 0.475520007778 0.000000000000 0.000000000000 0.000000000000 +t13 0.645073785447 0.645073785447 0.000000000000 0.000000000000 1.762679928914 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.645073785447 0.645073785447 1.450053893495 +t14 0.000000000000 0.000000000000 0.257950400701 0.107218391960 0.000000000000 0.697857100284 0.107218391960 0.000000000000 0.107218391960 0.429025762947 0.107218391960 0.257950400701 0.000000000000 0.107218391960 0.000000000000 0.000000000000 0.000000000000 +t15 0.000000000000 0.000000000000 0.107218391960 1.401385987177 0.000000000000 0.107218391960 2.149279252626 0.000000000000 0.475520007778 0.107218391960 1.401385987177 0.107218391960 0.000000000000 0.475520007778 0.000000000000 0.000000000000 0.000000000000 +t16 1.780427609337 1.022800418548 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 2.489130385220 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.022800418548 0.000000000000 1.022800418548 1.022800418548 0.645073785447 +t17 0.000000000000 0.000000000000 0.107218391960 0.475520007778 0.000000000000 0.107218391960 0.475520007778 0.000000000000 0.884996788111 0.107218391960 0.475520007778 0.107218391960 0.000000000000 0.748974374263 0.000000000000 0.000000000000 0.000000000000 +t2 0.000000000000 0.000000000000 0.257950400701 0.107218391960 0.000000000000 0.429025762947 0.107218391960 0.000000000000 0.107218391960 1.048476446653 0.107218391960 0.257950400701 0.000000000000 0.107218391960 0.000000000000 0.000000000000 0.000000000000 +t3 0.000000000000 0.000000000000 0.107218391960 1.835114753572 0.000000000000 0.107218391960 1.401385987177 0.000000000000 0.475520007778 0.107218391960 2.190816687187 0.107218391960 0.000000000000 0.475520007778 0.000000000000 0.000000000000 0.000000000000 +t4 0.000000000000 0.000000000000 0.996794874081 0.107218391960 0.000000000000 0.257950400701 0.107218391960 0.000000000000 0.107218391960 0.257950400701 0.107218391960 1.964608333772 0.000000000000 0.107218391960 0.000000000000 0.000000000000 0.000000000000 +t5 1.022800418548 2.107599585783 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.022800418548 0.000000000000 0.000000000000 0.000000000000 0.000000000000 3.003212999320 0.000000000000 2.107599585783 1.322111306246 0.645073785447 +t6 0.000000000000 0.000000000000 0.107218391960 0.475520007778 0.000000000000 0.107218391960 0.475520007778 0.000000000000 0.748974374263 0.107218391960 0.475520007778 0.107218391960 0.000000000000 1.257501092739 0.000000000000 0.000000000000 0.000000000000 +t7 1.022800418548 2.910439446801 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.022800418548 0.000000000000 0.000000000000 0.000000000000 0.000000000000 2.107599585783 0.000000000000 3.337722870987 1.322111306246 0.645073785447 +t8 1.022800418548 1.322111306246 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 1.022800418548 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.322111306246 0.000000000000 1.322111306246 1.673777386779 0.645073785447 +t9 0.645073785447 0.645073785447 0.000000000000 0.000000000000 1.450053893495 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.645073785447 0.000000000000 0.645073785447 0.645073785447 1.854699239135 diff --git a/tests/fixtures/tree03.nwk b/tests/fixtures/tree03.nwk new file mode 100644 index 0000000..26b5b97 --- /dev/null +++ b/tests/fixtures/tree03.nwk @@ -0,0 +1 @@ +((((t11:0.8683581008,t4:0.9678134597):0.7388444734,(t2:0.6194506837,t14:0.2688313373):0.1710753622):0.1507320087,((t6:0.5085267185,t17:0.1360224138):0.2734543665,((t12:0.1197570618,t3:0.3557019336):0.4337287664,t15:0.7478932654):0.9258659794):0.3683016158):0.107218392,(((t1:0.6947646956,t16:0.7087027759):0.7576271908,(t8:0.3516660805,((t7:0.4272834242,t10:0.8690852814):0.802839861,t5:0.8956134135):0.7854882795):0.2993108877):0.3777266331,(t9:0.4046453456,t13:0.3126260354):0.804980108):0.6450737854); diff --git a/tests/fixtures/tree03.patristic.tsv b/tests/fixtures/tree03.patristic.tsv new file mode 100644 index 0000000..9e184dd --- /dev/null +++ b/tests/fixtures/tree03.patristic.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 4.209116196027 4.340345279779 4.430064120330 2.947724662954 3.173049405217 4.624471557559 1.403467471479 3.360189093044 3.523668751586 4.666008992121 4.439800638705 3.432804467157 3.732693397673 3.767314338824 2.103368854616 3.039743973175 +t10 4.209116196027 0.000000000000 5.644677703036 5.734396543587 4.252057086211 4.477381828474 5.928803980816 4.223054276314 4.664521516301 4.828001174843 5.970341415377 5.744133061962 2.567538555944 5.037025820930 1.296368705574 2.809079502476 4.344076396432 +t11 4.340345279779 5.644677703036 0.000000000000 3.605588006321 3.627832903760 2.047109273728 3.799995443551 4.354283360066 2.535712979035 2.397728620097 3.841532878112 1.836171560455 4.868365974165 2.908217283664 5.202875845833 3.538930361625 3.719852213981 +t12 4.430064120330 5.734396543587 3.605588006321 0.000000000000 3.717551744310 2.438292131759 1.301379093667 4.444002200617 1.888828587951 2.788911478128 0.475458995439 3.705043365248 4.958084814716 2.261332892580 5.292594686383 3.628649202175 3.809571054531 +t13 2.947724662954 4.252057086211 3.627832903760 3.717551744310 0.000000000000 2.460537029197 3.911959181540 2.961662743241 2.647676717024 2.811156375566 3.953496616101 3.727288262686 3.475745357340 3.020181021653 3.810255229007 2.146309744800 0.717271381058 +t14 3.173049405217 4.477381828474 2.047109273728 2.438292131759 2.460537029197 0.000000000000 2.632699568989 3.186987485504 1.368417104473 0.888282021042 2.674237003550 2.146564632654 3.701070099603 1.740921409102 4.035579971271 2.371634487063 2.552556339419 +t15 4.624471557559 5.928803980816 3.799995443551 1.301379093667 3.911959181540 2.632699568989 0.000000000000 4.638409637846 2.083236025181 2.983318915358 1.537323965458 3.899450802477 5.152492251946 2.455740329809 5.487002123613 3.823056639405 4.003978491761 +t16 1.403467471479 4.223054276314 4.354283360066 4.444002200617 2.961662743241 3.186987485504 4.638409637846 0.000000000000 3.374127173331 3.537606831873 4.679947072407 4.453738718992 3.446742547443 3.746631477959 3.781252419110 2.117306934902 3.053682053462 +t17 3.360189093044 4.664521516301 2.535712979035 1.888828587951 2.647676717024 1.368417104473 2.083236025181 3.374127173331 0.000000000000 1.719036450842 2.124773459742 2.635168337962 3.888209787430 0.644549132325 4.222719659097 2.558774174890 2.739696027245 +t2 3.523668751586 4.828001174843 2.397728620097 2.788911478128 2.811156375566 0.888282021042 2.983318915358 3.537606831873 1.719036450842 0.000000000000 3.024856349919 2.497183979023 4.051689445972 2.091540755471 4.386199317640 2.722253833432 2.903175685788 +t3 4.666008992121 5.970341415377 3.841532878112 0.475458995439 3.953496616101 2.674237003550 1.537323965458 4.679947072407 2.124773459742 3.024856349919 0.000000000000 3.940988237038 5.194029686507 2.497277764371 5.528539558174 3.864594073966 4.045515926322 +t4 4.439800638705 5.744133061962 1.836171560455 3.705043365248 3.727288262686 2.146564632654 3.899450802477 4.453738718992 2.635168337962 2.497183979023 3.940988237038 0.000000000000 4.967821333092 3.007672642590 5.302331204759 3.638385720551 3.819307572907 +t5 3.432804467157 2.567538555944 4.868365974165 4.958084814716 3.475745357340 3.701070099603 5.152492251946 3.446742547443 3.888209787430 4.051689445972 5.194029686507 4.967821333092 0.000000000000 4.260714092059 2.125736698741 2.032767773606 3.567764667561 +t6 3.732693397673 5.037025820930 2.908217283664 2.261332892580 3.020181021653 1.740921409102 2.455740329809 3.746631477959 0.644549132325 2.091540755471 2.497277764371 3.007672642590 4.260714092059 0.000000000000 4.595223963726 2.931278479518 3.112200331874 +t7 3.767314338824 1.296368705574 5.202875845833 5.292594686383 3.810255229007 4.035579971271 5.487002123613 3.781252419110 4.222719659097 4.386199317640 5.528539558174 5.302331204759 2.125736698741 4.595223963726 0.000000000000 2.367277645273 3.902274539229 +t8 2.103368854616 2.809079502476 3.538930361625 3.628649202175 2.146309744800 2.371634487063 3.823056639405 2.117306934902 2.558774174890 2.722253833432 3.864594073966 3.638385720551 2.032767773606 2.931278479518 2.367277645273 0.000000000000 2.238329055021 +t9 3.039743973175 4.344076396432 3.719852213981 3.809571054531 0.717271381058 2.552556339419 4.003978491761 3.053682053462 2.739696027245 2.903175685788 4.045515926322 3.819307572907 3.567764667561 3.112200331874 3.902274539229 2.238329055021 0.000000000000 diff --git a/tests/fixtures/tree03.topology.tsv b/tests/fixtures/tree03.topology.tsv new file mode 100644 index 0000000..042bcbe --- /dev/null +++ b/tests/fixtures/tree03.topology.tsv @@ -0,0 +1,18 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 6.000000000000 8.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 2.000000000000 8.000000000000 8.000000000000 9.000000000000 8.000000000000 5.000000000000 8.000000000000 6.000000000000 4.000000000000 5.000000000000 +t10 6.000000000000 0.000000000000 10.000000000000 11.000000000000 7.000000000000 10.000000000000 10.000000000000 6.000000000000 10.000000000000 10.000000000000 11.000000000000 10.000000000000 3.000000000000 10.000000000000 2.000000000000 4.000000000000 7.000000000000 +t11 8.000000000000 10.000000000000 0.000000000000 7.000000000000 7.000000000000 4.000000000000 6.000000000000 8.000000000000 6.000000000000 4.000000000000 7.000000000000 2.000000000000 9.000000000000 6.000000000000 10.000000000000 8.000000000000 7.000000000000 +t12 9.000000000000 11.000000000000 7.000000000000 0.000000000000 8.000000000000 7.000000000000 3.000000000000 9.000000000000 5.000000000000 7.000000000000 2.000000000000 7.000000000000 10.000000000000 5.000000000000 11.000000000000 9.000000000000 8.000000000000 +t13 5.000000000000 7.000000000000 7.000000000000 8.000000000000 0.000000000000 7.000000000000 7.000000000000 5.000000000000 7.000000000000 7.000000000000 8.000000000000 7.000000000000 6.000000000000 7.000000000000 7.000000000000 5.000000000000 2.000000000000 +t14 8.000000000000 10.000000000000 4.000000000000 7.000000000000 7.000000000000 0.000000000000 6.000000000000 8.000000000000 6.000000000000 2.000000000000 7.000000000000 4.000000000000 9.000000000000 6.000000000000 10.000000000000 8.000000000000 7.000000000000 +t15 8.000000000000 10.000000000000 6.000000000000 3.000000000000 7.000000000000 6.000000000000 0.000000000000 8.000000000000 4.000000000000 6.000000000000 3.000000000000 6.000000000000 9.000000000000 4.000000000000 10.000000000000 8.000000000000 7.000000000000 +t16 2.000000000000 6.000000000000 8.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 0.000000000000 8.000000000000 8.000000000000 9.000000000000 8.000000000000 5.000000000000 8.000000000000 6.000000000000 4.000000000000 5.000000000000 +t17 8.000000000000 10.000000000000 6.000000000000 5.000000000000 7.000000000000 6.000000000000 4.000000000000 8.000000000000 0.000000000000 6.000000000000 5.000000000000 6.000000000000 9.000000000000 2.000000000000 10.000000000000 8.000000000000 7.000000000000 +t2 8.000000000000 10.000000000000 4.000000000000 7.000000000000 7.000000000000 2.000000000000 6.000000000000 8.000000000000 6.000000000000 0.000000000000 7.000000000000 4.000000000000 9.000000000000 6.000000000000 10.000000000000 8.000000000000 7.000000000000 +t3 9.000000000000 11.000000000000 7.000000000000 2.000000000000 8.000000000000 7.000000000000 3.000000000000 9.000000000000 5.000000000000 7.000000000000 0.000000000000 7.000000000000 10.000000000000 5.000000000000 11.000000000000 9.000000000000 8.000000000000 +t4 8.000000000000 10.000000000000 2.000000000000 7.000000000000 7.000000000000 4.000000000000 6.000000000000 8.000000000000 6.000000000000 4.000000000000 7.000000000000 0.000000000000 9.000000000000 6.000000000000 10.000000000000 8.000000000000 7.000000000000 +t5 5.000000000000 3.000000000000 9.000000000000 10.000000000000 6.000000000000 9.000000000000 9.000000000000 5.000000000000 9.000000000000 9.000000000000 10.000000000000 9.000000000000 0.000000000000 9.000000000000 3.000000000000 3.000000000000 6.000000000000 +t6 8.000000000000 10.000000000000 6.000000000000 5.000000000000 7.000000000000 6.000000000000 4.000000000000 8.000000000000 2.000000000000 6.000000000000 5.000000000000 6.000000000000 9.000000000000 0.000000000000 10.000000000000 8.000000000000 7.000000000000 +t7 6.000000000000 2.000000000000 10.000000000000 11.000000000000 7.000000000000 10.000000000000 10.000000000000 6.000000000000 10.000000000000 10.000000000000 11.000000000000 10.000000000000 3.000000000000 10.000000000000 0.000000000000 4.000000000000 7.000000000000 +t8 4.000000000000 4.000000000000 8.000000000000 9.000000000000 5.000000000000 8.000000000000 8.000000000000 4.000000000000 8.000000000000 8.000000000000 9.000000000000 8.000000000000 3.000000000000 8.000000000000 4.000000000000 0.000000000000 5.000000000000 +t9 5.000000000000 7.000000000000 7.000000000000 8.000000000000 2.000000000000 7.000000000000 7.000000000000 5.000000000000 7.000000000000 7.000000000000 8.000000000000 7.000000000000 6.000000000000 7.000000000000 7.000000000000 5.000000000000 0.000000000000 diff --git a/tests/fixtures/tree04.lmm.tsv b/tests/fixtures/tree04.lmm.tsv new file mode 100644 index 0000000..8ef2f84 --- /dev/null +++ b/tests/fixtures/tree04.lmm.tsv @@ -0,0 +1,19 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t18 t2 t3 t4 t5 t6 t7 t8 t9 +t1 1.826454006135 1.434058769606 1.323048059829 0.000000000000 0.000000000000 0.055704348022 0.055704348022 1.323048059829 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.960756540997 0.000000000000 0.960756540997 0.000000000000 +t10 1.434058769606 1.785461900057 1.323048059829 0.000000000000 0.000000000000 0.055704348022 0.055704348022 1.323048059829 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.960756540997 0.000000000000 0.960756540997 0.000000000000 +t11 1.323048059829 1.323048059829 1.958333726740 0.000000000000 0.000000000000 0.055704348022 0.055704348022 1.952479454223 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.960756540997 0.000000000000 0.960756540997 0.000000000000 +t12 0.000000000000 0.000000000000 0.000000000000 1.834822334116 0.517228086945 0.000000000000 0.000000000000 0.000000000000 0.517228086945 1.042361159576 0.492893308168 0.492893308168 0.492893308168 0.517228086945 0.000000000000 0.517228086945 0.000000000000 0.492893308168 +t13 0.000000000000 0.000000000000 0.000000000000 0.517228086945 3.687702669296 0.000000000000 0.000000000000 0.000000000000 1.509061724180 0.517228086945 0.492893308168 0.492893308168 0.492893308168 2.833347449545 0.000000000000 2.441255136626 0.000000000000 0.492893308168 +t14 0.055704348022 0.055704348022 0.055704348022 0.000000000000 0.000000000000 0.435066005681 0.104274603771 0.055704348022 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.055704348022 0.000000000000 0.055704348022 0.000000000000 +t15 0.055704348022 0.055704348022 0.055704348022 0.000000000000 0.000000000000 0.104274603771 0.457994617289 0.055704348022 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.055704348022 0.000000000000 0.055704348022 0.000000000000 +t16 1.323048059829 1.323048059829 1.952479454223 0.000000000000 0.000000000000 0.055704348022 0.055704348022 2.269571904792 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.960756540997 0.000000000000 0.960756540997 0.000000000000 +t17 0.000000000000 0.000000000000 0.000000000000 0.517228086945 1.509061724180 0.000000000000 0.000000000000 0.000000000000 1.601748950779 0.517228086945 0.492893308168 0.492893308168 0.492893308168 1.509061724180 0.000000000000 1.509061724180 0.000000000000 0.492893308168 +t18 0.000000000000 0.000000000000 0.000000000000 1.042361159576 0.517228086945 0.000000000000 0.000000000000 0.000000000000 0.517228086945 1.766559675103 0.492893308168 0.492893308168 0.492893308168 0.517228086945 0.000000000000 0.517228086945 0.000000000000 0.492893308168 +t2 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 1.831984268734 1.537219957914 0.760391999735 0.492893308168 0.000000000000 0.492893308168 0.000000000000 0.760391999735 +t3 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 1.537219957914 2.051610622788 0.760391999735 0.492893308168 0.000000000000 0.492893308168 0.000000000000 0.760391999735 +t4 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.760391999735 0.760391999735 1.824602956185 0.492893308168 0.000000000000 0.492893308168 0.000000000000 0.883239736082 +t5 0.000000000000 0.000000000000 0.000000000000 0.517228086945 2.833347449545 0.000000000000 0.000000000000 0.000000000000 1.509061724180 0.517228086945 0.492893308168 0.492893308168 0.492893308168 3.014986074297 0.000000000000 2.441255136626 0.000000000000 0.492893308168 +t6 0.960756540997 0.960756540997 0.960756540997 0.000000000000 0.000000000000 0.055704348022 0.055704348022 0.960756540997 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 2.235513422871 0.000000000000 1.956952624256 0.000000000000 +t7 0.000000000000 0.000000000000 0.000000000000 0.517228086945 2.441255136626 0.000000000000 0.000000000000 0.000000000000 1.509061724180 0.517228086945 0.492893308168 0.492893308168 0.492893308168 2.441255136626 0.000000000000 3.185927398736 0.000000000000 0.492893308168 +t8 0.960756540997 0.960756540997 0.960756540997 0.000000000000 0.000000000000 0.055704348022 0.055704348022 0.960756540997 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.956952624256 0.000000000000 2.867384099402 0.000000000000 +t9 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.000000000000 0.000000000000 0.000000000000 0.492893308168 0.492893308168 0.760391999735 0.760391999735 0.883239736082 0.492893308168 0.000000000000 0.492893308168 0.000000000000 0.979572141776 diff --git a/tests/fixtures/tree04.nwk b/tests/fixtures/tree04.nwk new file mode 100644 index 0000000..c5d1741 --- /dev/null +++ b/tests/fixtures/tree04.nwk @@ -0,0 +1 @@ +((((t17:0.0926872266,((t5:0.1816386248,t13:0.8543552198):0.3920923129,t7:0.7446722621):0.9321934124):0.9918336372,(t18:0.7241985155,t12:0.7924611745):0.5251330726):0.02433477878,((t4:0.9413632201,t9:0.09633240569):0.1228477363,(t2:0.2947643108,t3:0.5143906649):0.7768279582):0.2674986916):0.4928933082,(((t6:0.2785607986,t8:0.9104314751):0.9961960833,((t11:0.005854272516,t16:0.3170924506):0.6294313944,(t1:0.3923952365,t10:0.3514031305):0.1110107098):0.3622915188):0.905052193,(t15:0.3537200135,t14:0.3307914019):0.04857025575):0.05570434802); diff --git a/tests/fixtures/tree04.patristic.tsv b/tests/fixtures/tree04.patristic.tsv new file mode 100644 index 0000000..e88876d --- /dev/null +++ b/tests/fixtures/tree04.patristic.tsv @@ -0,0 +1,19 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t18 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 0.743798366981 1.138691613218 3.661276340252 5.514156675432 2.150111315772 2.173039927380 1.449929791270 3.428202956915 3.593013681239 3.658438274870 3.878064628923 3.651056962321 4.841440080432 2.140454347013 5.012381404871 2.772325023543 2.806026147911 +t10 0.743798366981 0.000000000000 1.097699507140 3.620284234174 5.473164569354 2.109119209694 2.132047821302 1.408937685192 3.387210850837 3.552021575160 3.617446168792 3.837072522845 3.610064856242 4.800447974354 2.099462240934 4.971389298793 2.731332917465 2.765034041833 +t11 1.138691613218 1.097699507140 0.000000000000 3.793156060856 5.646036396036 2.281991036376 2.304919647984 0.322946723085 3.560082677519 3.724893401843 3.790317995474 4.009944349527 3.782936682925 4.973319801036 2.272334067617 5.144261125475 2.904204744147 2.937905868515 +t12 3.661276340252 3.620284234174 3.793156060856 0.000000000000 4.488068829523 2.269888339797 2.292816951405 4.104394238908 2.402115111006 1.516659690067 2.681019986514 2.900646340568 2.673638673965 3.815352234524 4.070335756987 3.986293558963 4.702206433518 1.828607859556 +t13 5.514156675432 5.473164569354 5.646036396036 4.488068829523 0.000000000000 4.122768674977 4.145697286585 5.957274574088 2.271328171715 4.419806170510 4.533900321694 4.753526675748 4.526519009145 1.035993844504 5.923216092167 1.991119794780 6.555086768698 3.681488194736 +t14 2.150111315772 2.109119209694 2.281991036376 2.269888339797 4.122768674977 0.000000000000 0.684511415428 2.593229214428 2.036814956460 2.201625680784 2.267050274415 2.486676628469 2.259668961866 3.450052079977 2.559170732507 3.620993404416 3.191041409038 1.414638147457 +t15 2.173039927380 2.132047821302 2.304919647984 2.292816951405 4.145697286585 0.684511415428 0.000000000000 2.616157826036 2.059743568068 2.224554292392 2.289978886023 2.509605240077 2.282597573474 3.472980691586 2.582099344116 3.643922016025 3.213970020646 1.437566759065 +t16 1.449929791270 1.408937685192 0.322946723085 4.104394238908 5.957274574088 2.593229214428 2.616157826036 0.000000000000 3.871320855571 4.036131579895 4.101556173526 4.321182527579 4.094174860977 5.284557979088 2.583572245669 5.455499303527 3.215442922199 3.249144046567 +t17 3.428202956915 3.387210850837 3.560082677519 2.402115111006 2.271328171715 2.036814956460 2.059743568068 3.871320855571 0.000000000000 2.333852451993 2.447946603177 2.667572957231 2.440565290628 1.598611576715 3.837262373650 1.769552901154 4.469133050181 1.595534476219 +t18 3.593013681239 3.552021575160 3.724893401843 1.516659690067 4.419806170510 2.201625680784 2.224554292392 4.036131579895 2.333852451993 0.000000000000 2.612757327501 2.832383681554 2.605376014952 3.747089575510 4.002073097974 3.918030899949 4.633943774505 1.760345200542 +t2 3.658438274870 3.617446168792 3.790317995474 2.681019986514 4.533900321694 2.267050274415 2.289978886023 4.101556173526 2.447946603177 2.612757327501 0.000000000000 0.809154975694 2.135803225450 3.861183726694 4.067497691605 4.032125051133 4.699368368136 1.290772411041 +t3 3.878064628923 3.837072522845 4.009944349527 2.900646340568 4.753526675748 2.486676628469 2.509605240077 4.321182527579 2.667572957231 2.832383681554 0.809154975694 0.000000000000 2.355429579504 4.080810080748 4.287124045659 4.251751405187 4.918994722189 1.510398765095 +t4 3.651056962321 3.610064856242 3.782936682925 2.673638673965 4.526519009145 2.259668961866 2.282597573474 4.094174860977 2.440565290628 2.605376014952 2.135803225450 2.355429579504 0.000000000000 3.853802414145 4.060116379056 4.024743738584 4.691987055587 1.037695625797 +t5 4.841440080432 4.800447974354 4.973319801036 3.815352234524 1.035993844504 3.450052079977 3.472980691586 5.284557979088 1.598611576715 3.747089575510 3.861183726694 4.080810080748 3.853802414145 0.000000000000 5.250499497168 1.318403199781 5.882370173698 3.008771599736 +t6 2.140454347013 2.099462240934 2.272334067617 4.070335756987 5.923216092167 2.559170732507 2.582099344116 2.583572245669 3.837262373650 4.002073097974 4.067497691605 4.287124045659 4.060116379056 5.250499497168 0.000000000000 5.421440821607 1.188992273761 3.215085564647 +t7 5.012381404871 4.971389298793 5.144261125475 3.986293558963 1.991119794780 3.620993404416 3.643922016025 5.455499303527 1.769552901154 3.918030899949 4.032125051133 4.251751405187 4.024743738584 1.318403199781 5.421440821607 0.000000000000 6.053311498137 3.179712924175 +t8 2.772325023543 2.731332917465 2.904204744147 4.702206433518 6.555086768698 3.191041409038 3.213970020646 3.215442922199 4.469133050181 4.633943774505 4.699368368136 4.918994722189 4.691987055587 5.882370173698 1.188992273761 6.053311498137 0.000000000000 3.846956241177 +t9 2.806026147911 2.765034041833 2.937905868515 1.828607859556 3.681488194736 1.414638147457 1.437566759065 3.249144046567 1.595534476219 1.760345200542 1.290772411041 1.510398765095 1.037695625797 3.008771599736 3.215085564647 3.179712924175 3.846956241177 0.000000000000 diff --git a/tests/fixtures/tree04.topology.tsv b/tests/fixtures/tree04.topology.tsv new file mode 100644 index 0000000..93d57a3 --- /dev/null +++ b/tests/fixtures/tree04.topology.tsv @@ -0,0 +1,19 @@ + t1 t10 t11 t12 t13 t14 t15 t16 t17 t18 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 2.000000000000 4.000000000000 9.000000000000 11.000000000000 6.000000000000 6.000000000000 4.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 11.000000000000 5.000000000000 10.000000000000 5.000000000000 9.000000000000 +t10 2.000000000000 0.000000000000 4.000000000000 9.000000000000 11.000000000000 6.000000000000 6.000000000000 4.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 11.000000000000 5.000000000000 10.000000000000 5.000000000000 9.000000000000 +t11 4.000000000000 4.000000000000 0.000000000000 9.000000000000 11.000000000000 6.000000000000 6.000000000000 2.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 11.000000000000 5.000000000000 10.000000000000 5.000000000000 9.000000000000 +t12 9.000000000000 9.000000000000 9.000000000000 0.000000000000 6.000000000000 7.000000000000 7.000000000000 9.000000000000 4.000000000000 2.000000000000 6.000000000000 6.000000000000 6.000000000000 6.000000000000 8.000000000000 5.000000000000 8.000000000000 6.000000000000 +t13 11.000000000000 11.000000000000 11.000000000000 6.000000000000 0.000000000000 9.000000000000 9.000000000000 11.000000000000 4.000000000000 6.000000000000 8.000000000000 8.000000000000 8.000000000000 2.000000000000 10.000000000000 3.000000000000 10.000000000000 8.000000000000 +t14 6.000000000000 6.000000000000 6.000000000000 7.000000000000 9.000000000000 0.000000000000 2.000000000000 6.000000000000 7.000000000000 7.000000000000 7.000000000000 7.000000000000 7.000000000000 9.000000000000 5.000000000000 8.000000000000 5.000000000000 7.000000000000 +t15 6.000000000000 6.000000000000 6.000000000000 7.000000000000 9.000000000000 2.000000000000 0.000000000000 6.000000000000 7.000000000000 7.000000000000 7.000000000000 7.000000000000 7.000000000000 9.000000000000 5.000000000000 8.000000000000 5.000000000000 7.000000000000 +t16 4.000000000000 4.000000000000 2.000000000000 9.000000000000 11.000000000000 6.000000000000 6.000000000000 0.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 9.000000000000 11.000000000000 5.000000000000 10.000000000000 5.000000000000 9.000000000000 +t17 9.000000000000 9.000000000000 9.000000000000 4.000000000000 4.000000000000 7.000000000000 7.000000000000 9.000000000000 0.000000000000 4.000000000000 6.000000000000 6.000000000000 6.000000000000 4.000000000000 8.000000000000 3.000000000000 8.000000000000 6.000000000000 +t18 9.000000000000 9.000000000000 9.000000000000 2.000000000000 6.000000000000 7.000000000000 7.000000000000 9.000000000000 4.000000000000 0.000000000000 6.000000000000 6.000000000000 6.000000000000 6.000000000000 8.000000000000 5.000000000000 8.000000000000 6.000000000000 +t2 9.000000000000 9.000000000000 9.000000000000 6.000000000000 8.000000000000 7.000000000000 7.000000000000 9.000000000000 6.000000000000 6.000000000000 0.000000000000 2.000000000000 4.000000000000 8.000000000000 8.000000000000 7.000000000000 8.000000000000 4.000000000000 +t3 9.000000000000 9.000000000000 9.000000000000 6.000000000000 8.000000000000 7.000000000000 7.000000000000 9.000000000000 6.000000000000 6.000000000000 2.000000000000 0.000000000000 4.000000000000 8.000000000000 8.000000000000 7.000000000000 8.000000000000 4.000000000000 +t4 9.000000000000 9.000000000000 9.000000000000 6.000000000000 8.000000000000 7.000000000000 7.000000000000 9.000000000000 6.000000000000 6.000000000000 4.000000000000 4.000000000000 0.000000000000 8.000000000000 8.000000000000 7.000000000000 8.000000000000 2.000000000000 +t5 11.000000000000 11.000000000000 11.000000000000 6.000000000000 2.000000000000 9.000000000000 9.000000000000 11.000000000000 4.000000000000 6.000000000000 8.000000000000 8.000000000000 8.000000000000 0.000000000000 10.000000000000 3.000000000000 10.000000000000 8.000000000000 +t6 5.000000000000 5.000000000000 5.000000000000 8.000000000000 10.000000000000 5.000000000000 5.000000000000 5.000000000000 8.000000000000 8.000000000000 8.000000000000 8.000000000000 8.000000000000 10.000000000000 0.000000000000 9.000000000000 2.000000000000 8.000000000000 +t7 10.000000000000 10.000000000000 10.000000000000 5.000000000000 3.000000000000 8.000000000000 8.000000000000 10.000000000000 3.000000000000 5.000000000000 7.000000000000 7.000000000000 7.000000000000 3.000000000000 9.000000000000 0.000000000000 9.000000000000 7.000000000000 +t8 5.000000000000 5.000000000000 5.000000000000 8.000000000000 10.000000000000 5.000000000000 5.000000000000 5.000000000000 8.000000000000 8.000000000000 8.000000000000 8.000000000000 8.000000000000 10.000000000000 2.000000000000 9.000000000000 0.000000000000 8.000000000000 +t9 9.000000000000 9.000000000000 9.000000000000 6.000000000000 8.000000000000 7.000000000000 7.000000000000 9.000000000000 6.000000000000 6.000000000000 4.000000000000 4.000000000000 2.000000000000 8.000000000000 8.000000000000 7.000000000000 8.000000000000 0.000000000000 diff --git a/tests/fixtures/tree05.lmm.tsv b/tests/fixtures/tree05.lmm.tsv new file mode 100644 index 0000000..6be3e5c --- /dev/null +++ b/tests/fixtures/tree05.lmm.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 1.412819813471 0.000000000000 0.000000000000 0.963843473466 0.000000000000 1.018107262440 0.000000000000 0.000000000000 0.000000000000 1.018107262440 0.000000000000 0.000000000000 +t10 0.000000000000 0.682040252956 0.121796532068 0.000000000000 0.121796532068 0.000000000000 0.121796532068 0.121796532068 0.457185057225 0.000000000000 0.121796532068 0.121796532068 +t11 0.000000000000 0.121796532068 2.139318238711 0.000000000000 1.104086857522 0.000000000000 0.996908592293 1.723483712412 0.121796532068 0.000000000000 1.723483712412 0.996908592293 +t12 0.963843473466 0.000000000000 0.000000000000 1.052241293481 0.000000000000 0.963843473466 0.000000000000 0.000000000000 0.000000000000 0.963843473466 0.000000000000 0.000000000000 +t2 0.000000000000 0.121796532068 1.104086857522 0.000000000000 1.268882896286 0.000000000000 0.996908592293 1.104086857522 0.121796532068 0.000000000000 1.104086857522 0.996908592293 +t3 1.018107262440 0.000000000000 0.000000000000 0.963843473466 0.000000000000 1.379060709616 0.000000000000 0.000000000000 0.000000000000 1.259154318599 0.000000000000 0.000000000000 +t4 0.000000000000 0.121796532068 0.996908592293 0.000000000000 0.996908592293 0.000000000000 2.172244624468 0.996908592293 0.121796532068 0.000000000000 0.996908592293 1.303926248802 +t5 0.000000000000 0.121796532068 1.723483712412 0.000000000000 1.104086857522 0.000000000000 0.996908592293 2.642430364620 0.121796532068 0.000000000000 2.313111583935 0.996908592293 +t6 0.000000000000 0.457185057225 0.121796532068 0.000000000000 0.121796532068 0.000000000000 0.121796532068 0.121796532068 0.909587969305 0.000000000000 0.121796532068 0.121796532068 +t7 1.018107262440 0.000000000000 0.000000000000 0.963843473466 0.000000000000 1.259154318599 0.000000000000 0.000000000000 0.000000000000 1.890060830163 0.000000000000 0.000000000000 +t8 0.000000000000 0.121796532068 1.723483712412 0.000000000000 1.104086857522 0.000000000000 0.996908592293 2.313111583935 0.121796532068 0.000000000000 3.174793167273 0.996908592293 +t9 0.000000000000 0.121796532068 0.996908592293 0.000000000000 0.996908592293 0.000000000000 1.303926248802 0.996908592293 0.121796532068 0.000000000000 0.996908592293 1.668768494390 diff --git a/tests/fixtures/tree05.nwk b/tests/fixtures/tree05.nwk new file mode 100644 index 0000000..07c4e33 --- /dev/null +++ b/tests/fixtures/tree05.nwk @@ -0,0 +1 @@ +(((t1:0.394712551,(t3:0.119906391,t7:0.6309065116):0.2410470562):0.05426378897,t12:0.08839782001):0.9638434735,(((t4:0.8683183757,t9:0.3648422456):0.3070176565,(t2:0.1647960388,(t11:0.4158345263,(t5:0.3293187807,t8:0.8616815833):0.5896278715):0.6193968549):0.1071782652):0.8751120602,(t6:0.4524029121,t10:0.2248551957):0.3353885252):0.1217965321); diff --git a/tests/fixtures/tree05.patristic.tsv b/tests/fixtures/tree05.patristic.tsv new file mode 100644 index 0000000..8f489a6 --- /dev/null +++ b/tests/fixtures/tree05.patristic.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 2.094860066427 3.552138052182 0.537374160020 2.681702709757 0.755665998207 3.585064437939 4.055250178091 2.322407782776 1.266666118754 4.587612980744 3.081588307861 +t10 2.094860066427 0.000000000000 2.577765427530 1.734281546436 1.707330085104 2.061100962572 2.610691813286 3.080877553439 0.677258107811 2.572101083118 3.613240356091 2.107215683209 +t11 3.552138052182 2.577765427530 0.000000000000 3.191559532192 1.200027419953 3.518378948327 2.317745678592 1.334781178506 2.805313143879 4.029379068874 1.867143981159 1.814269548515 +t12 0.537374160020 1.734281546436 3.191559532192 0.000000000000 2.321124189766 0.503615056165 3.224485917948 3.694671658101 1.961829262786 1.014615176711 4.227034460753 2.721009787871 +t2 2.681702709757 1.707330085104 1.200027419953 2.321124189766 0.000000000000 2.647943605902 1.447310336167 1.703139545862 1.934877801454 3.158943726448 2.235502348514 0.943834206089 +t3 0.755665998207 2.061100962572 3.518378948327 0.503615056165 2.647943605902 0.000000000000 3.551305334084 4.021491074236 2.288648678921 0.750812902581 4.553853876889 3.047829204006 +t4 3.585064437939 2.610691813286 2.317745678592 3.224485917948 1.447310336167 3.551305334084 0.000000000000 2.820857804501 2.838239529636 4.062305454630 3.353220607154 1.233160621254 +t5 4.055250178091 3.080877553439 1.334781178506 3.694671658101 1.703139545862 4.021491074236 2.820857804501 0.000000000000 3.308425269788 4.532491194783 1.191000364022 2.317381674424 +t6 2.322407782776 0.677258107811 2.805313143879 1.961829262786 1.934877801454 2.288648678921 2.838239529636 3.308425269788 0.000000000000 2.799648799468 3.840788072441 2.334763399558 +t7 1.266666118754 2.572101083118 4.029379068874 1.014615176711 3.158943726448 0.750812902581 4.062305454630 4.532491194783 2.799648799468 0.000000000000 5.064853997435 3.558829324553 +t8 4.587612980744 3.613240356091 1.867143981159 4.227034460753 2.235502348514 4.553853876889 3.353220607154 1.191000364022 3.840788072441 5.064853997435 0.000000000000 2.849744477076 +t9 3.081588307861 2.107215683209 1.814269548515 2.721009787871 0.943834206089 3.047829204006 1.233160621254 2.317381674424 2.334763399558 3.558829324553 2.849744477076 0.000000000000 diff --git a/tests/fixtures/tree05.topology.tsv b/tests/fixtures/tree05.topology.tsv new file mode 100644 index 0000000..4028630 --- /dev/null +++ b/tests/fixtures/tree05.topology.tsv @@ -0,0 +1,13 @@ + t1 t10 t11 t12 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 6.000000000000 8.000000000000 3.000000000000 7.000000000000 3.000000000000 7.000000000000 9.000000000000 6.000000000000 3.000000000000 9.000000000000 7.000000000000 +t10 6.000000000000 0.000000000000 6.000000000000 5.000000000000 5.000000000000 7.000000000000 5.000000000000 7.000000000000 2.000000000000 7.000000000000 7.000000000000 5.000000000000 +t11 8.000000000000 6.000000000000 0.000000000000 7.000000000000 3.000000000000 9.000000000000 5.000000000000 3.000000000000 6.000000000000 9.000000000000 3.000000000000 5.000000000000 +t12 3.000000000000 5.000000000000 7.000000000000 0.000000000000 6.000000000000 4.000000000000 6.000000000000 8.000000000000 5.000000000000 4.000000000000 8.000000000000 6.000000000000 +t2 7.000000000000 5.000000000000 3.000000000000 6.000000000000 0.000000000000 8.000000000000 4.000000000000 4.000000000000 5.000000000000 8.000000000000 4.000000000000 4.000000000000 +t3 3.000000000000 7.000000000000 9.000000000000 4.000000000000 8.000000000000 0.000000000000 8.000000000000 10.000000000000 7.000000000000 2.000000000000 10.000000000000 8.000000000000 +t4 7.000000000000 5.000000000000 5.000000000000 6.000000000000 4.000000000000 8.000000000000 0.000000000000 6.000000000000 5.000000000000 8.000000000000 6.000000000000 2.000000000000 +t5 9.000000000000 7.000000000000 3.000000000000 8.000000000000 4.000000000000 10.000000000000 6.000000000000 0.000000000000 7.000000000000 10.000000000000 2.000000000000 6.000000000000 +t6 6.000000000000 2.000000000000 6.000000000000 5.000000000000 5.000000000000 7.000000000000 5.000000000000 7.000000000000 0.000000000000 7.000000000000 7.000000000000 5.000000000000 +t7 3.000000000000 7.000000000000 9.000000000000 4.000000000000 8.000000000000 2.000000000000 8.000000000000 10.000000000000 7.000000000000 0.000000000000 10.000000000000 8.000000000000 +t8 9.000000000000 7.000000000000 3.000000000000 8.000000000000 4.000000000000 10.000000000000 6.000000000000 2.000000000000 7.000000000000 10.000000000000 0.000000000000 6.000000000000 +t9 7.000000000000 5.000000000000 5.000000000000 6.000000000000 4.000000000000 8.000000000000 2.000000000000 6.000000000000 5.000000000000 8.000000000000 6.000000000000 0.000000000000 diff --git a/tests/fixtures/tree06.lmm.tsv b/tests/fixtures/tree06.lmm.tsv new file mode 100644 index 0000000..780dfec --- /dev/null +++ b/tests/fixtures/tree06.lmm.tsv @@ -0,0 +1,10 @@ + t1 t2 t3 t4 t5 t6 t7 t8 t9 +t1 2.987882709131 0.844335314119 1.340232875198 0.844335314119 2.126695765881 0.000000000000 0.000000000000 1.340232875198 0.000000000000 +t2 0.844335314119 1.951782006305 0.844335314119 1.352037202567 0.844335314119 0.000000000000 0.000000000000 0.844335314119 0.000000000000 +t3 1.340232875198 0.844335314119 2.459578693379 0.844335314119 1.340232875198 0.000000000000 0.000000000000 1.621566965710 0.000000000000 +t4 0.844335314119 1.352037202567 0.844335314119 2.301177664427 0.844335314119 0.000000000000 0.000000000000 0.844335314119 0.000000000000 +t5 2.126695765881 0.844335314119 1.340232875198 0.844335314119 2.127780230949 0.000000000000 0.000000000000 1.340232875198 0.000000000000 +t6 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 1.198197783902 0.203751023160 0.000000000000 0.114089301787 +t7 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.203751023160 0.346611253452 0.000000000000 0.114089301787 +t8 1.340232875198 0.844335314119 1.621566965710 0.844335314119 1.340232875198 0.000000000000 0.000000000000 2.261270712363 0.000000000000 +t9 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.000000000000 0.114089301787 0.114089301787 0.000000000000 0.766413139878 diff --git a/tests/fixtures/tree06.nwk b/tests/fixtures/tree06.nwk new file mode 100644 index 0000000..896fc35 --- /dev/null +++ b/tests/fixtures/tree06.nwk @@ -0,0 +1 @@ +(((t2:0.5997448037,t4:0.9491404619):0.5077018884,((t5:0.001084465068,t1:0.8611869432):0.7864628907,(t3:0.8380117277,t8:0.6397037467):0.2813340905):0.4958975611):0.8443353141,((t7:0.1428602303,t6:0.9944467607):0.08966172137,t9:0.6523238381):0.1140893018); diff --git a/tests/fixtures/tree06.patristic.tsv b/tests/fixtures/tree06.patristic.tsv new file mode 100644 index 0000000..ad66b40 --- /dev/null +++ b/tests/fixtures/tree06.patristic.tsv @@ -0,0 +1,10 @@ + t1 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 3.250994087197 2.766995652113 3.600389745319 0.862271408318 4.186080493033 3.334493962582 2.568687671097 3.754295849008 +t2 3.250994087197 0.000000000000 2.722690071445 1.548885265598 2.390891609015 3.149979790207 2.298393259756 2.524382090429 2.718195146183 +t3 2.766995652113 2.722690071445 0.000000000000 3.072085729567 1.906893173931 3.657776477281 2.806189946830 1.477715474321 3.225991833257 +t4 3.600389745319 1.548885265598 3.072085729567 0.000000000000 2.740287267137 3.499375448329 2.647788917879 2.873777748551 3.067590804305 +t5 0.862271408318 2.390891609015 1.906893173931 2.740287267137 0.000000000000 3.325978014851 2.474391484400 1.708585192915 2.894193370827 +t6 4.186080493033 3.149979790207 3.657776477281 3.499375448329 3.325978014851 0.000000000000 1.137306991033 3.459468496265 1.736432320205 +t7 3.334493962582 2.298393259756 2.806189946830 2.647788917879 2.474391484400 1.137306991033 0.000000000000 2.607881965814 0.884845789755 +t8 2.568687671097 2.524382090429 1.477715474321 2.873777748551 1.708585192915 3.459468496265 2.607881965814 0.000000000000 3.027683852240 +t9 3.754295849008 2.718195146183 3.225991833257 3.067590804305 2.894193370827 1.736432320205 0.884845789755 3.027683852240 0.000000000000 diff --git a/tests/fixtures/tree06.topology.tsv b/tests/fixtures/tree06.topology.tsv new file mode 100644 index 0000000..d70e425 --- /dev/null +++ b/tests/fixtures/tree06.topology.tsv @@ -0,0 +1,10 @@ + t1 t2 t3 t4 t5 t6 t7 t8 t9 +t1 0.000000000000 5.000000000000 4.000000000000 5.000000000000 2.000000000000 7.000000000000 7.000000000000 4.000000000000 6.000000000000 +t2 5.000000000000 0.000000000000 5.000000000000 2.000000000000 5.000000000000 6.000000000000 6.000000000000 5.000000000000 5.000000000000 +t3 4.000000000000 5.000000000000 0.000000000000 5.000000000000 4.000000000000 7.000000000000 7.000000000000 2.000000000000 6.000000000000 +t4 5.000000000000 2.000000000000 5.000000000000 0.000000000000 5.000000000000 6.000000000000 6.000000000000 5.000000000000 5.000000000000 +t5 2.000000000000 5.000000000000 4.000000000000 5.000000000000 0.000000000000 7.000000000000 7.000000000000 4.000000000000 6.000000000000 +t6 7.000000000000 6.000000000000 7.000000000000 6.000000000000 7.000000000000 0.000000000000 2.000000000000 7.000000000000 3.000000000000 +t7 7.000000000000 6.000000000000 7.000000000000 6.000000000000 7.000000000000 2.000000000000 0.000000000000 7.000000000000 3.000000000000 +t8 4.000000000000 5.000000000000 2.000000000000 5.000000000000 4.000000000000 7.000000000000 7.000000000000 0.000000000000 6.000000000000 +t9 6.000000000000 5.000000000000 6.000000000000 5.000000000000 6.000000000000 3.000000000000 3.000000000000 6.000000000000 0.000000000000 From 9755cab3271cafb0c35aaca600da5496ca48f42a Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:48:07 +0200 Subject: [PATCH 40/47] Split the tree logic out into a library src/main.rs held the whole tool: argument handling, the Newick parser, the LCA structure, the distance definitions and the output loop. Nothing outside the binary could reach any of it, which rules out fuzzing the parser, generating API documentation, and using distree from another Rust program. src/lib.rs is now the tree side (parser, tree, lca, midpoint, DistMode and compute_distance) and src/main.rs is the command line over the top: flags, input, warnings, formatting, output. The unit tests for the distance definitions moved with the code they test. No behaviour changes; the output is byte-identical. --- src/lib.rs | 277 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 276 +-------------------------------------------------- 2 files changed, 282 insertions(+), 271 deletions(-) create mode 100644 src/lib.rs diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..272c584 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,277 @@ +//! Distance matrices from a phylogeny. +//! +//! The tree side of `distree`: parsing Newick, flattening it into an +//! index-addressed array, answering most-recent-common-ancestor queries in +//! logarithmic time, midpoint rooting, and turning a pair of leaves into a +//! distance. The binary in `main.rs` is the command line over the top of it. + +pub mod lca; +pub mod midpoint; +pub mod parser; +#[cfg(test)] +mod testutil; +pub mod tree; + +/// Distance mode to compute. +/// +/// Selects how pairwise leaf distances are calculated from the tree. +#[derive(Clone, Copy, PartialEq)] +pub enum DistMode { + Patristic, + Topology, + Lmm, +} + +/// Compute the distance between two leaves according to `mode`. +/// +/// Returns the patristic distance, topological hop count, or LMM covariance depth. +#[inline] +pub fn compute_distance( + leaf_i: usize, + leaf_j: usize, + mode: DistMode, + lca_data: &lca::LcaData, +) -> f64 { + let m = lca_data.mrca(leaf_i, leaf_j); + match mode { + DistMode::Lmm => lca_data.depth_len[m], + DistMode::Topology => { + let d_i = lca_data.depth_top[leaf_i]; + let d_j = lca_data.depth_top[leaf_j]; + let d_m = lca_data.depth_top[m]; + ((d_i + d_j).saturating_sub(2 * d_m)) as f64 + } + DistMode::Patristic => { + let d_i = lca_data.depth_len[leaf_i]; + let d_j = lca_data.depth_len[leaf_j]; + let d_m = lca_data.depth_len[m]; + // Not clamped to zero. Rounding cannot make this negative: + // depth_len accumulates lengths from the root, so with + // non-negative branches d_i and d_j are both >= d_m, 2*d_m is + // exact, and rounding a non-negative exact result to nearest keeps + // it non-negative. The only way to get a negative value here is a + // negative branch length, which the tree is warned about, and + // rounding that up to zero would silently claim two distinct taxa + // sit on top of each other. + d_i + d_j - 2.0 * d_m + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lca::build_lca_structure; + use crate::parser::{flatten_raw, parse_newick}; + use crate::tree::Node; + use std::collections::HashSet; + + fn build_tree(newick: &str) -> (Vec, usize) { + let raw = parse_newick(newick).unwrap(); + let mut nodes = Vec::new(); + let root = flatten_raw(&raw, None, &mut nodes); + (nodes, root) + } + + fn get_leaf(nodes: &[Node], name: &str) -> usize { + nodes + .iter() + .position(|n| n.name.as_deref() == Some(name)) + .unwrap() + } + + fn patristic(nodes: &[Node], root: usize, a: &str, b: &str) -> f64 { + let lca = build_lca_structure(root, nodes); + let ai = get_leaf(nodes, a); + let bi = get_leaf(nodes, b); + compute_distance(ai, bi, DistMode::Patristic, &lca) + } + + #[test] + fn test_simple_patristic_distances() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + assert!((patristic(&nodes, root, "A", "B") - 3.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.5).abs() < 1e-10); + assert!((patristic(&nodes, root, "B", "C") - 5.5).abs() < 1e-10); + } + + #[test] + fn test_topology_distances() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let c = get_leaf(&nodes, "C"); + let d = compute_distance(a, c, DistMode::Topology, &lca); + assert_eq!(d as i64, 3); + } + + #[test] + fn test_lmm_matrix() { + let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let b = get_leaf(&nodes, "B"); + let c = get_leaf(&nodes, "C"); + // MRCA(A,B) depth = 0.5 + assert!((compute_distance(a, b, DistMode::Lmm, &lca) - 0.5).abs() < 1e-10); + // MRCA(A,C) depth = 0.0 (root) + assert!(compute_distance(a, c, DistMode::Lmm, &lca).abs() < 1e-10); + } + + #[test] + fn test_duplicate_leaves_detected() { + let (nodes, _root) = build_tree("(A:1.0,A:2.0);"); + let leaf_indices: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + let mut seen = HashSet::new(); + let has_dup = leaf_indices + .iter() + .any(|&i| !seen.insert(nodes[i].name.as_ref().unwrap().as_str())); + assert!(has_dup); + } + + #[test] + fn test_negative_branch_length_detected() { + let (nodes, _root) = build_tree("(A:-0.5,B:2.0);"); + assert!(nodes.iter().any(|n| n.length < 0.0)); + } + + #[test] + fn test_trifurcation() { + let (nodes, root) = build_tree("(A:1.0,B:2.0,C:3.0);"); + assert_eq!(nodes[root].children.len(), 3); + assert!((patristic(&nodes, root, "A", "B") - 3.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.0).abs() < 1e-10); + } + + #[test] + fn test_no_branch_lengths_all_zero() { + let (nodes, _root) = build_tree("(A,B,(C,D));"); + assert!(nodes.iter().all(|n| n.length == 0.0)); + } + + #[test] + fn test_single_leaf() { + let (nodes, _root) = build_tree("(A:1.0);"); + assert_eq!( + nodes + .iter() + .filter(|n| n.children.is_empty() && n.name.is_some()) + .count(), + 1 + ); + } + + #[test] + fn test_large_symmetric_tree() { + // ((A:1,B:1):1,(C:1,D:1):1) — all pairwise distances known + let (nodes, root) = build_tree("((A:1,B:1):1,(C:1,D:1):1);"); + assert!((patristic(&nodes, root, "A", "B") - 2.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "A", "C") - 4.0).abs() < 1e-10); + assert!((patristic(&nodes, root, "C", "D") - 2.0).abs() < 1e-10); + } + + #[test] + fn test_zero_distance_same_leaf() { + let (nodes, root) = build_tree("(A:1.0,B:2.0);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + assert!(compute_distance(a, a, DistMode::Patristic, &lca).abs() < 1e-10); + } + + #[test] + fn test_lower_triangle_row_counts() { + // In lower-triangle mode, row i has exactly i distance columns + let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); + let _lca = build_lca_structure(root, &nodes); + let mut leaf_pairs: Vec<(String, usize)> = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, n)| (n.name.clone().unwrap(), i)) + .collect(); + leaf_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + let sorted: Vec = leaf_pairs.iter().map(|(_, i)| *i).collect(); + + // Row 0 (A): 0 distance cols, Row 1 (B): 1 distance col, Row 2 (C): 2 distance cols + for (row_i, _) in sorted.iter().enumerate() { + let col_end = row_i; // lower triangle: number of distance values + assert_eq!(col_end, row_i); + } + } + + #[test] + fn test_patristic_clamped_nonnegative() { + // Self-distance must be exactly 0.0 (no negative FP artifacts) + let (nodes, root) = build_tree("(A:1.0000000000001,B:1.0000000000002);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let d = compute_distance(a, a, DistMode::Patristic, &lca); + assert_eq!(d, 0.0, "Self-distance must be exactly 0.0, got {}", d); + } + + #[test] + fn test_patristic_never_negative_on_random_trees() { + // Rounding must not produce a negative distance on any tree with + // non-negative branch lengths, and a leaf must be exactly 0 from itself + let mut rng = crate::testutil::Rng::new(0x5DEE_CE66_D000_0005); + + for _ in 0..200 { + let n_leaves = 2 + rng.below(30); + let newick = crate::testutil::random_newick(&mut rng, n_leaves); + let (nodes, root) = build_tree(&newick); + let lca = build_lca_structure(root, &nodes); + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + for &i in &leaves { + assert_eq!( + compute_distance(i, i, DistMode::Patristic, &lca), + 0.0, + "a leaf must be exactly 0.0 from itself in {}", + newick + ); + for &j in &leaves { + let d = compute_distance(i, j, DistMode::Patristic, &lca); + assert!(d >= 0.0, "negative distance {} in {}", d, newick); + } + } + } + } + + #[test] + fn test_negative_branch_length_gives_negative_distance() { + // A tree with negative branches is warned about; reporting 0.0 would + // claim two distinct taxa sit on top of each other + let (nodes, root) = build_tree("(A:-2.0,B:0.5);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let b = get_leaf(&nodes, "B"); + let d = compute_distance(a, b, DistMode::Patristic, &lca); + assert!((d - (-1.5)).abs() < 1e-12, "expected -1.5, got {}", d); + } + + #[test] + fn test_mode_conflict() { + // Just test the logic: LMM should be distinct from topology + let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); + let lca = build_lca_structure(root, &nodes); + let a = get_leaf(&nodes, "A"); + let c = get_leaf(&nodes, "C"); + let d_pat = compute_distance(a, c, DistMode::Patristic, &lca); + let d_top = compute_distance(a, c, DistMode::Topology, &lca); + let d_lmm = compute_distance(a, c, DistMode::Lmm, &lca); + // All three should give different values for this tree + assert!((d_pat - 8.0).abs() < 1e-10); // 1+3+4 + assert_eq!(d_top as i64, 3); // 2 edges from A + 1 from C + assert!(d_lmm.abs() < 1e-10); // MRCA(A,C) = root, depth 0 + } +} diff --git a/src/main.rs b/src/main.rs index 062f58c..03ef2d8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,3 @@ -mod lca; -mod midpoint; -mod parser; -#[cfg(test)] -mod testutil; -mod tree; - use clap::{Arg, ArgAction, Command}; use rayon::prelude::*; use std::collections::HashSet; @@ -12,20 +5,11 @@ use std::fs::File; use std::io::{self, BufWriter, Read, Write}; use std::process::ExitCode; -use lca::build_lca_structure; -use midpoint::midpoint_root; -use parser::{flatten_raw, parse_newick}; -use tree::Node; - -/// Distance mode to compute. -/// -/// Selects how pairwise leaf distances are calculated from the tree. -#[derive(Clone, Copy, PartialEq)] -enum DistMode { - Patristic, - Topology, - Lmm, -} +use distree::lca::build_lca_structure; +use distree::midpoint::midpoint_root; +use distree::parser::{flatten_raw, parse_newick}; +use distree::tree::Node; +use distree::{compute_distance, DistMode}; /// Largest accepted `--precision`. /// @@ -421,42 +405,6 @@ fn run() -> Result<(), Box> { Ok(()) } -/// Compute the distance between two leaves according to `mode`. -/// -/// Returns the patristic distance, topological hop count, or LMM covariance depth. -#[inline] -fn compute_distance( - leaf_i: usize, - leaf_j: usize, - mode: DistMode, - lca_data: &lca::LcaData, -) -> f64 { - let m = lca_data.mrca(leaf_i, leaf_j); - match mode { - DistMode::Lmm => lca_data.depth_len[m], - DistMode::Topology => { - let d_i = lca_data.depth_top[leaf_i]; - let d_j = lca_data.depth_top[leaf_j]; - let d_m = lca_data.depth_top[m]; - ((d_i + d_j).saturating_sub(2 * d_m)) as f64 - } - DistMode::Patristic => { - let d_i = lca_data.depth_len[leaf_i]; - let d_j = lca_data.depth_len[leaf_j]; - let d_m = lca_data.depth_len[m]; - // Not clamped to zero. Rounding cannot make this negative: - // depth_len accumulates lengths from the root, so with - // non-negative branches d_i and d_j are both >= d_m, 2*d_m is - // exact, and rounding a non-negative exact result to nearest keeps - // it non-negative. The only way to get a negative value here is a - // negative branch length, which the tree is warned about, and - // rounding that up to zero would silently claim two distinct taxa - // sit on top of each other. - d_i + d_j - 2.0 * d_m - } - } -} - /// Append a single distance value to a row buffer. /// /// Topology mode outputs integers; patristic and LMM use `precision` decimal @@ -471,217 +419,3 @@ fn format_distance(out: &mut Vec, dist: f64, mode: DistMode, precision: usiz }; result.expect("writing to a Vec cannot fail"); } - -#[cfg(test)] -mod tests { - use super::*; - - fn build_tree(newick: &str) -> (Vec, usize) { - let raw = parse_newick(newick).unwrap(); - let mut nodes = Vec::new(); - let root = flatten_raw(&raw, None, &mut nodes); - (nodes, root) - } - - fn get_leaf(nodes: &[Node], name: &str) -> usize { - nodes - .iter() - .position(|n| n.name.as_deref() == Some(name)) - .unwrap() - } - - fn patristic(nodes: &[Node], root: usize, a: &str, b: &str) -> f64 { - let lca = build_lca_structure(root, nodes); - let ai = get_leaf(nodes, a); - let bi = get_leaf(nodes, b); - compute_distance(ai, bi, DistMode::Patristic, &lca) - } - - #[test] - fn test_simple_patristic_distances() { - let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); - assert!((patristic(&nodes, root, "A", "B") - 3.0).abs() < 1e-10); - assert!((patristic(&nodes, root, "A", "C") - 4.5).abs() < 1e-10); - assert!((patristic(&nodes, root, "B", "C") - 5.5).abs() < 1e-10); - } - - #[test] - fn test_topology_distances() { - let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); - let lca = build_lca_structure(root, &nodes); - let a = get_leaf(&nodes, "A"); - let c = get_leaf(&nodes, "C"); - let d = compute_distance(a, c, DistMode::Topology, &lca); - assert_eq!(d as i64, 3); - } - - #[test] - fn test_lmm_matrix() { - let (nodes, root) = build_tree("((A:1.0,B:2.0):0.5,C:3.0);"); - let lca = build_lca_structure(root, &nodes); - let a = get_leaf(&nodes, "A"); - let b = get_leaf(&nodes, "B"); - let c = get_leaf(&nodes, "C"); - // MRCA(A,B) depth = 0.5 - assert!((compute_distance(a, b, DistMode::Lmm, &lca) - 0.5).abs() < 1e-10); - // MRCA(A,C) depth = 0.0 (root) - assert!(compute_distance(a, c, DistMode::Lmm, &lca).abs() < 1e-10); - } - - #[test] - fn test_duplicate_leaves_detected() { - let (nodes, _root) = build_tree("(A:1.0,A:2.0);"); - let leaf_indices: Vec = nodes - .iter() - .enumerate() - .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) - .map(|(i, _)| i) - .collect(); - let mut seen = HashSet::new(); - let has_dup = leaf_indices - .iter() - .any(|&i| !seen.insert(nodes[i].name.as_ref().unwrap().as_str())); - assert!(has_dup); - } - - #[test] - fn test_negative_branch_length_detected() { - let (nodes, _root) = build_tree("(A:-0.5,B:2.0);"); - assert!(nodes.iter().any(|n| n.length < 0.0)); - } - - #[test] - fn test_trifurcation() { - let (nodes, root) = build_tree("(A:1.0,B:2.0,C:3.0);"); - assert_eq!(nodes[root].children.len(), 3); - assert!((patristic(&nodes, root, "A", "B") - 3.0).abs() < 1e-10); - assert!((patristic(&nodes, root, "A", "C") - 4.0).abs() < 1e-10); - } - - #[test] - fn test_no_branch_lengths_all_zero() { - let (nodes, _root) = build_tree("(A,B,(C,D));"); - assert!(nodes.iter().all(|n| n.length == 0.0)); - } - - #[test] - fn test_single_leaf() { - let (nodes, _root) = build_tree("(A:1.0);"); - assert_eq!( - nodes - .iter() - .filter(|n| n.children.is_empty() && n.name.is_some()) - .count(), - 1 - ); - } - - #[test] - fn test_large_symmetric_tree() { - // ((A:1,B:1):1,(C:1,D:1):1) — all pairwise distances known - let (nodes, root) = build_tree("((A:1,B:1):1,(C:1,D:1):1);"); - assert!((patristic(&nodes, root, "A", "B") - 2.0).abs() < 1e-10); - assert!((patristic(&nodes, root, "A", "C") - 4.0).abs() < 1e-10); - assert!((patristic(&nodes, root, "C", "D") - 2.0).abs() < 1e-10); - } - - #[test] - fn test_zero_distance_same_leaf() { - let (nodes, root) = build_tree("(A:1.0,B:2.0);"); - let lca = build_lca_structure(root, &nodes); - let a = get_leaf(&nodes, "A"); - assert!(compute_distance(a, a, DistMode::Patristic, &lca).abs() < 1e-10); - } - - #[test] - fn test_lower_triangle_row_counts() { - // In lower-triangle mode, row i has exactly i distance columns - let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); - let _lca = build_lca_structure(root, &nodes); - let mut leaf_pairs: Vec<(String, usize)> = nodes - .iter() - .enumerate() - .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) - .map(|(i, n)| (n.name.clone().unwrap(), i)) - .collect(); - leaf_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); - let sorted: Vec = leaf_pairs.iter().map(|(_, i)| *i).collect(); - - // Row 0 (A): 0 distance cols, Row 1 (B): 1 distance col, Row 2 (C): 2 distance cols - for (row_i, _) in sorted.iter().enumerate() { - let col_end = row_i; // lower triangle: number of distance values - assert_eq!(col_end, row_i); - } - } - - #[test] - fn test_patristic_clamped_nonnegative() { - // Self-distance must be exactly 0.0 (no negative FP artifacts) - let (nodes, root) = build_tree("(A:1.0000000000001,B:1.0000000000002);"); - let lca = build_lca_structure(root, &nodes); - let a = get_leaf(&nodes, "A"); - let d = compute_distance(a, a, DistMode::Patristic, &lca); - assert_eq!(d, 0.0, "Self-distance must be exactly 0.0, got {}", d); - } - - #[test] - fn test_patristic_never_negative_on_random_trees() { - // Rounding must not produce a negative distance on any tree with - // non-negative branch lengths, and a leaf must be exactly 0 from itself - let mut rng = crate::testutil::Rng::new(0x5DEE_CE66_D000_0005); - - for _ in 0..200 { - let n_leaves = 2 + rng.below(30); - let newick = crate::testutil::random_newick(&mut rng, n_leaves); - let (nodes, root) = build_tree(&newick); - let lca = build_lca_structure(root, &nodes); - let leaves: Vec = nodes - .iter() - .enumerate() - .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) - .map(|(i, _)| i) - .collect(); - - for &i in &leaves { - assert_eq!( - compute_distance(i, i, DistMode::Patristic, &lca), - 0.0, - "a leaf must be exactly 0.0 from itself in {}", - newick - ); - for &j in &leaves { - let d = compute_distance(i, j, DistMode::Patristic, &lca); - assert!(d >= 0.0, "negative distance {} in {}", d, newick); - } - } - } - } - - #[test] - fn test_negative_branch_length_gives_negative_distance() { - // A tree with negative branches is warned about; reporting 0.0 would - // claim two distinct taxa sit on top of each other - let (nodes, root) = build_tree("(A:-2.0,B:0.5);"); - let lca = build_lca_structure(root, &nodes); - let a = get_leaf(&nodes, "A"); - let b = get_leaf(&nodes, "B"); - let d = compute_distance(a, b, DistMode::Patristic, &lca); - assert!((d - (-1.5)).abs() < 1e-12, "expected -1.5, got {}", d); - } - - #[test] - fn test_mode_conflict() { - // Just test the logic: LMM should be distinct from topology - let (nodes, root) = build_tree("((A:1,B:2):3,C:4);"); - let lca = build_lca_structure(root, &nodes); - let a = get_leaf(&nodes, "A"); - let c = get_leaf(&nodes, "C"); - let d_pat = compute_distance(a, c, DistMode::Patristic, &lca); - let d_top = compute_distance(a, c, DistMode::Topology, &lca); - let d_lmm = compute_distance(a, c, DistMode::Lmm, &lca); - // All three should give different values for this tree - assert!((d_pat - 8.0).abs() < 1e-10); // 1+3+4 - assert_eq!(d_top as i64, 3); // 2 edges from A + 1 from C - assert!(d_lmm.abs() < 1e-10); // MRCA(A,C) = root, depth 0 - } -} From 6ab96497d85aaf5aeeab7e2fd60d587e3bdb3df7 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:52:27 +0200 Subject: [PATCH 41/47] Reject branch lengths that make the arithmetic meaningless The torture test found this on its first run. 'A:1e910' parses: Rust returns infinity for an exponent past f64's range rather than an error. An infinite depth then makes every distance infinite and the diagonal NaN, since inf - inf is NaN, and the whole thing came out with exit code 0 as a matrix of inf and NaN. Two guards, because there are two ways in. The parser now refuses a branch length that is not finite, which is the case anyone will actually hit. And since a distance is d_i + d_j - 2*d_m, a depth past half of f64's range overflows that sum even when every individual length is finite, so the run also stops if the deepest leaf is far enough out for that: '((A:1e308,B:1e-308):1e300,C:0.0);' used to produce a NaN diagonal and now says to rescale the tree. Trees with large but usable lengths are untouched: 1e100 still works. --- fuzz/.gitignore | 4 + fuzz/Cargo.toml | 32 ++++ fuzz/fuzz_targets/full_pipeline.rs | 66 ++++++++ fuzz/fuzz_targets/parse_newick.rs | 28 ++++ src/main.rs | 15 ++ src/parser.rs | 16 +- tests/torture.rs | 243 +++++++++++++++++++++++++++++ 7 files changed, 402 insertions(+), 2 deletions(-) create mode 100644 fuzz/.gitignore create mode 100644 fuzz/Cargo.toml create mode 100644 fuzz/fuzz_targets/full_pipeline.rs create mode 100644 fuzz/fuzz_targets/parse_newick.rs create mode 100644 tests/torture.rs diff --git a/fuzz/.gitignore b/fuzz/.gitignore new file mode 100644 index 0000000..fe68c97 --- /dev/null +++ b/fuzz/.gitignore @@ -0,0 +1,4 @@ +target/ +corpus/ +artifacts/ +coverage/ diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 0000000..7fbc40c --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "distree-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.distree] +path = ".." + +# Detached from the parent workspace so `cargo build` at the repo root does not +# try to build a nightly-only crate. +[workspace] + +[[bin]] +name = "parse_newick" +path = "fuzz_targets/parse_newick.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "full_pipeline" +path = "fuzz_targets/full_pipeline.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/full_pipeline.rs b/fuzz/fuzz_targets/full_pipeline.rs new file mode 100644 index 0000000..a66b8c3 --- /dev/null +++ b/fuzz/fuzz_targets/full_pipeline.rs @@ -0,0 +1,66 @@ +#![no_main] + +//! Run whatever parses all the way through to a distance. +//! +//! The parser rejecting bad input is only half of it. A tree that parses can +//! still be a shape nothing downstream expects, and flattening, midpoint +//! rooting and the binary-lifting queries all index into flat arrays. This +//! target takes every input the parser accepts and pushes it through the rest +//! of the pipeline, in every mode, with and without rooting. + +use libfuzzer_sys::fuzz_target; + +use distree::lca::build_lca_structure; +use distree::midpoint::midpoint_root; +use distree::parser::{flatten_raw, parse_newick}; +use distree::tree::Node; +use distree::{compute_distance, DistMode}; + +fuzz_target!(|data: &[u8]| { + let Ok(text) = std::str::from_utf8(data) else { + return; + }; + if text.len() > 16 * 1024 { + return; + } + let Ok(raw) = parse_newick(text) else { + return; + }; + + for root_it in [false, true] { + let mut nodes: Vec = Vec::new(); + let mut root = flatten_raw(&raw, None, &mut nodes); + + // Midpoint rooting is undefined with negative branch lengths, and the + // binary warns rather than refusing, so keep those out of this arm. + if root_it { + if nodes.iter().any(|n| n.length < 0.0 || !n.length.is_finite()) { + continue; + } + root = midpoint_root(root, &mut nodes); + } + + let lca = build_lca_structure(root, &nodes); + + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + // The matrix is quadratic; a few hundred leaves is plenty to exercise + // the query paths without starving the fuzzer of executions. + let sample = leaves.len().min(24); + for &i in leaves.iter().take(sample) { + for &j in leaves.iter().take(sample) { + for mode in [DistMode::Patristic, DistMode::Topology, DistMode::Lmm] { + let d = compute_distance(i, j, mode, &lca); + if i == j && mode != DistMode::Lmm { + assert_eq!(d, 0.0, "a leaf must be exactly 0 from itself"); + } + } + } + } + } +}); diff --git a/fuzz/fuzz_targets/parse_newick.rs b/fuzz/fuzz_targets/parse_newick.rs new file mode 100644 index 0000000..2625b8b --- /dev/null +++ b/fuzz/fuzz_targets/parse_newick.rs @@ -0,0 +1,28 @@ +#![no_main] + +//! Throw arbitrary text at the Newick parser. +//! +//! The parser is hand-written, scans bytes, and is deliberately strict, which +//! is exactly the combination that hides an index that runs off the end or a +//! loop that fails to advance. Any input at all must come back as `Ok` or +//! `Err`, never a panic and never a hang. + +use libfuzzer_sys::fuzz_target; + +use distree::parser::parse_newick; + +fuzz_target!(|data: &[u8]| { + // The parser takes &str; anything that is not UTF-8 is rejected before it + // ever gets there, so there is nothing to learn from feeding it here. + let Ok(text) = std::str::from_utf8(data) else { + return; + }; + + // Very long inputs only slow the fuzzer down; the interesting behaviour is + // in the structure, not the size. + if text.len() > 64 * 1024 { + return; + } + + let _ = parse_newick(text); +}); diff --git a/src/main.rs b/src/main.rs index 03ef2d8..cb8df19 100644 --- a/src/main.rs +++ b/src/main.rs @@ -221,6 +221,21 @@ fn run() -> Result<(), Box> { // Build LCA let lca_data = build_lca_structure(root_idx, &nodes); + // A distance is d_i + d_j - 2*d_m, so a root-to-tip depth past half of + // f64's range overflows that sum to infinity, and inf - inf is NaN. The + // matrix comes out as inf with NaN down the diagonal, which is not a + // failure any downstream tool would notice. + let deepest = lca_data.depth_len.iter().copied().fold(0.0_f64, f64::max); + if !(deepest * 2.0).is_finite() { + return Err(format!( + "Branch lengths are too large to compute distances with: the deepest leaf is {:e} \ + from the root, and summing two such depths overflows a 64-bit float. Rescale the \ + tree.", + deepest + ) + .into()); + } + // Collect leaves let leaf_indices: Vec = nodes .iter() diff --git a/src/parser.rs b/src/parser.rs index cdc4b54..d0968f1 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -355,9 +355,21 @@ fn parse_length_bytes(bytes: &[u8], pos: &mut usize) -> Result { } let numstr = std::str::from_utf8(&bytes[start..*pos]) .map_err(|_| "Invalid UTF-8 in branch length".to_string())?; - numstr + let value = numstr .parse::() - .map_err(|e: ParseFloatError| format!("Failed to parse branch length '{}': {}", numstr, e)) + .map_err(|e: ParseFloatError| { + format!("Failed to parse branch length '{}': {}", numstr, e) + })?; + // Rust parses an exponent past f64's range as infinity rather than as an + // error, and an infinite branch length turns the whole matrix into inf with + // NaN down the diagonal, since inf - inf is NaN. + if !value.is_finite() { + return Err(format!( + "Branch length '{}' at position {} is not a finite number.", + numstr, start + )); + } + Ok(value) } /// Flatten a `RawNode` tree into a flat `Vec` iteratively (no stack overflow). diff --git a/tests/torture.rs b/tests/torture.rs new file mode 100644 index 0000000..1671c2b --- /dev/null +++ b/tests/torture.rs @@ -0,0 +1,243 @@ +//! Push malformed and mutated input through the parser and the pipeline. +//! +//! This is the fuzzing that runs on every push. It is not coverage-guided, so +//! it will not find what `cargo fuzz` finds given hours, but it is +//! deterministic, needs no nightly toolchain, and covers the mutations that +//! actually happen to a Newick file: truncation partway through a download, a +//! byte flipped in transfer, a chunk duplicated, a delimiter dropped. +//! +//! The contract under test is narrow and absolute: **any** input at all comes +//! back as `Ok` or `Err`. Never a panic, never an out-of-bounds index, never a +//! loop that does not advance. Whatever parses then has to survive flattening, +//! rooting and every distance mode. +//! +//! For the deeper, coverage-guided version: +//! +//! ```text +//! cargo +nightly fuzz run parse_newick +//! cargo +nightly fuzz run full_pipeline +//! ``` + +use distree::lca::build_lca_structure; +use distree::midpoint::midpoint_root; +use distree::parser::{flatten_raw, parse_newick}; +use distree::tree::Node; +use distree::{compute_distance, DistMode}; + +/// xorshift64, so a failure is reproducible from the seed in the message. +struct Rng(u64); + +impl Rng { + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, n: usize) -> usize { + if n == 0 { + return 0; + } + (self.next_u64() % n as u64) as usize + } +} + +/// The bytes that mean something in Newick, plus a few that do not, so the +/// mutations land on structure rather than on label text. +const ALPHABET: &[u8] = b"(),:;[]'\"0123456789.eE+- \t\nABxy"; + +fn seed_trees() -> Vec { + vec![ + "(A:1.0,B:2.0);".to_string(), + "((A:1.0,B:2.0):0.5,C:3.0);".to_string(), + "(A:1,B:2,C:3);".to_string(), + "[&R] ((A:0.1[&&NHX:S=human],B:0.2):0.3,C:0.4);".to_string(), + "('Taxon A':1.0,\"Taxon B\":2.0)'my clade':0.5;".to_string(), + "(A:1.5e-3,B:2.0E+1,(C:0.0,D:-0.5):1e10);".to_string(), + "((((A:1):1):1):1);".to_string(), + "(it''s:1,B:2);".to_string(), + "(A,B,(C,D));".to_string(), + ";".to_string(), + ] +} + +/// Apply one random mutation to `bytes`. +fn mutate(rng: &mut Rng, bytes: &mut Vec) { + if bytes.is_empty() { + bytes.push(ALPHABET[rng.below(ALPHABET.len())]); + return; + } + match rng.below(6) { + // Truncate: an interrupted write or a partial download + 0 => { + let at = rng.below(bytes.len()); + bytes.truncate(at); + } + // Flip one byte to another meaningful one + 1 => { + let at = rng.below(bytes.len()); + bytes[at] = ALPHABET[rng.below(ALPHABET.len())]; + } + // Insert + 2 => { + let at = rng.below(bytes.len() + 1); + bytes.insert(at, ALPHABET[rng.below(ALPHABET.len())]); + } + // Delete + 3 => { + let at = rng.below(bytes.len()); + bytes.remove(at); + } + // Duplicate a run + 4 => { + let start = rng.below(bytes.len()); + let end = (start + 1 + rng.below(bytes.len() - start)).min(bytes.len()); + let chunk = bytes[start..end].to_vec(); + let at = rng.below(bytes.len() + 1); + for (k, b) in chunk.into_iter().enumerate() { + if bytes.len() < 4096 { + bytes.insert(at + k, b); + } + } + } + // Swap two bytes + _ => { + let a = rng.below(bytes.len()); + let b = rng.below(bytes.len()); + bytes.swap(a, b); + } + } +} + +/// Everything the binary does after a successful parse. +fn exercise_pipeline(text: &str) { + let Ok(raw) = parse_newick(text) else { + return; + }; + + for do_midpoint in [false, true] { + let mut nodes: Vec = Vec::new(); + let mut root = flatten_raw(&raw, None, &mut nodes); + + if do_midpoint { + // Midpoint rooting assumes non-negative, finite lengths; the binary + // warns about the rest rather than refusing. + if nodes.iter().any(|n| n.length < 0.0 || !n.length.is_finite()) { + continue; + } + root = midpoint_root(root, &mut nodes); + } + + let lca = build_lca_structure(root, &nodes); + + // The binary refuses a tree whose depths are too large to subtract, + // because d_i + d_j overflows and inf - inf is NaN. Hold the pipeline + // to the same precondition rather than asserting past it. + let deepest = lca.depth_len.iter().copied().fold(0.0_f64, f64::max); + if !(deepest * 2.0).is_finite() { + continue; + } + + let leaves: Vec = nodes + .iter() + .enumerate() + .filter(|(_, n)| n.children.is_empty() && n.name.is_some()) + .map(|(i, _)| i) + .collect(); + + for &i in leaves.iter().take(16) { + for &j in leaves.iter().take(16) { + for mode in [DistMode::Patristic, DistMode::Topology, DistMode::Lmm] { + let d = compute_distance(i, j, mode, &lca); + if i == j && mode != DistMode::Lmm { + assert_eq!(d, 0.0, "a leaf must be exactly 0.0 from itself"); + } + } + } + } + } +} + +#[test] +fn test_mutated_trees_never_panic() { + let seeds = seed_trees(); + + for (s, seed_tree) in seeds.iter().enumerate() { + let mut rng = Rng(0x243F_6A88_85A3_08D3 ^ (s as u64) << 32); + + for round in 0..4_000 { + let mut bytes = seed_tree.as_bytes().to_vec(); + // One to four mutations: one finds the shallow cases, four gets far + // enough from a valid tree to be interesting. + for _ in 0..=rng.below(4) { + mutate(&mut rng, &mut bytes); + } + // Every byte in ALPHABET is ASCII, so the result is still UTF-8 + let text = match std::str::from_utf8(&bytes) { + Ok(t) => t, + Err(_) => continue, + }; + + // A panic here fails the test with the seed and round that caused it + let result = std::panic::catch_unwind(|| { + let _ = parse_newick(text); + exercise_pipeline(text); + }); + assert!( + result.is_ok(), + "panicked on seed {} round {} with input {:?}", + s, + round, + text + ); + } + } +} + +#[test] +fn test_random_bytes_never_panic() { + let mut rng = Rng(0x13198A2E_03707344); + + for round in 0..4_000 { + let len = rng.below(64); + let bytes: Vec = (0..len).map(|_| ALPHABET[rng.below(ALPHABET.len())]).collect(); + let text = std::str::from_utf8(&bytes).expect("ALPHABET is ASCII"); + + let result = std::panic::catch_unwind(|| { + let _ = parse_newick(text); + exercise_pipeline(text); + }); + assert!(result.is_ok(), "panicked on round {} with input {:?}", round, text); + } +} + +#[test] +fn test_pathological_shapes_never_panic() { + // Shapes that are valid enough to reach the pipeline but are not what any + // of it was written with in mind. + let cases: Vec = vec![ + // A ladder, which makes the LCA depth equal to the node count + format!("{}A:1{};", "(".repeat(2_000), ")".repeat(2_000)), + // One node with thousands of children + format!("({});", (0..2_000).map(|i| format!("L{}:0.1", i)).collect::>().join(",")), + // Every branch zero, so the diameter is zero and midpoint has nothing to find + "((A:0,B:0):0,(C:0,D:0):0);".to_string(), + // Enormous and tiny lengths in the same tree + "((A:1e308,B:1e-308):1e300,C:0.0);".to_string(), + // A single leaf, and a single leaf with no length + "(A:1.0);".to_string(), + "A;".to_string(), + // Internal nodes carrying the labels instead of the tips + "((:1,:2)inner:3,:4)root;".to_string(), + // Deeply nested comments + format!("(A:1[{}],B:2);", "[".repeat(500) + &"]".repeat(500)), + ]; + + for (i, case) in cases.iter().enumerate() { + let result = std::panic::catch_unwind(|| exercise_pipeline(case)); + assert!(result.is_ok(), "panicked on pathological case {}", i); + } +} From b883c6eabbf140a1a495fd4b575dbeb7c2ebd612 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:57:36 +0200 Subject: [PATCH 42/47] Fuzz the parser and the pipeline Two ways in, because they catch different things. tests/torture.rs runs on every push and needs no extra toolchain. It takes valid trees and truncates, flips, duplicates and swaps bytes in them, the way a real Newick file gets damaged, then asserts only that nothing panics and nothing hangs. It found the infinite-branch-length bug on its first run. fuzz/ is the coverage-guided version, for when there is time to leave it going. Two targets: the parser alone, and everything that parses pushed through flattening, rooting and all three distance modes. 12.4M runs on the first and 5.1M on the second, no crashes, starting from the seeds in fuzz/seeds/. --- docs/about/contributing.md | 46 ++++++ fuzz/Cargo.lock | 325 +++++++++++++++++++++++++++++++++++++ fuzz/seeds/01.nwk | 1 + fuzz/seeds/02.nwk | 1 + fuzz/seeds/03.nwk | 1 + fuzz/seeds/04.nwk | 1 + fuzz/seeds/05.nwk | 1 + fuzz/seeds/06.nwk | 1 + fuzz/seeds/07.nwk | 1 + fuzz/seeds/08.nwk | 1 + fuzz/seeds/09.nwk | 1 + fuzz/seeds/10.nwk | 1 + 12 files changed, 381 insertions(+) create mode 100644 fuzz/Cargo.lock create mode 100644 fuzz/seeds/01.nwk create mode 100644 fuzz/seeds/02.nwk create mode 100644 fuzz/seeds/03.nwk create mode 100644 fuzz/seeds/04.nwk create mode 100644 fuzz/seeds/05.nwk create mode 100644 fuzz/seeds/06.nwk create mode 100644 fuzz/seeds/07.nwk create mode 100644 fuzz/seeds/08.nwk create mode 100644 fuzz/seeds/09.nwk create mode 100644 fuzz/seeds/10.nwk diff --git a/docs/about/contributing.md b/docs/about/contributing.md index 5997a29..aec7ecd 100644 --- a/docs/about/contributing.md +++ b/docs/about/contributing.md @@ -49,6 +49,52 @@ The suite is in three places: binary-lifting MRCA agrees with walking up from both nodes, over every pair of nodes; and a patristic distance is never negative on a tree with non-negative branch lengths. +- **Cross-validation against ape** in `tests/crossvalidation.rs`, checked + against reference matrices committed under `tests/fixtures/`. +- **Mutation testing** in `tests/torture.rs`, which throws truncated, flipped + and duplicated Newick at the parser and the pipeline. + +### Cross-validating against ape + +Everything above is written against the same understanding of the problem as +the code, so none of it would catch a distance being *defined* wrongly. ape is +the independent check. With R, `ape` and `phangorn` installed: + +```bash +cargo build --release +Rscript scripts/crossvalidate.R 250 target/release/distree +``` + +It compares all four modes against `cophenetic.phylo`, `vcv.phylo`, +`cophenetic.phylo` over unit branch lengths, and `phangorn::midpoint`. The +expected result is agreement to under 1e-9, which is the 12-decimal text +round-trip, and exactly zero for edge counts. + +To refresh the committed fixtures after a deliberate change: + +```bash +Rscript scripts/crossvalidate.R 6 target/release/distree --write-fixtures +``` + +The same comparison runs as a weekly GitHub Actions job, and on demand from the +Actions tab. + +### Fuzzing + +`tests/torture.rs` runs on every push and needs nothing extra. The +coverage-guided version needs a nightly toolchain: + +```bash +cargo +nightly install cargo-fuzz +cargo +nightly fuzz run parse_newick fuzz/seeds +cargo +nightly fuzz run full_pipeline fuzz/seeds +``` + +`parse_newick` throws arbitrary text at the parser; `full_pipeline` takes +everything that parses and pushes it through flattening, rooting and every +distance mode. Both assert only that nothing panics and nothing hangs, which is +a low bar and has already caught a real bug: a branch length of `1e910` parsed +as infinity and turned the matrix into `inf` with `NaN` down the diagonal. A parser change wants a case in `src/parser.rs` for what should now parse **and** one for what should still be rejected. The failures worth guarding are the ones diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock new file mode 100644 index 0000000..da9473a --- /dev/null +++ b/fuzz/Cargo.lock @@ -0,0 +1,325 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "distree" +version = "1.0.1" +dependencies = [ + "clap", + "rayon", +] + +[[package]] +name = "distree-fuzz" +version = "0.0.0" +dependencies = [ + "distree", + "libfuzzer-sys", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/fuzz/seeds/01.nwk b/fuzz/seeds/01.nwk new file mode 100644 index 0000000..ddcd782 --- /dev/null +++ b/fuzz/seeds/01.nwk @@ -0,0 +1 @@ +(A:1.0,B:2.0); \ No newline at end of file diff --git a/fuzz/seeds/02.nwk b/fuzz/seeds/02.nwk new file mode 100644 index 0000000..5e03591 --- /dev/null +++ b/fuzz/seeds/02.nwk @@ -0,0 +1 @@ +((A:1.0,B:2.0):0.5,C:3.0); \ No newline at end of file diff --git a/fuzz/seeds/03.nwk b/fuzz/seeds/03.nwk new file mode 100644 index 0000000..83128a8 --- /dev/null +++ b/fuzz/seeds/03.nwk @@ -0,0 +1 @@ +(A:1,B:2,C:3); \ No newline at end of file diff --git a/fuzz/seeds/04.nwk b/fuzz/seeds/04.nwk new file mode 100644 index 0000000..c782586 --- /dev/null +++ b/fuzz/seeds/04.nwk @@ -0,0 +1 @@ +[&R] ((A:0.1[&&NHX:S=human],B:0.2):0.3,C:0.4); \ No newline at end of file diff --git a/fuzz/seeds/05.nwk b/fuzz/seeds/05.nwk new file mode 100644 index 0000000..e79adbe --- /dev/null +++ b/fuzz/seeds/05.nwk @@ -0,0 +1 @@ +('Taxon A':1.0,"Taxon B":2.0)'my clade':0.5; \ No newline at end of file diff --git a/fuzz/seeds/06.nwk b/fuzz/seeds/06.nwk new file mode 100644 index 0000000..b739e14 --- /dev/null +++ b/fuzz/seeds/06.nwk @@ -0,0 +1 @@ +(A:1.5e-3,B:2.0E+1,(C:0.0,D:-0.5):1e10); \ No newline at end of file diff --git a/fuzz/seeds/07.nwk b/fuzz/seeds/07.nwk new file mode 100644 index 0000000..4cd4be6 --- /dev/null +++ b/fuzz/seeds/07.nwk @@ -0,0 +1 @@ +((((A:1):1):1):1); \ No newline at end of file diff --git a/fuzz/seeds/08.nwk b/fuzz/seeds/08.nwk new file mode 100644 index 0000000..444840b --- /dev/null +++ b/fuzz/seeds/08.nwk @@ -0,0 +1 @@ +('it''s':1,B:2); \ No newline at end of file diff --git a/fuzz/seeds/09.nwk b/fuzz/seeds/09.nwk new file mode 100644 index 0000000..0fb07c9 --- /dev/null +++ b/fuzz/seeds/09.nwk @@ -0,0 +1 @@ +(A,B,(C,D)); \ No newline at end of file diff --git a/fuzz/seeds/10.nwk b/fuzz/seeds/10.nwk new file mode 100644 index 0000000..2ca2e8a --- /dev/null +++ b/fuzz/seeds/10.nwk @@ -0,0 +1 @@ +(A:1.0,B:2.0) \ No newline at end of file From 1bd7e8020f888a98f14cd19620233d6a37038788 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:01:55 +0200 Subject: [PATCH 43/47] Read gzip directly, and add --taxa and --stats Three things people kept having to work around. Large trees arrive compressed, and distree made you pipe them through gunzip. It now decompresses them itself, detected by magic bytes rather than by extension so it works from stdin too. --taxa FILE restricts the matrix to the labels listed in a file, one per line, blanks and # comments skipped. It filters the output rather than pruning the tree, which is the point: the path between two leaves does not depend on which other leaves are in the matrix, so a subset carries the same distances as the full run. A label that is not in the tree is an error naming it, since a quietly smaller matrix is the kind of thing nobody notices until much later. --stats prints a summary to stderr: leaves, nodes, mode, cells, and the minimum, maximum and mean off the diagonal. The workers accumulate it as they go, so it costs one pass and nothing measurable. --- Cargo.lock | 42 ++++++++ Cargo.toml | 2 + src/main.rs | 222 ++++++++++++++++++++++++++++++++++++++----- tests/integration.rs | 104 +++++++++++++++++++- 4 files changed, 343 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 868efb3..b7723e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "anstream" version = "1.0.0" @@ -116,6 +122,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -146,6 +161,7 @@ name = "distree" version = "1.0.1" dependencies = [ "clap", + "flate2", "rayon", "tempfile", ] @@ -178,6 +194,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -278,6 +304,16 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -405,6 +441,12 @@ dependencies = [ "zmij", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "strsim" version = "0.11.1" diff --git a/Cargo.toml b/Cargo.toml index 3c1020b..d3abc4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,9 @@ edition = "2021" [dependencies] clap = { version = "4.1", features = ["derive"] } +flate2 = "1.1.9" rayon = "1.5" [dev-dependencies] +flate2 = "1.1.9" tempfile = "3" diff --git a/src/main.rs b/src/main.rs index cb8df19..66b9169 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,11 @@ use clap::{Arg, ArgAction, Command}; +use flate2::read::GzDecoder; use rayon::prelude::*; use std::collections::HashSet; use std::fs::File; use std::io::{self, BufWriter, Read, Write}; use std::process::ExitCode; +use std::time::Instant; use distree::lca::build_lca_structure; use distree::midpoint::midpoint_root; @@ -44,6 +46,7 @@ fn main() -> ExitCode { } fn run() -> Result<(), Box> { + let started = Instant::now(); let matches = Command::new("distree") .version(env!("CARGO_PKG_VERSION")) .author("Paula Ruiz-Rodriguez") @@ -100,6 +103,18 @@ fn run() -> Result<(), Box> { .help("Output PHYLIP lower-triangle format (taxa count header, row labels, no diagonal)") .action(ArgAction::SetTrue), ) + .arg( + Arg::new("taxa") + .long("taxa") + .help("Restrict the matrix to the leaf labels listed in FILE, one per line") + .value_name("FILE"), + ) + .arg( + Arg::new("stats") + .long("stats") + .help("Print a summary of the run to stderr") + .action(ArgAction::SetTrue), + ) .get_matches(); let tree_path = matches @@ -158,17 +173,15 @@ fn run() -> Result<(), Box> { .read_to_end(&mut raw_input)?; } - // Large trees are usually shipped compressed, and a gzip file reaching the - // UTF-8 check produced "stream did not contain valid UTF-8", which says - // nothing about what to do next. + // Large trees are usually shipped compressed, so read gzip directly rather + // than making the caller pipe it through gunzip. Detected by magic bytes + // rather than by extension, so it works from stdin too. if raw_input.starts_with(&[0x1f, 0x8b]) { - return Err(format!( - "{} is gzip-compressed. distree reads plain text; decompress it on the way in:\n\ - \x20 gunzip -c {} | distree -", - source, - if tree_path == "-" { "FILE.nwk.gz" } else { tree_path.as_str() } - ) - .into()); + let mut decoded = Vec::new(); + GzDecoder::new(&raw_input[..]) + .read_to_end(&mut decoded) + .map_err(|e| format!("{} is gzip-compressed but could not be read: {}", source, e))?; + raw_input = decoded; } let newick_str = String::from_utf8(raw_input).map_err(|_| { @@ -309,6 +322,42 @@ fn run() -> Result<(), Box> { .iter() .map(|&i| (nodes[i].name.clone().expect("leaf has name"), i)) .collect(); + + // Restrict to the requested subset, if any. The distances themselves are + // unaffected: the path between two leaves does not depend on which other + // leaves are in the matrix, so this filters the output rather than pruning + // the tree. + if let Some(path) = matches.get_one::("taxa") { + let wanted = read_taxa_list(path)?; + let present: HashSet<&str> = leaf_label_pairs.iter().map(|(l, _)| l.as_str()).collect(); + + let missing: Vec<&str> = wanted + .iter() + .filter(|w| !present.contains(w.as_str())) + .map(String::as_str) + .collect(); + if !missing.is_empty() { + // Silently dropping these would give a matrix smaller than asked + // for, which is the kind of thing nobody notices until much later. + let shown = missing.iter().take(5).cloned().collect::>().join(", "); + return Err(format!( + "{} of the {} labels in '{}' are not leaves of the tree: {}{}", + missing.len(), + wanted.len(), + path, + shown, + if missing.len() > 5 { ", ..." } else { "" } + ) + .into()); + } + + let keep: HashSet<&str> = wanted.iter().map(String::as_str).collect(); + leaf_label_pairs.retain(|(label, _)| keep.contains(label.as_str())); + if leaf_label_pairs.is_empty() { + return Err(format!("'{}' selected no leaves.", path).into()); + } + } + leaf_label_pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0)); let sorted_labels: Vec<&str> = leaf_label_pairs @@ -378,33 +427,78 @@ fn run() -> Result<(), Box> { // leaving it in the serial write loop capped the whole run: at six decimals // it was about two thirds of the work no number of cores could touch. let batch_rows = (BATCH_CELLS / n_leaves).clamp(1, n_leaves); - let mut batch: Vec> = vec![Vec::new(); batch_rows]; + let mut batch: Vec<(Vec, Stats)> = vec![(Vec::new(), Stats::default()); batch_rows]; let row_width = |row_i: usize| if do_lower { row_i } else { n_leaves }; + let mut totals = Stats::default(); let mut first_row = 0; while first_row < n_leaves { let rows = (n_leaves - first_row).min(batch_rows); - batch[..rows].par_iter_mut().enumerate().for_each(|(k, out)| { - let row_i = first_row + k; - let leaf_i = sorted_leaf_indices[row_i]; - out.clear(); - out.extend_from_slice(sorted_labels[row_i].as_bytes()); - for &leaf_j in &sorted_leaf_indices[..row_width(row_i)] { - let dist = compute_distance(leaf_i, leaf_j, mode, &lca_data); - out.push(b'\t'); - format_distance(out, dist, mode, precision); - } - out.push(b'\n'); - }); + batch[..rows] + .par_iter_mut() + .enumerate() + .for_each(|(k, (out, stats))| { + let row_i = first_row + k; + let leaf_i = sorted_leaf_indices[row_i]; + out.clear(); + out.extend_from_slice(sorted_labels[row_i].as_bytes()); + *stats = Stats::default(); + for (col, &leaf_j) in sorted_leaf_indices[..row_width(row_i)].iter().enumerate() { + let dist = compute_distance(leaf_i, leaf_j, mode, &lca_data); + out.push(b'\t'); + format_distance(out, dist, mode, precision); + // The diagonal is zero by construction in the distance + // modes and the leaf's own depth under --lmm, so it would + // only drag the summary around. + if col != row_i { + stats.add(dist); + } + } + out.push(b'\n'); + }); - for row in &batch[..rows] { + for (row, stats) in &batch[..rows] { writer.write_all(row)?; + totals.merge(stats); } first_row += rows; } + if *matches.get_one::("stats").unwrap_or(&false) { + let unit = match mode { + DistMode::Patristic => "patristic distance", + DistMode::Topology => "edge count", + DistMode::Lmm => "root-to-MRCA depth", + }; + eprintln!("--- Statistics ---"); + eprintln!("Leaves in matrix: {}", n_leaves); + if n_leaves != leaf_indices.len() { + eprintln!("Leaves in tree: {}", leaf_indices.len()); + } + eprintln!("Nodes in tree: {}", nodes.len()); + eprintln!("Mode: {}", unit); + if do_midpoint && mode != DistMode::Topology { + eprintln!("Rooting: midpoint"); + } + eprintln!("Cells written: {}", totals.count + n_leaves as u64 * u64::from(!do_lower)); + if totals.count > 0 { + // Off the diagonal, which is zero by construction in the distance + // modes and would only pull the summary towards it. + if mode == DistMode::Topology { + eprintln!("Minimum: {}", totals.min as i64); + eprintln!("Maximum: {}", totals.max as i64); + eprintln!("Mean: {:.3}", totals.mean()); + } else { + eprintln!("Minimum: {:.*}", precision, totals.min); + eprintln!("Maximum: {:.*}", precision, totals.max); + eprintln!("Mean: {:.*}", precision, totals.mean()); + } + } + eprintln!("Time: {:.3}s", started.elapsed().as_secs_f64()); + } + // BufWriter flushes on drop but discards whatever error it hits, so a full // disk or a failing filesystem produced a truncated matrix and exit code 0. if let Err(e) = writer.flush() { @@ -420,6 +514,86 @@ fn run() -> Result<(), Box> { Ok(()) } +/// Running summary of the off-diagonal cells, for `--stats`. +/// +/// Accumulated per row by the worker that computed it, then merged in the +/// writing loop, so the numbers cost one pass and no second traversal. +#[derive(Clone, Copy)] +struct Stats { + min: f64, + max: f64, + sum: f64, + count: u64, +} + +impl Default for Stats { + fn default() -> Self { + Stats { min: f64::INFINITY, max: f64::NEG_INFINITY, sum: 0.0, count: 0 } + } +} + +impl Stats { + #[inline] + fn add(&mut self, value: f64) { + if value < self.min { + self.min = value; + } + if value > self.max { + self.max = value; + } + self.sum += value; + self.count += 1; + } + + fn merge(&mut self, other: &Stats) { + if other.count == 0 { + return; + } + if other.min < self.min { + self.min = other.min; + } + if other.max > self.max { + self.max = other.max; + } + self.sum += other.sum; + self.count += other.count; + } + + fn mean(&self) -> f64 { + if self.count == 0 { + f64::NAN + } else { + self.sum / self.count as f64 + } + } +} + +/// Read the leaf labels for `--taxa`: one per line, blanks and `#` comments +/// skipped, duplicates collapsed. +fn read_taxa_list(path: &str) -> Result, Box> { + let text = std::fs::read_to_string(path) + .map_err(|e| format!("Cannot read the taxa list '{}': {}", path, e))?; + + let mut seen = HashSet::new(); + let mut wanted = Vec::new(); + for line in text.lines() { + // Trailing \r survives a file written on Windows and would stop every + // label matching. + let label = line.trim_end_matches(['\r', '\n']).trim(); + if label.is_empty() || label.starts_with('#') { + continue; + } + if seen.insert(label.to_string()) { + wanted.push(label.to_string()); + } + } + + if wanted.is_empty() { + return Err(format!("The taxa list '{}' is empty.", path).into()); + } + Ok(wanted) +} + /// Append a single distance value to a row buffer. /// /// Topology mode outputs integers; patristic and LMM use `precision` decimal diff --git a/tests/integration.rs b/tests/integration.rs index 692bcea..7863aca 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -199,17 +199,115 @@ fn test_binary_warns_lower_drops_lmm_diagonal() { assert!(!stderr.contains("diagonal"), "stderr: {}", stderr); } +fn gzip(bytes: &[u8]) -> Vec { + use flate2::write::GzEncoder; + use flate2::Compression; + let mut enc = GzEncoder::new(Vec::new(), Compression::default()); + enc.write_all(bytes).unwrap(); + enc.finish().unwrap() +} + +#[test] +fn test_binary_reads_gzip_input() { + let dir = tempfile::tempdir().unwrap(); + let plain = dir.path().join("t.nwk"); + let packed = dir.path().join("t.nwk.gz"); + let newick = "((A:1.0,B:2.0):0.5,C:3.0);"; + std::fs::write(&plain, newick).unwrap(); + std::fs::write(&packed, gzip(newick.as_bytes())).unwrap(); + + let (code, from_plain, _) = run(&[plain.to_str().unwrap()], None); + assert_eq!(code, 0); + let (code, from_gz, _) = run(&[packed.to_str().unwrap()], None); + assert_eq!(code, 0); + assert_eq!(from_plain, from_gz, "gzip input must give the same matrix"); +} + #[test] -fn test_binary_names_gzip_input() { +fn test_binary_reads_gzip_from_stdin() { + // Detection is by magic bytes, so it works with no filename to go on + let packed = gzip(b"(A:1.0,B:3.0);"); + let mut cmd = Command::new(bin()); + cmd.arg("-").stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = cmd.spawn().unwrap(); + child.stdin.take().unwrap().write_all(&packed).unwrap(); + let out = child.wait_with_output().unwrap(); + + assert_eq!(out.status.code(), Some(0)); + let stdout = String::from_utf8_lossy(&out.stdout); + assert_eq!(stdout.lines().count(), 3, "header + 2 rows: {}", stdout); +} + +#[test] +fn test_binary_reports_corrupt_gzip() { let dir = tempfile::tempdir().unwrap(); let tree = dir.path().join("t.nwk.gz"); - // Gzip magic bytes are enough; the file never gets decompressed std::fs::write(&tree, [0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00]).unwrap(); let (code, _stdout, stderr) = run(&[tree.to_str().unwrap()], None); assert_ne!(code, 0); assert!(stderr.contains("gzip"), "stderr should name gzip: {}", stderr); - assert!(stderr.contains("gunzip -c"), "and say what to do: {}", stderr); +} + +#[test] +fn test_binary_taxa_subset() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1.0,B:2.0):0.5,(C:3.0,D:4.0):0.5);").unwrap(); + let taxa = dir.path().join("keep.txt"); + // Out of order, with a blank line, a comment and a duplicate + std::fs::write(&taxa, "D\n\n# keep these\nA\nA\n").unwrap(); + + let (code, subset, _) = run(&["--taxa", taxa.to_str().unwrap(), tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let lines: Vec<&str> = subset.lines().collect(); + assert_eq!(lines.len(), 3, "header + 2 rows: {:?}", lines); + assert_eq!(lines[0], "\tA\tD"); + + // The distance must be the one from the full tree: a subset filters the + // output, it does not prune the tree + let (_, full, _) = run(&[tree.to_str().unwrap()], None); + let full_ad = full + .lines() + .find(|l| l.starts_with("A\t")) + .unwrap() + .split('\t') + .nth(4) + .unwrap() + .to_string(); + let subset_ad = lines[1].split('\t').nth(2).unwrap(); + assert_eq!(subset_ad, full_ad, "A-D differs between subset and full matrix"); +} + +#[test] +fn test_binary_taxa_rejects_unknown_labels() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "(A:1.0,B:2.0);").unwrap(); + let taxa = dir.path().join("keep.txt"); + std::fs::write(&taxa, "A\nNOT_IN_TREE\n").unwrap(); + + let (code, _stdout, stderr) = run(&["--taxa", taxa.to_str().unwrap(), tree.to_str().unwrap()], None); + assert_ne!(code, 0, "an unknown label should be an error, not a smaller matrix"); + assert!(stderr.contains("NOT_IN_TREE"), "stderr should name it: {}", stderr); +} + +#[test] +fn test_binary_stats_goes_to_stderr() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("t.nwk"); + std::fs::write(&tree, "((A:1.0,B:2.0):0.5,C:3.0);").unwrap(); + + let (code, with_stats, stderr) = run(&["--stats", "-p", "3", tree.to_str().unwrap()], None); + assert_eq!(code, 0); + let (_, without, _) = run(&["-p", "3", tree.to_str().unwrap()], None); + + assert_eq!(with_stats, without, "--stats must not touch the matrix"); + assert!(stderr.contains("Leaves in matrix: 3"), "stderr: {}", stderr); + assert!(stderr.contains("Nodes in tree: 5"), "stderr: {}", stderr); + // Off-diagonal distances here are 3.0, 4.5 and 5.5 + assert!(stderr.contains("Minimum: 3.000"), "stderr: {}", stderr); + assert!(stderr.contains("Maximum: 5.500"), "stderr: {}", stderr); } #[test] From 646beff852c02a67aab5e6976f088cde8fd74c2c Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:07:12 +0200 Subject: [PATCH 44/47] Add --npy: write the matrix as a NumPy array Text is an expensive way to move a large matrix. At twelve decimals a cell is fourteen bytes and most of the run is spent producing them; as a 64-bit float it is eight bytes, exact, and costs nothing to write. On an 8,000-leaf tree, to a real file: --lower -p 12 0.26s 458 MB --lower --npy 0.10s 244 MB -p 12 0.56s 916 MB --npy 0.19s 488 MB so 2.6x faster and 47% smaller, at full precision rather than twelve decimals. numpy.load reads it directly. --lower switches it to the condensed vector rather than PHYLIP's lower triangle, because those are not the same triangle: SciPy reads the upper one row by row, and emitting PHYLIP's order would hand squareform the right values in the wrong places. Checked against squareform in the tests. .npy has nowhere to put labels, so they go to .labels.txt in row order, which is why --npy needs -o. --- src/main.rs | 105 ++++++++++++++++++++++++++++++++++++++++--- tests/integration.rs | 86 +++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index 66b9169..36dea7c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -115,6 +115,12 @@ fn run() -> Result<(), Box> { .help("Print a summary of the run to stderr") .action(ArgAction::SetTrue), ) + .arg( + Arg::new("npy") + .long("npy") + .help("Write a NumPy .npy array instead of text (requires -o; labels go to .labels.txt)") + .action(ArgAction::SetTrue), + ) .get_matches(); let tree_path = matches @@ -128,6 +134,21 @@ fn run() -> Result<(), Box> { let output_path = matches.get_one::("output"); let precision = *matches.get_one::("precision").unwrap_or(&10); + let do_npy = *matches.get_one::("npy").unwrap_or(&false); + + if do_npy && output_path.is_none() { + return Err( + "--npy writes binary, so it needs a file: add -o FILE. The labels go to \ + FILE.labels.txt beside it." + .into(), + ); + } + if do_npy && do_topology && !do_lmm { + // Edge counts are integers; an f64 array of them is honest but odd, so + // say what is happening rather than let it surprise anyone. + eprintln!("Warning: --npy writes 64-bit floats, so --topology edge counts are stored as floats."); + } + if precision > MAX_PRECISION { return Err(format!( "--precision must be between 0 and {}, got {}. A 64-bit float holds about \ @@ -399,7 +420,28 @@ fn run() -> Result<(), Box> { } // Print header - if do_lower { + if do_npy { + // NumPy needs the shape up front, which is known, so the array still + // streams out a row at a time behind this. + let shape = if do_lower { + // The condensed form scipy.spatial.distance.squareform expects + format!("({},)", n_leaves * (n_leaves - 1) / 2) + } else { + format!("({}, {})", n_leaves, n_leaves) + }; + writer.write_all(&npy_header(&shape))?; + + // .npy has nowhere to put the labels, so they go beside it in the + // order of the rows. + let labels_path = format!("{}.labels.txt", output_path.expect("--npy requires -o")); + let mut labels = String::with_capacity(n_leaves * 12); + for lab in &sorted_labels { + labels.push_str(lab); + labels.push('\n'); + } + std::fs::write(&labels_path, labels) + .map_err(|e| format!("Cannot write the labels to '{}': {}", labels_path, e))?; + } else if do_lower { // PHYLIP format: first line is the number of taxa writeln!(writer, "{}", n_leaves)?; } else { @@ -428,7 +470,19 @@ fn run() -> Result<(), Box> { // it was about two thirds of the work no number of cores could touch. let batch_rows = (BATCH_CELLS / n_leaves).clamp(1, n_leaves); let mut batch: Vec<(Vec, Stats)> = vec![(Vec::new(), Stats::default()); batch_rows]; - let row_width = |row_i: usize| if do_lower { row_i } else { n_leaves }; + + // Which columns a row carries. The two triangular forms are not the same + // triangle: PHYLIP wants row i to hold columns 0..i, while the condensed + // vector SciPy reads wants columns i+1..n, which concatenated over the rows + // is its (0,1), (0,2), ... (1,2), ... ordering. Emitting the PHYLIP order + // into a .npy would hand squareform the right values in the wrong places. + let row_range = |row_i: usize| -> std::ops::Range { + match (do_lower, do_npy) { + (false, _) => 0..n_leaves, + (true, false) => 0..row_i, + (true, true) => (row_i + 1).min(n_leaves)..n_leaves, + } + }; let mut totals = Stats::default(); let mut first_row = 0; @@ -442,12 +496,21 @@ fn run() -> Result<(), Box> { let row_i = first_row + k; let leaf_i = sorted_leaf_indices[row_i]; out.clear(); - out.extend_from_slice(sorted_labels[row_i].as_bytes()); + if !do_npy { + out.extend_from_slice(sorted_labels[row_i].as_bytes()); + } *stats = Stats::default(); - for (col, &leaf_j) in sorted_leaf_indices[..row_width(row_i)].iter().enumerate() { + let cols = row_range(row_i); + let first_col = cols.start; + for (offset, &leaf_j) in sorted_leaf_indices[cols].iter().enumerate() { + let col = first_col + offset; let dist = compute_distance(leaf_i, leaf_j, mode, &lca_data); - out.push(b'\t'); - format_distance(out, dist, mode, precision); + if do_npy { + out.extend_from_slice(&dist.to_le_bytes()); + } else { + out.push(b'\t'); + format_distance(out, dist, mode, precision); + } // The diagonal is zero by construction in the distance // modes and the leaf's own depth under --lmm, so it would // only drag the summary around. @@ -455,7 +518,9 @@ fn run() -> Result<(), Box> { stats.add(dist); } } - out.push(b'\n'); + if !do_npy { + out.push(b'\n'); + } }); for (row, stats) in &batch[..rows] { @@ -514,6 +579,32 @@ fn run() -> Result<(), Box> { Ok(()) } +/// Build the header of a NumPy `.npy` file (format 1.0) for a little-endian +/// f64 array of the given shape. +/// +/// Magic, version, then a two-byte header length and an ASCII dict, the whole +/// thing padded to a multiple of 64 bytes so the array data lands aligned. +fn npy_header(shape: &str) -> Vec { + let dict = format!( + "{{'descr': ' (String, Vec) { + let bytes = std::fs::read(path).expect("npy file"); + assert_eq!(&bytes[..6], b"\x93NUMPY", "npy magic"); + assert_eq!(&bytes[6..8], &[1, 0], "npy version 1.0"); + let header_len = u16::from_le_bytes([bytes[8], bytes[9]]) as usize; + let header = String::from_utf8(bytes[10..10 + header_len].to_vec()).unwrap(); + assert!(header.contains("'descr': ' Date: Sun, 26 Jul 2026 21:11:06 +0200 Subject: [PATCH 45/47] Format the numbers by hand where the answer is not in doubt write!("{:.p$}") expands the float to its exact decimal form and goes through core::fmt to print it. That is correct and it was about half the cost of a text run. Scaling by a power of ten and emitting the digits directly is roughly seven times faster at the formatting itself: --lower -p 6 0.22s -> 0.13s --lower -p 10 0.24s -> 0.15s -p 10 (square) 0.47s -> 0.29s The reason a shortcut like this is normally a bad idea is that multiplying by 10^p rounds once, and where that carries the product across the nearest .5 boundary there is no way to tell from here which way the exact value rounds. So it does not guess: it measures its distance from the boundary against the error bound, and anything inside that goes to the standard formatter. Roughly one value in a thousand of real distance data takes that path, and every exact tie does. Verified rather than assumed. The unit tests check the fast path against the standard formatter over 200,000 random values across 24 orders of magnitude, straight down the middle of the rounding boundaries at every precision, over distance-shaped values, and on subnormals, 2^53, signed zero and the rest. Separately, 612 matrices across 11 trees, 17 precisions and both float modes are byte-identical to the old output. The first version of this got signed zero wrong: a small negative value that rounds to zero prints as -0.00, and taking the sign from the rounded integer loses it. The boundary test caught it. --- src/main.rs | 210 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 204 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 36dea7c..adc6420 100644 --- a/src/main.rs +++ b/src/main.rs @@ -685,6 +685,76 @@ fn read_taxa_list(path: &str) -> Result, Box> Ok(wanted) } +/// Powers of ten that are exact as f64, which is all of them up to 1e22. +const POW10: [f64; 16] = [ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, +]; + +/// Write `value` with `precision` decimals, or return false and write nothing. +/// +/// `write!("{:.p$}")` goes through `core::fmt` and an exact decimal expansion +/// of the float, which is correct and slow: it was around half the cost of a +/// text run. Scaling by a power of ten and emitting the digits by hand is +/// roughly seven times faster. +/// +/// It is only safe where the answer is not in doubt. Multiplying by 10^p +/// rounds once, by at most `|scaled| * 2^-53`, and if that is enough to carry +/// the product across the nearest `.5` boundary then which way the exact value +/// rounds is unknowable from here. Those cases return false and go to the +/// formatter, which is why this can be fast without being approximate. +#[inline] +fn write_fixed_fast(out: &mut Vec, value: f64, precision: usize) -> bool { + if precision >= POW10.len() || !value.is_finite() { + return false; + } + let scaled = value * POW10[precision]; + // Past 2^53 the integers are no longer consecutive, so the digits below + // would be invented. + if scaled.abs() >= 9.007_199_254_740_992e15 { + return false; + } + + let distance_from_tie = ((scaled - scaled.floor()) - 0.5).abs(); + let rounding_error = scaled.abs() * f64::EPSILON + f64::MIN_POSITIVE; + if distance_from_tie <= rounding_error { + return false; + } + + let rounded = scaled.round() as i64; + // The sign comes from the input rather than the rounded integer, so a small + // negative value that rounds to zero still prints as "-0.00", matching the + // standard formatter. + let negative = value.is_sign_negative(); + + let mut buf = [0u8; 32]; + let mut at = buf.len(); + let mut digits = rounded.unsigned_abs(); + for _ in 0..precision { + at -= 1; + buf[at] = b'0' + (digits % 10) as u8; + digits /= 10; + } + if precision > 0 { + at -= 1; + buf[at] = b'.'; + } + if digits == 0 { + at -= 1; + buf[at] = b'0'; + } + while digits > 0 { + at -= 1; + buf[at] = b'0' + (digits % 10) as u8; + digits /= 10; + } + if negative { + at -= 1; + buf[at] = b'-'; + } + out.extend_from_slice(&buf[at..]); + true +} + /// Append a single distance value to a row buffer. /// /// Topology mode outputs integers; patristic and LMM use `precision` decimal @@ -692,10 +762,138 @@ fn read_taxa_list(path: &str) -> Result, Box> /// inside the parallel row builder. #[inline] fn format_distance(out: &mut Vec, dist: f64, mode: DistMode, precision: usize) { - let result = if mode == DistMode::Topology { - write!(out, "{}", dist as i64) - } else { - write!(out, "{:.prec$}", dist, prec = precision) - }; - result.expect("writing to a Vec cannot fail"); + if mode == DistMode::Topology { + write!(out, "{}", dist as i64).expect("writing to a Vec cannot fail"); + return; + } + if write_fixed_fast(out, dist, precision) { + return; + } + write!(out, "{:.prec$}", dist, prec = precision).expect("writing to a Vec cannot fail"); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// xorshift64, so a failure is reproducible. + struct Rng(u64); + + impl Rng { + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn unit(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 + } + } + + /// The fast path either produces exactly what the standard formatter would, + /// or declines. Anything else is a silently wrong number in the matrix. + fn assert_agrees(value: f64, precision: usize) { + let mut fast = Vec::new(); + if !write_fixed_fast(&mut fast, value, precision) { + return; // declined, so the formatter handles it + } + let slow = format!("{:.prec$}", value, prec = precision); + assert_eq!( + String::from_utf8(fast).unwrap(), + slow, + "fast path disagrees for {:e} at {} decimals", + value, + precision + ); + } + + #[test] + fn test_fast_formatter_matches_std_on_random_values() { + let mut rng = Rng(0x9E37_79B9_7F4A_7C15); + for _ in 0..200_000 { + let bits = rng.next_u64(); + let magnitude = (bits % 24) as i32 - 12; + let sign = if bits & 1 == 0 { 1.0 } else { -1.0 }; + let value = rng.unit() * 10f64.powi(magnitude) * sign; + assert_agrees(value, (rng.next_u64() % 16) as usize); + } + } + + #[test] + fn test_fast_formatter_matches_std_near_rounding_boundaries() { + // Exact halves are where a scaled-integer shortcut goes wrong, so walk + // straight down the middle of them at every precision. + for precision in 0..16usize { + let step = 10f64.powi(-(precision as i32)); + for i in 0..4_000u64 { + for delta in [0.0, 0.5, -0.5, 0.499_999_999, 0.500_000_001, 1.0 / 3.0] { + assert_agrees((i as f64 + delta) * step, precision); + assert_agrees(-(i as f64 + delta) * step, precision); + } + } + } + } + + #[test] + fn test_fast_formatter_matches_std_on_distance_shaped_values() { + // What distree actually emits: small positive distances, at the + // precisions people use. + let mut rng = Rng(0x2545_F491_4F6C_DD1D); + for _ in 0..100_000 { + let value = rng.unit() * 1e-4; + for precision in [0usize, 3, 6, 9, 10, 12, 15] { + assert_agrees(value, precision); + assert_agrees(value * 4_411_532.0, precision); + } + } + } + + #[test] + fn test_fast_formatter_edge_cases() { + let awkward = [ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + -0.5, + 0.05, + 0.005, + f64::MIN_POSITIVE, + -f64::MIN_POSITIVE, + 5e-324, // the smallest subnormal + 1e-300, + 1e300, + 9.007_199_254_740_992e15, // 2^53 + 9.007_199_254_740_991e15, + 0.1, + 0.2, + 0.3, + 1.0 / 3.0, + 2.0 / 3.0, + 1e15, + 1e16, + 123_456.789_012_345, + ]; + for &value in &awkward { + for precision in 0..=MAX_PRECISION { + assert_agrees(value, precision); + } + } + } + + #[test] + fn test_fast_formatter_declines_rather_than_guesses() { + // Anything it cannot do exactly must be refused, not approximated + let mut out = Vec::new(); + assert!(!write_fixed_fast(&mut out, 1.0, 16), "precision past the table"); + assert!(!write_fixed_fast(&mut out, f64::NAN, 6), "NaN"); + assert!(!write_fixed_fast(&mut out, f64::INFINITY, 6), "infinity"); + assert!(!write_fixed_fast(&mut out, 1e300, 6), "past 2^53 once scaled"); + assert!(out.is_empty(), "a declined value must write nothing"); + } } From 593627b518e83c53b53b18305c3371b5118cd695 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:15:10 +0200 Subject: [PATCH 46/47] Document the new flags and the performance work Covers --npy, --taxa, --stats and gzipped input across the CLI reference, input, output and recipes pages, and refreshes the benchmark tables now that the formatter is faster. Also records what the fuzzing and the ape cross-validation are for, since they are the part of the suite that would otherwise look like more of the same. --- CHANGELOG.md | 15 +++++- README.md | 5 +- docs/guide/cli.md | 81 +++++++++++++++++++++++++++++--- docs/guide/input.md | 20 +++++++- docs/guide/output.md | 42 +++++++++++++++++ docs/how-it-works/performance.md | 49 ++++++++++++++----- docs/index.md | 9 +++- docs/recipes.md | 73 ++++++++++++++++++++++++++++ 8 files changed, 269 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04863fe..2df4505 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## [1.0.1] - 2026-04-04 ### Fixed +- Branch lengths that are not finite are rejected. `A:1e910` parses as infinity rather than failing, and an infinite depth made every distance `inf` with `NaN` down the diagonal, reported with a success exit code. The run also stops when finite depths are large enough that `d_i + d_j` overflows +- A leaf label containing a space is rejected in `--lower` mode: PHYLIP readers treat whitespace as the end of the name, so every row after it landed a column out +- `--lower` with `--lmm` now warns that the omitted diagonal holds each leaf's root-to-tip length rather than zeros - Parallel computation now speeds the run up instead of slowing it down. One parallel job per row could not pay for synchronising the thread pool, and float formatting sat in the serial write loop; rows are now batched and each worker formats its own. A 20,000-tip matrix went from 10.6 s to 1.6 s, and an 8,000-tip one now scales from 1.26 s on one core to 0.21 s on eight - Trees prefixed with a `[&R]` / `[&U]` rooting marker, or carrying a comment before a label, no longer fail to parse - Non-ASCII leaf labels (accents, Greek, CJK) are preserved instead of being mangled into mojibake @@ -35,12 +38,20 @@ - Stdin support: use `-` as the phylogeny argument to read from stdin - Warning when no branch lengths are detected in patristic mode - Warning when negative branch lengths are found in the tree +- `--npy` writes the matrix as a NumPy array of 64-bit floats instead of text: exact, half the size, and 2.6x faster. With `--lower` it writes the condensed vector SciPy reads. Labels go to `.labels.txt` +- `--taxa FILE` restricts the matrix to a list of leaf labels, keeping the distances the full tree gives rather than pruning it +- `--stats` prints a summary to stderr: leaves, nodes, mode, cells, and the minimum, maximum and mean off the diagonal +- Gzipped input is read directly, detected by content rather than by file name, so it works from stdin too - CITATION.cff with DOI -- MkDocs Material documentation site, published to GitHub Pages -- Comprehensive test suite (73 tests), including randomised checks of midpoint rooting and of MRCA queries against a brute-force walk +- MkDocs Material documentation site with a tutorial, published to GitHub Pages +- Comprehensive test suite (96 tests), including randomised checks of midpoint rooting and of MRCA queries against a brute-force walk ### Changed - Codebase split into modules: `parser.rs`, `tree.rs`, `lca.rs`, `midpoint.rs` +- Fixed-precision float formatting is done by hand where the rounding is unambiguous and handed to the standard formatter where it is not, which cut a text run by a third to a half with byte-identical output +- Cross-validated against R's ape (`cophenetic.phylo`, `vcv.phylo`) and `phangorn::midpoint` over 250 generated trees; reference matrices for six of them are committed so `cargo test` checks the same thing without R +- The parser and the whole pipeline are fuzzed, in the test suite on every push and with `cargo fuzz` for longer runs +- The tree logic moved into a library target, so it can be fuzzed, documented and used from other Rust code - LCA binary-lifting table stores plain `usize` rather than `Option`, halving the memory it needs - Output buffer raised from 8 KB to 1 MB, so a multi-gigabyte matrix is not hundreds of thousands of write syscalls - The input text and the recursive parse tree are released once the flat node array is built, rather than held to the end of the run diff --git a/README.md b/README.md index ad9e817..b40a8b2 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,10 @@ __and Mireia Coscolla1__ * **LMM (var-covar) matrix**: Outputs the depth (in branch-length units) of the lowest common ancestor (MRCA) for each pair of leaves. * **Midpoint rooting**: Optionally re-roots the tree at its midpoint before computing distances. * **Low memory footprint**: Streams each row to output immediately, without holding the full matrix in RAM. -* **Parallel computation**: Uses multiple CPU cores to compute each row in parallel, reducing runtime for large datasets. +* **Parallel computation**: Uses multiple CPU cores, with the distances and their formatting both spread across them. +* **NumPy output**: `--npy` writes an exact 64-bit float array instead of text, 2.6x faster and half the size. +* **Subsets**: `--taxa` restricts the matrix to a list of tips, keeping the distances the full tree gives. +* **Gzipped input**: reads a compressed tree directly, detected by content rather than by file name. ## Typical Applications diff --git a/docs/guide/cli.md b/docs/guide/cli.md index e610d6e..05022f2 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -8,9 +8,9 @@ Usage: distree [OPTIONS] `` -: Path to the tree file in Newick format. Use `-` to read from stdin. The file - must hold exactly one tree, and its tip labels must be unique. See - [Input](input.md). +: Path to the tree file in Newick format, plain or gzipped. Use `-` to read + from stdin. The file must hold exactly one tree, and its tip labels must be + unique. See [Input](input.md). ## Options @@ -19,15 +19,62 @@ Usage: distree [OPTIONS] | `--midpoint` | off | Midpoint-root the tree before computing distances. Ignored with `--topology`. See [Midpoint rooting](rooting.md) | | `--lmm` | off | Write the variance-covariance matrix: each cell is the root-to-MRCA distance. Takes precedence over `--topology` | | `--topology` | off | Ignore branch lengths and count edges. Values are written as integers | -| `--lower` | off | Write a PHYLIP lower triangle: a taxa count, then one row per taxon with no diagonal | +| `--lower` | off | Write a PHYLIP lower triangle: a taxa count, then one row per taxon with no diagonal. With `--npy`, the condensed vector instead | +| `--npy` | off | Write a NumPy `.npy` array of 64-bit floats instead of text. Requires `-o`; labels go to `.labels.txt` | +| `--taxa FILE` | all tips | Restrict the matrix to the labels listed in `FILE`, one per line | +| `--stats` | off | Print a summary of the run to stderr | | `-o, --output FILE` | stdout | Write the matrix to a file. Not created until the tree has parsed | -| `-p, --precision N` | `10` | Decimal places, from 0 to 30. Ignored by `--topology` | +| `-p, --precision N` | `10` | Decimal places, from 0 to 30. Ignored by `--topology` and by `--npy` | | `-t, --threads N` | all cores | Threads for the parallel row computation. Must be at least 1 | | `-h, --help` | | Print help | | `-V, --version` | | Print version | Without `--lmm` or `--topology`, distree computes patristic distances. +## Selecting a subset + +```bash +distree tree.nwk --taxa keep.txt -o subset.tsv +``` + +`keep.txt` holds one leaf label per line. Blank lines and lines starting with +`#` are skipped, and duplicates are collapsed. Order does not matter: the matrix +comes out alphabetical as always. + +A label that is not a leaf of the tree is an error naming it, rather than a +matrix quietly smaller than the list. + +The distances themselves are the ones from the full tree. A subset filters the +output, it does not prune the tree, so the path between two tips is the same +whether or not the tips in between are in the matrix. That is the opposite of +what pruning the tree first would give you. + +## The run summary + +```bash +distree tree.nwk --stats -o distances.tsv +``` + +``` +--- Statistics --- +Leaves in matrix: 7 +Nodes in tree: 13 +Mode: patristic distance +Cells written: 49 +Minimum: 0.0000004534 +Maximum: 0.0000612030 +Mean: 0.0000358688 +Time: 0.001s +``` + +It goes to stderr, so a piped run is unaffected. The minimum, maximum and mean +are taken off the diagonal, which is zero by construction in the distance modes +and would only pull the summary towards it. When `--taxa` is in use, the summary +reports both counts. + +The numbers are accumulated by the workers as they compute, so the summary costs +one pass and nothing measurable. + ## How the modes interact | Combination | Result | @@ -38,6 +85,11 @@ Without `--lmm` or `--topology`, distree computes patristic distances. | `--midpoint` alone | Applies, and changes nothing: a patristic matrix is the same under any rooting | | `--lower` with any mode | Applies to all three | | `-p` with `--topology` | Ignored; edge counts are integers | +| `--npy` without `-o` | An error. Binary output needs a file | +| `--npy` with `--lower` | Writes the condensed vector SciPy reads, not PHYLIP's triangle | +| `--npy` with `-p` | The precision is ignored; a 64-bit float is exact | +| `--npy` with `--topology` | Works, with a warning that integers are stored as floats | +| `--taxa` with any mode | Applies to all three, and to both output formats | ## Exit codes @@ -55,7 +107,14 @@ Everything below goes to stderr, never into the matrix. | Message | Cause | |:--|:--| | `Cannot open '': ...` | The tree file is missing or unreadable | +| `... is gzip-compressed but could not be read: ...` | The gzip stream is truncated or corrupt | +| `... is not valid UTF-8 text` | Not a text file, and not gzip either | | `Empty input: no Newick tree found.` | The file is empty or only whitespace | +| `Branch length '' at position N is not a finite number.` | An exponent past the range of a 64-bit float, which would make every distance infinite | +| `Branch lengths are too large to compute distances with` | The depths are finite but summing two of them overflows | +| ` of the labels in '' are not leaves of the tree` | `--taxa` names something the tree does not have | +| `Cannot read the taxa list '': ...` | `--taxa` file missing or unreadable | +| `--npy writes binary, so it needs a file` | `--npy` without `-o` | | `Failed to parse Newick tree: ...` | Malformed tree. The rest of the message names the position | | `No labeled leaves found in the tree.` | The tree parsed but no tip carries a label | | `Duplicate leaf name '' found.` | Two tips share a label; the matrix could not be indexed | @@ -75,6 +134,8 @@ Everything below goes to stderr, never into the matrix. | `N leaf/leaves have no label and were excluded from the matrix` | Unlabelled tips cannot be named in a row, so they are left out | | `--lmm and --topology are mutually exclusive. Using --lmm.` | Both were passed | | `--midpoint is ignored in --topology mode.` | Both were passed | +| `--lower omits the diagonal, which in --lmm mode holds each leaf's root-to-tip length` | The lower triangle drops data that is not zeros under `--lmm` | +| `--npy writes 64-bit floats, so --topology edge counts are stored as floats` | Both were passed | ## Examples @@ -91,8 +152,14 @@ distree tree.nwk --midpoint --lmm -p 8 -o varcovar.tsv # Edge counts from a cladogram distree cladogram.nwk --topology -o topo.tsv -# From stdin, capped at 4 threads -gunzip -c tree.nwk.gz | distree - -t 4 -o distances.tsv +# A gzipped tree, capped at 4 threads +distree tree.nwk.gz -t 4 -o distances.tsv + +# Just the 200 samples in a list, as a NumPy array +distree tree.nwk --taxa cohort.txt --npy -o cohort.npy + +# The condensed vector scipy.cluster.hierarchy takes directly +distree tree.nwk --lower --npy -o condensed.npy # Just the header, to check the tip ordering distree tree.nwk | head -1 | tr '\t' '\n' | tail -n +2 diff --git a/docs/guide/input.md b/docs/guide/input.md index a4fec06..26f73b8 100644 --- a/docs/guide/input.md +++ b/docs/guide/input.md @@ -1,12 +1,16 @@ # Input -distree reads one Newick tree, from a file or from stdin: +distree reads one Newick tree, from a file or from stdin, plain or gzipped: ```bash distree tree.nwk +distree tree.nwk.gz gunzip -c tree.nwk.gz | distree - ``` +Compression is detected from the file's leading bytes rather than its name, so +it works from stdin and on a `.gz` file that was renamed. + ## What is accepted **Branch lengths** in any form Rust's float parser takes, including scientific @@ -59,6 +63,7 @@ position, and nothing is written to `-o`. | `('unclosed:1,B:2);` | `Unclosed quote starting at position 1.` | | `(A:1,B:2)[oops;` | `Unclosed comment starting at position 9: no matching ']'.` | | `(A:,B:2);` | `Expected a numeric branch length` | +| `(A:1e910,B:2);` | `Branch length '1e910' ... is not a finite number.` | | `(A:1,A:2);` | `Duplicate leaf name 'A' found. Leaf names must be unique.` | | A label holding a tab, newline or carriage return | `contains a tab character, which would corrupt the output by splitting the row` | | A tree with no labelled tips | `No labeled leaves found in the tree.` | @@ -76,6 +81,19 @@ position, and nothing is written to `-o`. for t in tree_*; do distree "$t" -o "${t}.tsv"; done ``` +!!! warning "Branch lengths at the edge of what a float can hold" + + An exponent past the range of a 64-bit float, `A:1e910`, does not fail to + parse: Rust returns infinity for it. An infinite depth then makes every + distance infinite and the diagonal `NaN`, because `inf - inf` is `NaN`, and + the whole matrix used to come out that way with a success exit code. Such a + branch length is now rejected outright. + + A distance is `d_i + d_j - 2·d_m`, so the same overflow can happen with + every individual length finite, if the depths are past half of the float + range. The run stops for that too, saying to rescale the tree. Ordinary + large values are unaffected: `1e100` still works. + !!! question "Truncated trees" A tree cut short by an interrupted download or a full disk used to parse: diff --git a/docs/guide/output.md b/docs/guide/output.md index 938f05c..5383fa1 100644 --- a/docs/guide/output.md +++ b/docs/guide/output.md @@ -94,6 +94,48 @@ and is what those readers expect. relaxed form that whitespace-splitting readers, including modern PHYLIP builds, accept. +## NumPy arrays + +```bash +distree tree.nwk --npy -o distances.npy +``` + +Text is an expensive way to move a large matrix. At twelve decimals a cell is +fourteen bytes and producing them is most of the run; as a 64-bit float it is +eight bytes, exact, and costs nothing to write. On an 8,000-tip tree: + +| | Time | Size | +|:--|--:|--:| +| `--lower -p 12` | 0.26 s | 458 MB | +| `--lower --npy` | 0.10 s | 244 MB | +| `-p 12` | 0.56 s | 916 MB | +| `--npy` | 0.19 s | 488 MB | + +`.npy` has nowhere to put labels, so they go to `.labels.txt` in row +order. That is why `--npy` needs `-o` rather than writing to stdout. + +```python +import numpy as np +m = np.load("distances.npy") +labels = open("distances.npy.labels.txt").read().split() +``` + +With `--lower`, the array is the **condensed vector** rather than PHYLIP's +lower triangle, because those are two different triangles: SciPy reads the +upper one row by row. It goes straight into `scipy`: + +```python +import numpy as np +from scipy.cluster.hierarchy import linkage, fcluster +from scipy.spatial.distance import squareform + +v = np.load("condensed.npy") # distree tree.nwk --lower --npy -o condensed.npy +z = linkage(v, method="single") # takes the condensed form directly +full = squareform(v) # or expand it to the square matrix +``` + +`-p` has no effect here; a 64-bit float carries the value exactly. + ## Precision ```bash diff --git a/docs/how-it-works/performance.md b/docs/how-it-works/performance.md index 182c704..12d360d 100644 --- a/docs/how-it-works/performance.md +++ b/docs/how-it-works/performance.md @@ -29,9 +29,9 @@ Apple M4 Pro with 14 cores: |--:|--:|--:|--:| | 1,000 | 0.5 M | 0.00 s | 11 MB | | 2,000 | 2.0 M | 0.01 s | 24 MB | -| 4,000 | 8.0 M | 0.06 s | 35 MB | -| 8,000 | 32 M | 0.23 s | 32 MB | -| 20,000 | 200 M | 1.8 s | 35 MB | +| 4,000 | 8.0 M | 0.03 s | 35 MB | +| 8,000 | 32 M | 0.13 s | 32 MB | +| 20,000 | 200 M | 1.02 s | 35 MB | Time is quadratic in the tips, as it must be. Memory is not: it flattens out around 35 MB, because past a few thousand tips it is the fixed batch buffer plus @@ -43,11 +43,11 @@ the LCA table rather than anything that grows with the matrix. | `-t` | Time | Speedup | |--:|--:|--:| -| 1 | 1.52 s | 1.0x | -| 2 | 0.66 s | 2.3x | -| 4 | 0.36 s | 4.2x | -| 8 | 0.24 s | 6.3x | -| 14 | 0.23 s | 6.6x | +| 1 | 0.56 s | 1.0x | +| 2 | 0.31 s | 1.8x | +| 4 | 0.18 s | 3.1x | +| 8 | 0.12 s | 4.7x | +| 14 | 0.13 s | 4.3x | Each worker computes and formats whole rows, so the parallel section covers both halves of the cost and the curve holds up until it runs into memory bandwidth @@ -65,6 +65,30 @@ and the single writer. Set `-t` when you are sharing a machine, or when the run is part of a pipeline that is already parallel. Leaving it unset is right for a dedicated node. +## Where the time goes + +For a text run, roughly half of it used to be turning floats into decimal +digits, which is more expensive than computing the distance being printed. +`write!("{:.p$}")` expands the float to its exact decimal form and goes through +`core::fmt`; distree instead scales by a power of ten and emits the digits +directly, which is about seven times faster at that step and cut the whole run +by a third to a half: + +| | Before | After | +|:--|--:|--:| +| `--lower -p 6` | 0.22 s | 0.13 s | +| `--lower -p 10` | 0.24 s | 0.15 s | +| `-p 10` square | 0.47 s | 0.29 s | + +The shortcut is only taken where the answer is not in doubt. Multiplying by +`10^p` rounds once, and where that could carry the product across the nearest +`.5` boundary the value goes to the standard formatter instead, so the output is +byte-identical either way. About one value in a thousand takes that path. + +If none of the time should go on formatting at all, [`--npy`](../guide/output.md#numpy-arrays) +writes raw 64-bit floats and skips it entirely: another 2.6x, at full precision +and half the file size. + ## Memory Nothing in distree holds the matrix. The peak is: @@ -103,10 +127,11 @@ on it. At `-p 10` a cell is 12 to 14 bytes; at `-p 6` it is 8 to 10. | 10,000 | 1.3 GB | 900 MB | 450 MB | | 50,000 | 33 GB | 22 GB | 11 GB | -Two things follow. Use `--lower` when the reader accepts it, which halves the -file. And do not ask for more decimals than the branch lengths carry: `-p 6` -against the default `-p 10` is a third off the file for no loss on any realistic -tree. +Three things follow. Use `--lower` when the reader accepts it, which halves the +file. Do not ask for more decimals than the branch lengths carry: `-p 6` against +the default `-p 10` is a third off the file for no loss on any realistic tree. +And if the reader is Python, [`--npy`](../guide/output.md#numpy-arrays) is 8 +bytes a cell at full precision, which beats text at any setting. Rows are written as they are computed, so a pipeline that consumes them as they arrive never holds the whole matrix either: diff --git a/docs/index.md b/docs/index.md index 5b171cc..365796b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,6 +53,10 @@ memory tracks the number of tips, not the square of it. The TSV layout, PHYLIP lower triangle, precision, and reading it back in. +- :material-school: **[Tutorial](tutorial.md)** + + A worked outbreak investigation, from the tree to transmission clusters. + - :material-cog: **[How it works](how-it-works/algorithm.md)** The parser, the LCA structure, and the streaming loop. @@ -82,8 +86,9 @@ decimal places, across eight threads. Swap `--topology` for edge counts, | **Topological** | The number of edges on that path, ignoring branch lengths | | **Variance-covariance** | The root-to-MRCA distance for each pair, the `C` matrix of a PGLS or a phylogenetic mixed model | | **Rooting** | Optional midpoint rooting, which matters for `--lmm` and cannot matter for the other two | -| **Formats** | Square TSV with labels, or a PHYLIP lower triangle | -| **Input** | Newick from a file or stdin, with quoted labels, polytomies, NHX and BEAST comments, and UTF-8 tip names | +| **Formats** | Square TSV with labels, a PHYLIP lower triangle, or a NumPy array | +| **Input** | Newick from a file or stdin, plain or gzipped, with quoted labels, polytomies, NHX and BEAST comments, and UTF-8 tip names | +| **Subsets** | `--taxa` restricts the matrix to a list of tips, with the distances the full tree gives | | **Scale** | Memory grows with the number of tips, not with the matrix; rows stream out as they are computed | | **Parallelism** | Each row is computed across all cores, or as many as `-t` allows | diff --git a/docs/recipes.md b/docs/recipes.md index 97da5c3..9256282 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -162,6 +162,50 @@ distree cladogram.nwk --topology -o topo.tsv The same applies when the lengths exist but are not comparable across the tree, for instance from concatenated loci with different rates. +## Straight into SciPy, without the text + +For anything Python is going to read, `--npy` skips the text round trip +entirely: exact 64-bit floats, half the file, and none of the formatting cost. + +```bash +distree tree.nwk --lower --npy -o condensed.npy +``` + +```python +import numpy as np +from scipy.cluster.hierarchy import linkage, fcluster + +v = np.load("condensed.npy") # the condensed form scipy wants +labels = open("condensed.npy.labels.txt").read().split() + +z = linkage(v, method="single") +clusters = fcluster(z, t=12 / 4411532, criterion="distance") +``` + +Drop `--lower` for the square matrix, which is what `MDS(dissimilarity= +"precomputed")` and `umap.UMAP(metric="precomputed")` take. + +## One cohort out of a big tree + +```bash +distree big.nwk --taxa cohort.txt -p 10 -o cohort.tsv +``` + +`cohort.txt` is one label per line, with `#` comments allowed. The distances are +the ones from the full tree, which is the point: the path between two isolates +does not change because the isolates between them were left out of the matrix. +Pruning the tree first would give you different numbers. + +Building the list from a metadata table: + +```bash +awk -F'\t' 'NR > 1 && $3 == "Valencia" { print $1 }' metadata.tsv > cohort.txt +distree big.nwk --taxa cohort.txt --stats -o valencia.tsv +``` + +`--stats` then reports both counts, so it is obvious how much of the tree the +cohort was. + ## Very large trees Rows are written as they are computed, so nothing needs the whole matrix in one @@ -171,6 +215,12 @@ piece. Compress on the way out: distree big.nwk -p 6 --lower | gzip > distances.phy.gz ``` +The input can be gzipped too, and does not need decompressing first: + +```bash +distree big.nwk.gz --lower --npy -o distances.npy +``` + Or filter as it streams, keeping only the pairs that matter: ```bash @@ -194,6 +244,29 @@ ls trees/*.nwk | xargs -P 8 -I{} sh -c \ `-t 1` matters here: without it every distree would try to use every core and they would fight each other. +## Checking a run at a glance + +```bash +distree tree.nwk --stats -o distances.tsv +``` + +``` +--- Statistics --- +Leaves in matrix: 1284 +Nodes in tree: 2567 +Mode: patristic distance +Cells written: 1648656 +Minimum: 0.0000002267 +Maximum: 0.0004821553 +Mean: 0.0001839204 +Time: 0.042s +``` + +Worth a look before anything downstream. A leaf count that is not what you +expected usually means the tree was not the one you meant; a maximum in the +wrong order of magnitude usually means the branch lengths are not in the units +you assumed. + ## Splitting a multi-tree file distree takes one tree per file, and refuses a file holding several rather than From b836bb5f9efa2c80125f96520024b98ada51b997 Mon Sep 17 00:00:00 2001 From: Paururo <50167687+Paururo@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:59:53 +0200 Subject: [PATCH 47/47] Cut the README down to a landing page It was 329 lines and had become a second copy of the documentation site: the full flag reference, the output formats, five worked use cases, the troubleshooting list. Two copies of the same thing only stay in step while someone is watching, and this one had already drifted. Its Usage block listed neither --npy nor --taxa nor --stats, and said nothing about reading gzip, so anyone reading the README on GitHub got a flag reference that the site contradicted. What is left is what a landing page is for: what the tool does, where the documentation is, how to install it, eight commands worth copying, a performance table and how to cite it. The detail lives on the site, which is the only copy now. Every command in the quick start was run as written. --- README.md | 292 ++++++++++++++---------------------------------------- 1 file changed, 75 insertions(+), 217 deletions(-) diff --git a/README.md b/README.md index b40a8b2..f7f38f1 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,11 @@ __and Mireia Coscolla1__
1. I2SysBio, University of Valencia-CSIC, FISABIO Joint Research Unit Infection and Public Health, Valencia, Spain -`distree` is a command-line tool written in Rust that extracts a distance matrix from a phylogenetic tree in Newick format. It is designed to handle large trees with thousands of sequences by using a low-memory, parallelized approach. +`distree` reads a phylogenetic tree in Newick format and writes the pairwise +distance between every pair of tips: **patristic** (summed branch lengths), +**topological** (edge counts), or the **variance-covariance** matrix a +comparative model wants. Rows stream out as they are computed, so memory grows +with the number of tips rather than with the square of it. 📖 **Full documentation: ** @@ -26,259 +30,113 @@ __and Mireia Coscolla1__ | **[Input](https://pathogenomics-lab.github.io/distree/guide/input/)** | What Newick is accepted, what is rejected, and why | | **[Distance modes](https://pathogenomics-lab.github.io/distree/guide/distances/)** | Patristic, topological and var-covar, on one worked example | | **[Midpoint rooting](https://pathogenomics-lab.github.io/distree/guide/rooting/)** | When the root changes the answer, and when it cannot | -| **[Output](https://pathogenomics-lab.github.io/distree/guide/output/)** | TSV, PHYLIP lower triangle, precision, reading it back | +| **[Output](https://pathogenomics-lab.github.io/distree/guide/output/)** | TSV, PHYLIP lower triangle, NumPy arrays, precision | | **[CLI reference](https://pathogenomics-lab.github.io/distree/guide/cli/)** | Every flag, every default, every message | | **[How it works](https://pathogenomics-lab.github.io/distree/how-it-works/algorithm/)** | The parser, the LCA structure, the streaming loop | | **[Performance](https://pathogenomics-lab.github.io/distree/how-it-works/performance/)** | Measured speed, memory and thread scaling | | **[Recipes](https://pathogenomics-lab.github.io/distree/recipes/)** | PCoA, transmission clusters, PGLS, neighbour-joining | -## Features - -* **Patristic distances**: Computes the sum of branch lengths between every pair of leaves (taxa). -* **Topological distances**: Ignores branch lengths and calculates the number of edges (nodes) between leaves. -* **LMM (var-covar) matrix**: Outputs the depth (in branch-length units) of the lowest common ancestor (MRCA) for each pair of leaves. -* **Midpoint rooting**: Optionally re-roots the tree at its midpoint before computing distances. -* **Low memory footprint**: Streams each row to output immediately, without holding the full matrix in RAM. -* **Parallel computation**: Uses multiple CPU cores, with the distances and their formatting both spread across them. -* **NumPy output**: `--npy` writes an exact 64-bit float array instead of text, 2.6x faster and half the size. -* **Subsets**: `--taxa` restricts the matrix to a list of tips, keeping the distances the full tree gives. -* **Gzipped input**: reads a compressed tree directly, detected by content rather than by file name. - -## Typical Applications - -1. **Comparative Genomics & Evolutionary Studies** - - * **Patristic Distances**: When analyzing evolutionary divergence, the sum of branch lengths between two sequences reflects their accumulated genetic change. This is useful for constructing distance-based phylogenetic trees (e.g., Neighbor-Joining), clustering sequences into operational taxonomic units (OTUs), or computing pairwise divergence metrics for downstream analyses. - * **Topological Distances**: In cases where branch-length estimates are unreliable or not meaningful (e.g., when raw sequence counts are compared), using topological distances (number of nodes between leaves) can provide a rough measure of relatedness. This can be helpful for broad clustering or when using simpler distance-based algorithms that only require tree shape. - -2. **Epidemiology & Public Health** - - * **Rapid Outbreak Tracking**: For pathogens such as bacteria or viruses, building a phylogenetic tree from whole-genome data can be computationally expensive to revisit. Extracting a distance matrix allows quick pairwise comparisons to identify clusters of closely related strains (e.g., potential transmission clusters) without recomputing distances from raw alignments. - * **Contact Tracing & Transmission Networks**: If you have a large outbreak dataset, computing patristic distances between samples quickly helps identify subclusters or "clades" of interest (e.g., to infer likely transmission chains). Similarly, topological distances might approximate epidemiological closeness if branch-length estimates vary widely. - -3. **Microbiome & Environmental Sequencing** - - * **OTU/ASV Clustering**: In 16S rRNA amplicon studies, one often constructs a phylogenetic tree of all amplicon sequence variants (ASVs). A distance matrix (patristic or topological) can feed into beta-diversity metrics (e.g., UniFrac requires branch lengths). Here, `distree` can quickly compute pairwise distances after the tree is built, facilitating ordination (PCoA) or clustering of samples by their phylogenetic composition. - * **Phylogenetic Diversity**: Calculating the sum of branch lengths between taxa supports metrics like Faith's Phylogenetic Diversity or UniFrac. `distree` can produce the matrix needed for those algorithms without re-traversing the tree multiple times. - -4. **Machine Learning & Dimensionality Reduction** - - * **Input for MDS/t-SNE/UMAP**: Many machine learning or visualization methods (Multidimensional Scaling, t-SNE, UMAP) accept a distance matrix as input. Converting a large phylogeny into a pairwise distance matrix enables embedding taxa into low-dimensional space, highlighting evolutionary relationships or clusters. - * **Distance-Based Feature Engineering**: In trait-based prediction models (e.g., predicting phenotype from genotype), patristic distances can be used as kernel features. `distree` can produce the kernel matrix needed for Gaussian Process Regression or support vector machines (SVM) with a custom kernel. - -5. **Benchmarking & Simulation Studies** - - * **Comparing Tree-Building Methods**: When evaluating different phylogenetic inference methods, it is often necessary to compute distances from each resulting tree. `distree` can generate distance matrices for multiple trees rapidly, enabling head-to-head comparisons. - * **Simulation Validation**: In simulation frameworks (e.g., Seq-Gen), one generates a tree, simulates sequence data, reconstructs a tree, and compares distance matrices. Having a fast, consistent tool to extract matrices accelerates simulation benchmarks. +--- ## Installation -### 🐍 Using conda -``` -conda install -c bioconda distree -``` -### 🐍 Using mamba -``` -mamba install -c bioconda distree -``` -### Compilation -1. Ensure you have [Rust](https://www.rust-lang.org/tools/install) installed. - -2. Clone or download the `distree` repository. - -3. In the project root, run: - - ```bash - cargo build --release - ``` - -4. The optimized binary will be available at: - - ```bash - target/release/distree - ``` - -## Usage - ```bash -Usage: distree [OPTIONS] - -Arguments: - Path to the tree file in Newick format (use '-' for stdin) - -Options: - --midpoint Midpoint-root the tree before computing distances - --lmm Produce the var-covar matrix C (depth of the MRCA) - --topology Ignore branch lengths; use purely topological distances - --lower Output PHYLIP lower-triangle format (taxa count, row labels, no diagonal) - -o, --output Path to write the TSV output file (defaults to stdout) - -p, --precision Number of decimal places for output [default: 10] - -t, --threads Number of parallel threads (default: all cores) - -h, --help Print help information - -V, --version Print version information -``` - -### Argument & Option Details - -* ``: Path to the input tree in Newick format. Use `-` to read from stdin. Leaf labels must be unique, and the file must hold exactly one tree. - -* `--midpoint`: Re-root at the midpoint of the longest path before computing distances. Ignored with `--topology`, where the number of edges between two leaves does not depend on where the tree is rooted. - -* `--lmm`: Var-covar matrix for Phylogenetic Comparative Methods. Each entry (i, j) = depth of MRCA(i, j). Mutually exclusive with `--topology`. - -* `--topology`: Number of edges between leaves (integers). Ignores branch lengths. - -* `--lower`: PHYLIP lower-triangle format. Outputs a header line with the number of taxa, followed by one row per taxon with its label and the lower triangle of distances (no diagonal). Compatible with PHYLIP, Mash, and tools expecting this standard format. - -* `-p, --precision `: Decimal places in output (default: 10, maximum: 30). Applies to patristic and LMM modes. A 64-bit float holds about 17 significant digits, so anything past that is padding. - -* `-t, --threads `: Thread count for parallel computation. Must be at least 1; omit the flag to use all available cores. - -* `-o, --output `: Write TSV to file instead of stdout. - -## Output Format - -The tool outputs a tab-separated values (TSV) matrix. - -1. The first line is a header with leaf labels sorted alphabetically, preceded by an empty cell (for row labels). - -2. Each subsequent line begins with a leaf label (sorted alphabetically), followed by N columns of distances (depending on the chosen mode) to every other leaf in the same sorted order. - -All three examples below are the real output of `distree -p 3` for one tree: - -``` -((LeafA:1.95,LeafB:3.25):0.35,(LeafC:0.80,LeafD:1.20):0.50); -``` - -Patristic distances (default), the sum of branch lengths along the path: - -``` - LeafA LeafB LeafC LeafD -LeafA 0.000 5.200 3.600 4.000 -LeafB 5.200 0.000 4.900 5.300 -LeafC 3.600 4.900 0.000 2.000 -LeafD 4.000 5.300 2.000 0.000 -``` - -Topological distances (`--topology`), the number of edges along the path: - -``` - LeafA LeafB LeafC LeafD -LeafA 0 2 4 4 -LeafB 2 0 4 4 -LeafC 4 4 0 2 -LeafD 4 4 2 0 -``` - -LMM depths (`--lmm`), the root-to-MRCA distance. The diagonal is each leaf's -own root-to-tip length, and pairs meeting at the root score 0: - -``` - LeafA LeafB LeafC LeafD -LeafA 2.300 0.350 0.000 0.000 -LeafB 0.350 3.600 0.000 0.000 -LeafC 0.000 0.000 1.300 0.500 -LeafD 0.000 0.000 0.500 1.700 -``` - -PHYLIP lower triangle (`--lower`): taxa count, then one row per taxon: - -``` -4 -LeafA -LeafB 5.200 -LeafC 3.600 4.900 -LeafD 4.000 5.300 2.000 -``` - -## Detailed Use Cases - -### 1. Producing a Patristic Distance Matrix - -**Scenario**: You have a large set of bacterial genomes, build a phylogenetic tree with reliable branch lengths (e.g., using RAxML or IQ-TREE), and now need the pairwise patristic distances to feed into clustering, PCoA, or hierarchical analyses. - -**Command**: +# Bioconda +conda install -c bioconda distree -```bash -./distree tree.nwk -o patristic.tsv +# From source +cargo build --release ``` -**Why**: Downstream tools like SciKit-Learn (for MDS) or R's `ape::cmdscale()` expect a distance matrix. Patristic distances reflect evolutionary time or change. - -### 2. Computing Topological Distances Only - -**Scenario**: You have a phylogenetic tree where branch lengths come from different sources or are not directly comparable (e.g., concatenated multi-locus data). You prefer to measure closeness by number of shared nodes instead. +Prebuilt binaries for Linux and macOS, x86-64 and arm64, are on the +[releases page](https://github.com/PathoGenOmics-Lab/distree/releases). -**Command**: +## Quick start ```bash -./distree --topology tree.nwk -o topo_distances.tsv -``` +# Patristic distances, as a tab-separated matrix +distree tree.nwk -o distances.tsv -**Why**: Topological distances emphasize tree structure without scaling by substitution rate or time. Useful when comparing tree shapes or for rapid, coarse clustering when exact branch lengths are noisy. +# Six decimals across eight threads, PHYLIP lower triangle +distree tree.nwk --lower -p 6 -t 8 -o distances.phy -### 3. Generating an LMM (Var-Covar) Matrix for Comparative Methods +# Edge counts instead of branch lengths +distree tree.nwk --topology -o topology.tsv -**Scenario**: You are performing phylogenetic generalized least squares (PGLS) in R (`caper::pgls` or `nlme::gls`) and need the phylogenetic variance-covariance matrix. Each entry C\[i,j] is the distance from root to MRCA(i, j). +# The variance-covariance matrix for PGLS, midpoint-rooted +distree tree.nwk --midpoint --lmm -o varcovar.tsv -**Command**: +# A NumPy array: exact, half the size of text, and 2.6x faster +distree tree.nwk --npy -o distances.npy -```bash -./distree --lmm tree.nwk -o varcovar.tsv -``` - -**Why**: In comparative models, traits shared due to common ancestry introduce covariance. This LMM matrix directly encodes that covariance structure for all taxa. - -### 4. Using Midpoint Rooting Before Distance Extraction +# The condensed vector scipy.cluster.hierarchy reads directly +distree tree.nwk --lower --npy -o condensed.npy -**Scenario**: Your input tree is unrooted or rooted arbitrarily (e.g., by outgroup choice). You want a symmetric distance matrix that does not depend on outgroup, so you midpoint-root the tree first. +# Just the samples in a list, from a gzipped tree +distree tree.nwk.gz --taxa cohort.txt --stats -o cohort.tsv -**Command**: - -```bash -./distree --midpoint tree.nwk -o midrooted_distances.tsv +# From stdin +gunzip -c tree.nwk.gz | distree - -o distances.tsv ``` -**Why**: Midpoint rooting places the root at a balanced position. Downstream patristic or LMM calculations become more interpretable if the root is centrally placed. +## What it does -### 5. Downstream Dimensionality Reduction & Visualization +| | | +|:--|:--| +| **Patristic** | The sum of branch lengths on the path between two tips | +| **Topological** | The number of edges on that path, ignoring branch lengths | +| **Variance-covariance** | The root-to-MRCA distance for each pair, the `C` matrix of a PGLS or phylogenetic mixed model | +| **Rooting** | Optional midpoint rooting, which matters for `--lmm` and cannot matter for the other two | +| **Formats** | Square TSV, PHYLIP lower triangle, or a NumPy `.npy` array | +| **Input** | Newick from a file or stdin, plain or gzipped, with quoted labels, polytomies, NHX and BEAST comments, and UTF-8 tip names | +| **Subsets** | `--taxa` restricts the matrix to a list of tips, with the distances the full tree gives | +| **Scale** | Rows stream out as they are computed; memory tracks the tips, not the matrix | +| **Parallelism** | Distances and their formatting both spread across cores | -**Scenario**: You intend to visualize relationships among taxa via MDS or t-SNE. You need a distance matrix as input. +It works on the tree you give it: it does not build one, does not read an +alignment, and does not cluster the matrix for you. It also does not guess. A +truncated tree, a file holding several trees, an unclosed quote or a label with +a tab in it are rejected with the position, rather than turned into a matrix +that looks right and is not. -**Command**: +## Performance -```bash -./distree tree.nwk | Rscript -e "dist <- as.matrix(read.table('file:///dev/stdin', header=TRUE, row.names=1)); mds <- cmdscale(dist); plot(mds)" -``` +Balanced trees, `-p 6 --lower`, release build on an Apple M4 Pro with 14 cores. -**Why**: Feeding the distance matrix directly into R for MDS (multidimensional scaling) or UMAP allows you to view clustering of taxa in two or three dimensions. +| Tips | Cells | Time | Peak memory | +|--:|--:|--:|--:| +| 1,000 | 0.5 M | 0.00 s | 11 MB | +| 8,000 | 32 M | 0.13 s | 32 MB | +| 20,000 | 200 M | 1.02 s | 35 MB | -## Performance and Resource Considerations +Memory flattens out because nothing holds the matrix: past a few thousand tips +it is the LCA table plus one batch of rows. `--npy` is another 2.6x on top, at +full precision and half the file size. Full numbers, thread scaling and the +memory arithmetic are +[here](https://pathogenomics-lab.github.io/distree/how-it-works/performance/). -* **Memory**: `distree` streams one row at a time. At any given moment, only a single vector of length N (number of leaves) resides in memory, plus O(M log M) for LCA structures, where M is the total number of nodes. For trees with tens of thousands of taxa, memory usage remains low. +## Correctness -* **Parallelism**: Each row's distance computations are parallelized across available CPU cores via Rayon. For N taxa, computing N rows (each of size N) takes O(N^2 / #cores) time. +Distances are cross-validated against R's +[ape](https://cran.r-project.org/package=ape) over generated trees: +`cophenetic.phylo` for patristic, `vcv.phylo` for variance-covariance, +`cophenetic.phylo` over unit branch lengths for edge counts, and +`phangorn::midpoint` for the rooting. All four agree to under 1e-9, which is the +text round-trip, and exactly for edge counts. The parser and the pipeline are +fuzzed. See +[Contributing](https://pathogenomics-lab.github.io/distree/about/contributing/). -* **Disk I/O**: If writing to a file via `--output`, a buffered writer (`BufWriter`) minimizes I/O calls. Streaming directly to stdout also remains efficient. +## Citation -## Troubleshooting and Tips +> Ruiz-Rodriguez P, Coscolla M. *distree: distance matrices from a phylogeny.* +> PathoGenOmics Lab. [doi:10.5281/zenodo.16811766](https://doi.org/10.5281/zenodo.16811766) -* **Invalid Newick**: `distree` errors out rather than guessing, and the message names the offending position. It rejects unbalanced parentheses (a truncated file would otherwise produce a matrix with quietly wrong branch lengths), unclosed quotes and comments, and anything left over after the tree. The trailing semicolon is optional. -* **One tree per file**: A file holding several trees (bootstrap replicates, a posterior sample) is rejected instead of silently using the first one. Split it first. -* **Whitespace in Labels**: Leaf labels may contain spaces if they are enclosed in single or double quotes in the Newick file (e.g., `'Taxon A':1.0`). A doubled quote inside a quoted label is the escape for a literal one (`'it''s'` → `it's`). Tabs, newlines and carriage returns in labels are rejected outright, as they would split a row and corrupt the output; replace them with underscores before running distree. -* **Non-ASCII labels**: Accents, Greek letters and CJK text are passed through unchanged, so labels keep matching the sample names used downstream. -* **NHX and BEAST annotations**: Bracket-enclosed metadata (`[&&NHX:...]`, `[&rate=...]`) is silently skipped, before or after the label, as is the `[&R]` / `[&U]` rooting marker that IQ-TREE, MrBayes and BEAST write at the start of the file. Branch lengths and labels are preserved. -* **Negative branch lengths**: Reported as a warning and carried through to the output, so a neighbour-joining tree can yield a negative distance. They also make `--midpoint` unreliable, since locating the longest path assumes non-negative lengths. -* **Choosing Distance Type**: +Please record the version and the mode you used: a patristic matrix, a +topological one and a variance-covariance matrix are three different objects. - * Use `--lmm` if performing phylogenetic comparative analyses (e.g., trait evolution, PGLS). - * Use default patristic distances for standard evolutionary distance-based methods (e.g., Neighbor-Joining, clustering). - * Use `--topology` for quick, coarse tree-shape comparisons when branch lengths are unreliable. -* **Large Trees**: For very large trees (e.g., >50,000 leaves), ensure you have enough CPU cores. You may run `distree` on a compute node or multi-core server to exploit parallel speedups. - ---- +## License -*distree* is a versatile tool for extracting various phylogenetic distance matrices from large trees. By combining midpoint rooting, multiple distance modes, and parallel streaming, it caters to evolutionary, comparative, and clustering analyses without overwhelming memory. +[GPL-3.0](https://github.com/PathoGenOmics-Lab/distree/blob/main/LICENSE). ---