From 029dedd2c8664b614ba64599879a789c0988f3d8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:34:09 +0100 Subject: [PATCH 01/19] refactor(ancestry): og archaic and og ancient in Rust, with a chain liftover og archaic table|strict|genotypes|report and og ancient select|lift|person-geno|align|report port bin/archaic_report.py and bin/ancient_report.py. og-intervals gains chain, the UCSC chain reader with a single-position lift. Bead: pgp-bs9u.21 --- rust/Cargo.lock | 1 + rust/crates/og-ancestry/Cargo.toml | 3 +- rust/crates/og-ancestry/src/ancient.rs | 1569 +++++++++++++++++++++ rust/crates/og-ancestry/src/archaic.rs | 1728 ++++++++++++++++++++++++ rust/crates/og-ancestry/src/lib.rs | 2 + rust/crates/og-intervals/src/chain.rs | 376 ++++++ rust/crates/og-intervals/src/lib.rs | 3 + rust/og/src/main.rs | 12 + 8 files changed, 3693 insertions(+), 1 deletion(-) create mode 100644 rust/crates/og-ancestry/src/ancient.rs create mode 100644 rust/crates/og-ancestry/src/archaic.rs create mode 100644 rust/crates/og-intervals/src/chain.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 86841ba6c..c25f174d4 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -601,6 +601,7 @@ name = "og-ancestry" version = "0.1.0" dependencies = [ "flate2", + "og-intervals", "og-io", "og-tsv", "pycompat", diff --git a/rust/crates/og-ancestry/Cargo.toml b/rust/crates/og-ancestry/Cargo.toml index b27b4bbfa..4406d01ba 100644 --- a/rust/crates/og-ancestry/Cargo.toml +++ b/rust/crates/og-ancestry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-ancestry" -description = "Step 26b, fine-grained ancestry: og ancestry-fine panel-map, check-map, order, sample-bed, chroms and report" +description = "Ancestry steps: og ancestry-fine (26b), og archaic (43) and og ancient (44)" version.workspace = true edition.workspace = true license.workspace = true @@ -9,6 +9,7 @@ publish.workspace = true [dependencies] pycompat.workspace = true og-io.workspace = true +og-intervals.workspace = true og-tsv.workspace = true flate2.workspace = true diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs new file mode 100644 index 000000000..612a71212 --- /dev/null +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -0,0 +1,1569 @@ +//! Step 44, ancient similarity (`docs/44-ancient.md`): where a genome falls +//! among ancient genomes on a PCA of present-day West Eurasians (smartpca, +//! EIGENSOFT), with the ancient samples of the Allen Ancient DNA Resource +//! (AADR) projected onto the same axes, as `og ancient `. +//! +//! - `select` (setup): reads the AADR `.anno` and `.ind` files and chooses +//! the people: the present-day West Eurasians of [`MODERN_GROUPS`] build +//! the axes, the ancient West Eurasians with enough sites are projected. +//! - `lift` (setup): moves the 1240K sites of the AADR `.snp` file from +//! GRCh37 to GRCh38 with UCSC's chain file ([`og_intervals::chain`]) and +//! keeps a site only when one of its two alleles is the GRCh38 base. +//! - `person-geno` (step 44): the person's genotype at each site, appended +//! as one more column to the present-day panel's EIGENSTRAT `.geno`, +//! streamed row by row. +//! - `align` (setup, step 44): checks that the axes of two smartpca runs on +//! the same panel agree, up to the sign of each. +//! - `report` (step 44): the nearest ancient populations, the summary and +//! the plot. +//! +//! Each command writes what `bin/ancient_report.py` wrote (checked by +//! `tests/parity`). Deliberate changes: every output is written under a +//! temporary name and renamed into place (the Python wrote the population +//! list, `ancients.tsv`, the stats and the text report in place); a gzip +//! input is told by its first bytes, not its name; a `.fai` that does not +//! match its FASTA stops `lift`; and malformed input is an `ERROR:` line +//! and exit 1, never a traceback. + +use super::{ArgError, Args, Fail, Pending, Result, Spec, cannot_read, int, repr, split_ws}; +use og_intervals::chain::Chain; +use og_intervals::fasta::Indexed; +use og_io::{Lines, Utf8}; +use pycompat::fmt::{fixed, int_grouped, int_str, ljust, rjust, round_int}; +use pycompat::text::{html_escape, strip, upper}; +use std::collections::{BTreeMap, HashMap}; +use std::io::{BufRead, Write}; +use std::path::{Path, PathBuf}; + +/// Present-day groups of the AADR 1240K file (Group ID, as written in the +/// `.anno` file of v66.p1) that build the axes: Europe, the Caucasus and +/// the Near East. North African groups and groups with recent East +/// Eurasian ancestry are left out, so the axes describe West Eurasian +/// variation only. +pub const MODERN_GROUPS: [&str; 41] = [ + "Abkhasian", + "Adygei", + "Albanian", + "Armenian", + "Basque", + "BedouinA", + "BedouinB", + "Bulgarian", + "CEU", + "Chechen", + "Czech", + "Druze", + "English", + "Estonian", + "FIN", + "Finnish", + "French", + "GBR", + "Georgian", + "Greek", + "Greek_Crete", + "Hungarian", + "IBS", + "Icelandic", + "Iranian", + "Italian_North", + "Jew_Iraqi", + "Jordanian", + "Lezgin", + "Norwegian", + "Orcadian", + "Ossetian", + "Palestinian", + "Polish", + "Russian", + "Sardinian", + "Scottish", + "Spanish", + "Spanish_North", + "TSI", + "Turkish", +]; +const MODERN_DATA_TYPE: &str = "Shotgun.diploid"; +/// Ancient samples projected: dated, passing the AADR's own assessment, +/// with this many of the 1240K autosomal sites covered, inside West Eurasia. +const MIN_ANCIENT_SNPS: i64 = 100_000; +const LAT: (f64, f64) = (25.0, 75.0); +const LON: (f64, f64) = (-25.0, 75.0); +const PASSING: [&str; 3] = ["PASS", "PROVISIONAL_PASS", "MERGE_PASS"]; +/// The label of a projected ancient person in the `.ind` and `.evec` files. +pub const ANCIENT_LABEL: &str = "Ancient"; +/// The label of the person in the run's `.evec`. +pub const PERSON_LABEL: &str = "Person"; +/// PCs the distance uses, each weighted by the square root of its eigenvalue. +const DISTANCE_PCS: i64 = 4; +const TOP: usize = 20; +/// Two runs on the same panel give the same axes; a PC whose present-day +/// coordinates correlate less than this between the two is a mismatch. +const MIN_AXIS_CORRELATION: f64 = 0.999; +const CAVEAT: &str = "Closeness on a PCA is genetic similarity, not descent: a genome near an ancient group \ +shares ancestry with it, it does not descend from those people."; +const QPADM_NOTE: &str = "qpAdm models of ancestry proportions are not part of this step."; + +/// The `.anno` columns read, matched by the start of the header cell (the +/// AADR writes long explanations after the name). +const ANNO_COLUMNS: [(Col, &str); 13] = [ + (Col::Id, "Genetic ID"), + (Col::Individual, "Individual ID"), + (Col::DateBp, "Date mean in BP"), + (Col::DateSd, "Date standard deviation in BP"), + (Col::FullDate, "Full Date"), + (Col::Group, "Group ID"), + (Col::Locality, "Locality"), + (Col::Country, "Political Entity"), + (Col::Lat, "Latitude"), + (Col::Lon, "Longitude"), + (Col::DataType, "Data type"), + ( + Col::Snps, + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", + ), + (Col::Assessment, "ASSESSMENT"), +]; + +/// A column of the `.anno` file this tool reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Col { + Id, + Individual, + DateBp, + DateSd, + FullDate, + Group, + Locality, + Country, + Lat, + Lon, + DataType, + Snps, + Assessment, +} + +/// One `.anno` row: the 13 cells read, stripped, in [`ANNO_COLUMNS`] order. +#[derive(Debug, Clone)] +struct Anno([String; 13]); + +impl Anno { + fn get(&self, c: Col) -> &str { + &self.0[c as usize] + } +} + +/// A text file read line by line as Python's text mode reads it. +struct Text { + path: PathBuf, + lines: Lines>, + line: String, +} + +impl Text { + fn open(path: &str, utf8: Utf8) -> Result { + let path = PathBuf::from(path); + let input = og_io::open(&path).map_err(|e| cannot_read(&path, &e))?; + Ok(Text { + path, + lines: Lines::new(input, utf8), + line: String::new(), + }) + } + + /// The next line with its `\n`, or `None` at the end. + fn raw(&mut self) -> Result> { + let more = self + .lines + .read_line(&mut self.line) + .map_err(|e| cannot_read(&self.path, &e))?; + Ok(more.then_some(self.line.as_str())) + } + + /// The next line without its `\n`. + fn next(&mut self) -> Result> { + Ok(self.raw()?.map(|l| l.strip_suffix('\n').unwrap_or(l))) + } +} + +/// Python's `float(s)` of a finite value, or `None`. +fn to_float(s: &str) -> Option { + pycompat::num::parse_float(s).ok().filter(|v| v.is_finite()) +} + +/// Python's `float(s)`, or the input error that names the file. +fn need_float(s: &str, path: &Path) -> Result { + pycompat::num::parse_float(s) + .map_err(|_| Fail(format!("{}: {} is not a number", path.display(), repr(s)))) +} + +/// Python's `f >= i` for a float and an int: exact, whatever their sizes. +fn float_ge_int(f: f64, i: i64) -> bool { + let fi = i as f64; + if f != fi { + return f > fi; + } + // f equals the float nearest i, so it is a whole number. + (f as i128) >= i128::from(i) +} + +/// `" ".join(str(s).replace("\t", " ").split()) or "."`. +fn clean(s: &str) -> String { + let out = split_ws(s).collect::>().join(" "); + if out.is_empty() { ".".to_owned() } else { out } +} + +// --- select ----------------------------------------------------------------------- + +/// The index of each read column in the `.anno` header. +fn anno_columns(header: &[&str]) -> Result<[usize; 13]> { + let cells: Vec<&str> = header + .iter() + .map(|c| strip(strip(strip(c, None), Some("\"")), None)) + .collect(); + let mut out = [0usize; 13]; + for (k, (col, name)) in ANNO_COLUMNS.iter().enumerate() { + // ASSESSMENT must not match ASSESSMENT WARNINGS: the exact cell first. + let exact = cells.iter().position(|c| c == name); + let hit = if *col == Col::Assessment { + exact + } else { + exact.or_else(|| cells.iter().position(|c| c.starts_with(name))) + }; + out[k] = hit.ok_or_else(|| Fail(format!("the .anno file has no column '{name}'")))?; + } + Ok(out) +} + +fn passing(assessment: &str) -> bool { + let a = upper(strip(assessment, None)); + PASSING.iter().any(|p| a.starts_with(p)) +} + +/// What [`choose`] keeps of one genetic id. +#[derive(Debug, Clone)] +enum Chosen { + /// A present-day reference person, with their group. + Modern(String), + /// An ancient person to project, with their row. + Ancient(Box), +} + +/// The present-day reference people and the ancient people to project, +/// by genetic id, one genetic id per individual (the one with the most +/// sites). Rows are streamed from the `.anno` file. +fn choose(path: &str, min_snps: i64) -> Result> { + let mut t = Text::open(path, Utf8::Replace)?; + let Some(head) = t.next()? else { + return Err(Fail(format!("{path} is empty"))); + }; + let header: Vec<&str> = head.split('\t').collect(); + let col = anno_columns(&header)?; + let widest = *col.iter().max().expect("13 columns"); + // (kind is modern, individual) -> (sites, row), in first-seen order. + let mut index: HashMap<(bool, String), usize> = HashMap::new(); + let mut best: Vec<(f64, Anno, bool)> = Vec::new(); + while let Some(line) = t.next()? { + let cells: Vec<&str> = line.split('\t').collect(); + if line.is_empty() || cells.len() <= widest { + continue; + } + let r = Anno(col.map(|i| strip(cells[i], None).to_owned())); + if !passing(r.get(Col::Assessment)) { + continue; + } + let bp = to_float(r.get(Col::DateBp)); + let snps = to_float(r.get(Col::Snps)).unwrap_or(0.0); + let modern = if bp == Some(0.0) + && MODERN_GROUPS.contains(&r.get(Col::Group)) + && r.get(Col::DataType) == MODERN_DATA_TYPE + { + true + } else if bp.is_some_and(|b| b > 0.0) && float_ge_int(snps, min_snps) { + let (Some(lat), Some(lon)) = (to_float(r.get(Col::Lat)), to_float(r.get(Col::Lon))) + else { + continue; + }; + if !(LAT.0 <= lat && lat <= LAT.1 && LON.0 <= lon && lon <= LON.1) { + continue; + } + false + } else { + continue; + }; + let who = if r.get(Col::Individual).is_empty() { + r.get(Col::Id) + } else { + r.get(Col::Individual) + }; + let key = (modern, who.to_owned()); + match index.get(&key) { + None => { + index.insert(key, best.len()); + best.push((snps, r, modern)); + } + Some(&i) if snps > best[i].0 => best[i] = (snps, r, modern), + Some(_) => {} + } + } + let mut out = HashMap::new(); + for (_, r, modern) in best { + let id = r.get(Col::Id).to_owned(); + let c = if modern { + Chosen::Modern(r.get(Col::Group).to_owned()) + } else { + Chosen::Ancient(Box::new(r)) + }; + out.insert(id, c); + } + Ok(out) +} + +const SELECT: Spec = Spec { + verb: "select", + usage: "usage: og ancient select --anno ANNO --ind IND --out-ind OUT_IND --poplist POPLIST --ancients ANCIENTS [--min-snps MIN_SNPS]\n", + options: &[ + "--anno", + "--ind", + "--out-ind", + "--poplist", + "--ancients", + "--min-snps", + ], + required: &["--anno", "--ind", "--out-ind", "--poplist", "--ancients"], +}; + +fn cmd_select(a: &Args, out: &mut dyn Write) -> Result<()> { + let min_snps = a + .int("--min-snps", MIN_ANCIENT_SNPS) + .unwrap_or(MIN_ANCIENT_SNPS); + let chosen = choose(a.req("--anno"), min_snps)?; + let (mut n_mod, mut n_anc) = (0u64, 0u64); + let mut groups: std::collections::BTreeSet = Default::default(); + let mut ind = String::new(); + let mut t = Text::open(a.req("--ind"), Utf8::Strict)?; + while let Some(line) = t.next()? { + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 3 { + continue; + } + let label = match chosen.get(c[0]) { + Some(Chosen::Modern(g)) => { + n_mod += 1; + groups.insert(g.clone()); + g.as_str() + } + Some(Chosen::Ancient(_)) => { + n_anc += 1; + ANCIENT_LABEL + } + None => "Ignore", + }; + ind += &format!("{}\t{}\t{label}\n", c[0], c[1]); + } + let missing: Vec<&str> = { + let mut m: Vec<&str> = MODERN_GROUPS + .iter() + .copied() + .filter(|g| !groups.contains(*g)) + .collect(); + m.sort_unstable(); + m + }; + if n_mod < 50 || n_anc < 1 { + return Err(Fail(format!( + "the .ind file matches {n_mod} present-day reference people and {n_anc} ancient ones; \ + is it the .ind file of the same release as the .anno file?" + ))); + } + let mut files = Pending::new(); + files.put(PathBuf::from(a.req("--out-ind")), ind.as_bytes())?; + let pops: String = groups.iter().map(|g| format!("{g}\n")).collect(); + files.put(PathBuf::from(a.req("--poplist")), pops.as_bytes())?; + let mut anc = + String::from("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n"); + let sorted: BTreeMap<&String, &Chosen> = chosen.iter().collect(); + for c in sorted.values() { + let Chosen::Ancient(r) = c else { continue }; + let cells: Vec = [ + Col::Id, + Col::Group, + Col::DateBp, + Col::DateSd, + Col::FullDate, + Col::Locality, + Col::Country, + Col::Lat, + Col::Lon, + Col::Snps, + ] + .iter() + .map(|k| clean(r.get(*k))) + .collect(); + anc += &cells.join("\t"); + anc.push('\n'); + } + files.put(PathBuf::from(a.req("--ancients")), anc.as_bytes())?; + files.commit_all()?; + let suffix = if !missing.is_empty() && missing.len() <= 10 { + format!(" (not in this file: {})", missing.join(", ")) + } else if !missing.is_empty() { + format!( + " ({} of the {} groups are not in this file)", + missing.len(), + MODERN_GROUPS.len() + ) + } else { + String::new() + }; + let _ = writeln!( + out, + "Present-day reference people: {n_mod} in {} groups{suffix}", + groups.len() + ); + let _ = writeln!( + out, + "Ancient people projected: {n_anc} (at least {} sites, West Eurasia)", + int_grouped(i128::from(min_snps)) + ); + Ok(()) +} + +// --- lift ------------------------------------------------------------------------- + +/// A base's complement, for the four bases. +fn complement(b: &str) -> Option<&'static str> { + match b { + "A" => Some("T"), + "C" => Some("G"), + "G" => Some("C"), + "T" => Some("A"), + _ => None, + } +} + +/// The chromosome column of an EIGENSOFT `.snp` line: 1 to 22, or `None`. +fn eigen_chrom(c: &str) -> Option { + int(c).filter(|n| (1..=22).contains(n)) +} + +const LIFT: Spec = Spec { + verb: "lift", + usage: "usage: og ancient lift --snp SNP --chain CHAIN --fasta FASTA --out OUT\n", + options: &["--snp", "--chain", "--fasta", "--out"], + required: &["--snp", "--chain", "--fasta", "--out"], +}; + +fn cmd_lift(a: &Args, out: &mut dyn Write) -> Result<()> { + let chain_path = a.req("--chain"); + let input = og_io::open(chain_path).map_err(|e| cannot_read(Path::new(chain_path), &e))?; + let chain = Chain::read(input).map_err(|e| Fail(format!("{chain_path}: {e}")))?; + let fasta_path = a.req("--fasta"); + let mut fasta = Indexed::open(fasta_path).map_err(|e| Fail(format!("{fasta_path}: {e}")))?; + let mut kept: Vec<(i64, i64, String, &'static str, &'static str)> = Vec::new(); + let mut dropped: BTreeMap<&str, u64> = BTreeMap::new(); + let mut n: u64 = 0; + let snp_path = a.req("--snp"); + let mut t = Text::open(snp_path, Utf8::Replace)?; + let mut lineno = 0u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 6 { + continue; + } + n += 1; + let Some(chrom) = eigen_chrom(c[1]) else { + *dropped.entry("not an autosome").or_default() += 1; + continue; + }; + let (u1, u2) = (upper(c[4]), upper(c[5])); + let (Some(k1), Some(k2)) = (complement(&u1), complement(&u2)) else { + *dropped + .entry("alleles not two different bases") + .or_default() += 1; + continue; + }; + if u1 == u2 { + *dropped + .entry("alleles not two different bases") + .or_default() += 1; + continue; + } + let pos = pycompat::num::parse_float(c[3]) + .ok() + .filter(|p| p.is_finite()) + .map(f64::trunc) + .filter(|p| p.abs() < 9.0e18) + .ok_or_else(|| { + Fail(format!( + "{snp_path}: line {lineno}: position {} is not a number", + repr(c[3]) + )) + })? as i64; + let name = format!("chr{chrom}"); + let Some(hit) = chain.lift(&name, pos) else { + *dropped.entry("no single place on GRCh38").or_default() += 1; + continue; + }; + if hit.chrom != name { + *dropped.entry("moves to another chromosome").or_default() += 1; + continue; + } + let (a1, a2) = if hit.strand == "-" { + (k1, k2) + } else { + ( + complement(k1).expect("a base"), + complement(k2).expect("a base"), + ) + }; + let q_pos = hit.pos; + let base = if q_pos >= 1 { + fasta + .base(hit.chrom, (q_pos - 1) as u64) + .map_err(|e| Fail(format!("{fasta_path}: {e}")))? + } else { + None + }; + let base = base.map(|b| b.to_ascii_uppercase()); + if !base.is_some_and(|b| b == a1.as_bytes()[0] || b == a2.as_bytes()[0]) { + *dropped + .entry("neither allele is the GRCh38 base") + .or_default() += 1; + continue; + } + kept.push((chrom, q_pos, c[0].to_owned(), a1, a2)); + } + kept.sort_unstable(); + let mut text = String::from("snp\tchrom\tpos\tallele1\tallele2\n"); + for (chrom, pos, sid, a1, a2) in &kept { + text += &format!("{sid}\tchr{chrom}\t{pos}\t{a1}\t{a2}\n"); + } + let mut files = Pending::new(); + files.put(PathBuf::from(a.req("--out")), text.as_bytes())?; + files.commit_all()?; + let _ = writeln!( + out, + "Sites of the .snp file: {}; on GRCh38: {}", + int_grouped(i128::from(n)), + int_grouped(kept.len() as i128) + ); + for (why, k) in &dropped { + let _ = writeln!(out, " dropped, {why}: {}", int_grouped(i128::from(*k))); + } + if kept.is_empty() { + return Err(Fail( + "no site could be placed on GRCh38: check the chain file and the reference".into(), + )); + } + Ok(()) +} + +// --- person-geno ------------------------------------------------------------------ + +/// Copies of allele1 in a bcftools `%TGT` genotype, or `None` (missing, a +/// haploid or half call, or an allele that is neither of the two). +fn dosage(gt: &str, a1: &str, a2: &str) -> Option { + let mut parts = gt.split(['/', '|']); + let (Some(x), Some(y), None) = (parts.next(), parts.next(), parts.next()) else { + return None; + }; + let mut d = 0; + for al in [x, y] { + if al == a1 { + d += 1; + } else if al != a2 { + return None; + } + } + Some(d) +} + +/// The panel's GRCh38 sites: per (chromosome, position), the sites there +/// (id, allele1, allele2), packed in one array. +struct Sites { + chroms: HashMap, + at: HashMap<(u32, i64), Vec>, + sites: Vec<(Box, Box, Box)>, +} + +fn read_sites(path: &str) -> Result { + let mut s = Sites { + chroms: HashMap::new(), + at: HashMap::new(), + sites: Vec::new(), + }; + let mut t = Text::open(path, Utf8::Strict)?; + let _ = t.next()?; + let mut lineno = 1u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + continue; + } + let pos = int(c[2]).ok_or_else(|| { + Fail(format!( + "{path}: line {lineno}: position {} is not a whole number", + repr(c[2]) + )) + })?; + let next = s.chroms.len() as u32; + let chrom = *s.chroms.entry(c[1].to_owned()).or_insert(next); + s.at.entry((chrom, pos)) + .or_default() + .push(s.sites.len() as u32); + s.sites.push((c[0].into(), c[3].into(), c[4].into())); + } + Ok(s) +} + +const PERSON_GENO: Spec = Spec { + verb: "person-geno", + usage: "usage: og ancient person-geno --sites SITES --genotypes GENOTYPES --snp SNP --moderns-geno MODERNS_GENO --out OUT --stats STATS\n", + options: &[ + "--sites", + "--genotypes", + "--snp", + "--moderns-geno", + "--out", + "--stats", + ], + required: &[ + "--sites", + "--genotypes", + "--snp", + "--moderns-geno", + "--out", + "--stats", + ], +}; + +fn cmd_person_geno(a: &Args, out: &mut dyn Write) -> Result<()> { + let sites = read_sites(a.req("--sites"))?; + // The person's dosage by site id: a later line for the same site wins. + let mut dos: HashMap<&str, u8> = HashMap::new(); + let (mut called, mut other) = (0u64, 0u64); + let gpath = a.req("--genotypes"); + let mut t = Text::open(gpath, Utf8::Strict)?; + let mut lineno = 0u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + continue; + } + let pos = int(c[1]).ok_or_else(|| { + Fail(format!( + "{gpath}: line {lineno}: position {} is not a whole number", + repr(c[1]) + )) + })?; + let Some(&chrom) = sites.chroms.get(c[0]) else { + continue; + }; + let Some(here) = sites.at.get(&(chrom, pos)) else { + continue; + }; + let gt = upper(c[4]); + // quirk: a position the AADR holds two SNPs at is counted once per + // SNP in sites_genotyped, as the Python counted it. + for &i in here { + let (sid, a1, a2) = &sites.sites[i as usize]; + match dosage(>, a1, a2) { + None => other += 1, + Some(d) => { + dos.insert(sid, d); + called += 1; + } + } + } + } + let snp_path = a.req("--snp"); + let geno_path = a.req("--moderns-geno"); + let mut snp = Text::open(snp_path, Utf8::Strict)?; + let mut geno = Text::open(geno_path, Utf8::Strict)?; + let mut files = Pending::new(); + let out_path = PathBuf::from(a.req("--out")); + let shown = out_path.clone(); + let o = files.create(out_path)?; + let (mut n, mut used) = (0u64, 0u64); + let mut width: Option = None; + let mut row = String::new(); + while let Some(sline) = snp.next()? { + let Some(id) = split_ws(sline).next() else { + continue; + }; + let d = dos.get(id).copied(); + let g = geno.next()?.unwrap_or(""); + if g.is_empty() { + return Err(Fail(format!("{geno_path} has fewer lines than {snp_path}"))); + } + let w = g.chars().count(); + match width { + None => width = Some(w), + Some(want) if w != want => { + return Err(Fail(format!( + "{geno_path} line {} has {w} genotypes, not {want}", + n + 1 + ))); + } + Some(_) => {} + } + if d.is_some() { + used += 1; + } + row.clear(); + row.push_str(g); + match d { + None => row.push('9'), + Some(d) => row.push(char::from(b'0' + d)), + } + row.push('\n'); + o.write_all(row.as_bytes()) + .map_err(|e| super::cannot_write(&shown, &e))?; + n += 1; + } + if geno.raw()?.is_some() { + return Err(Fail(format!("{geno_path} has more lines than {snp_path}"))); + } + let genotyped = called + other; + files.put( + PathBuf::from(a.req("--stats")), + format!( + "panel_sites\t{n}\nsites_genotyped\t{genotyped}\nother_allele\t{other}\nsites_used\t{used}\n" + ) + .as_bytes(), + )?; + files.commit_all()?; + let _ = writeln!( + out, + "Sites of the panel: {}; genotyped from the gVCF: {}; used: {} (an allele that is neither of a site's two: {})", + int_grouped(i128::from(n)), + int_grouped(i128::from(genotyped)), + int_grouped(i128::from(used)), + int_grouped(i128::from(other)) + ); + if used == 0 { + return Err(Fail( + "none of the panel's sites was genotyped: is the gVCF on GRCh38 with chr names?".into(), + )); + } + Ok(()) +} + +// --- evec, align ------------------------------------------------------------------ + +/// A smartpca `.evec`: the eigenvalues, and each row's label and PCs in +/// file order (a repeated id keeps its first place and its last values). +struct Evec { + evals: Vec, + rows: Vec<(String, String, Vec)>, + index: HashMap, +} + +impl Evec { + fn get(&self, id: &str) -> Option<(&str, &[f64])> { + self.index.get(id).map(|&i| { + let r = &self.rows[i]; + (r.1.as_str(), r.2.as_slice()) + }) + } +} + +fn read_evec(path: &str) -> Result { + let mut e = Evec { + evals: Vec::new(), + rows: Vec::new(), + index: HashMap::new(), + }; + let p = Path::new(path); + let mut t = Text::open(path, Utf8::Strict)?; + while let Some(line) = t.next()? { + let c: Vec<&str> = split_ws(line).collect(); + let Some(first) = c.first() else { continue }; + if first.starts_with('#') { + if *first == "#eigvals:" { + e.evals = c[1..] + .iter() + .map(|x| need_float(x, p)) + .collect::>()?; + } + continue; + } + let label = c[c.len() - 1].to_owned(); + let pcs: Vec = if c.len() > 1 { + c[1..c.len() - 1] + .iter() + .map(|x| need_float(x, p)) + .collect::>()? + } else { + Vec::new() + }; + match e.index.get(*first) { + Some(&i) => e.rows[i] = ((*first).to_owned(), label, pcs), + None => { + e.index.insert((*first).to_owned(), e.rows.len()); + e.rows.push(((*first).to_owned(), label, pcs)); + } + } + } + if e.evals.is_empty() || e.rows.is_empty() { + return Err(Fail(format!( + "{path} is not a smartpca .evec file (no #eigvals: line or no rows)" + ))); + } + Ok(e) +} + +/// The Pearson correlation of two equal-length series, 0 when either is +/// flat; means by `fmean`, sums by the builtin `sum`. +fn correlation(x: &[f64], y: &[f64]) -> f64 { + let mx = pycompat::stats::fmean(x.iter().copied()).unwrap_or(f64::NAN); + let my = pycompat::stats::fmean(y.iter().copied()).unwrap_or(f64::NAN); + let sxy = pycompat::num::sum(x.iter().zip(y).map(|(a, b)| (a - mx) * (b - my))); + let sx = pycompat::num::sum(x.iter().map(|a| (a - mx) * (a - mx))).sqrt(); + let sy = pycompat::num::sum(y.iter().map(|b| (b - my) * (b - my))).sqrt(); + if sx > 0.0 && sy > 0.0 { + sxy / (sx * sy) + } else { + 0.0 + } +} + +/// The k-th PC of a row, or the error that names the row. +fn pc(pcs: &[f64], k: usize, id: &str, path: &str) -> Result { + pcs.get(k) + .copied() + .ok_or_else(|| Fail(format!("{path}: {id} has no PC{}", k + 1))) +} + +/// The sign that turns each of the run's PCs into the panel's, from the +/// present-day people both runs used to build the axes. +fn axis_signs( + panel: &Evec, + panel_path: &str, + run: &Evec, + run_path: &str, + n_pcs: usize, +) -> Result> { + let mut shared: Vec<&str> = run + .rows + .iter() + .filter(|(id, lab, _)| { + lab != ANCIENT_LABEL + && lab != PERSON_LABEL + && panel.get(id).is_some_and(|(pl, _)| pl == lab) + }) + .map(|(id, ..)| id.as_str()) + .collect(); + shared.sort_unstable(); + if shared.len() < 10 { + return Err(Fail(format!( + "only {} present-day people are in both runs; the panel and this run do not match", + shared.len() + ))); + } + let mut signs = Vec::with_capacity(n_pcs); + for k in 0..n_pcs { + let mut x = Vec::with_capacity(shared.len()); + let mut y = Vec::with_capacity(shared.len()); + for id in &shared { + x.push(pc(panel.get(id).expect("shared").1, k, id, panel_path)?); + y.push(pc(run.get(id).expect("shared").1, k, id, run_path)?); + } + let r = correlation(&x, &y); + if r.abs() < MIN_AXIS_CORRELATION { + return Err(Fail(format!( + "PC{} of this run correlates {} with the panel's over {} present-day people: \ + the axes differ. Install the panel again (setup.sh --ancient-panel).", + k + 1, + fixed(r, 4), + shared.len() + ))); + } + signs.push(if r > 0.0 { 1.0 } else { -1.0 }); + } + Ok(signs) +} + +const ALIGN: Spec = Spec { + verb: "align", + usage: "usage: og ancient align --panel-evec PANEL_EVEC --run-evec RUN_EVEC [--pcs PCS]\n", + options: &["--panel-evec", "--run-evec", "--pcs"], + required: &["--panel-evec", "--run-evec"], +}; + +fn cmd_align(a: &Args, out: &mut dyn Write) -> Result<()> { + let (pp, rp) = (a.req("--panel-evec"), a.req("--run-evec")); + let panel = read_evec(pp)?; + let run = read_evec(rp)?; + let pcs = a.int("--pcs", DISTANCE_PCS).unwrap_or(DISTANCE_PCS); + let signs = axis_signs(&panel, pp, &run, rp, usize::try_from(pcs).unwrap_or(0))?; + let words: Vec = signs + .iter() + .enumerate() + .map(|(i, s)| format!("PC{}{}", i + 1, if *s > 0.0 { '+' } else { '-' })) + .collect(); + let _ = writeln!(out, "Axes agree with the panel's: {}", words.join(" ")); + Ok(()) +} + +// --- report ----------------------------------------------------------------------- + +/// A whole float as Python prints the int of it. +fn whole(x: f64) -> String { + int_str(x) +} + +/// `int(round(x))` printed. +fn rounded(x: f64) -> String { + whole(round_int(x).unwrap_or(f64::NAN)) +} + +/// A date in years before 1950 as `N BCE` or `N CE`, the AADR's own +/// reading: BCE = BP - 1950 and CE = 1950 - BP, with 1950 BP as 1 BCE. +fn year_label(bp: f64) -> String { + let y = 1950.0 - round_int(bp).unwrap_or(0.0); + if y >= 1.0 { + format!("{} CE", whole(y)) + } else { + format!("{} BCE", whole(if -y > 1.0 { -y } else { 1.0 })) + } +} + +/// The span of a population's dates, oldest first: `5300-4900 BCE`, +/// `200-450 CE`, or `300 BCE-200 CE`. +fn range_label(old_bp: f64, young_bp: f64) -> String { + let (a, b) = (year_label(old_bp), year_label(young_bp)); + if a == b { + return a; + } + let (na, ea) = a.split_once(' ').expect("N ERA"); + let (nb, eb) = b.split_once(' ').expect("N ERA"); + if ea == eb { + format!("{na}-{nb} {ea}") + } else { + format!("{a}-{b}") + } +} + +/// An `ancients.tsv` row as `csv.DictReader` gives it: a short row's +/// missing cells are `None`. +struct AncientRow { + group: String, + date_bp: Option, + country: Option, + locality: Option, +} + +/// `ancients.tsv` by id, in file order (a repeated id keeps its first +/// place and its last values). +fn read_ancients(path: &str) -> Result> { + let mut t = Text::open(path, Utf8::Strict)?; + let mut rows: Vec<(String, AncientRow)> = Vec::new(); + let mut index: HashMap = HashMap::new(); + // DictReader: the first non-empty row is the header. + let mut names: Option> = None; + while let Some(line) = t.next()? { + if line.is_empty() { + continue; + } + let cells: Vec<&str> = line.split('\t').collect(); + let Some(names) = names.as_ref() else { + names = Some(cells.iter().map(|s| (*s).to_owned()).collect()); + continue; + }; + // The last column of a repeated name wins, as in a dict. + let field = |key: &str| -> Option { + let k = names.iter().rposition(|n| n == key)?; + cells.get(k).map(|s| (*s).to_owned()) + }; + if !names.iter().any(|n| n == "id") { + return Err(Fail(format!("{path} has no id column"))); + } + let id = field("id").unwrap_or_else(|| "None".to_owned()); + let r = AncientRow { + group: field("group").unwrap_or_else(|| "None".to_owned()), + date_bp: field("date_bp"), + country: field("country"), + locality: field("locality"), + }; + match index.get(&id) { + Some(&i) => rows[i].1 = r, + None => { + index.insert(id.clone(), rows.len()); + rows.push((id, r)); + } + } + } + Ok(rows) +} + +/// The most common non-empty value other than `.`, the first in sort +/// order on a tie; `.` when there is none. +fn most_common<'a>(values: impl Iterator>) -> String { + let mut counts: BTreeMap<&str, u64> = BTreeMap::new(); + for v in values.flatten() { + if !v.is_empty() && v != "." { + *counts.entry(v).or_default() += 1; + } + } + let top = counts.values().copied().max().unwrap_or(0); + counts + .into_iter() + .find(|(_, k)| *k == top) + .map_or_else(|| ".".to_owned(), |(v, _)| v.to_owned()) +} + +/// One ancient group, nearest first. +struct Near { + rank: usize, + group: String, + n: usize, + distance: f64, + pc1: f64, + pc2: f64, + date_bp: Option, + dates: String, + country: String, + locality: String, +} + +/// The ancient groups ranked by the weighted distance from the person to +/// the group's mean position on the first `n_pcs` PCs; the top [`TOP`] and +/// the number of groups. +fn nearest_populations( + person: &[f64], + panel: &Evec, + ancients: &[(String, AncientRow)], + n_pcs: usize, +) -> Result<(Vec, usize)> { + let w: Vec = panel + .evals + .iter() + .take(n_pcs) + .map(|v| if 0.0 > *v { 0.0 } else { *v }.sqrt()) + .collect(); + // Groups in first-seen order, each with its members. + let mut order: Vec<&str> = Vec::new(); + let mut groups: HashMap<&str, Vec<(&[f64], &AncientRow)>> = HashMap::new(); + for (gid, r) in ancients { + if let Some((label, pcs)) = panel.get(gid) + && label == ANCIENT_LABEL + { + let g = groups.entry(r.group.as_str()).or_insert_with(|| { + order.push(r.group.as_str()); + Vec::new() + }); + g.push((pcs, r)); + } + } + let mut rows = Vec::with_capacity(order.len()); + for g in order { + let members = &groups[g]; + let k = members[0].0.len().min(n_pcs); + let mut centre = Vec::with_capacity(k); + for i in 0..k { + let mut xs = Vec::with_capacity(members.len()); + for (pcs, _) in members { + xs.push(*pcs.get(i).ok_or_else(|| { + Fail(format!( + "a member of the ancient group {g} has no PC{}", + i + 1 + )) + })?); + } + centre.push(pycompat::stats::fmean(xs).map_err(|e| Fail(e.to_string()))?); + } + let Some(&pc1) = centre.first() else { + return Err(Fail(format!("the ancient group {g} has no PCs"))); + }; + let distance = pycompat::num::sum(person.iter().take(n_pcs).zip(¢re).zip(&w).map( + |((a, b), wi)| { + let d = wi * (a - b); + d * d + }, + )) + .sqrt(); + let dates: Vec = members + .iter() + .filter_map(|(_, r)| r.date_bp.as_deref().and_then(to_float)) + .collect(); + let (date_bp, dates_text) = if dates.is_empty() { + (None, ".".to_owned()) + } else { + let old = dates + .iter() + .copied() + .fold(dates[0], |m, d| if d > m { d } else { m }); + let young = dates + .iter() + .copied() + .fold(dates[0], |m, d| if d < m { d } else { m }); + ( + Some( + pycompat::stats::median(dates.iter().copied()) + .map_err(|e| Fail(e.to_string()))?, + ), + range_label(old, young), + ) + }; + rows.push(Near { + rank: 0, + group: g.to_owned(), + n: members.len(), + distance, + pc1, + pc2: centre.get(1).copied().unwrap_or(0.0), + date_bp, + dates: dates_text, + country: most_common(members.iter().map(|(_, r)| r.country.as_deref())), + locality: most_common(members.iter().map(|(_, r)| r.locality.as_deref())), + }); + } + rows.sort_by(|a, b| { + a.distance + .partial_cmp(&b.distance) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.group.cmp(&b.group)) + }); + for (i, r) in rows.iter_mut().enumerate() { + r.rank = i + 1; + } + let n = rows.len(); + rows.truncate(TOP); + Ok((rows, n)) +} + +/// Age bins of the plot: (oldest BP, colour, legend), oldest first. +const AGE_BINS: [(f64, &str, &str); 4] = [ + (f64::INFINITY, "#5aa9e6", "before 8000 BCE"), + (9950.0, "#7fc97f", "8000-4000 BCE"), + (5950.0, "#f0a35e", "4000-1000 BCE"), + (2950.0, "#e377c2", "after 1000 BCE"), +]; + +fn age_colour(bp: Option) -> &'static str { + let mut colour = AGE_BINS[0].1; + for (limit, c, _) in AGE_BINS { + if bp.is_some_and(|b| b < limit) { + colour = c; + } + } + colour +} + +/// Python's `min()` and `max()` of a non-empty list of floats. +fn py_min_max(xs: &[f64]) -> (f64, f64) { + let (mut lo, mut hi) = (xs[0], xs[0]); + for &x in &xs[1..] { + if x < lo { + lo = x; + } + if x > hi { + hi = x; + } + } + (lo, hi) +} + +/// The PC1/PC2 plane: present-day people grey, ancient people coloured by +/// age, the person a large ring, on a dark background. +fn svg_plot( + person: &[f64], + panel: &Evec, + ancients: &HashMap, + sample: &str, + nearest: &[Near], +) -> Result { + const WIDTH: i64 = 900; + const HEIGHT: i64 = 720; + const PAD: f64 = 0.04; + const LEFT: i64 = 60; + const RIGHT: i64 = 20; + const TOPM: i64 = 50; + const BOTTOM: i64 = 110; + let two = |id: &str, pcs: &[f64]| -> Result<(f64, f64)> { + match pcs { + [x, y, ..] => Ok((*x, *y)), + _ => Err(Fail(format!("{id} has fewer than two PCs to plot"))), + } + }; + let mut xs = Vec::with_capacity(panel.rows.len() + 1); + let mut ys = Vec::with_capacity(panel.rows.len() + 1); + for (id, _, pcs) in &panel.rows { + let (x, y) = two(id, pcs)?; + xs.push(x); + ys.push(y); + } + let (px0, py0) = two(sample, person)?; + xs.push(px0); + ys.push(py0); + let (x0, x1) = py_min_max(&xs); + let (y0, y1) = py_min_max(&ys); + let or_one = |d: f64| if d == 0.0 { 1.0 } else { d }; + let (dx, dy) = (or_one(x1 - x0), or_one(y1 - y0)); + let (pw, ph) = (WIDTH - LEFT - RIGHT, HEIGHT - TOPM - BOTTOM); + let (left, top, pwf, phf) = (LEFT as f64, TOPM as f64, pw as f64, ph as f64); + let x = |v: f64| left + ((v - x0) / dx * (1.0 - 2.0 * PAD) + PAD) * pwf; + let y = |v: f64| top + (1.0 - ((v - y0) / dy * (1.0 - 2.0 * PAD) + PAD)) * phf; + let e = |s: &str| html_escape(s).into_owned(); + let mut out = vec![ + format!( + "" + ), + "" + .to_owned(), + format!(""), + format!( + "{} among ancient West Eurasians (AADR, projected onto present-day PCs)", + e(sample) + ), + format!( + "" + ), + ]; + for (_, label, pcs) in &panel.rows { + if label != ANCIENT_LABEL { + out.push(format!( + "", + fixed(x(pcs[0]), 1), + fixed(y(pcs[1]), 1) + )); + } + } + for (gid, label, pcs) in &panel.rows { + if label == ANCIENT_LABEL + && let Some(r) = ancients.get(gid) + { + let c = age_colour(r.date_bp.as_deref().and_then(to_float)); + out.push(format!( + "", + fixed(x(pcs[0]), 1), + fixed(y(pcs[1]), 1) + )); + } + } + for r in nearest.iter().take(5) { + out.push(format!( + "{}", + fixed(x(r.pc1) + 6.0, 1), + fixed(y(r.pc2) - 4.0, 1), + e(&r.group) + )); + } + let (px, py) = (x(px0), y(py0)); + out.push(format!( + "", + fixed(px, 1), + fixed(py, 1) + )); + out.push(format!( + "", + fixed(px, 1), + fixed(py, 1) + )); + out.push(format!( + "{}", + fixed(px + 12.0, 1), + fixed(py + 4.0, 1), + e(sample) + )); + out.push(format!( + "PC1", + fixed(left + pwf / 2.0, 0), + TOPM + ph + 22 + )); + let mid = fixed(top + phf / 2.0, 0); + out.push(format!( + "PC2" + )); + let (mut lx, ly) = (LEFT, TOPM + ph + 48); + out.push(format!( + "present day", + lx + 5, + ly - 4, + lx + 14 + )); + lx += 120; + for (_, c, name) in AGE_BINS { + out.push(format!( + "{}", + lx + 5, + ly - 4, + lx + 14, + e(name) + )); + lx += 150; + } + out.push(format!( + "{}", + ly + 26, + e(CAVEAT) + )); + out.push("".to_owned()); + Ok(out.join("\n") + "\n") +} + +const REPORT: Spec = Spec { + verb: "report", + usage: "usage: og ancient report --panel-evec PANEL_EVEC --run-evec RUN_EVEC --ancients ANCIENTS --sample SAMPLE --release RELEASE [--stats STATS] --out-tsv OUT_TSV --out-info OUT_INFO --out-txt OUT_TXT --out-svg OUT_SVG\n", + options: &[ + "--panel-evec", + "--run-evec", + "--ancients", + "--sample", + "--release", + "--stats", + "--out-tsv", + "--out-info", + "--out-txt", + "--out-svg", + ], + required: &[ + "--panel-evec", + "--run-evec", + "--ancients", + "--sample", + "--release", + "--out-tsv", + "--out-info", + "--out-txt", + "--out-svg", + ], +}; + +fn cmd_report(a: &Args, out: &mut dyn Write) -> Result<()> { + let (pp, rp) = (a.req("--panel-evec"), a.req("--run-evec")); + let panel = read_evec(pp)?; + let run = read_evec(rp)?; + let sample = a.req("--sample"); + let Some((PERSON_LABEL, pcs)) = run.get(sample) else { + return Err(Fail(format!( + "{rp} has no row for {sample} labelled {PERSON_LABEL}" + ))); + }; + let n_pcs = (DISTANCE_PCS as usize) + .min(panel.evals.len()) + .min(pcs.len()); + let signs = axis_signs(&panel, pp, &run, rp, n_pcs.max(2))?; + let person: Vec = signs.iter().zip(pcs).map(|(s, v)| s * v).collect(); + let ancients = read_ancients(a.req("--ancients"))?; + let (nearest, n_groups) = nearest_populations(&person, &panel, &ancients, n_pcs)?; + let mut stats: HashMap = HashMap::new(); + if let Some(sp) = a.get("--stats").filter(|s| !s.is_empty()) { + let mut t = Text::open(sp, Utf8::Strict)?; + while let Some(line) = t.next()? { + let (k, v) = line.split_once('\t').unwrap_or((line, "")); + stats.insert(k.to_owned(), v.to_owned()); + } + } + let stat = |k: &str| stats.get(k).map_or(".", String::as_str).to_owned(); + let n_anc = panel.rows.iter().filter(|r| r.1 == ANCIENT_LABEL).count(); + let n_mod = panel + .rows + .iter() + .filter(|r| r.1 != ANCIENT_LABEL && r.1 != PERSON_LABEL) + .count(); + let mut files = Pending::new(); + let mut tsv = + String::from("rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n"); + for r in &nearest { + tsv += &format!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n", + r.rank, + clean(&r.group), + r.n, + fixed(r.distance, 5), + fixed(r.pc1, 5), + fixed(r.pc2, 5), + r.date_bp.map_or_else(|| ".".to_owned(), rounded), + r.dates, + clean(&r.country), + clean(&r.locality) + ); + } + files.put(PathBuf::from(a.req("--out-tsv")), tsv.as_bytes())?; + let release = a.req("--release"); + let p1 = person.first().copied().unwrap_or(f64::NAN); + let p2 = person.get(1).copied().unwrap_or(f64::NAN); + let info: [(&str, String); 9] = [ + ("release", release.to_owned()), + ("sites_used", stat("sites_used")), + ("panel_sites", stat("panel_sites")), + ("present_day_people", n_mod.to_string()), + ("ancient_people", n_anc.to_string()), + ("ancient_groups", n_groups.to_string()), + ("pcs_in_distance", n_pcs.to_string()), + ("pc1", fixed(p1, 5)), + ("pc2", fixed(p2, 5)), + ]; + let mut text = String::from("key\tvalue\n"); + for (k, v) in &info { + text += &format!("{k}\t{v}\n"); + } + files.put(PathBuf::from(a.req("--out-info")), text.as_bytes())?; + let by_id: HashMap = + ancients.iter().map(|(id, r)| (id.clone(), r)).collect(); + let svg = svg_plot(&person, &panel, &by_id, sample, &nearest)?; + let mut lines = vec![ + format!( + "Ancient populations nearest to {sample} (AADR {release}, smartpca with lsqproject)" + ), + format!( + "Sites used: {} of {}; axes from {n_mod} present-day West Eurasians; {n_anc} ancient people in {n_groups} groups projected.", + stat("sites_used"), + stat("panel_sites") + ), + format!( + "Distance: the first {n_pcs} PCs, each weighted by the square root of its eigenvalue." + ), + String::new(), + ]; + for r in nearest.iter().take(10) { + lines.push(format!( + " {}. {} {} {} n={} d={}", + rjust(&r.rank.to_string(), 2), + ljust(&r.group, 45), + ljust(&r.dates, 22), + ljust(&r.country, 20), + ljust(&r.n.to_string(), 4), + fixed(r.distance, 4) + )); + } + lines.extend([String::new(), CAVEAT.to_owned(), QPADM_NOTE.to_owned()]); + let body = lines.join("\n") + "\n"; + files.put(PathBuf::from(a.req("--out-svg")), svg.as_bytes())?; + files.put(PathBuf::from(a.req("--out-txt")), body.as_bytes())?; + files.commit_all()?; + let _ = out.write_all(body.as_bytes()); + Ok(()) +} + +// --- the command line ------------------------------------------------------------- + +const USAGE: &str = "usage: og ancient {select,lift,person-geno,align,report} ...\n"; + +type Run = fn(&Args, &mut dyn Write) -> Result<()>; + +/// `og ancient [args]`: the exit code. Results go to `out`, +/// messages to `err`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og ancient: error: the following arguments are required: verb" + ); + return 2; + }; + let (spec, run): (&Spec, Run) = match verb.as_str() { + "select" => (&SELECT, cmd_select), + "lift" => (&LIFT, cmd_lift), + "person-geno" => (&PERSON_GENO, cmd_person_geno), + "align" => (&ALIGN, cmd_align), + "report" => (&REPORT, cmd_report), + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og ancient: error: argument verb: invalid choice: {}", + repr(other) + ); + return 2; + } + }; + let parsed = Args::parse(spec, &args[1..]).and_then(|a| { + for name in ["--min-snps", "--pcs"] { + if spec.options.contains(&name) { + a.int(name, 0)?; + } + } + Ok(a) + }); + let a = match parsed { + Ok(a) => a, + Err(ArgError::Help) => { + let _ = write!(out, "{}", spec.usage); + return 0; + } + Err(ArgError::Usage(msg)) => { + let _ = write!(err, "{}", spec.usage); + let _ = writeln!(err, "og ancient {}: error: {msg}", spec.verb); + return 2; + } + }; + match run(&a, out) { + Ok(()) => 0, + Err(Fail(msg)) => { + let _ = out.flush(); + let _ = writeln!(err, "ERROR: {msg}"); + 1 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dosages() { + assert_eq!(dosage("A/G", "A", "G"), Some(1)); + assert_eq!(dosage("A|A", "A", "G"), Some(2)); + assert_eq!(dosage("G/G", "A", "G"), Some(0)); + assert_eq!(dosage("A/T", "A", "G"), None); + assert_eq!(dosage("./.", "A", "G"), None); + assert_eq!(dosage("A", "A", "G"), None); + } + + #[test] + fn year_labels() { + assert_eq!(year_label(4250.0), "2300 BCE"); + assert_eq!(year_label(1950.0), "1 BCE"); + assert_eq!(year_label(1949.0), "1 CE"); + assert_eq!(year_label(1750.5), "200 CE"); + assert_eq!(range_label(7250.0, 6850.0), "5300-4900 BCE"); + assert_eq!(range_label(1750.0, 1500.0), "200-450 CE"); + assert_eq!(range_label(2250.0, 1750.0), "300 BCE-200 CE"); + assert_eq!(range_label(2250.0, 2250.0), "300 BCE"); + } + + #[test] + fn most_common_breaks_ties_by_name() { + let v = [ + Some("b"), + Some("a"), + Some("b"), + Some("a"), + Some("."), + None, + Some(""), + ]; + assert_eq!(most_common(v.into_iter()), "a"); + assert_eq!(most_common([Some("."), None].into_iter()), "."); + } + + #[test] + fn exact_int_compare() { + assert!(float_ge_int(100_000.0, 100_000)); + assert!(!float_ge_int(99_999.5, 100_000)); + assert!(!float_ge_int( + 9_007_199_254_740_992.0, + 9_007_199_254_740_993 + )); + assert!(!float_ge_int(f64::NAN, 0)); + } + + #[test] + fn clean_cells() { + assert_eq!(clean(" a\t b "), "a b"); + assert_eq!(clean(" \t "), "."); + } +} diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs new file mode 100644 index 000000000..37bdf7624 --- /dev/null +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -0,0 +1,1728 @@ +//! Step 43, archaic ancestry (`docs/43-archaic.md`): the commands around +//! IBDmix (Chen et al. 2020), as `og archaic `. +//! +//! IBDmix compares a modern genome with an archaic one site by site and +//! adds up a log-odds score (LOD) that a stretch of the modern genome shares +//! a recent ancestor with the archaic genome. It takes each site's allele +//! frequency from the modern people in its input, and one genome alone +//! gives no frequency, so `genotypes` adds stand-in columns whose genotypes +//! carry the 1000 Genomes allele count of the chosen population. +//! +//! - `table` (setup): one GRCh37 chromosome of the archaic table, a +//! sites-only VCF that Picard lifts to GRCh38. The 1000 Genomes SNVs and +//! the three archaic genomes' calls are merged by position in one pass; +//! each genome is read inside its own mask, and a call outside it (or at +//! an excluded site) is kept as masked, code 10 plus its ALT count. +//! - `strict` (setup): the lifted rows of one chromosome, once per +//! position, with every call outside the 1000 Genomes strict mask marked +//! masked. +//! - `genotypes` (step 43): IBDmix's genotype file of each chromosome, each +//! archaic genome's mask of masked positions, and the spans table. +//! - `report` (step 43): the segments at LOD 4 and 50 kb or longer, merged +//! into Neanderthal and Denisovan ones, the shares, the ten longest and +//! their protein-coding genes. +//! +//! Each command writes what `bin/archaic_report.py` wrote (checked by +//! `tests/parity`). Deliberate changes: every output is written under a +//! temporary name and renamed into place; a gzip input is told by its first +//! bytes, not its name; `strict` reads its rows in position order and stops +//! (exit 2) on a row that goes back, instead of keeping every position of a +//! chromosome in memory; and malformed input is an `ERROR:` line, never a +//! traceback. +//! +//! The table's bytes are part of the archaic data version: a change to what +//! `table` or `strict` writes must change `ARCHAIC_TABLE_FORMAT` in +//! `scripts/setup.sh`, so existing installs rebuild their table. + +use super::{ArgError, Args, Fail, Result, Spec, cannot_read, cannot_write, int, repr, split_ws}; +use flate2::Compression; +use flate2::write::GzEncoder; +use og_io::{AtomicFile, Lines, Utf8}; +use pycompat::fmt::{fixed, general, zero_pad}; +use std::cmp::Reverse; +use std::collections::BinaryHeap; +use std::io::{self, BufRead, Write}; +use std::path::{Path, PathBuf}; + +pub const ARCHAICS: [&str; 3] = ["Altai", "Vindija", "Denisova"]; +const NEANDERTHALS: [&str; 2] = ["Altai", "Vindija"]; +const POPS: [&str; 6] = ["ALL", "AFR", "AMR", "EAS", "EUR", "SAS"]; +const TABLE_HEADER: [&str; 13] = [ + "chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "AF_ALL", "AF_AFR", "AF_AMR", + "AF_EAS", "AF_EUR", "AF_SAS", +]; +/// Table code: MASKED plus the ALT count, a call outside its genome's mask. +const MASKED: u8 = 10; +/// Stand-in modern people: 200 alleles, about one 1000 Genomes population. +const PSEUDO: i64 = 100; +/// Chen et al. 2020: LOD 4 and 50 kb. +const MIN_LOD: f64 = 4.0; +const MIN_LENGTH: i64 = 50_000; +const TOP: usize = 10; +const SEGMENT_HEADER: &str = "chrom\tstart\tend\tlength_bp\tancestry\tmatched\tlod\tgenes"; +const CAVEAT: &str = "IBDmix finds stretches where your genome matches a Neanderthal or the Denisovan genome more \ +closely than the allele frequencies of a modern population explain. A stretch counts when either \ +of your two copies carries it, and the share is of the analysed span of the autosomes. Some of what \ +it finds in anyone is old ancestry shared with the archaic genomes rather than gene flow from them: \ +Chen et al. 2020 find Neanderthal sequence in every population they studied, Africans included. \ +The numbers move with the cutoffs (LOD 4, 50 kb) and with the frequencies used. An estimate, not \ +a measurement."; + +/// Why a command stopped: its exit code (2 for a wrong option value the +/// command checks itself, 1 otherwise) and the message after `ERROR: `. +#[derive(Debug)] +struct Stop(u8, String); + +impl From for Stop { + fn from(f: Fail) -> Self { + Stop(1, f.0) + } +} + +fn die(msg: impl Into) -> std::result::Result { + Err(Stop(1, msg.into())) +} + +fn usage(msg: impl Into) -> std::result::Result { + Err(Stop(2, msg.into())) +} + +type Out = std::result::Result; + +/// A text file read line by line as Python's text mode reads it (universal +/// newlines, strict UTF-8; gzip read through), keeping the raw line for +/// messages that quote it. +struct Text { + path: PathBuf, + lines: Lines>, + line: String, +} + +impl Text { + fn open(path: &str) -> Result { + let path = PathBuf::from(path); + let input = og_io::open(&path).map_err(|e| cannot_read(&path, &e))?; + Ok(Text { + path, + lines: Lines::new(input, Utf8::Strict), + line: String::new(), + }) + } + + /// The next line with its `\n`, or `None` at the end. + fn raw(&mut self) -> Result> { + let more = self + .lines + .read_line(&mut self.line) + .map_err(|e| cannot_read(&self.path, &e))?; + Ok(more.then_some(self.line.as_str())) + } + + /// The next line without its `\n` (`line.rstrip("\n")`). + fn next(&mut self) -> Result> { + Ok(self.raw()?.map(|l| l.strip_suffix('\n').unwrap_or(l))) + } +} + +/// Python's `int(s)`, or the message of the ValueError it raises. +fn need_int(s: &str) -> Out { + int(s).ok_or_else(|| { + Stop( + 1, + format!("invalid literal for int() with base 10: {}", repr(s)), + ) + }) +} + +/// Python's `float(s)`, or the message of the ValueError it raises. +fn need_float(s: &str) -> Out { + pycompat::num::parse_float(s) + .map_err(|_| Stop(1, format!("could not convert string to float: {}", repr(s)))) +} + +/// `repr(line[:80])`. +fn head80(line: &str) -> String { + let end = line.char_indices().nth(80).map_or(line.len(), |(i, _)| i); + repr(&line[..end]) +} + +/// `os.path.join(a, b)` for a relative `b`. +fn join(a: &str, b: &str) -> String { + if a.is_empty() || a.ends_with('/') { + format!("{a}{b}") + } else { + format!("{a}/{b}") + } +} + +/// `'0/1'`, `'1|1'`, `'0/0'` to 1, 2, 0; a missing, partial or +/// multi-allelic call to `None`. +fn gt_code(gt: &str) -> Option { + let mut parts = gt.split(['/', '|']); + let (Some(a), Some(b), None) = (parts.next(), parts.next(), parts.next()) else { + return None; + }; + let allele = |x: &str| match x { + "0" => Some(0), + "1" => Some(1), + _ => None, + }; + Some(allele(a)? + allele(b)?) +} + +/// Python's `max(a, b)` and `min(a, b)` of floats: the first unless the +/// second compares greater (smaller), so a NaN first stays. +fn py_max(a: f64, b: f64) -> f64 { + if b > a { b } else { a } +} + +fn py_min(a: f64, b: f64) -> f64 { + if b < a { b } else { a } +} + +// --- table (setup) ---------------------------------------------------------------- + +/// One mask's intervals on one chromosome, sorted and merged: 0-based, +/// half-open, so `[s, e)` holds the 1-based positions `s + 1` to `e`. The +/// chromosome is matched with and without the `chr` prefix. +fn read_mask(path: &str, chrom: &str) -> Out> { + let num = chrom.strip_prefix("chr").unwrap_or(chrom); + let with_chr = format!("chr{num}"); + let mut iv = Vec::new(); + let mut t = Text::open(path)?; + while let Some(line) = t.raw()? { + if line.starts_with('#') || line.starts_with("track") || line.starts_with("browser") { + continue; + } + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 3 || (c[0] != num && c[0] != with_chr) { + continue; + } + let (s, e) = (need_int(c[1])?, need_int(c[2])?); + if e <= s { + continue; // an empty or reversed interval holds no position + } + iv.push((s, e)); + } + iv.sort_unstable(); + let mut out: Vec<(i64, i64)> = Vec::with_capacity(iv.len()); + for (s, e) in iv { + match out.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => out.push((s, e)), + } + } + Ok(out) +} + +/// Whether a 1-based position is inside a mask, for positions asked in +/// ascending order: the walk over the intervals never goes back. +struct Mask { + iv: Vec<(i64, i64)>, + i: usize, +} + +impl Mask { + fn holds(&mut self, pos: i64) -> bool { + let p = pos - 1; + while self.i < self.iv.len() && self.iv[self.i].1 <= p { + self.i += 1; + } + self.i < self.iv.len() && self.iv[self.i].0 <= p + } +} + +/// One record of a table input: a 1000 Genomes SNV (REF, ALT and the six +/// frequencies as written) or an archaic call (REF, ALT, GT). +#[derive(Debug, Clone)] +enum Rec { + Modern { + r#ref: String, + alt: String, + afs: Vec, + }, + Archaic { + r#ref: String, + alt: String, + gt: String, + }, +} + +/// One input of the merge: its lines, read on demand. +struct Stream { + text: Text, + /// 0 for the 1000 Genomes lines, 1 + i for archaic genome i. + src: usize, +} + +impl Stream { + /// The next record, `(pos, rec)`; 1000 Genomes lines that are not + /// SNVs are skipped. + fn next(&mut self) -> Out> { + loop { + let path = self.text.path.display().to_string(); + let Some(raw) = self.text.raw()? else { + return Ok(None); + }; + let line = raw.strip_suffix('\n').unwrap_or(raw); + let c: Vec<&str> = line.split('\t').collect(); + if self.src == 0 { + if c.len() < 9 { + return die(format!( + "{path}: a line has {} columns, want 9: {}", + c.len(), + head80(raw) + )); + } + if c[1].chars().count() != 1 || c[2].chars().count() != 1 { + continue; + } + let pos = need_int(c[0])?; + return Ok(Some(( + pos, + Rec::Modern { + r#ref: c[1].to_owned(), + alt: c[2].to_owned(), + afs: c[3..9].iter().map(|s| (*s).to_owned()).collect(), + }, + ))); + } + if c.len() < 4 { + return die(format!( + "{path}: a line has {} columns, want 4: {}", + c.len(), + head80(raw) + )); + } + let pos = need_int(c[0])?; + return Ok(Some(( + pos, + Rec::Archaic { + r#ref: c[1].to_owned(), + alt: c[2].to_owned(), + gt: c[3].to_owned(), + }, + ))); + } + } +} + +/// An archaic call at one position: (REF, ALT, GT). +type Call<'a> = Option<(&'a str, &'a str, &'a str)>; + +/// The ALT count of one archaic record at a row with this REF and ALT, or 9. +fn archaic_code(rec: Call<'_>, r: &str, alt: &str) -> u8 { + let Some((aref, aalt, gt)) = rec else { + return 9; + }; + if aref != r { + return 9; + } + let Some(code) = gt_code(gt) else { + return 9; + }; + if aalt == "." || aalt == "<*>" || code == 0 { + return if code == 0 { 0 } else { 9 }; + } + if aalt == alt { code } else { 9 } +} + +/// The row of one position: REF, ALT, the three codes and the six +/// frequencies as written, or `None` when nothing is known there. +fn table_row<'a>( + modern: Option<(&'a str, &'a str, &'a [String])>, + arch: [Call<'a>; 3], +) -> Option<(&'a str, &'a str, [u8; 3], Vec)> { + const ZEROS: [&str; 6] = ["0"; 6]; + let (r, alt, afs): (&str, &str, Vec<&str>) = match modern { + Some((r, alt, afs)) => (r, alt, afs.iter().map(String::as_str).collect()), + None => { + // A site no modern person varies at: the ALT of the first + // archaic genome with a non-reference call, every frequency 0. + let (r, alt, _) = arch.iter().flatten().find(|(r, a, gt)| { + !matches!(gt_code(gt), None | Some(0)) + && r.chars().count() == 1 + && a.chars().count() == 1 + })?; + (r, alt, ZEROS.to_vec()) + } + }; + let codes = arch.map(|a| archaic_code(a, r, alt)); + if codes.iter().all(|&c| c == 9) { + return None; + } + if modern.is_none() && !codes.iter().any(|&c| c == 1 || c == 2) { + return None; + } + let clean = afs + .iter() + .map(|v| { + let x = pycompat::num::parse_float(v).unwrap_or(0.0); + general(py_min(py_max(x, 0.0), 1.0), 6) + }) + .collect(); + Some((r, alt, codes, clean)) +} + +/// `NAME=FILE` with NAME one of the archaic genomes. +fn named<'a>(spec: &'a str, option: &str) -> Out<(&'a str, &'a str)> { + let (name, path) = spec.split_once('=').unwrap_or((spec, "")); + if !ARCHAICS.contains(&name) || path.is_empty() { + return usage(format!( + "{option} takes NAME=FILE with NAME one of {}, got {}", + ARCHAICS.join(", "), + repr(spec) + )); + } + Ok((name, path)) +} + +fn cmd_table(a: &Args, err: &mut dyn Write) -> Out<()> { + let chrom_arg = a.req("--chrom"); + let mut names = Vec::new(); + let mut paths = Vec::new(); + for spec in a.all("--archaic") { + let (name, path) = named(spec, "--archaic")?; + names.push(name); + paths.push(path); + } + if names != ARCHAICS { + return usage(format!( + "--archaic must name {} in that order, got {}", + ARCHAICS.join(", "), + names.join(", ") + )); + } + let mut masks: [Option; 3] = [None, None, None]; + for spec in a.all("--mask") { + let (name, path) = named(spec, "--mask")?; + let k = ARCHAICS.iter().position(|n| *n == name).expect("checked"); + if masks[k].is_some() { + return usage(format!("--mask names {name} twice")); + } + let iv = read_mask(path, chrom_arg)?; + if iv.is_empty() { + return die(format!( + "{path} holds no interval on chromosome {chrom_arg}: a mask that matches nothing \ + would make every site unknown" + )); + } + masks[k] = Some(Mask { iv, i: 0 }); + } + let mut excluded = Vec::new(); + for path in a.all("--exclude") { + let iv = read_mask(path, chrom_arg)?; + if iv.is_empty() { + return die(format!( + "{path} holds no interval on chromosome {chrom_arg}: an empty exclusion list is a \ + download that went wrong, not a chromosome without segmental duplications" + )); + } + excluded.push(Mask { iv, i: 0 }); + } + let chrom = if chrom_arg.starts_with("chr") { + chrom_arg.to_owned() + } else { + format!("chr{chrom_arg}") + }; + let out_path = PathBuf::from(a.req("--out")); + let mut out = AtomicFile::create(&out_path).map_err(|e| cannot_write(&out_path, &e))?; + let mut head = format!("##fileformat=VCFv4.2\n##contig=\n"); + for n in ARCHAICS { + head += &format!( + "##INFO=\n" + ); + } + for p in POPS { + head += &format!( + "##INFO=\n" + ); + } + head += "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"; + let w = |out: &mut AtomicFile, s: &str| { + out.write_all(s.as_bytes()) + .map_err(|e| Stop::from(cannot_write(&out_path, &e))) + }; + w(&mut out, &head)?; + + // The k-way merge of heapq.merge: the head of each input on a heap + // keyed (position, input), each input advanced only when its head is + // taken, so the order is the same as the Python's on any input. + let mut streams = vec![Stream { + text: Text::open(a.req("--modern"))?, + src: 0, + }]; + for (i, p) in paths.iter().enumerate() { + streams.push(Stream { + text: Text::open(p)?, + src: 1 + i, + }); + } + let mut heads: Vec> = vec![None; streams.len()]; + let mut heap = BinaryHeap::new(); + for (i, s) in streams.iter_mut().enumerate() { + if let Some((pos, rec)) = s.next()? { + heads[i] = Some(rec); + heap.push(Reverse((pos, i))); + } + } + let mut rows: u64 = 0; + let mut cur: Option = None; + let mut modern: Option = None; + let mut arch: [Option; 3] = [None, None, None]; + let mut last = -1i64; + let mut flush = |cur: i64, + modern: &Option, + arch: &[Option; 3], + out: &mut AtomicFile| + -> Out<()> { + // Every mask is walked for every position, before any early + // return, because the walk only goes forward. + let mut excl = false; + for m in excluded.iter_mut() { + excl |= m.holds(cur); + } + let mut outside = [false; 3]; + for (k, m) in masks.iter_mut().enumerate() { + outside[k] = excl || m.as_mut().is_some_and(|m| !m.holds(cur)); + } + let modern = modern.as_ref().map(|m| match m { + Rec::Modern { r#ref, alt, afs } => (r#ref.as_str(), alt.as_str(), afs.as_slice()), + Rec::Archaic { .. } => unreachable!("only the modern input gives Modern"), + }); + let calls = [0, 1, 2].map(|k| match &arch[k] { + Some(Rec::Archaic { r#ref, alt, gt }) => { + Some((r#ref.as_str(), alt.as_str(), gt.as_str())) + } + _ => None, + }); + let Some((r, alt, codes, afs)) = table_row(modern, calls) else { + return Ok(()); + }; + // A call outside its genome's mask, or at an excluded site, is kept + // but marked masked: step 43 hands IBDmix the call and a mask of + // those positions, as Chen et al. do. + let mut info = String::new(); + for k in 0..3 { + let c = codes[k]; + let c = if outside[k] && c != 9 { c + MASKED } else { c }; + info += &format!("{}={c};", ARCHAICS[k]); + } + for (k, p) in POPS.iter().enumerate() { + info += &format!("AF_{p}={}", afs[k]); + if k + 1 < POPS.len() { + info.push(';'); + } + } + out.write_all(format!("{chrom}\t{cur}\t.\t{r}\t{alt}\t.\t.\t{info}\n").as_bytes()) + .map_err(|e| Stop::from(cannot_write(&out_path, &e)))?; + rows += 1; + Ok(()) + }; + while let Some(Reverse((pos, i))) = heap.pop() { + let rec = heads[i].take().expect("a head for each heap entry"); + if pos < last { + return die(format!("input not sorted by position at {pos}")); + } + if cur != Some(pos) { + if let Some(c) = cur { + flush(c, &modern, &arch, &mut out)?; + } + cur = Some(pos); + modern = None; + arch = [None, None, None]; + } + last = pos; + match rec { + Rec::Modern { .. } => { + // A second modern record at one position (a split + // multi-allelic site) makes the row ambiguous: keep neither. + modern = Some(match modern { + None => rec, + Some(_) => Rec::Modern { + r#ref: "N".into(), + alt: "N".into(), + afs: vec!["0".into(); POPS.len()], + }, + }); + } + Rec::Archaic { .. } => arch[streams[i].src - 1] = Some(rec), + } + // As heapq.merge does, the input whose record was taken is read on + // only now. + if let Some((p, r)) = streams[i].next()? { + heads[i] = Some(r); + heap.push(Reverse((p, i))); + } + } + if let Some(c) = cur { + flush(c, &modern, &arch, &mut out)?; + } + out.commit().map_err(|e| cannot_write(&out_path, &e))?; + let _ = writeln!(err, "{chrom}: {rows} rows"); + Ok(()) +} + +// --- strict (setup) --------------------------------------------------------------- + +/// The lifted GRCh38 rows of one chromosome (no header, the 13 table +/// columns), once per position, with the calls of every row outside the +/// 1000 Genomes strict mask marked masked (MASKED plus the ALT count; 9 +/// stays 9). The strict mask is the same for the three genomes, so all +/// three are marked. +fn cmd_strict(a: &Args, err: &mut dyn Write) -> Out<()> { + let chrom = a.req("--chrom"); + let strict = a.req("--strict"); + let iv = read_mask(strict, chrom)?; + if iv.is_empty() { + return die(format!( + "{strict} holds no interval on chromosome {chrom}: a mask that matches nothing \ + would mark every site masked" + )); + } + let rows_path = a.req("--rows"); + let mut t = Text::open(rows_path)?; + let out_path = PathBuf::from(a.req("--out")); + let mut out = AtomicFile::create(&out_path).map_err(|e| cannot_write(&out_path, &e))?; + let (mut rows, mut marked) = (0u64, 0u64); + let mut last: Option = None; + let mut n = 0u64; + while let Some(raw) = t.raw()? { + n += 1; + let line = raw.strip_suffix('\n').unwrap_or(raw); + let mut c: Vec = line.split('\t').map(str::to_owned).collect(); + if c.len() != TABLE_HEADER.len() { + return die(format!( + "{rows_path}: a row has {} columns, want {}: {}", + c.len(), + TABLE_HEADER.len(), + head80(raw) + )); + } + let pos = need_int(&c[1])?; + // The rows come sorted (Picard sorts its output), so a position + // seen before is the one just before: a repeat keeps its first row. + match last { + Some(l) if pos == l => continue, + Some(l) if pos < l => { + return usage(format!( + "{rows_path}: line {n}: position {pos} comes after {l}; the lifted rows must be sorted by position" + )); + } + _ => {} + } + last = Some(pos); + let p = pos - 1; + let i = iv.partition_point(|&(s, _)| s <= p); + let inside = i > 0 && p < iv[i - 1].1; + if !inside { + let mut hit = false; + for cell in &mut c[4..7] { + if let Some(v) = ["0", "1", "2"].iter().position(|x| x == cell) { + *cell = (v as u8 + MASKED).to_string(); + hit = true; + } + } + marked += u64::from(hit); + } + out.write_all((c.join("\t") + "\n").as_bytes()) + .map_err(|e| cannot_write(&out_path, &e))?; + rows += 1; + } + out.commit().map_err(|e| cannot_write(&out_path, &e))?; + let _ = writeln!( + err, + "{chrom}: {rows} rows, {marked} outside the strict mask and marked masked" + ); + Ok(()) +} + +// --- genotypes (step 43) ---------------------------------------------------------- + +/// The stand-in columns for each ALT allele count 0..2n: as many 2s and +/// one 1 as the count needs, then 0s. +fn pseudo_columns(n: usize) -> Vec { + (0..=2 * n) + .map(|ac| { + let mut g = vec!["2"; ac / 2]; + if ac % 2 == 1 { + g.push("1"); + } + g.resize(n, "0"); + g.join("\t") + }) + .collect() +} + +/// A call of the sample's VCF: REF, ALT (`None` for an insertion, a +/// deletion or a multi-allelic record) and the code (0, 1, 2, or 9 when it +/// cannot be read). +#[derive(Debug, Clone, PartialEq, Eq)] +struct Sample { + r#ref: String, + alt: Option, + code: u8, +} + +/// What [`pick_call`] keeps. +#[derive(Debug, PartialEq, Eq)] +enum Picked { + Call(Sample), + Clash, +} + +/// The call to keep when a position comes twice, as in a VCF split with +/// bcftools norm -m-. An SNV wins over an insertion or a multi-allelic +/// record, in either order. Of one SNV twice, a usable record wins over +/// code 9, and two different usable codes clash. Of two different SNVs, +/// the one that carries its ALT wins over a 0/0 or unusable one; both 0/0 +/// is the reference; both carrying their ALT, or neither readable, clash +/// (unknown, 9). +fn pick_call(prev: Option, call: Sample) -> Picked { + let Some(prev) = prev else { + return Picked::Call(call); + }; + if prev.alt.is_none() || call.alt.is_none() { + return Picked::Call(if prev.alt.is_none() && call.alt.is_some() { + call + } else { + prev + }); + } + if call.alt == prev.alt { + if call.code == 9 || call.code == prev.code { + return Picked::Call(prev); + } + return if prev.code == 9 { + Picked::Call(call) + } else { + Picked::Clash + }; + } + let carries = |c: u8| c == 1 || c == 2; + let (has_prev, has_call) = (carries(prev.code), carries(call.code)); + if has_prev != has_call { + return Picked::Call(if has_prev { prev } else { call }); + } + if !has_prev && prev.code == 0 && call.code == 0 { + return Picked::Call(Sample { + r#ref: prev.r#ref, + alt: None, + code: 0, + }); + } + Picked::Clash +} + +/// One record of the sample's VCF that `genotypes` reads. +struct VcfRec { + chrom: String, + pos: i64, + call: Sample, +} + +/// The biallelic SNVs (and the other records, as code 9) of a one-sample +/// VCF, in file order. A record whose FILTER is not PASS or `.` gives +/// code 9 (not read), so a DeepVariant RefCall is neither your allele nor +/// the reference. +struct Vcf { + text: Text, +} + +impl Vcf { + fn next(&mut self) -> Out> { + loop { + let path = self.text.path.display().to_string(); + let Some(raw) = self.text.raw()? else { + return Ok(None); + }; + if raw.starts_with("##") { + continue; + } + let line = raw.strip_suffix('\n').unwrap_or(raw); + let c: Vec<&str> = line.split('\t').collect(); + if raw.starts_with('#') { + if c.len() != 10 { + return die(format!( + "{path} holds {} samples; step 43 reads a one-sample VCF \ + (keep one with bcftools view -s ).", + c.len().saturating_sub(9) + )); + } + continue; + } + if c.len() < 10 || c[3].chars().count() != 1 || c[4] == "." { + continue; + } + if c[4].contains(',') || c[4].chars().count() != 1 { + return Ok(Some(VcfRec { + chrom: c[0].to_owned(), + pos: need_int(c[1])?, + call: Sample { + r#ref: c[3].to_owned(), + alt: None, + code: 9, + }, + })); + } + let fmt: Vec<&str> = c[8].split(':').collect(); + let Some(k) = fmt.iter().position(|f| *f == "GT") else { + continue; + }; + let gt = c[9].split(':').nth(k).unwrap_or("."); + let mut code = gt_code(gt).unwrap_or(9); + if c[6] != "PASS" && c[6] != "." { + code = 9; + } + return Ok(Some(VcfRec { + chrom: c[0].to_owned(), + pos: need_int(c[1])?, + call: Sample { + r#ref: c[3].to_owned(), + alt: Some(c[4].to_owned()), + code, + }, + })); + } + } +} + +/// The sample's calls on one chromosome, one per position after +/// [`pick_call`], sorted by position: a compact array, not a map. +fn settle(mut recs: Vec<(i64, Sample)>) -> Vec<(i64, Sample)> { + // A stable sort keeps the records of one position in file order. + recs.sort_by_key(|(p, _)| *p); + let mut out: Vec<(i64, Sample)> = Vec::new(); + let mut clashed = false; + for (pos, call) in recs { + match out.last_mut() { + Some((p, kept)) if *p == pos => { + if clashed { + continue; + } + let r#ref = call.r#ref.clone(); + match pick_call(Some(kept.clone()), call) { + Picked::Call(c) => *kept = c, + Picked::Clash => { + *kept = Sample { + r#ref, + alt: None, + code: 9, + }; + clashed = true; + } + } + } + _ => { + clashed = false; + out.push((pos, call)); + } + } + } + out +} + +/// `chrN.tsv.gz` with N from 1 to 22: the chromosome's name. +fn table_chrom(name: &str) -> Option<&str> { + let chrom = name.strip_suffix(".tsv.gz")?; + let num = chrom.strip_prefix("chr")?; + let ok = match num.len() { + 1 => matches!(num.as_bytes()[0], b'1'..=b'9'), + 2 => { + let (a, b) = (num.as_bytes()[0], num.as_bytes()[1]); + (a == b'1' && b.is_ascii_digit()) || (a == b'2' && (b'0'..=b'2').contains(&b)) + } + _ => false, + }; + ok.then_some(chrom) +} + +/// A file written whole through a temporary name. +fn put(path: &Path, data: &[u8]) -> Result<()> { + let mut f = AtomicFile::create(path).map_err(|e| cannot_write(path, &e))?; + f.write_all(data).map_err(|e| cannot_write(path, &e))?; + f.commit().map_err(|e| cannot_write(path, &e)) +} + +fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { + let pseudo_n = a + .int("--pseudo", PSEUDO) + .map_err(|_| Stop(2, String::new()))?; + if pseudo_n < 1 { + return usage("--pseudo must be 1 or more"); + } + let pop = a.get("--pop").unwrap_or("ALL"); + let Some(af_col) = POPS.iter().position(|p| *p == pop).map(|k| 7 + k) else { + return usage(format!( + "--pop must be one of {}, got {}", + POPS.join(", "), + repr(pop) + )); + }; + let table_dir = a.req("--table-dir"); + let rd = std::fs::read_dir(table_dir).map_err(|e| cannot_read(Path::new(table_dir), &e))?; + let mut names: Vec = Vec::new(); + for e in rd { + let e = e.map_err(|e| cannot_read(Path::new(table_dir), &e))?; + if let Ok(n) = e.file_name().into_string() { + names.push(n); + } + } + names.sort(); + let tables: Vec<(String, String)> = names + .iter() + .filter_map(|n| table_chrom(n).map(|c| (c.to_owned(), join(table_dir, n)))) + .collect(); + if tables.is_empty() { + return die(format!( + "{table_dir} holds no chrN.tsv.gz: install the archaic data with scripts/setup.sh --archaic" + )); + } + let out_dir = a.req("--out-dir"); + std::fs::create_dir_all(out_dir).map_err(|e| cannot_write(Path::new(out_dir), &e))?; + let n = pseudo_n as usize; + let pseudo = pseudo_columns(n); + let two_n = (2 * n) as f64; + let mut head = String::from("chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tSAMPLE"); + let mut samples = String::from("SAMPLE\n"); + for i in 1..=n { + let id = format!("P{}", zero_pad(i as i128, 3)); + head += &format!("\t{id}"); + samples += &format!("{id}\n"); + } + head.push('\n'); + put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; + + let vcf_path = a.req("--vcf"); + let mut vcf = Vcf { + text: Text::open(vcf_path)?, + }; + let mut spans: Vec = Vec::new(); + let mut done: Vec = Vec::new(); + let mut pending = vcf.next()?; + while let Some(first) = pending.take() { + let chrom = first.chrom.clone(); + if done.contains(&chrom) { + return die(format!( + "{vcf_path} is not sorted: {chrom} comes back after another chromosome" + )); + } + done.push(chrom.clone()); + let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); + // The sample's calls on this chromosome; on a chromosome the table + // does not hold they are read past. + let mut recs = Vec::new(); + if table.is_some() { + recs.push((first.pos, first.call)); + } + loop { + match vcf.next()? { + Some(r) if r.chrom == chrom => { + if table.is_some() { + recs.push((r.pos, r.call)); + } + } + other => { + pending = other; + break; + } + } + } + let Some(table) = table else { + continue; + }; + let calls = settle(recs); + if let Some(span) = write_chrom( + &chrom, table, &calls, &head, &pseudo, af_col, two_n, out_dir, + )? { + spans.push(span); + } + } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; + if spans.is_empty() { + let names: Vec<&str> = tables.iter().map(|(c, _)| c.as_str()).collect(); + let mut sorted = names.clone(); + sorted.sort_unstable(); + return die(format!( + "{vcf_path} has no records on a chromosome of the archaic table ({})", + sorted.join(", ") + )); + } + let sites: u64 = spans.iter().map(|s| s.3).sum(); + let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); + Ok(()) +} + +/// A chromosome's row of `spans.tsv`: chromosome, first and last +/// position, sites, sites with your ALT, sites with another allele. +type Span = (String, i64, i64, u64, u64, u64); + +/// IBDmix's genotype file of one chromosome, and each archaic genome's +/// mask; the chromosome's span row, or `None` when no row was written. +#[allow(clippy::too_many_arguments)] +fn write_chrom( + chrom: &str, + table: &str, + calls: &[(i64, Sample)], + head: &str, + pseudo: &[String], + af_col: usize, + two_n: f64, + out_dir: &str, +) -> Out> { + let num = &chrom[3..]; + let path = Path::new(out_dir).join(format!("{chrom}.gt.gz")); + let file = AtomicFile::create(&path).map_err(|e| cannot_write(&path, &e))?; + let mut gz = GzEncoder::new(file, Compression::new(1)); + let werr = |e: io::Error| Stop::from(cannot_write(&path, &e)); + gz.write_all(head.as_bytes()).map_err(werr)?; + let mut t = Text::open(table)?; + let header = t.next()?.unwrap_or(""); + if header.split('\t').ne(TABLE_HEADER.iter().copied()) { + return die(format!( + "{table} does not start with the table header ({})", + TABLE_HEADER.join(" ") + )); + } + let (mut n, mut first, mut last, mut dropped, mut carried) = (0u64, 0i64, 0i64, 0u64, 0u64); + // Per archaic genome, the runs of written rows whose call is masked: + // [first, last] positions, closed at the next unmasked row. + let mut runs: [Vec<(i64, i64)>; 3] = [Vec::new(), Vec::new(), Vec::new()]; + let mut open_run: [Option<(i64, i64)>; 3] = [None; 3]; + let mut row = String::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 7 || c.len() <= af_col { + return die(format!( + "{table}: a row has {} columns, want {}", + c.len(), + TABLE_HEADER.len() + )); + } + let pos = need_int(c[1])?; + let (r, alt) = (c[2], c[3]); + let mut arch = [0u8; 3]; + for k in 0..3 { + let v = c[4 + k]; + arch[k] = match v { + "0" | "1" | "2" | "9" | "10" | "11" | "12" => v.parse().expect("a code"), + _ => { + return die(format!( + "{table}: {chrom}:{pos} has {} code {}; the table holds 0, 1, 2, 9 or {MASKED} to {}", + ARCHAICS[k], + repr(v), + MASKED + 2 + )); + } + }; + } + let code = match calls.binary_search_by_key(&pos, |(p, _)| *p) { + Err(_) => 0, + Ok(i) => { + let call = &calls[i].1; + if call.r#ref != r || call.alt.as_ref().is_some_and(|x| x != alt) { + // Another allele than the row's: IBDmix's generate_gt + // drops the line. + dropped += 1; + continue; + } + call.code + } + }; + if code == 1 || code == 2 { + carried += 1; + } + let ac = match pycompat::num::parse_float(c[af_col]) { + Err(_) => 0.0, + Ok(x) => match pycompat::fmt::round_int(x * two_n) { + Ok(v) => v, + Err(pycompat::fmt::RoundError::Nan) => 0.0, + Err(e) => return die(format!("{table}: {chrom}:{pos}: {e}")), + }, + }; + let ac = ac.clamp(0.0, two_n) as usize; + for k in 0..3 { + if arch[k] >= MASKED { + match &mut open_run[k] { + None => open_run[k] = Some((pos, pos)), + Some(run) => run.1 = pos, + } + } else if let Some(run) = open_run[k].take() { + runs[k].push(run); + } + } + row.clear(); + row += &format!("{num}\t{pos}\t{r}\t{alt}"); + for v in arch { + let v = if v >= MASKED { v - MASKED } else { v }; + row += &format!("\t{v}"); + } + row += &format!("\t{code}\t{}\n", pseudo[ac]); + gz.write_all(row.as_bytes()).map_err(werr)?; + if n == 0 { + first = pos; + } + last = pos; + n += 1; + } + if n == 0 { + drop(gz); + match std::fs::remove_file(&path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => { + return Err(Stop::from(cannot_write(&path, &e))); + } + _ => {} + } + return Ok(None); + } + gz.finish().and_then(AtomicFile::commit).map_err(werr)?; + // IBDmix's mask (-r): the chromosome as in the genotype file, and a + // line covers the positions start + 1 to end, as in a BED. + for k in 0..3 { + if let Some(run) = open_run[k] { + runs[k].push(run); + } + let text: String = runs[k] + .iter() + .map(|(s, e)| format!("{num}\t{}\t{e}\n", s - 1)) + .collect(); + put( + &Path::new(out_dir).join(format!("{chrom}.{}.mask.bed", ARCHAICS[k])), + text.as_bytes(), + )?; + } + Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) +} + +// --- report (step 43) ------------------------------------------------------------- + +/// A segment: chromosome, start, end, LOD. +type Seg = (String, i64, i64, f64); + +/// IBDmix's output rows for SAMPLE, kept at LOD 4 and 50 kb or longer. +fn read_segments(path: &str) -> Out> { + let mut t = Text::open(path)?; + let head: Vec = t + .next()? + .unwrap_or("") + .split('\t') + .map(str::to_owned) + .collect(); + let first5: Vec<&str> = head.iter().take(5).map(String::as_str).collect(); + if first5 != ["ID", "chrom", "start", "end", "slod"] { + return die(format!( + "{path} is not IBDmix output (header {})", + first5.join(" ") + )); + } + let mut segs = Vec::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 || c[0] != "SAMPLE" { + continue; + } + let (s, e, lod) = (need_int(c[2])?, need_int(c[3])?, need_float(c[4])?); + if lod >= MIN_LOD && e - s >= MIN_LENGTH { + let chrom = if c[1].starts_with("chr") { + c[1].to_owned() + } else { + format!("chr{}", c[1]) + }; + segs.push((chrom, s, e, lod)); + } + } + Ok(segs) +} + +/// The order of chromosomes in the report: the number, or 99 for any +/// other name (so chrX and chrY sort together, by position). +fn chrom_key(c: &str) -> i128 { + let c = c.strip_prefix("chr").unwrap_or(c); + if pycompat::text::isdigit(c) { + pycompat::num::parse_int(c).unwrap_or(99) + } else { + 99 + } +} + +/// A merged segment: chromosome, start, end, the highest LOD, and which +/// archaic genomes it holds. +type Merged = (String, i64, i64, f64, Vec<&'static str>); + +/// Overlapping or touching segments of one class as one. +fn merge(mut segs: Vec<(Seg, &'static str)>) -> Vec { + segs.sort_by(|(a, _), (b, _)| (chrom_key(&a.0), a.1, a.2).cmp(&(chrom_key(&b.0), b.1, b.2))); + let mut out: Vec = Vec::new(); + for ((chrom, start, end, lod), who) in segs { + match out.last_mut() { + Some(m) if m.0 == chrom && start <= m.2 => { + m.2 = m.2.max(end); + m.3 = py_max(m.3, lod); + if !m.4.contains(&who) { + m.4.push(who); + } + } + _ => out.push((chrom, start, end, lod, vec![who])), + } + } + out +} + +fn overlaps(chrom: &str, s: i64, e: i64, others: &[Merged]) -> bool { + others.iter().any(|o| o.0 == chrom && o.1 < e && s < o.2) +} + +/// A row of the report: chromosome, start, end, ancestry, matched, LOD. +type Row = (String, i64, i64, &'static str, Vec<&'static str>, f64); + +/// Neanderthal: the union of the Altai and Vindija segments. Denisovan: +/// the Denisova segments that overlap no Neanderthal one (a stretch both +/// match is counted as Neanderthal, the more common source outside +/// Oceania). +fn classify(per: &[Vec; 3]) -> Vec { + let nea = merge( + NEANDERTHALS + .iter() + .enumerate() + .flat_map(|(k, n)| per[k].iter().map(move |s| (s.clone(), *n))) + .collect(), + ); + let den = merge(per[2].iter().map(|s| (s.clone(), "Denisova")).collect()); + let mut rows: Vec = Vec::new(); + for (c, s, e, l, mut who) in nea.iter().cloned() { + who.sort_unstable(); + if overlaps(&c, s, e, &den) { + who.push("Denisova"); + } + rows.push((c, s, e, "Neanderthal", who, l)); + } + for (c, s, e, l, _) in den.iter().cloned() { + if !overlaps(&c, s, e, &nea) { + rows.push((c, s, e, "Denisovan", vec!["Denisova"], l)); + } + } + rows.sort_by(|a, b| (chrom_key(&a.0), a.1).cmp(&(chrom_key(&b.0), b.1))); + rows +} + +/// Genes per chromosome: (start, end, name), sorted. +type Genes = std::collections::HashMap>; + +/// Protein-coding genes of a GENCODE gene GTF, per chromosome, sorted by +/// (start, end, name). +fn read_genes(path: &str) -> Out { + let mut genes = Genes::new(); + let mut t = Text::open(path)?; + while let Some(line) = t.next()? { + if line.starts_with('#') { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 9 || c[2] != "gene" || !c[8].contains("gene_type \"protein_coding\"") { + continue; + } + if let Some(name) = gene_name(c[8]) { + let (s, e) = (need_int(c[3])?, need_int(c[4])?); + genes + .entry(c[0].to_owned()) + .or_default() + .push((s, e, name.to_owned())); + } + } + for v in genes.values_mut() { + v.sort(); + } + Ok(genes) +} + +/// The first `gene_name "X"` with a non-empty X (the regex +/// `gene_name "([^"]+)"`). +fn gene_name(attrs: &str) -> Option<&str> { + const KEY: &str = "gene_name \""; + attrs.match_indices(KEY).find_map(|(i, _)| { + let rest = &attrs[i + KEY.len()..]; + let end = rest.find('"')?; + (end > 0).then(|| &rest[..end]) + }) +} + +/// The names of the genes overlapping `[start, end)`, once each, in order. +/// IBDmix's start is the segment's first site and its end the next site, +/// past the segment; a GTF gene is `[s, e]`, 1-based. +fn genes_in<'a>(genes: &'a Genes, chrom: &str, start: i64, end: i64) -> Vec<&'a str> { + let name = if chrom.starts_with("chr") { + chrom.to_owned() + } else { + format!("chr{chrom}") + }; + let mut seen: Vec<&str> = Vec::new(); + for (s, e, n) in genes.get(&name).map_or(&[][..], Vec::as_slice) { + if *s >= end { + break; // sorted by start: no later gene starts before the end + } + if *e >= start && !seen.contains(&n.as_str()) { + seen.push(n); + } + } + seen +} + +fn cmd_report(a: &Args, out: &mut dyn Write) -> Out<()> { + let spans_path = a.req("--spans"); + let mut t = Text::open(spans_path)?; + let head = t.next()?.unwrap_or("").to_owned(); + if head + .split('\t') + .take(4) + .ne(["chrom", "first", "last", "sites"]) + { + return die(format!( + "{spans_path} is not the spans table of the genotypes step" + )); + } + let mut spans: Vec<(String, i64, i64, i64)> = Vec::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 4 { + return die(format!( + "{spans_path}: a row has {} columns, want 4", + c.len() + )); + } + spans.push(( + c[0].to_owned(), + need_int(c[1])?, + need_int(c[2])?, + need_int(c[3])?, + )); + } + if spans.is_empty() { + return die(format!("{spans_path} lists no chromosome")); + } + let spans_bp: i64 = spans.iter().map(|(_, f, l, _)| l - f + 1).sum(); + let ibdmix = a.req("--ibdmix-dir"); + let mut per: [Vec; 3] = [Vec::new(), Vec::new(), Vec::new()]; + for (chrom, ..) in &spans { + for (k, n) in ARCHAICS.iter().enumerate() { + let p = join(ibdmix, &format!("{chrom}.{n}.txt")); + if !Path::new(&p).is_file() { + return die(format!("no IBDmix output {p}")); + } + per[k].extend(read_segments(&p)?); + } + } + let rows = classify(&per); + let genes_arg = a.get("--genes").unwrap_or(""); + let genes = if genes_arg.is_empty() { + Default::default() + } else { + read_genes(genes_arg)? + }; + let genes_of = |c: &str, s: i64, e: i64| -> Vec<&str> { + if genes_arg.is_empty() { + Vec::new() + } else { + genes_in(&genes, c, s, e) + } + }; + let sample = a.req("--sample"); + let pop = a.get("--pop").unwrap_or("ALL"); + let out_dir = a.req("--out-dir"); + std::fs::create_dir_all(out_dir).map_err(|e| cannot_write(Path::new(out_dir), &e))?; + + let seg_name = format!("{sample}_archaic_segments.tsv"); + let mut text = format!("{SEGMENT_HEADER}\n"); + for (c, s, e, anc, who, lod) in &rows { + let g = genes_of(c, *s, *e); + text += &format!( + "{c}\t{s}\t{e}\t{}\t{anc}\t{}\t{}\t{}\n", + e - s, + who.join(","), + fixed(*lod, 1), + if g.is_empty() { + ".".to_owned() + } else { + g.join(",") + } + ); + } + put(&Path::new(out_dir).join(&seg_name), text.as_bytes())?; + + // The summary of each ancestry. + let mut summary: Vec<(String, String)> = Vec::new(); + for anc in ["Neanderthal", "Denisovan"] { + let r: Vec<&Row> = rows.iter().filter(|x| x.3 == anc).collect(); + let bp: i64 = r.iter().map(|x| x.2 - x.1).sum(); + let key = anc.to_lowercase(); + summary.push((format!("{key}_segments"), r.len().to_string())); + summary.push((format!("{key}_bp"), bp.to_string())); + summary.push(( + format!("{key}_percent"), + if spans_bp != 0 { + fixed(100.0 * bp as f64 / spans_bp as f64, 2) + } else { + "0.00".to_owned() + }, + )); + summary.push(( + format!("{key}_longest_bp"), + r.iter().map(|x| x.2 - x.1).max().unwrap_or(0).to_string(), + )); + } + let sv = |k: &str| { + summary + .iter() + .find(|(key, _)| key == k) + .map(|(_, v)| v.as_str()) + .expect("a summary key") + }; + let mut longest: Vec<&Row> = rows.iter().collect(); + longest.sort_by(|a, b| { + (-(a.2 - a.1), chrom_key(&a.0), a.1).cmp(&(-(b.2 - b.1), chrom_key(&b.0), b.1)) + }); + longest.truncate(TOP); + let sites: i64 = spans.iter().map(|s| s.3).sum(); + let mut text = String::from("key\tvalue\n"); + let meta: [(&str, String); 10] = [ + ("sample", sample.to_owned()), + ("method", "IBDmix".to_owned()), + ( + "archaic_genomes", + "Altai Neanderthal, Vindija 33.19 Neanderthal, Altai Denisovan".to_owned(), + ), + ("population", pop.to_owned()), + ("min_lod", general(MIN_LOD, 6)), + ("min_length_bp", MIN_LENGTH.to_string()), + ("analysed_bp", spans_bp.to_string()), + ("analysed_chromosomes", spans.len().to_string()), + ("analysed_sites", sites.to_string()), + ( + "genes", + if genes_arg.is_empty() { + "none (no gene annotation installed)" + } else { + "GENCODE protein-coding" + } + .to_owned(), + ), + ]; + for (k, v) in meta.iter() { + text += &format!("{k}\t{v}\n"); + } + for (k, v) in &summary { + text += &format!("{k}\t{v}\n"); + } + let genes_text = |g: &[&str]| { + if g.is_empty() { + "none".to_owned() + } else { + g.join(", ") + } + }; + for (i, (c, s, e, anc, who, _)) in longest.iter().enumerate() { + let g = genes_of(c, *s, *e); + text += &format!( + "longest_{}\t{c}:{s}-{e} {anc} {} Mb ({}); genes: {}\n", + i + 1, + fixed((e - s) as f64 / 1e6, 2), + who.join("/"), + genes_text(&g) + ); + } + put( + &Path::new(out_dir).join(format!("{sample}_archaic.tsv")), + text.as_bytes(), + )?; + let mb = |k: &str| fixed(need_int(sv(k)).unwrap_or(0) as f64 / 1e6, 1); + let mut lines = vec![ + format!( + "Archaic ancestry of {sample} (IBDmix against the Altai and Vindija Neanderthals and the Altai Denisovan)" + ), + format!( + " Analysed: {} autosomes, {} Mb, {sites} sites; modern frequencies: 1000 Genomes {pop}", + spans.len(), + fixed(spans_bp as f64 / 1e6, 1) + ), + format!( + " Neanderthal: {}% ({} Mb in {} segments)", + sv("neanderthal_percent"), + mb("neanderthal_bp"), + sv("neanderthal_segments") + ), + format!( + " Denisovan: {}% ({} Mb in {} segments)", + sv("denisovan_percent"), + mb("denisovan_bp"), + sv("denisovan_segments") + ), + ]; + if !longest.is_empty() { + lines.push(format!(" The {} longest segments:", longest.len())); + for (c, s, e, anc, who, lod) in &longest { + let g = genes_of(c, *s, *e); + lines.push(format!( + " {c}:{s}-{e} {} Mb {anc} ({}, LOD {}) genes: {}", + fixed((e - s) as f64 / 1e6, 2), + who.join("/"), + fixed(*lod, 0), + genes_text(&g) + )); + } + } + lines.push(format!(" {CAVEAT}")); + lines.push(format!(" Every segment: {seg_name}")); + let body = lines.join("\n") + "\n"; + put( + &Path::new(out_dir).join(format!("{sample}_archaic.txt")), + body.as_bytes(), + )?; + let _ = out.write_all(body.as_bytes()); + Ok(()) +} + +// --- the command line ------------------------------------------------------------- + +const USAGE: &str = "usage: og archaic {table,strict,genotypes,report} ...\n"; + +const TABLE: Spec = Spec { + verb: "table", + usage: "usage: og archaic table --chrom CHROM --modern MODERN [--archaic NAME=FILE] [--mask NAME=BED] [--exclude BED] --out OUT\n", + options: &[ + "--chrom", + "--modern", + "--archaic", + "--mask", + "--exclude", + "--out", + ], + required: &["--chrom", "--modern", "--out"], +}; +const STRICT: Spec = Spec { + verb: "strict", + usage: "usage: og archaic strict --chrom CHROM --strict STRICT --rows ROWS --out OUT\n", + options: &["--chrom", "--strict", "--rows", "--out"], + required: &["--chrom", "--strict", "--rows", "--out"], +}; +const GENOTYPES: Spec = Spec { + verb: "genotypes", + usage: "usage: og archaic genotypes --table-dir TABLE_DIR --vcf VCF [--pop POP] [--pseudo PSEUDO] --out-dir OUT_DIR\n", + options: &["--table-dir", "--vcf", "--pop", "--pseudo", "--out-dir"], + required: &["--table-dir", "--vcf", "--out-dir"], +}; +const REPORT: Spec = Spec { + verb: "report", + usage: "usage: og archaic report --sample SAMPLE --ibdmix-dir IBDMIX_DIR --spans SPANS [--genes GENES] [--pop POP] --out-dir OUT_DIR\n", + options: &[ + "--sample", + "--ibdmix-dir", + "--spans", + "--genes", + "--pop", + "--out-dir", + ], + required: &["--sample", "--ibdmix-dir", "--spans", "--out-dir"], +}; + +/// `og archaic [args]`: the exit code. Results go to `out`, +/// messages to `err`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og archaic: error: the following arguments are required: verb" + ); + return 2; + }; + let spec = match verb.as_str() { + "table" => &TABLE, + "strict" => &STRICT, + "genotypes" => &GENOTYPES, + "report" => &REPORT, + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og archaic: error: argument verb: invalid choice: {}", + repr(other) + ); + return 2; + } + }; + let a = match Args::parse(spec, &args[1..]).and_then(|a| { + if spec.verb == "genotypes" { + a.int("--pseudo", PSEUDO)?; + } + Ok(a) + }) { + Ok(a) => a, + Err(ArgError::Help) => { + let _ = write!(out, "{}", spec.usage); + return 0; + } + Err(ArgError::Usage(msg)) => { + let _ = write!(err, "{}", spec.usage); + let _ = writeln!(err, "og archaic {}: error: {msg}", spec.verb); + return 2; + } + }; + let done = match spec.verb { + "table" => cmd_table(&a, err), + "strict" => cmd_strict(&a, err), + "genotypes" => cmd_genotypes(&a, err), + _ => cmd_report(&a, out), + }; + match done { + Ok(()) => 0, + Err(Stop(code, msg)) => { + let _ = out.flush(); + let _ = writeln!(err, "ERROR: {msg}"); + code + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s(r: &str, alt: Option<&str>, code: u8) -> Sample { + Sample { + r#ref: r.into(), + alt: alt.map(Into::into), + code, + } + } + + fn code(p: Picked) -> u8 { + match p { + Picked::Call(c) => c.code, + Picked::Clash => 9, + } + } + + #[test] + fn gt_codes() { + assert_eq!(gt_code("0/1"), Some(1)); + assert_eq!(gt_code("1|1"), Some(2)); + assert_eq!(gt_code("0/0"), Some(0)); + assert_eq!(gt_code("./."), None); + assert_eq!(gt_code("1/2"), None); + assert_eq!(gt_code("1"), None); + assert_eq!(gt_code("0/1/1"), None); + } + + #[test] + fn pick_call_is_symmetric_in_its_code() { + let pairs = [ + (s("A", Some("G"), 1), s("A", None, 9)), + (s("A", Some("G"), 1), s("A", Some("T"), 0)), + (s("A", Some("G"), 9), s("A", Some("G"), 2)), + (s("A", Some("G"), 0), s("A", Some("T"), 0)), + (s("A", Some("G"), 1), s("A", Some("T"), 2)), + (s("A", Some("G"), 9), s("A", Some("T"), 0)), + (s("A", Some("G"), 1), s("A", Some("G"), 2)), + ]; + for (x, y) in pairs { + assert_eq!( + code(pick_call(Some(x.clone()), y.clone())), + code(pick_call(Some(y), x)) + ); + } + assert_eq!( + pick_call(Some(s("A", Some("G"), 0)), s("A", Some("T"), 0)), + Picked::Call(s("A", None, 0)) + ); + } + + #[test] + fn a_clash_ends_its_position() { + let recs = vec![ + (4000, s("T", Some("C"), 1)), + (4000, s("T", Some("G"), 1)), + (4000, s("T", Some("C"), 1)), + (10, s("A", Some("G"), 2)), + ]; + assert_eq!( + settle(recs), + vec![(10, s("A", Some("G"), 2)), (4000, s("T", None, 9))] + ); + } + + #[test] + fn pseudo_columns_carry_the_count() { + let p = pseudo_columns(2); + assert_eq!(p, ["0\t0", "1\t0", "2\t0", "2\t1", "2\t2"]); + } + + #[test] + fn table_names() { + assert_eq!(table_chrom("chr1.tsv.gz"), Some("chr1")); + assert_eq!(table_chrom("chr22.tsv.gz"), Some("chr22")); + assert_eq!(table_chrom("chr19.tsv.gz"), Some("chr19")); + assert_eq!(table_chrom("chr23.tsv.gz"), None); + assert_eq!(table_chrom("chr0.tsv.gz"), None); + assert_eq!(table_chrom("chrX.tsv.gz"), None); + assert_eq!(table_chrom("chr01.tsv.gz"), None); + assert_eq!(table_chrom("chr1.tsv"), None); + } + + #[test] + fn frequencies_clamp_like_python() { + let afs: Vec = ["1e-05", "nan", "inf", "-0.0", "x", "1.5"] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + let (_, _, _, clean) = table_row( + Some(("A", "G", &afs)), + [Some(("A", "G", "0/1")), None, None], + ) + .unwrap(); + assert_eq!(clean, ["1e-05", "nan", "1", "-0", "0", "1"]); + } + + #[test] + fn genes_touching_the_ends() { + let mut genes = std::collections::HashMap::new(); + let mut v = vec![ + (90, 100, "ENDS_AT_START".to_owned()), + (100, 120, "INSIDE".to_owned()), + (200, 210, "STARTS_AT_END".to_owned()), + (50, 99, "BEFORE".to_owned()), + ]; + v.sort(); + genes.insert("chr22".to_owned(), v); + assert_eq!( + genes_in(&genes, "22", 100, 200), + ["ENDS_AT_START", "INSIDE"] + ); + } + + #[test] + fn gene_names() { + assert_eq!(gene_name("gene_id \"G\"; gene_name \"ABC\";"), Some("ABC")); + assert_eq!(gene_name("gene_name \"\"; gene_name \"B\";"), Some("B")); + assert_eq!(gene_name("gene_name \"open"), None); + } + + #[test] + fn merge_keeps_the_highest_lod() { + let m = merge(vec![ + (("chr22".into(), 1_000_000, 1_300_000, 40.0), "Altai"), + (("chr22".into(), 1_200_000, 1_500_000, 50.0), "Vindija"), + (("chr22".into(), 1_500_000, 1_600_000, 7.0), "Altai"), + ]); + assert_eq!(m.len(), 1); + assert_eq!((m[0].1, m[0].2, m[0].3), (1_000_000, 1_600_000, 50.0)); + assert_eq!(m[0].4, ["Altai", "Vindija"]); + } +} diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 0594164a9..74febd042 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -22,6 +22,8 @@ //! A group name is ours: a set of reference populations we put together //! and named, not a nationality. +pub mod ancient; +pub mod archaic; pub mod chroms; pub mod order; pub mod panel; diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs new file mode 100644 index 000000000..559a4d71e --- /dev/null +++ b/rust/crates/og-intervals/src/chain.rs @@ -0,0 +1,376 @@ +//! UCSC chain files, for lifting single positions from one assembly to +//! another (`og ancient lift`). +//! +//! A chain file is a list of chains. Each starts with a header line, +//! `chain score tName tSize tStrand tStart tEnd qName qSize qStrand qStart +//! qEnd id`, followed by block lines `size dt dq` (the last one `size` +//! alone) and a blank line. The `t` side is the assembly a position is +//! lifted from (the source), the `q` side the one it is lifted to. Each +//! block is an ungapped stretch of `size` bases; `dt` and `dq` are the gaps +//! before the next block on each side. +//! +//! [`Chain::lift`] maps a position only when the blocks that cover it all +//! belong to one chain; a position in a gap, or covered by two chains, has +//! no single place and gives `None`. + +use og_io::{Lines, Utf8}; +use pycompat::num::{parse_int, py_isspace}; +use std::collections::HashMap; +use std::fmt; +use std::io::{self, BufRead}; + +/// Why a chain file could not be read. +#[derive(Debug)] +pub enum ChainError { + Io(io::Error), + /// A chain whose source strand is not `+`: the chain's id (or its + /// 0-based number when the header has no id) and the strand. + SourceStrand { + chain: String, + strand: String, + }, + /// A block line before any chain header, or after a chain's last block. + Orphan(String), + /// A line that cannot be used: its number (1-based) and why. + Line { + line: usize, + why: String, + }, +} + +impl fmt::Display for ChainError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ChainError::Io(e) => write!(f, "{e}"), + ChainError::SourceStrand { chain, strand } => { + write!(f, "chain {chain} has source strand {strand}") + } + ChainError::Orphan(line) => write!(f, "a block line outside a chain: {line}"), + ChainError::Line { line, why } => write!(f, "line {line}: {why}"), + } + } +} + +impl std::error::Error for ChainError {} + +impl From for ChainError { + fn from(e: io::Error) -> Self { + ChainError::Io(e) + } +} + +/// One ungapped block: `[start, end)` on the source, 0-based. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Block { + start: u32, + end: u32, + /// The chain's number in the file (0-based). + chain: u32, + /// The target sequence's name, an index into [`Chain::names`]. + name: u32, + /// The block's first base on the target, counted on its strand. + to_start: u32, + /// The target strand, an index into [`Chain::names`]. + strand: u32, + /// The target sequence's length. + to_size: u32, +} + +/// The blocks of one source sequence, sorted, with the furthest end of +/// each prefix, so a lookup stops where no earlier block can reach. +#[derive(Debug, Default)] +struct Source { + blocks: Vec, + reach: Vec, +} + +/// A chain file, indexed by source sequence. +#[derive(Debug, Default)] +pub struct Chain { + /// Target names and strands, interned. + names: Vec, + sources: HashMap, +} + +/// Where [`Chain::lift`] puts a position. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Lifted<'a> { + /// The target sequence. + pub chrom: &'a str, + /// The 1-based position on it. + pub pos: i64, + /// The target strand as the file writes it (`+` or `-`). + pub strand: &'a str, +} + +/// The header fields a block line needs, kept while the chain's blocks run. +struct Open { + source: String, + at: i64, + name: u32, + to_at: i64, + strand: u32, + to_size: i64, + chain: u32, +} + +impl Chain { + /// Reads a chain file (universal newlines; invalid UTF-8 is replaced). + /// Fields are split on whitespace, so a header with tabs reads too. + pub fn read(input: impl BufRead) -> Result { + let mut lines = Lines::new(input, Utf8::Replace); + let mut text = String::new(); + let mut out = Chain::default(); + let mut interned: HashMap = HashMap::new(); + let mut intern = |names: &mut Vec, s: &str| -> u32 { + if let Some(&i) = interned.get(s) { + return i; + } + let i = names.len() as u32; + names.push(s.to_owned()); + interned.insert(s.to_owned(), i); + i + }; + let mut cur: Option = None; + let mut chains: u32 = 0; + let mut n = 0; + while lines.read_line(&mut text)? { + n += 1; + let c: Vec<&str> = text.split(py_isspace).filter(|f| !f.is_empty()).collect(); + let bad = |why: String| ChainError::Line { line: n, why }; + if c.is_empty() { + cur = None; + continue; + } + if c[0] == "chain" { + if c.len() < 11 { + return Err(bad(format!( + "a chain header of {} fields, want 13", + c.len() + ))); + } + let number = chains; + chains += 1; + if c[4] != "+" { + return Err(ChainError::SourceStrand { + chain: c + .get(12) + .map_or_else(|| number.to_string(), |s| (*s).to_owned()), + strand: c[4].to_owned(), + }); + } + let num = |k: usize| coord(c[k], n, k + 1); + cur = Some(Open { + source: c[2].to_owned(), + at: num(5)?, + name: intern(&mut out.names, c[7]), + to_size: num(8)?, + strand: intern(&mut out.names, c[9]), + to_at: num(10)?, + chain: number, + }); + continue; + } + let Some(open) = cur.as_mut() else { + return Err(ChainError::Orphan( + pycompat::num::py_strip(&text).to_owned(), + )); + }; + let size = coord(c[0], n, 1)?; + let fit = |v: i64| { + u32::try_from(v).map_err(|_| bad(format!("position {v} does not fit 32 bits"))) + }; + let block = Block { + start: fit(open.at)?, + end: fit(open.at + size)?, + chain: open.chain, + name: open.name, + to_start: fit(open.to_at)?, + strand: open.strand, + to_size: fit(open.to_size)?, + }; + out.sources + .entry(open.source.clone()) + .or_default() + .blocks + .push(block); + if c.len() >= 3 { + open.at += size + coord(c[1], n, 2)?; + open.to_at += size + coord(c[2], n, 3)?; + } else { + cur = None; + } + } + for s in out.sources.values_mut() { + let names = &out.names; + s.blocks.sort_by(|a, b| { + (a.start, a.end, a.chain) + .cmp(&(b.start, b.end, b.chain)) + .then_with(|| names[a.name as usize].cmp(&names[b.name as usize])) + .then_with(|| a.to_start.cmp(&b.to_start)) + .then_with(|| names[a.strand as usize].cmp(&names[b.strand as usize])) + .then_with(|| a.to_size.cmp(&b.to_size)) + }); + let mut far = 0; + s.reach = s + .blocks + .iter() + .map(|b| { + far = far.max(b.end); + far + }) + .collect(); + } + Ok(out) + } + + /// How many blocks were read. + pub fn blocks(&self) -> usize { + self.sources.values().map(|s| s.blocks.len()).sum() + } + + /// The 1-based position `pos` of source sequence `chrom` on the target, + /// or `None` when no block covers it, or blocks of more than one chain + /// do. On a `-` target strand the position is mirrored: + /// `size - 1 - offset`, 0-based. + pub fn lift(&self, chrom: &str, pos: i64) -> Option> { + let src = self.sources.get(chrom)?; + let p = pos - 1; + // Blocks that start at or before p, walked back while one of them + // may still reach past p. + let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); + let mut first: Option<&Block> = None; + while j > 0 && i64::from(src.reach[j - 1]) > p { + j -= 1; + let b = &src.blocks[j]; + if i64::from(b.end) > p { + match first { + None => first = Some(b), + Some(f) if f.chain != b.chain => return None, + Some(_) => {} + } + } + } + let b = first?; + let mut q = i64::from(b.to_start) + (p - i64::from(b.start)); + let strand = self.names[b.strand as usize].as_str(); + if strand == "-" { + q = i64::from(b.to_size) - 1 - q; + } + Some(Lifted { + chrom: &self.names[b.name as usize], + pos: q + 1, + strand, + }) + } +} + +/// A whole-number field of line `line`, column `col`: Python's `int()`. +fn coord(field: &str, line: usize, col: usize) -> Result { + parse_int(field) + .ok() + .and_then(|v| i64::try_from(v).ok()) + .ok_or_else(|| ChainError::Line { + line, + why: format!("field {col} '{field}' is not a whole number"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The chain of tests/test_ancient_report.py: chr1 with a gap, chr2 on + /// the minus strand, chr3 covered twice. + const TEST: &str = "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1 +100 50 50 +50 + +chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2 +100 + +chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3 +100 + +chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4 +100 +"; + + fn chain() -> Chain { + Chain::read(TEST.as_bytes()).unwrap() + } + + fn lift(c: &Chain, chrom: &str, pos: i64) -> Option<(String, i64, String)> { + c.lift(chrom, pos) + .map(|l| (l.chrom.to_owned(), l.pos, l.strand.to_owned())) + } + + fn at(chrom: &str, pos: i64, strand: &str) -> Option<(String, i64, String)> { + Some((chrom.to_owned(), pos, strand.to_owned())) + } + + #[test] + fn blocks_move_by_their_offset() { + let c = chain(); + assert_eq!(c.blocks(), 5); + assert_eq!(lift(&c, "chr1", 1051), at("chr1", 2051, "+")); + assert_eq!(lift(&c, "chr1", 1160), at("chr1", 2160, "+")); + assert_eq!(lift(&c, "chr1", 1120), None, "a gap has no place"); + assert_eq!(lift(&c, "chr9", 1051), None, "an unknown sequence"); + } + + #[test] + fn block_ends_are_half_open() { + let c = chain(); + // The first block holds 0-based 1000..1100: 1-based 1001 to 1100. + assert_eq!(lift(&c, "chr1", 1001), at("chr1", 2001, "+")); + assert_eq!(lift(&c, "chr1", 1100), at("chr1", 2100, "+")); + assert_eq!(lift(&c, "chr1", 1000), None); + assert_eq!(lift(&c, "chr1", 1101), None); + assert_eq!(lift(&c, "chr1", 1151), at("chr1", 2151, "+")); + assert_eq!(lift(&c, "chr1", 1200), at("chr1", 2200, "+")); + assert_eq!(lift(&c, "chr1", 1201), None); + assert_eq!(lift(&c, "chr1", 0), None); + assert_eq!(lift(&c, "chr1", -5), None); + } + + #[test] + fn minus_strand_mirrors() { + // 0-based 510 is 310 on the reverse strand: 10000 - 1 - 310 = 9689. + assert_eq!(lift(&chain(), "chr2", 511), at("chr2", 9690, "-")); + } + + #[test] + fn two_chains_over_a_position_give_none() { + let c = chain(); + assert_eq!(lift(&c, "chr3", 171), None); + assert_eq!(lift(&c, "chr3", 121), at("chr3", 121, "+")); + assert_eq!(lift(&c, "chr3", 221), at("chr3", 5221, "+")); + } + + #[test] + fn a_long_block_far_back_still_counts() { + // A long block of chain 1 starts long before a short block of + // chain 2; a position inside both is covered twice. + let text = "chain 1 c 9000 + 0 5000 c 9000 + 0 5000 1\n5000\n\n\ + chain 1 c 9000 + 4000 4010 d 9000 + 0 10 2\n10\n"; + let c = Chain::read(text.as_bytes()).unwrap(); + assert_eq!(lift(&c, "c", 4005), None); + assert_eq!(lift(&c, "c", 3000), at("c", 3000, "+")); + } + + #[test] + fn errors() { + let e = Chain::read("chain 1 c 10 - 0 5 c 10 + 0 5 7\n5\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "chain 7 has source strand -"); + let e = Chain::read("chain 1 c 10 - 0 5 c 10 + 0 5\n5\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "chain 0 has source strand -"); + let e = Chain::read(" 5 1 1 \n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "a block line outside a chain: 5 1 1"); + // After a block line with one field the chain is over. + let e = Chain::read("chain 1 c 10 + 0 5 c 10 + 0 5 1\n5\n6\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "a block line outside a chain: 6"); + let e = Chain::read("chain 1 c 10 + x 5 c 10 + 0 5 1\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "line 1: field 6 'x' is not a whole number"); + } +} diff --git a/rust/crates/og-intervals/src/lib.rs b/rust/crates/og-intervals/src/lib.rs index 3e3104345..2a5008bd1 100644 --- a/rust/crates/og-intervals/src/lib.rs +++ b/rust/crates/og-intervals/src/lib.rs @@ -10,8 +10,11 @@ //! search, BED files such as the PAR regions read into them) and an //! overlap index with reciprocal overlap. //! - [`fasta`]: random access into a FASTA through its `.fai`. +//! - [`chain`]: UCSC chain files, to lift single positions to another +//! assembly. pub mod bed; +pub mod chain; pub mod fasta; pub mod mask; pub mod sets; diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index dd69a5991..35edc30d6 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -57,6 +57,8 @@ const USAGE: &str = "usage: og [args] og family ibd [args] og consumer build-sites|export|check [args] og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report [args] + og archaic table|strict|genotypes|report [args] + og ancient select|lift|person-geno|align|report [args] og vntyper report [args] og locityper report [args] og mocha report [args] @@ -252,6 +254,16 @@ fn main() -> ExitCode { &mut std::io::stdout().lock(), &mut std::io::stderr().lock(), )), + "archaic" => ExitCode::from(og_ancestry::archaic::cli( + rest, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )), + "ancient" => ExitCode::from(og_ancestry::ancient::cli( + rest, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )), "str" => match rest.first().map(String::as_str) { #[cfg(feature = "bam")] Some("motif") => ExitCode::from(og_bam::motif::cli( From f1d1aaeb9a116a35e7de7820468dedd4dc0f0971 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:45:48 +0100 Subject: [PATCH 02/19] refactor(ancestry): steps 43 and 44 and their setup run og archaic and og ancient The parity cases compare og with bin/archaic_report.py and bin/ancient_report.py on synthetic inputs. setup.sh stamps the archaic table with a format id instead of the Python's hash, and takes a table the last Python built as current. Bead: pgp-bs9u.21 --- modules/local/ancient/main.nf | 8 +- modules/local/archaic/main.nf | 8 +- scripts/43-archaic.sh | 8 +- scripts/44-ancient.sh | 4 +- scripts/setup.sh | 32 ++- tests/parity/cases.tsv | 51 +++++ tests/parity/gen/ancient-inputs.sh | 211 ++++++++++++++++++ tests/parity/gen/archaic-inputs.sh | 187 ++++++++++++++++ .../long-geno/o.geno.tmp.absent | 0 .../short-geno/o.geno.tmp.absent | 0 .../wide-geno/o.geno.tmp.absent | 0 .../bad-code/gt/chr22.gt.gz.absent | 0 .../golden/archaic-strict/unsorted/exit | 1 + .../archaic-strict/unsorted/s.tsv.absent | 0 .../golden/archaic-strict/unsorted/stderr | 1 + .../archaic-table/unsorted/t.vcf.absent | 0 tests/parity/oracle.ref | 9 + 17 files changed, 499 insertions(+), 21 deletions(-) create mode 100644 tests/parity/gen/ancient-inputs.sh create mode 100644 tests/parity/gen/archaic-inputs.sh create mode 100644 tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent create mode 100644 tests/parity/golden/archaic-strict/unsorted/exit create mode 100644 tests/parity/golden/archaic-strict/unsorted/s.tsv.absent create mode 100644 tests/parity/golden/archaic-strict/unsorted/stderr create mode 100644 tests/parity/golden/archaic-table/unsorted/t.vcf.absent diff --git a/modules/local/ancient/main.nf b/modules/local/ancient/main.nf index 839f6ace7..a5b83de3b 100644 --- a/modules/local/ancient/main.nf +++ b/modules/local/ancient/main.nf @@ -6,11 +6,11 @@ reference block over a site becomes a record with the reference base, and a site without reads has no line ANCIENT_GENO the sample as one more column of the panel's present-day - EIGENSTRAT genotypes (bin/ancient_report.py person-geno) + EIGENSTRAT genotypes (og ancient person-geno) ANCIENT_PCA smartpca: axes from the present-day West Eurasians, the sample projected (lsqproject) ANCIENT_REPORT the nearest ancient groups, the summary and the plot, on - the panel's axes (bin/ancient_report.py report) + the panel's axes (og ancient report) The panel is the folder setup.sh --ancient-panel writes (reference/ancient/aadr_). Runs for the samples with a gVCF only; @@ -78,7 +78,7 @@ process ANCIENT_GENO { script: """ - ancient_report.py person-geno --sites ${panel}/sites_grch38.tsv --genotypes ${genotypes} \\ + og ancient person-geno --sites ${panel}/sites_grch38.tsv --genotypes ${genotypes} \\ --snp ${panel}/moderns.snp --moderns-geno ${panel}/moderns.geno --out merged.geno --stats stats.tsv { cat ${panel}/moderns.ind; printf '%s\\tU\\tPerson\\n' '${meta.id}'; } > merged.ind @@ -148,7 +148,7 @@ process ANCIENT_REPORT { script: """ release=\$(awk -F'\\t' '\$1 == "release" {print \$2}' ${panel}/DONE) - ancient_report.py report --panel-evec ${panel}/panel.evec --run-evec ${evec} --ancients ${panel}/ancients.tsv \\ + og ancient report --panel-evec ${panel}/panel.evec --run-evec ${evec} --ancients ${panel}/ancients.tsv \\ --sample ${meta.id} --release "\$release" --stats ${stats} \\ --out-tsv ${meta.id}_ancient.tsv --out-info ${meta.id}_ancient_info.tsv \\ --out-txt ${meta.id}_ancient.txt --out-svg ${meta.id}_ancient_pca.svg diff --git a/modules/local/archaic/main.nf b/modules/local/archaic/main.nf index ec31864df..bf68a4653 100644 --- a/modules/local/archaic/main.nf +++ b/modules/local/archaic/main.nf @@ -4,13 +4,13 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Opt-in ('archaic' in --tools, with --archaic_data). Three processes, as in scripts/43-archaic.sh: - ARCHAIC_GENOTYPES bin/archaic_report.py genotypes: IBDmix's genotype file of + ARCHAIC_GENOTYPES og archaic genotypes: IBDmix's genotype file of each chromosome (the archaic genomes, the sample's VCF and the 1000 Genomes stand-ins of --archaic_pop), and each archaic genome's mask ARCHAIC_IBDMIX IBDmix against each archaic genome, LOD 4, the options of Chen et al. 2020, with that genome's mask - ARCHAIC_REPORT bin/archaic_report.py report: the segments of 50 kb or + ARCHAIC_REPORT og archaic report: the segments of 50 kb or more, the share, the ten longest with their genes Equivalent to: scripts/43-archaic.sh @@ -34,7 +34,7 @@ process ARCHAIC_GENOTYPES { script: """ - archaic_report.py genotypes --table-dir ${archaic_data} --vcf ${vcf} --pop ${params.archaic_pop} --out-dir gt + og archaic genotypes --table-dir ${archaic_data} --vcf ${vcf} --pop ${params.archaic_pop} --out-dir gt cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -126,7 +126,7 @@ process ARCHAIC_REPORT { script: def genes_arg = genes ? "--genes ${genes}" : '' """ - archaic_report.py report --sample ${meta.id} --ibdmix-dir ${ibdmix} --spans ${ibdmix}/spans.tsv \\ + og archaic report --sample ${meta.id} --ibdmix-dir ${ibdmix} --spans ${ibdmix}/spans.tsv \\ --pop ${params.archaic_pop} ${genes_arg} --out-dir . cat <<-END_VERSIONS > versions.yml diff --git a/scripts/43-archaic.sh b/scripts/43-archaic.sh index c6bcb62a3..9a98b9392 100755 --- a/scripts/43-archaic.sh +++ b/scripts/43-archaic.sh @@ -6,14 +6,14 @@ # high-coverage archaic genomes, the Altai and Vindija 33.19 Neanderthals and # the Altai Denisovan, and calls the stretches you share with one of them. # It needs the archaic data setup.sh --archaic builds (reference/archaic/): -# 1. bin/archaic_report.py genotypes writes IBDmix's genotype file of each +# 1. og archaic genotypes writes IBDmix's genotype file of each # chromosome: the archaic genotypes, yours from the VCF, and 100 # stand-ins carrying the 1000 Genomes allele count of ARCHAIC_POP at each # site (IBDmix takes the modern frequency from the modern columns, and # one genome alone gives it none; docs/43-archaic.md says why this is # the same score); # 2. IBDmix scores your column against each archaic genome (LOD 4); -# 3. bin/archaic_report.py report keeps the segments of 50 kb or more, +# 3. og archaic report keeps the segments of 50 kb or more, # merges the two Neanderthals, and writes the share, the segments and # the ten longest with their protein-coding genes. # @@ -61,7 +61,7 @@ W=$(cpath "$WORK") echo "[1/3] Writing IBDmix's genotype files (your VCF, the archaic genomes, the stand-ins)..." run_in --cpus 1 --memory 4g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py genotypes --table-dir "$D" --vcf "/genome/${SAMPLE}/vcf/${SAMPLE}.vcf.gz" \ + /pgp-bin/og archaic genotypes --table-dir "$D" --vcf "/genome/${SAMPLE}/vcf/${SAMPLE}.vcf.gz" \ --pop "$POP" --out-dir "${W}/gt" # One IBDmix run per chromosome and archaic genome, THREADS at a time, with the @@ -101,7 +101,7 @@ else echo " No GENCODE gene file (${GENES}): the segments are listed without their genes (setup.sh installs it)." fi run_in --cpus 1 --memory 2g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py report --sample "$SAMPLE" --ibdmix-dir "${W}/ibdmix" \ + /pgp-bin/og archaic report --sample "$SAMPLE" --ibdmix-dir "${W}/ibdmix" \ --spans "${W}/gt/spans.tsv" --pop "$POP" ${GARG[@]+"${GARG[@]}"} --out-dir "$(cpath "$OUT")" for f in "${OUT}/${SAMPLE}_archaic.tsv" "${OUT}/${SAMPLE}_archaic_segments.tsv"; do [ -s "$f" ] || { echo "ERROR: step 43 wrote no ${f}" >&2; exit 1; } diff --git a/scripts/44-ancient.sh b/scripts/44-ancient.sh index 28cc87a79..3206fa8f4 100755 --- a/scripts/44-ancient.sh +++ b/scripts/44-ancient.sh @@ -78,7 +78,7 @@ run_in --cpus 2 --memory 4g "$BCFTOOLS_IMAGE" \ echo "[2/4] Adding ${SAMPLE} to the present-day panel..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py person-geno --sites "${P}/sites_grch38.tsv" --genotypes "${W}/genotypes.tsv" \ + /pgp-bin/og ancient person-geno --sites "${P}/sites_grch38.tsv" --genotypes "${W}/genotypes.tsv" \ --snp "${P}/moderns.snp" --moderns-geno "${P}/moderns.geno" --out "${W}/merged.geno" --stats "${W}/stats.tsv" { cat "${PANEL}/moderns.ind"; printf '%s\tU\tPerson\n' "$SAMPLE"; } > "${WORK}/merged.ind" @@ -93,7 +93,7 @@ run_in --cpus "$THREADS" --memory 8g "$EIGENSOFT_IMAGE" smartpca -p "${W}/run.pa echo "[4/4] The nearest ancient groups..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py report --panel-evec "${P}/panel.evec" --run-evec "${W}/run.evec" \ + /pgp-bin/og ancient report --panel-evec "${P}/panel.evec" --run-evec "${W}/run.evec" \ --ancients "${P}/ancients.tsv" --sample "$SAMPLE" --release "$RELEASE" --stats "${W}/stats.tsv" \ --out-tsv "${O}/${SAMPLE}_ancient.tsv" --out-info "${O}/${SAMPLE}_ancient_info.tsv" \ --out-txt "${O}/${SAMPLE}_ancient.txt" --out-svg "${O}/${SAMPLE}_ancient_pca.svg" diff --git a/scripts/setup.sh b/scripts/setup.sh index 88b6c4943..326dfeddd 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -801,7 +801,7 @@ consumer_sites_pin_check() { # id, each checked against the md5 Dataverse publishes) become the panel of # step 44 in reference/ancient/aadr_: # panel.ind, poplist.txt, ancients.tsv who builds the axes (the present-day -# West Eurasians of bin/ancient_report.py's MODERN_GROUPS) and who is +# West Eurasians of og ancient's MODERN_GROUPS) and who is # projected (dated West Eurasians with 100,000 sites or more); # sites_grch38.tsv the autosomal sites on GRCh38 (UCSC's hg19ToHg38 chain, # each kept only when one of its alleles is the GRCh38 base); @@ -830,6 +830,7 @@ install_ancient_panel() { done echo "Installing the ancient populations panel, AADR ${AADR_VERSION} 1240K (step 44) in ${dir}" ensure_image EIGENSOFT_IMAGE 44 || return 1 + need_og "chooses the ancient panel's people and places its sites" || return 1 mkdir -p "$raw" for e in anno ind snp geno; do f="${raw}/aadr.${e}" @@ -868,9 +869,9 @@ install_ancient_panel() { # shellcheck disable=SC2016 # $1 to $4 belong to the inner bash if ! run_in --rw "$part" -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 4g "$PYTHON_IMAGE" \ bash -euo pipefail -c ' - python3 /pgp-bin/ancient_report.py select --anno "$1/aadr.anno" --ind "$1/aadr.ind" \ + /pgp-bin/og ancient select --anno "$1/aadr.anno" --ind "$1/aadr.ind" \ --out-ind "$2/panel.ind" --poplist "$2/poplist.txt" --ancients "$2/ancients.tsv" - python3 /pgp-bin/ancient_report.py lift --snp "$1/aadr.snp" --chain "$3" --fasta "$4" --out "$2/sites_grch38.tsv"' \ + /pgp-bin/og ancient lift --snp "$1/aadr.snp" --chain "$3" --fasta "$4" --out "$2/sites_grch38.tsv"' \ _ "$R" "$P" "$C" "$REF_FASTA_C"; then echo "[WARN] Could not choose the panel's people or place its sites (the lines above say why)." return 1 @@ -894,7 +895,7 @@ install_ancient_panel() { return 1 fi if ! run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py align --panel-evec "${P}/panel.evec" --run-evec "${P}/check.evec"; then + /pgp-bin/og ancient align --panel-evec "${P}/panel.evec" --run-evec "${P}/check.evec"; then echo "[WARN] The two smartpca runs on the same present-day people gave different axes (logs in ${part})." return 1 fi @@ -1836,6 +1837,13 @@ archaic_filterbed_sha() { return 1 } +# The version of the table og archaic table and strict write: a change to +# their bytes changes it, and every install rebuilds its table. +ARCHAIC_TABLE_FORMAT=archaic-table-v1 +# What code= held when bin/archaic_report.py built the table: the sha256 of +# its part above "# --- genotypes (step 43)", at its last version. +ARCHAIC_TABLE_PYTHON_SHA256=6bdc4c99a8d80d96179551e37022a3cd5324f650577b9fd64e27d84383e0f1d7 + # install_archaic: the table of step 43 in reference/archaic/ARCHAIC_DATA_VERSION # (ARCHAIC_DIR: another folder inside GENOME_DIR), one chrN.tsv.gz per # autosome, GRCh38: position, REF, ALT, the ALT count of the Altai, Vindija @@ -1858,6 +1866,7 @@ install_archaic() { cpath "$dir" >/dev/null || return 1 echo "Installing the archaic data ${ARCHAIC_DATA_VERSION} (step 43) in ${dir}" for v in BCFTOOLS_IMAGE PICARD_IMAGE PYTHON_IMAGE IBDMIX_IMAGE; do ensure_image "$v" 43 || return 1; done + need_og "builds the archaic table" || return 1 [ -s "$REF_DICT" ] || { echo "[WARN] ${REF_DICT} is missing: run $0 ${GENOME_DIR} first (it writes the sequence dictionary)"; return 1; } mkdir -p "${dir}/md5" "${dir}/download" "${dir}/work" "${dir}/mask" "${dir}/exclude" for v in "Altai|${ARCHAIC_ALTAI_MD5SUMS_SHA256}" "Vindija33.19|${ARCHAIC_VINDIJA_MD5SUMS_SHA256}" "Denisova|${ARCHAIC_DENISOVA_MD5SUMS_SHA256}"; do @@ -1893,9 +1902,18 @@ install_archaic() { chroms=$(seq 1 22 | tr '\n' ' ') fi [ -n "${chroms// /}" ] || { echo "[WARN] ARCHAIC_REGIONS=${regions} names no autosome (1 to 22)"; return 1; } - stamp="data=${ARCHAIC_DATA_VERSION} regions=${rstamp} bcftools=${BCFTOOLS_IMAGE} picard=${PICARD_IMAGE} masks=$(_digest sha256 <(printf '%s\n%s\n%s\n%s\n%s\n' "$ARCHAIC_STRICT_MASK_SHA256" "$ARCHAIC_SEGDUPS_SHA256" "$ARCHAIC_FILTERBED_SHA256_ALTAI" "$ARCHAIC_FILTERBED_SHA256_VINDIJA" "$ARCHAIC_FILTERBED_SHA256_DENISOVA")) code=$(_digest sha256 <(sed '/^# --- genotypes (step 43)/,$d' "${PGP_ROOT}/bin/archaic_report.py"))" + stamp="data=${ARCHAIC_DATA_VERSION} regions=${rstamp} bcftools=${BCFTOOLS_IMAGE} picard=${PICARD_IMAGE} masks=$(_digest sha256 <(printf '%s\n%s\n%s\n%s\n%s\n' "$ARCHAIC_STRICT_MASK_SHA256" "$ARCHAIC_SEGDUPS_SHA256" "$ARCHAIC_FILTERBED_SHA256_ALTAI" "$ARCHAIC_FILTERBED_SHA256_VINDIJA" "$ARCHAIC_FILTERBED_SHA256_DENISOVA")) code=${ARCHAIC_TABLE_FORMAT}" + # Until og archaic replaced bin/archaic_report.py, code= held the sha256 of + # that file's table half. The last Python wrote the same table bytes as + # og archaic table and strict (tests/parity), so a table it built counts as + # built here: its stamp is accepted once and rewritten. Remove this one + # release after 1.0.0. + local legacy="${stamp% code=*} code=${ARCHAIC_TABLE_PYTHON_SHA256}" local have=1 for c in $chroms; do [ -s "${dir}/chr${c}.tsv.gz" ] || have=0; done + for v in "${dir}/DONE" "${dir}"/chr*.stamp; do + [ "$(cat "$v" 2>/dev/null)" = "$legacy" ] && echo "$stamp" > "$v" + done if [ "$have" = 1 ] && [ "$(cat "${dir}/DONE" 2>/dev/null)" = "$stamp" ]; then echo "[OK] archaic data ${ARCHAIC_DATA_VERSION} (step 43): ${dir}" return 0 @@ -1975,7 +1993,7 @@ install_archaic() { echo "[WARN] Could not read the archaic genotypes of chr${c}"; return 1 fi if ! run_in --rw "$dir" -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py table --chrom "$c" --modern "$(cpath "${wk}/modern.tsv")" \ + /pgp-bin/og archaic table --chrom "$c" --modern "$(cpath "${wk}/modern.tsv")" \ --archaic "Altai=$(cpath "${wk}/Altai.tsv")" --archaic "Vindija=$(cpath "${wk}/Vindija33.19.tsv")" \ --archaic "Denisova=$(cpath "${wk}/Denisova.tsv")" \ --mask "Altai=$(cpath "${dir}/mask/Altai.chr${c}.bed.gz")" \ @@ -2009,7 +2027,7 @@ install_archaic() { echo "[WARN] the GRCh38 strict mask has no interval on chr${c}: ${dir}/strict_GRCh38.bed"; return 1 fi if ! run_in --rw "$dir" -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py strict --chrom "$c" --strict "$(cpath "${wk}/strict.bed")" \ + /pgp-bin/og archaic strict --chrom "$c" --strict "$(cpath "${wk}/strict.bed")" \ --rows "$(cpath "${wk}/lifted.tsv")" --out "$(cpath "${wk}/marked.tsv")"; then echo "[WARN] Could not mark the strict mask on chr${c}"; return 1 fi diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 586dfda38..6ec8e03fa 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -917,3 +917,54 @@ family-ibd mapbp gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree mapbp/pedigree.ts family-ibd nomap gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree nomap/pedigree.tsv --chroms nomap/chroms.txt --maps nomap/maps --phased nomap/phased --ibd nomap/ibd --out-dir out - content PYTHON_IMAGE family-ibd short gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree short/pedigree.tsv --chroms short/chroms.txt --maps short/maps --phased short/phased --ibd short/ibd --out-dir out - content PYTHON_IMAGE family-ibd twice gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree twice/pedigree.tsv --chroms twice/chroms.txt --maps twice/maps --phased twice/phased --ibd twice/ibd --out-dir out - content PYTHON_IMAGE +# Step 43, og archaic (bin/archaic_report.py) +archaic-table masked gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=altai.bed --mask Vindija=vindija.bed --exclude excl.bed --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table plain gen:archaic-inputs.sh --chrom chr22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table gz gen:archaic-inputs.sh --chrom 22 --modern modern.tsv.gz --archaic Altai=altai.tsv.gz --archaic Vindija=vindija.tsv.gz --archaic Denisova=denisova.tsv.gz --mask Denisova=altai.bed --exclude excl.bed --exclude vindija.bed --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table unsorted gen:archaic-inputs.sh --chrom 22 --modern unsorted.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table order gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Vindija=vindija.tsv --archaic Altai=altai.tsv --archaic Denisova=denisova.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table bad-name gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Neanderthal=altai.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table mask-twice gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=altai.bed --mask Altai=vindija.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-table empty-mask gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=empty.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-table empty-exclude gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --exclude empty.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-strict plain gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows.tsv --out s.tsv s.tsv bytes PYTHON_IMAGE +archaic-strict unsorted gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows-unsorted.tsv --out s.tsv - bytes PYTHON_IMAGE +archaic-strict empty-mask gen:archaic-inputs.sh --chrom 22 --strict empty.bed --rows rows.tsv --out s.tsv - bytes PYTHON_IMAGE +archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr22.Altai.mask.bed *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes default gen:archaic-inputs.sh --table-dir . --vcf s.vcf.gz --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes afr-one gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop AFR --pseudo 1 --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes two-samples gen:archaic-inputs.sh --table-dir . --vcf two.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes comes-back gen:archaic-inputs.sh --table-dir . --vcf back.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes no-table-chrom gen:archaic-inputs.sh --table-dir . --vcf none.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes bad-code gen:archaic-inputs.sh --table-dir badcode --vcf s.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes bad-pop gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop XYZ --out-dir gt - bytes PYTHON_IMAGE +archaic-genotypes bad-pseudo gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pseudo 0 --out-dir gt - bytes PYTHON_IMAGE +archaic-genotypes no-table gen:archaic-inputs.sh --table-dir emptydir --vcf s.vcf --out-dir gt - bytes PYTHON_IMAGE +archaic-report genes gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans spans.tsv --genes genes.gtf --pop EUR --out-dir rep rep/S_archaic.tsv,rep/S_archaic.txt,rep/S_archaic_segments.tsv bytes PYTHON_IMAGE +archaic-report no-genes gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans spans.tsv --out-dir rep rep/S_archaic.tsv,rep/S_archaic.txt,rep/S_archaic_segments.tsv bytes PYTHON_IMAGE +archaic-report missing-output gen:archaic-inputs.sh --sample S --ibdmix-dir rep --spans spans.tsv --out-dir rep - bytes PYTHON_IMAGE +archaic-report not-spans gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans genes.gtf --out-dir rep - bytes PYTHON_IMAGE +# Step 44, og ancient (bin/ancient_report.py) +ancient-select plain gen:ancient-inputs.sh --anno aadr.anno --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv o.ind,pops.txt,anc.tsv bytes PYTHON_IMAGE +ancient-select min-snps gen:ancient-inputs.sh --anno aadr.anno --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv --min-snps 140000 o.ind,pops.txt,anc.tsv bytes PYTHON_IMAGE +ancient-select too-few gen:ancient-inputs.sh --anno aadr.anno --ind few.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv - bytes PYTHON_IMAGE +ancient-select no-column gen:ancient-inputs.sh --anno aadr.ind --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv - bytes PYTHON_IMAGE +ancient-lift plain gen:ancient-inputs.sh --snp aadr.snp --chain hg19.chain --fasta hg38.fa --out sites.out.tsv sites.out.tsv bytes PYTHON_IMAGE +ancient-lift gz-chain gen:ancient-inputs.sh --snp aadr.snp --chain hg19.chain.gz --fasta hg38.fa --out sites.out.tsv sites.out.tsv bytes PYTHON_IMAGE +ancient-lift minus-source gen:ancient-inputs.sh --snp aadr.snp --chain minus-source.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-lift orphan-block gen:ancient-inputs.sh --snp aadr.snp --chain orphan.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-lift nothing-placed gen:ancient-inputs.sh --snp panel.geno --chain hg19.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-person-geno plain gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno panel.geno --out o.geno --stats o.stats o.geno,o.stats bytes PYTHON_IMAGE +ancient-person-geno short-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno short.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno long-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno long.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno wide-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno wide.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno none-used gen:ancient-inputs.sh --sites sites.tsv --genotypes nowhere.tsv --snp panel.snp --moderns-geno panel.geno --out o.geno --stats o.stats o.geno,o.stats bytes PYTHON_IMAGE +ancient-align flipped gen:ancient-inputs.sh --panel-evec panel.evec --run-evec flipped.evec - bytes PYTHON_IMAGE +ancient-align five-pcs gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --pcs 5 - bytes PYTHON_IMAGE +ancient-align differ gen:ancient-inputs.sh --panel-evec panel.evec --run-evec bad.evec - bytes PYTHON_IMAGE +ancient-align too-few gen:ancient-inputs.sh --panel-evec panel.evec --run-evec few.evec - bytes PYTHON_IMAGE +ancient-align not-evec gen:ancient-inputs.sh --panel-evec stats.tsv --run-evec run.evec - bytes PYTHON_IMAGE +ancient-report plain gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S1 --release v66.p1 --stats stats.tsv --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg o.tsv,o.info,o.txt,o.svg bytes PYTHON_IMAGE +ancient-report no-stats gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S1 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg o.tsv,o.info,o.txt,o.svg bytes PYTHON_IMAGE +ancient-report not-person gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S2 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg - bytes PYTHON_IMAGE +ancient-report axes-differ gen:ancient-inputs.sh --panel-evec panel.evec --run-evec bad.evec --ancients ancients.tsv --sample S1 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg - bytes PYTHON_IMAGE diff --git a/tests/parity/gen/ancient-inputs.sh b/tests/parity/gen/ancient-inputs.sh new file mode 100644 index 000000000..74d6a9a35 --- /dev/null +++ b/tests/parity/gen/ancient-inputs.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# The inputs of the ancient cases of step 44 (ancient-select, ancient-lift, +# ancient-person-geno, ancient-align, ancient-report), all synthetic, +# written with awk: +# aadr.anno, aadr.ind, few.ind +# an AADR-style annotation (long header cells, a quoted one, +# ASSESSMENT WARNINGS before ASSESSMENT) with present-day people +# of the reference groups, one individual under two genetic +# ids, ancient people inside and outside West Eurasia, below the +# site floor, undated, failing the assessment; the .ind file +# (unknown ids, short lines) and one that matches too few +# hg19.chain, hg19.chain.gz, minus-source.chain, orphan.chain, hg38.fa(.fai), stale.fa(.fai) +# a chain with a gap, a minus-strand block, two chains over one +# stretch and a block that moves to another chromosome; a +# reference with lower-case lines; one whose .fai is wrong +# aadr.snp sites in blocks, in gaps, on block edges, on the minus +# strand, off the autosomes, with equal or non-base alleles and +# with a fractional position +# sites.tsv, genotypes.tsv, panel.snp, panel.geno, short.geno, +# long.geno, wide.geno, nowhere.tsv +# person-geno: two SNPs at one position, other alleles, missing +# and haploid calls; a .geno that is short, long or uneven +# panel.evec, run.evec, flipped.evec, bad.evec, few.evec, ancients.tsv, stats.tsv +# two smartpca runs on one panel (the second with PC2's sign +# flipped), one whose axes differ, one with too few present-day +# people, and the ancient people's table (names to escape, a +# long name, undated people, dates on both sides of year 1) +set -euo pipefail + +# --- select ------------------------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + print "Genetic ID", "Master ID", "Individual ID (PubMed)", \ + "Date mean in BP in years before 1950 CE [OxCal mu for a direct radiocarbon date]", \ + "Date standard deviation in BP [OxCal sigma]", "Full Date One of two formats", \ + "Group ID", "\"Locality\"", "Political Entity", "Latitude", "Longitude", "Data type", \ + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", \ + "ASSESSMENT WARNINGS (Xcontam interval is listed if lower bound is >0.005)", " ASSESSMENT " > "aadr.anno" + split("CEU French Sardinian Basque GBR IBS TSI Druze Armenian Polish Mozabite Saami", M, " ") + for (i = 1; i <= 90; i++) { + g = M[i % 12 + 1] + a = (i % 9 == 0) ? "QUESTIONABLE" : (i % 10 == 0 ? " pass (low coverage)" : "PASS") + dt = (i % 15 == 0) ? "1240K" : "Shotgun.diploid" + print "M" i, "M" i, "Ind_M" i, (i % 17 == 0 ? "-0.0" : "0"), "0", "present", g, "Paris\tFrance", "48.8", "2.3", dt, 900000 + i, "", a >> "aadr.anno" + } + print "M91", "M91", "Ind_M5", "0", "0", "present", "CEU", "Utah", "USA", "40", "-111", "Shotgun.diploid", 999999999, "", "PASS" >> "aadr.anno" + split("Germany_CordedWare England_BellBeaker Turkey_N Italy_Imperial_Rome Hungary_LBA_with_a_very_long_name_that_runs_past_45 Türkiye_Neolithic", G, " ") + for (i = 1; i <= 60; i++) { + g = G[i % 7 + 1] + bp = (i % 13 == 0) ? ".." : 500 + 137 * i + lat = (i % 11 == 0) ? 10 : 30 + i % 40 + lon = (i % 19 == 0) ? "" : -20 + i + snps = (i % 8 == 0) ? 99999 : 100000 + 1000 * i + a = (i % 14 == 0) ? "FAIL" : (i % 6 == 0 ? "PROVISIONAL_PASS" : "MERGE_PASS") + ind = (i % 10 == 0) ? "" : "Ind_A" int(i / 2) + print "A" i, "A" i, ind, bp, "50", " 2500-2000 BCE ", g, "Site " (i % 3), (i % 4 == 0 ? "" : "Country" (i % 2)), lat, lon, "1240K", snps, "x", a >> "aadr.anno" + } + print "short", "row" >> "aadr.anno" + for (i = 1; i <= 91; i++) print "M" i, (i % 2 ? "M" : "F"), "Pop" > "aadr.ind" + for (i = 1; i <= 60; i++) print " A" i, "U", "Pop" >> "aadr.ind" + print "X1 M" >> "aadr.ind" + print "Unknown", "M", "Pop" >> "aadr.ind" + for (i = 1; i <= 20; i++) print "M" i, "M", "Pop" > "few.ind" +}' + +# --- lift --------------------------------------------------------------------------- +cat > hg19.chain <<'EOF' +chain 1000 chr1 100000 + 1000 1600 chr1 3000 + 2000 2600 1 +200 100 100 +300 + +chain 900 chr2 50000 + 500 900 chr2 3000 - 100 500 2 +400 + +chain 800 chr3 50000 + 100 400 chr3 3000 + 100 400 3 +300 + +chain 700 chr3 50000 + 300 500 chr3 3000 + 1300 1500 4 +200 + +chain 600 chr4 50000 + 0 500 chr5 3000 + 0 500 5 +500 + +chain 500 chr5 50000 + 100 1100 chr5 3000 + 1100 2100 6 +1000 +EOF +gzip -n -c hg19.chain > hg19.chain.gz +printf 'chain 1 chr1 100 - 0 10 chr1 100 + 0 10 9\n10\n' > minus-source.chain +printf 'chain 1 chr1 100 + 0 10 chr1 100 + 0 10 9\n10\n5\n' > orphan.chain +awk 'BEGIN { + s = "ACGTTGCAGC"; off = 0 + for (c = 1; c <= 5; c++) { + h = ">chr" c; printf "%s\n", h > "hg38.fa"; off += length(h) + 1 + printf "chr%d\t3000\t%d\t60\t61\n", c, off > "hg38.fa.fai" + printf "chr%d\t3000\t%d\t60\t61\n", c, off + 1 > "stale.fa.fai" + line = "" + for (i = 0; i < 3000; i++) { + line = line substr(s, (i * 7 + c * 3) % 10 + 1, 1) + if (length(line) == 60) { + if (int(i / 60) % 3 == 1) line = tolower(line) + printf "%s\n", line > "hg38.fa"; off += 61; line = "" + } + } + } +}' +cp hg38.fa stale.fa +awk 'BEGIN { + OFS = " " + split("A C A G A T C G C T G T", P, " ") + n = 0 + pos[1] = "999 1000 1001 1100 1200 1201 1250 1300 1301 1450 1600 1601" + pos[2] = "500 501 600 750 900 901" + pos[3] = "100 101 250 300 301 350 400 401 450 500 501" + pos[4] = "1 250 500" + pos[5] = "101 500 1100 1101" + for (c = 1; c <= 5; c++) { + k = split(pos[c], Q, " ") + for (j = 1; j <= k; j++) { + for (p = 1; p <= 6; p++) { + n++ + a1 = P[2 * p - 1]; a2 = P[2 * p] + if (n % 5 == 0) { t = a1; a1 = a2; a2 = t } + if (n % 23 == 0) { a1 = tolower(a1) } + q = (n % 29 == 0) ? Q[j] ".0" : Q[j] + printf " rs%d\t%d 0.0123 %s %s %s\n", n, c, q, a1, a2 > "aadr.snp" + } + } + } + print "rsA", 1, "0", 1150, "A", "A" > "aadr.snp" + print "rsN", 1, "0", 1150, "N", "G" > "aadr.snp" + print "rsX", 23, "0", 1150, "A", "G" > "aadr.snp" + print "rsY", "X", "0", 1150, "A", "G" > "aadr.snp" + print "rs0", 0, "0", 1150, "A", "G" > "aadr.snp" + print "short", 1, "0", 1150, "A" > "aadr.snp" + print "rsDup", 1, "0", 1150, "A", "G" > "aadr.snp" + print "rsDup0", 1, "0", 1150, "A", "G" > "aadr.snp" +}' + +# --- person-geno -------------------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("A G C T A C G T", P, " ") + print "snp", "chrom", "pos", "allele1", "allele2" > "sites.tsv" + for (i = 1; i <= 200; i++) { + pos = (i == 51) ? 500 : 10 * i + print "rs" i, "chr1", pos, P[i % 4 * 2 + 1], P[i % 4 * 2 + 2] > "sites.tsv" + } + print "rs999", "chr1" > "sites.tsv" + split("A/G A|A G/G ./. C/T A a/g G|A A/G/G", T, " ") + for (i = 1; i <= 220; i += 3) { + a1 = P[i % 4 * 2 + 1]; a2 = P[i % 4 * 2 + 2] + g = T[i % 10 + 1] + gsub(/A/, "1", g); gsub(/G/, "2", g); gsub(/a/, "3", g); gsub(/g/, "4", g) + gsub(/1/, a1, g); gsub(/2/, a2, g); gsub(/3/, tolower(a1), g); gsub(/4/, tolower(a2), g) + print "chr1", 10 * i, a1, a2, g > "genotypes.tsv" + } + print "chr1", 500, "A", "G", "A/G" > "genotypes.tsv" + print "chr1", 510, "C", "T", "T/T" > "genotypes.tsv" + print "chr1", 510, "C", "T", "C/T" > "genotypes.tsv" + print "1", 20, "A", "G", "A/G" > "genotypes.tsv" + print "chr1", 30 > "genotypes.tsv" + print "chr2", 10, "A", "G", "A/G" > "nowhere.tsv" + for (i = 1; i <= 230; i++) { + if (i == 100) print "" > "panel.snp" + print "rs" i, 1, "0.0", 10 * i, "A", "G" > "panel.snp" + g = sprintf("%d%d%d%d%d", i % 3, (i + 1) % 3, 9, i % 2, 2) + print g > "panel.geno" + if (i < 230) print g > "short.geno" + print g > "long.geno" + print (i == 77 ? g "0" : g) > "wide.geno" + } + print "" > "long.geno" +}' + +# --- align and report --------------------------------------------------------------- +awk 'BEGIN { + split("French Sardinian CEU Basque GBR", M, " ") + printf " #eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "panel.evec" + printf "#eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "run.evec" + printf "#eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "bad.evec" + printf "#eigvals: 5.512 3.201\n" > "few.evec" + for (i = 1; i <= 40; i++) { + p1 = sin(i * 1.7) * 0.05; p2 = cos(i * 0.9) * 0.04; p3 = sin(i * 0.31) * 0.03; p4 = cos(i * 2.3) * 0.02; p5 = sin(i) * 0.01 + lab = M[i % 5 + 1] + printf "%12s %10.6f %10.6f %10.6f %10.6f %10.6f %15s\n", "M" i, p1, p2, p3, p4, p5, lab > "panel.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M" i, p1, -p2, p3, -p4, p5, lab > "run.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M" i, (i % 2 ? p1 : -p1), p2, p3, p4, p5, lab > "bad.evec" + if (i <= 9) printf "%s %.6f %.6f %s\n", "M" i, p1, p2, lab > "few.evec" + } + for (i = 1; i <= 60; i++) { + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "A" i, sin(i * 0.77) * 0.06, cos(i * 1.31) * 0.05, sin(i * 0.5) * 0.02, cos(i) * 0.02, 0.001, "Ancient" > "panel.evec" + } + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M3", 0.01, 0.02, 0.03, 0.04, 0.05, "Other" > "panel.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S1", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Person" > "run.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S1", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Person" > "bad.evec" + printf "%s %.6f %.6f %s\n", "S1", 0.0123, -0.0045, "Person" > "few.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S2", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Ancient" > "run.evec" + OFS = "\t" + print "id", "group", "date_bp", "date_sd", "full_date", "locality", "country", "lat", "lon", "snps" > "ancients.tsv" + split("Germany_CordedWare England_BellBeaker Turkey_N Italy_Imperial_Rome Hungary_LBA_with_a_very_long_name_that_runs_past_45 Türkiye_Neolithic", G, " ") + split("1650 1950 1949 2450.5 4250 7300 9960 6000 3000 .", D, " ") + for (i = 1; i <= 60; i++) { + loc = (i % 3 == 0) ? "." : "Site " (i % 4) + print "A" i, G[i % 7 + 1], D[i % 10 + 1], "50", ".", loc, "Country" (i % 2), "45", "10", 200000 > "ancients.tsv" + } + print "A99", "Nowhere", "5000", "50", ".", ".", ".", "45", "10", 200000 > "ancients.tsv" + print "A7", "Germany_CordedWare", "5100", "50", ".", "Halle", "Germany" > "ancients.tsv" + print "" > "ancients.tsv" + printf "panel_sites\t230\nsites_used\t61\nother\n" > "stats.tsv" +}' +cp run.evec flipped.evec diff --git a/tests/parity/gen/archaic-inputs.sh b/tests/parity/gen/archaic-inputs.sh new file mode 100644 index 000000000..5d179c532 --- /dev/null +++ b/tests/parity/gen/archaic-inputs.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# The inputs of the archaic cases of step 43 (archaic-table, archaic-strict, +# archaic-genotypes, archaic-report), all synthetic, written with awk: +# modern.tsv, altai.tsv, vindija.tsv, denisova.tsv +# bcftools query lines of one GRCh37 chromosome (22): 1000 Genomes +# SNVs with their six frequencies (indels, a split multi-allelic +# site, frequencies that need :.6g's exponent form, NaN, out of +# range and unreadable values) and the three genomes' calls +# (missing, partial, multi-allelic, gVCF <*> and REF-mismatch +# calls, and sites only an archaic genome carries), also gzipped +# altai.bed, vindija.bed, excl.bed, empty.bed +# the masks: chromosome 22 named with and without chr, unsorted, +# overlapping, reversed and empty intervals, track lines +# unsorted.tsv a modern file that goes back +# rows.tsv, rows-unsorted.tsv, strict.bed +# lifted GRCh38 rows (repeated positions) and the strict mask +# chr21.tsv.gz, chr22.tsv.gz, chrX.tsv.gz +# the GRCh38 table step 43 reads (chrX is not a table name), +# with masked codes and frequencies that round half to even +# s.vcf, s.vcf.gz, two.vcf, back.vcf, none.vcf, badcode/ +# the sample's VCF: split records, clashes, indels, a +# multi-allelic record, RefCall, missing GT, a chromosome the +# table lacks; a two-sample VCF, one whose chromosome comes +# back, one with no table chromosome +# chr21.Altai.txt ... chr22.Denisova.txt, spans.tsv, genes.gtf +# IBDmix's segments (short, weak, overlapping and touching +# ones, other IDs) and a GENCODE-style gene GTF +set -euo pipefail + +# --- table (GRCh37 inputs) ---------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0.5 1e-05 0.000123 . nan 1.5 -0.1 0.123456789 1 0 2.5e-7 x", F, " ") + for (i = 1; i <= 60; i++) { + pos = 1000 + 10 * i + r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + if (i % 11 == 0) { print pos, r "T", a, F[1], F[2], F[3], F[4], F[5], F[6] > "modern.tsv"; continue } + if (i % 13 == 0) { + print pos, r, a, "0.1", "0.2", "0.3", "0.4", "0.5", "0.6" > "modern.tsv" + print pos, r, B[(i + 2) % 4 + 1], "0.1", "0.2", "0.3", "0.4", "0.5", "0.6" > "modern.tsv" + continue + } + if (i % 7 == 3) continue + print pos, r, a, F[i % 12 + 1], F[(i + 1) % 12 + 1], F[(i + 2) % 12 + 1], F[(i + 3) % 12 + 1], F[(i + 4) % 12 + 1], F[(i + 5) % 12 + 1] > "modern.tsv" + } + split("0/0 0/1 1/1 ./. 1/2 0|1 1 0/1/1 . 1|1", G, " ") + for (i = 1; i <= 64; i++) { + pos = 1000 + 10 * i + (i % 9 == 0 ? 5 : 0) + r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + if (i % 3 != 0) print pos, r, (i % 10 == 4 ? "<*>" : a), G[i % 10 + 1] > "altai.tsv" + if (i % 4 != 1) print pos, (i % 17 == 0 ? "C" : r), (i % 6 == 0 ? "." : a), G[(i + 3) % 10 + 1] > "vindija.tsv" + if (i % 5 != 2) print pos, r, (i % 8 == 0 ? B[(i + 2) % 4 + 1] : a), G[(i + 7) % 10 + 1] > "denisova.tsv" + } +}' +for f in modern altai vindija denisova; do gzip -n -c "${f}.tsv" > "${f}.tsv.gz"; done +printf '1200\tA\tG\t0.1\t0.1\t0.1\t0.1\t0.1\t0.1\n1100\tA\tG\t0.1\t0.1\t0.1\t0.1\t0.1\t0.1\n' > unsorted.tsv +cat > altai.bed <<'EOF' +track name=altai +# a comment +22 1300 1500 +22 1000 1100 +22 1080 1250 +22 1600 1600 +22 1700 1650 +chr21 0 100000 +EOF +printf 'browser position chr22\nchr22\t1000\t1400\nchr22\t1450\t1700\n' > vindija.bed +printf 'chr22\t1195\t1215\n22\t1500\t1520\n' > excl.bed +printf 'chr21\t0\t1000\n' > empty.bed + +# --- strict (lifted rows) ----------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("0 1 2 9 10 11 12", C, " ") + for (i = 1; i <= 40; i++) { + pos = 5000 + 7 * i + row = "chr22" OFS pos OFS "A" OFS "G" OFS C[i % 7 + 1] OFS C[(i + 2) % 7 + 1] OFS C[(i + 4) % 7 + 1] OFS "0.1" OFS "0.2" OFS "0.3" OFS "0.4" OFS "0.5" OFS "0.6" + print row > "rows.tsv" + if (i % 9 == 0) print "chr22" OFS pos OFS "C" OFS "T" OFS "1" OFS "1" OFS "1" OFS "0" OFS "0" OFS "0" OFS "0" OFS "0" OFS "0" > "rows.tsv" + } +}' +{ head -5 rows.tsv; printf 'chr22\t5001\tA\tG\t1\t1\t1\t0\t0\t0\t0\t0\t0\n'; tail -3 rows.tsv; } > rows-unsorted.tsv +printf 'chr22\t5010\t5100\n22\t5150\t5200\nchr22\t5240\t5260\n' > strict.bed + +# --- genotypes (the GRCh38 table and the sample) ------------------------------------ +HEAD='chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS' +table() { # CHROM START N + awk -v c="$1" -v s="$2" -v n="$3" 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0 1 2 9 10 11 12", C, " ") + split("0.0025 0.0075 0.5 1e-05 . nan 0.99 1.2 -0.3 0.0125 0.333333", F, " ") + for (i = 1; i <= n; i++) { + pos = s + 10 * i + print c, pos, B[i % 4 + 1], B[(i + 1) % 4 + 1], C[i % 7 + 1], C[(i + 3) % 7 + 1], C[(i * 5) % 7 + 1], \ + F[i % 11 + 1], F[(i + 1) % 11 + 1], F[(i + 2) % 11 + 1], F[(i + 3) % 11 + 1], F[(i + 4) % 11 + 1], F[(i + 5) % 11 + 1] + } + }' +} +{ printf '%b\n' "$HEAD"; table chr21 1000 30; } | gzip -n > chr21.tsv.gz +{ printf '%b\n' "$HEAD"; table chr22 2000 50; } | gzip -n > chr22.tsv.gz +{ printf '%b\n' "$HEAD"; table chrX 3000 5; } | gzip -n > chrX.tsv.gz +mkdir -p badcode emptydir +{ printf '%b\n' "$HEAD"; table chr22 2000 4 | awk -F'\t' -v OFS='\t' 'NR == 3 { $5 = 3 } { print }'; } | gzip -n > badcode/chr22.tsv.gz +vcf_head() { + printf '##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "$1" +} +{ + vcf_head S + awk 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0/1 1/1 0/0 ./. 0|1 1|1 0/2 1", G, " ") + print "chr1", 50, ".", "A", "G", ".", "PASS", ".", "GT", "0/1" + for (i = 1; i <= 30; i += 2) { + pos = 1000 + 10 * i; r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + print "chr21", pos, ".", r, a, ".", (i % 5 == 0 ? "RefCall" : "PASS"), ".", "GT:GQ", G[i % 8 + 1] ":30" + } + for (i = 1; i <= 50; i++) { + pos = 2000 + 10 * i; r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1]; o = B[(i + 2) % 4 + 1] + k = i % 12 + if (k == 0) print "chr22", pos, ".", r, a, ".", ".", ".", "GT", G[i % 8 + 1] + else if (k == 1) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/0" } + else if (k == 2) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/0"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/0" } + else if (k == 3) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "1/1" } + else if (k == 4) { print "chr22", pos, ".", r, a "," o, ".", "PASS", ".", "GT", "1/2" } + else if (k == 5) { print "chr22", pos, ".", r "T", r, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "1/1" } + else if (k == 6) { print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "1/1" } + else if (k == 7) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GQ", "20" } + else if (k == 8) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT:DP", "./.:3" } + else if (k == 9) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "./." } + else if (k == 10) { print "chr22", pos, ".", r, ".", ".", "PASS", ".", "GT", "0/0" } + else print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT:AD", "1/1:0,9" + } + print "chr22", 9999, ".", "A", "G", ".", "PASS", ".", "GT", "0/1" + }' +} > s.vcf +gzip -n -c s.vcf > s.vcf.gz +{ printf '##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n'; printf 'chr22\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\t0/0\n'; } > two.vcf +{ vcf_head S; printf 'chr22\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchr21\t1010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t2020\t.\tG\tT\t.\tPASS\t.\tGT\t0/1\n'; } > back.vcf +{ vcf_head S; printf 'chr1\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchrX\t3010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\n'; } > none.vcf + +# --- report (IBDmix output) --------------------------------------------------------- +printf 'chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\nchr21\t1000000\t46000000\t3000\t400\t2\nchr22\t16000000\t50800000\t2500\t300\t1\n' > spans.tsv +seg() { printf 'ID\tchrom\tstart\tend\tslod\n'; cat; } +seg > chr21.Altai.txt <<'EOF' +SAMPLE 21 1000000 1200000 12.5 +SAMPLE 21 1150000 1400000 7.25 +SAMPLE 21 2000000 2040000 30 +SAMPLE 21 3000000 3100000 3.99 +P001 21 5000000 9000000 99 +SAMPLE 21 9000000 9500000 4 +EOF +seg > chr21.Vindija.txt <<'EOF' +SAMPLE 21 1400000 1600000 8.5 +SAMPLE 21 9000000 9300000 45.5 +SAMPLE chr21 20000000 20060000 4.5 +EOF +seg > chr21.Denisova.txt <<'EOF' +SAMPLE 21 1100000 1300000 6 +SAMPLE 21 30000000 30500000 14.5 +SAMPLE 21 30400000 30700000 5.5 +EOF +seg > chr22.Altai.txt <<'EOF' +SAMPLE 22 17000000 17800000 22.5 +SAMPLE 22 40000000 40050000 9 +EOF +seg > chr22.Vindija.txt <<'EOF' +SAMPLE 22 17500000 18100000 10.5 +EOF +seg > chr22.Denisova.txt <<'EOF' +SAMPLE 22 25000000 25900000 8.5 +SAMPLE 22 17000000 17050000 15 +EOF +cat > genes.gtf <<'EOF' +##description: synthetic +chr21 HAVANA gene 1000 1000000 . + . gene_id "G1"; gene_type "protein_coding"; gene_name "TOUCHES_START"; +chr21 HAVANA gene 1199999 1250000 . + . gene_id "G2"; gene_type "protein_coding"; gene_name "INSIDE"; +chr21 HAVANA gene 1100000 1300000 . + . gene_id "G3"; gene_type "lncRNA"; gene_name "NOT_CODING"; +chr21 HAVANA transcript 1100000 1300000 . + . gene_id "G2"; gene_type "protein_coding"; gene_name "INSIDE"; +chr21 HAVANA gene 1600000 1700000 . + . gene_id "G4"; gene_type "protein_coding"; gene_name "AT_END"; +chr21 HAVANA gene 30500000 30600000 . - . gene_id "G5"; gene_type "protein_coding"; gene_name "DENI"; +chr21 HAVANA gene 1300000 1350000 . - . gene_id "G6"; gene_type "protein_coding"; gene_name "INSIDE"; +chr22 HAVANA gene 17700000 17900000 . + . gene_id "G7"; gene_type "protein_coding"; gene_name "ON22"; +chr22 HAVANA gene 17000000 17100000 . + . gene_id "G8"; gene_type "protein_coding"; gene_id_only "x"; +EOF diff --git a/tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent b/tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-strict/unsorted/exit b/tests/parity/golden/archaic-strict/unsorted/exit new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/tests/parity/golden/archaic-strict/unsorted/exit @@ -0,0 +1 @@ +2 diff --git a/tests/parity/golden/archaic-strict/unsorted/s.tsv.absent b/tests/parity/golden/archaic-strict/unsorted/s.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-strict/unsorted/stderr b/tests/parity/golden/archaic-strict/unsorted/stderr new file mode 100644 index 000000000..075caa55f --- /dev/null +++ b/tests/parity/golden/archaic-strict/unsorted/stderr @@ -0,0 +1 @@ +ERROR: rows-unsorted.tsv: line 6: position 5001 comes after 5035; the lifted rows must be sorted by position diff --git a/tests/parity/golden/archaic-table/unsorted/t.vcf.absent b/tests/parity/golden/archaic-table/unsorted/t.vcf.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index 1604b265c..481cec78f 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -62,3 +62,12 @@ y-yleaf-positions 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/yleaf_run.py lookup 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py python3 lookup lookup-query 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py bash ./oracle.sh lookup query family-ibd 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/ibd_report.py sh ./ibd_oracle.sh family ibd +archaic-table fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" table "$@"' archaic table +archaic-strict fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" strict "$@"' archaic strict +archaic-genotypes fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" genotypes "$@"' archaic genotypes +archaic-report fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" report "$@"' archaic report +ancient-select fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" select "$@"' ancient select +ancient-lift fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" lift "$@"' ancient lift +ancient-person-geno fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" person-geno "$@"' ancient person-geno +ancient-align fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" align "$@"' ancient align +ancient-report fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" report "$@"' ancient report From cae6e3e598e1a09541f4f4a0ece847490853f564 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:24:21 +0100 Subject: [PATCH 03/19] refactor(ancestry): delete archaic_report.py and ancient_report.py; tests, demo and evidence run og The step 43 and 44 tests drive og archaic and og ancient as the pipeline does and join Guard's OG_REQUIRED list; their unit checks of internals moved to the Rust tests. The docs demo writes step 44 with og ancient report, archaic_compare.py keeps its own copy of the IBDmix reader, and the archaic-share evidence job builds og. The table and VCF readers no longer build a path string per line. Bead: pgp-bs9u.21 --- .github/workflows/guard.yml | 2 +- .github/workflows/public-evidence.yml | 12 +- bin/ancient_report.py | 753 ------------------------- bin/archaic_compare.py | 26 +- bin/archaic_report.py | 745 ------------------------ bin/collect_summary.py | 2 +- rust/crates/og-ancestry/src/ancient.rs | 3 + rust/crates/og-ancestry/src/archaic.rs | 19 +- scripts/ci/e2e-case-paths.tsv | 4 +- tests/demo/make_demo_sample.py | 81 +-- tests/e2e/ancient-similarity-1.sh | 2 +- tests/test_ancient_report.py | 228 ++++---- tests/test_archaic_report.py | 62 +- 13 files changed, 237 insertions(+), 1702 deletions(-) delete mode 100755 bin/ancient_report.py delete mode 100755 bin/archaic_report.py diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 5a209a286..e22eba988 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -327,7 +327,7 @@ jobs: run: | fail=0 for t in tests/test_annotsv_sv_inputs.py tests/test_pgx_refblock_fill.py tests/test_pgx_mito_merge.py tests/test_run_delta.py \ - tests/test_ancestry_fine_report.py \ + tests/test_ancestry_fine_report.py tests/test_archaic_report.py tests/test_ancient_report.py \ tests/test_annotsv_sv_inputs.sh tests/test_annotsv_postfilter.sh tests/test_annotsv_benign_mane.sh \ tests/test_str_catalog.py tests/test_str_ranges.py tests/test_og_setup_zip.py tests/test_setup_fine_maps.sh \ tests/test_render_report.py tests/test_collect_summary.py tests/test_chip_report.py \ diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 11b39d5a4..d44d44de8 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -227,7 +227,7 @@ jobs: # The positions of this part where HG002 has two records or more, one # of them an SNV, as GIAB writes the VCF and split with bcftools # norm -m-, and how many of them are rows of the table (the rows the - # same-position rule of archaic_report.py decides). + # same-position rule of og archaic genotypes decides). docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools norm -m- -Oz -o /g/HG002/split.vcf.gz /g/HG002/vcf/HG002.vcf.gz python3 -I - "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" <<'PY' @@ -470,7 +470,7 @@ jobs: archaic-share: needs: [archaic-sites, archaic-hg002] runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -492,6 +492,12 @@ jobs: mv parts "lone-${c}" && mkdir parts && mv "lone-${c}" "parts/archaic-part-${c}" fi ls parts + - name: Build og (og archaic report) + working-directory: rust + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og - name: One report per population, over the autosomes that ran env: WANT: ${{ needs.archaic-sites.outputs.count }} @@ -517,7 +523,7 @@ jobs: mkdir -p "${d}/ibdmix" cp parts/archaic-part-*/"${pop}"/chr*.txt "${d}/ibdmix/" { printf 'chrom\tfirst\tlast\tsites\n'; for s in parts/archaic-part-*/"${pop}"/spans.tsv; do tail -n +2 "$s" | cut -f1-4; done; } > "${d}/spans.tsv" - python3 bin/archaic_report.py report --sample HG002 --ibdmix-dir "${d}/ibdmix" --spans "${d}/spans.tsv" \ + rust/target/release/og archaic report --sample HG002 --ibdmix-dir "${d}/ibdmix" --spans "${d}/spans.tsv" \ --pop "$pop" --out-dir "${d}/out" kv() { awk -F'\t' -v k="$1" '$1 == k {print $2}' "${d}/out/HG002_archaic.tsv"; } [ "$(kv analysed_chromosomes)" -eq "$WANT" ] || { echo "${pop}: $(kv analysed_chromosomes) chromosomes, want ${WANT}"; exit 1; } diff --git a/bin/ancient_report.py b/bin/ancient_report.py deleted file mode 100755 index 82f0002f0..000000000 --- a/bin/ancient_report.py +++ /dev/null @@ -1,753 +0,0 @@ -#!/usr/bin/env python3 -"""Step 44: where a genome falls among ancient genomes on a PCA of present-day -West Eurasians (smartpca, EIGENSOFT), with the ancient samples of the Allen -Ancient DNA Resource (AADR) projected onto the same axes. - -Subcommands, in the order they run: - - select (setup.sh) read the AADR .anno and .ind files and choose the - people: the present-day West Eurasians of MODERN_GROUPS build - the axes, and the ancient West Eurasians with enough sites - covered are projected. Writes the relabelled .ind (everyone - else is Ignore), the population list and ancients.tsv. - lift (setup.sh) move the 1240K sites of the AADR .snp file from - GRCh37 to GRCh38 with UCSC's chain file, and keep a site only - when one of its two alleles is the GRCh38 reference base. - person-geno (step 44) the person's genotype at each site, from bcftools' - output on the gVCF, appended as one more column to the - present-day panel's EIGENSTRAT genotype file. - align (setup.sh, step 44) check that the axes of two smartpca runs - on the same present-day panel agree, up to the sign of each. - report (step 44) the nearest ancient populations, the summary and - the plot. - -Only the Python standard library is used. Every number the report shows comes -from smartpca's .evec files and the AADR annotation. -""" -import argparse -import bisect -import collections -import csv -import gzip -import html -import math -import os -import statistics -import sys - -# Present-day groups of the AADR 1240K file (Group ID, as written in the -# .anno file of v66.p1) that build the axes: Europe, the Caucasus and the Near -# East, from the 1000 Genomes Project, HGDP and SGDP shotgun genomes. North -# African groups (Mozabite) and groups with recent East Eurasian ancestry -# (Saami, Bashkir, Tatar, Yakut) are left out, so the axes describe West -# Eurasian variation only. -MODERN_GROUPS = ( - "Abkhasian", "Adygei", "Albanian", "Armenian", "Basque", "BedouinA", "BedouinB", - "Bulgarian", "CEU", "Chechen", "Czech", "Druze", "English", "Estonian", "FIN", - "Finnish", "French", "GBR", "Georgian", "Greek", "Greek_Crete", "Hungarian", "IBS", - "Icelandic", "Iranian", "Italian_North", "Jew_Iraqi", "Jordanian", "Lezgin", - "Norwegian", "Orcadian", "Ossetian", "Palestinian", "Polish", "Russian", "Sardinian", - "Scottish", "Spanish", "Spanish_North", "TSI", "Turkish", -) -MODERN_DATA_TYPE = "Shotgun.diploid" -# Ancient samples projected: dated, passing the AADR's own assessment, with -# this many of the 1240K autosomal sites covered, inside this box. -MIN_ANCIENT_SNPS = 100_000 -WEST_EURASIA = {"lat": (25.0, 75.0), "lon": (-25.0, 75.0)} -PASSING = ("PASS", "PROVISIONAL_PASS", "MERGE_PASS") -ANCIENT_LABEL = "Ancient" -PERSON_LABEL = "Person" -# PCs the distance uses, each weighted by the square root of its eigenvalue. -DISTANCE_PCS = 4 -TOP = 20 -# Two runs on the same panel give the same axes; any PC whose present-day -# coordinates correlate less than this between the two is a panel mismatch. -MIN_AXIS_CORRELATION = 0.999 -CAVEAT = ("Closeness on a PCA is genetic similarity, not descent: a genome near an ancient group " - "shares ancestry with it, it does not descend from those people.") -QPADM_NOTE = "qpAdm models of ancestry proportions are not part of this step." - -# Header cells of the .anno file this script reads, matched by their start -# (the AADR writes long explanations after the name). -ANNO_COLUMNS = { - "id": "Genetic ID", - "individual": "Individual ID", - "date_bp": "Date mean in BP", - "date_sd": "Date standard deviation in BP", - "full_date": "Full Date", - "group": "Group ID", - "locality": "Locality", - "country": "Political Entity", - "lat": "Latitude", - "lon": "Longitude", - "data_type": "Data type", - "snps": "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", - "assessment": "ASSESSMENT", -} -COMPLEMENT = {"A": "T", "C": "G", "G": "C", "T": "A"} - - -class Fail(Exception): - pass - - -def opener(path): - return gzip.open(path, "rt", errors="replace") if path.endswith(".gz") else open(path, errors="replace") - - -# --- select --------------------------------------------------------------------- - -def anno_columns(header): - cells = [c.strip().strip('"').strip() for c in header] - out = {} - for key, name in ANNO_COLUMNS.items(): - # ASSESSMENT must not match ASSESSMENT WARNINGS: the exact cell first. - hits = [i for i, c in enumerate(cells) if c == name] or [i for i, c in enumerate(cells) if c.startswith(name)] - if key == "assessment": - hits = [i for i in hits if cells[i] == name] - if not hits: - raise Fail(f"the .anno file has no column '{name}'") - out[key] = hits[0] - return out - - -def to_float(s): - try: - v = float(s) - except (TypeError, ValueError): - return None - return v if math.isfinite(v) else None - - -def passing(assessment): - return assessment.strip().upper().startswith(PASSING) - - -def read_anno(path): - with opener(path) as f: - rows = list(csv.reader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - if not rows: - raise Fail(f"{path} is empty") - col = anno_columns(rows[0]) - out = [] - for r in rows[1:]: - if len(r) <= max(col.values()): - continue - out.append({k: r[i].strip() for k, i in col.items()}) - return out - - -def choose(anno, min_snps=MIN_ANCIENT_SNPS): - """Return {genetic id: ('modern', group) | ('ancient', row)}: the - present-day reference people and the ancient people to project, one - genetic id per individual (the one with the most sites).""" - modern_groups = set(MODERN_GROUPS) - best = {} - for r in anno: - if not passing(r["assessment"]): - continue - bp = to_float(r["date_bp"]) - snps = to_float(r["snps"]) or 0 - if bp == 0 and r["group"] in modern_groups and r["data_type"] == MODERN_DATA_TYPE: - kind = "modern" - elif bp is not None and bp > 0 and snps >= min_snps: - lat, lon = to_float(r["lat"]), to_float(r["lon"]) - if lat is None or lon is None: - continue - if not (WEST_EURASIA["lat"][0] <= lat <= WEST_EURASIA["lat"][1] - and WEST_EURASIA["lon"][0] <= lon <= WEST_EURASIA["lon"][1]): - continue - kind = "ancient" - else: - continue - key = (kind, r["individual"] or r["id"]) - if key not in best or snps > best[key][0]: - best[key] = (snps, r) - out = {} - for (kind, _), (_, r) in best.items(): - out[r["id"]] = (kind, r["group"] if kind == "modern" else r) - return out - - -def cmd_select(a): - anno = read_anno(a.anno) - chosen = choose(anno, a.min_snps) - n_mod = n_anc = 0 - groups = set() - with open(a.ind) as f, open(a.out_ind + ".tmp", "w") as o: - for line in f: - c = line.split() - if len(c) < 3: - continue - kind, val = chosen.get(c[0], (None, None)) - if kind == "modern": - label = val - n_mod += 1 - groups.add(val) - elif kind == "ancient": - label = ANCIENT_LABEL - n_anc += 1 - else: - label = "Ignore" - o.write(f"{c[0]}\t{c[1]}\t{label}\n") - missing = sorted(set(MODERN_GROUPS) - groups) - if n_mod < 50 or n_anc < 1: - os.remove(a.out_ind + ".tmp") - raise Fail(f"the .ind file matches {n_mod} present-day reference people and {n_anc} ancient ones; " - "is it the .ind file of the same release as the .anno file?") - os.replace(a.out_ind + ".tmp", a.out_ind) - with open(a.poplist, "w") as o: - for g in sorted(groups): - o.write(g + "\n") - with open(a.ancients, "w") as o: - o.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for gid, (kind, r) in sorted(chosen.items()): - if kind != "ancient": - continue - o.write("\t".join(clean(r[k]) for k in ("id", "group", "date_bp", "date_sd", "full_date", - "locality", "country", "lat", "lon", "snps")) + "\n") - print(f"Present-day reference people: {n_mod} in {len(groups)} groups" - + (f" (not in this file: {', '.join(missing)})" if 0 < len(missing) <= 10 else - f" ({len(missing)} of the {len(MODERN_GROUPS)} groups are not in this file)" if missing else "")) - print(f"Ancient people projected: {n_anc} (at least {a.min_snps:,} sites, West Eurasia)") - return 0 - - -def clean(s): - return " ".join(str(s).replace("\t", " ").split()) or "." - - -# --- lift ----------------------------------------------------------------------- - -def read_chain(path): - """UCSC chain file -> {source chrom: sorted [(start, end, chain no, target - chrom, target start, target strand, target size)]}, 0-based half-open - ungapped blocks.""" - blocks = collections.defaultdict(list) - n = -1 - with opener(path) as f: - cur = None - for line in f: - c = line.split() - if not c: - cur = None - continue - if c[0] == "chain": - n += 1 - t_name, t_strand, t_start = c[2], c[4], int(c[5]) - q_name, q_size, q_strand, q_start = c[7], int(c[8]), c[9], int(c[10]) - if t_strand != "+": - raise Fail(f"{path}: chain {c[12] if len(c) > 12 else n} has source strand {t_strand}") - cur = [t_name, t_start, q_name, q_start, q_strand, q_size, n] - continue - if cur is None: - raise Fail(f"{path}: a block line outside a chain: {line.strip()}") - size = int(c[0]) - t_name, t_pos, q_name, q_pos, q_strand, q_size, k = cur - blocks[t_name].append((t_pos, t_pos + size, k, q_name, q_pos, q_strand, q_size)) - if len(c) >= 3: - cur[1] = t_pos + size + int(c[1]) - cur[3] = q_pos + size + int(c[2]) - else: - cur = None - for v in blocks.values(): - v.sort() - return blocks - - -def index_blocks(blocks): - """read_chain's blocks -> {chrom: (blocks, their starts, longest block)}.""" - return {k: (v, [x[0] for x in v], max(x[1] - x[0] for x in v)) for k, v in blocks.items()} - - -def lift_point(index, chrom, pos1): - """1-based GRCh37 position -> (chrom, 1-based position, strand) on the - target, or None when no block, or blocks of more than one chain, cover it.""" - if chrom not in index: - return None - b, starts, longest = index[chrom] - p = pos1 - 1 - hits = [] - # Blocks of different chains can overlap: every block that starts at or - # before p, back to the longest block's length, may still reach it. - j = bisect.bisect_right(starts, p) - 1 - while j >= 0 and b[j][0] > p - longest: - if b[j][1] > p: - hits.append(b[j]) - j -= 1 - if len({h[2] for h in hits}) != 1: - return None - s, _, _, q_name, q_pos, q_strand, q_size = hits[0] - q = q_pos + (p - s) - if q_strand == "-": - q = q_size - 1 - q - return q_name, q + 1, q_strand - - -class Fasta: - """Random access to a FASTA with a samtools .fai index.""" - - def __init__(self, path): - self.f = open(path, "rb") - self.idx = {} - with open(path + ".fai") as fai: - for line in fai: - c = line.split("\t") - self.idx[c[0]] = (int(c[1]), int(c[2]), int(c[3]), int(c[4])) - - def base(self, chrom, pos1): - if chrom not in self.idx: - return None - length, offset, line_bases, line_width = self.idx[chrom] - if not 1 <= pos1 <= length: - return None - p = pos1 - 1 - self.f.seek(offset + (p // line_bases) * line_width + p % line_bases) - return self.f.read(1).decode("ascii", "replace").upper() - - -def eigen_chrom(c): - """The chromosome column of an EIGENSOFT .snp line -> 1..22, or None.""" - try: - n = int(c) - except ValueError: - return None - return n if 1 <= n <= 22 else None - - -def cmd_lift(a): - blocks = index_blocks(read_chain(a.chain)) - fasta = Fasta(a.fasta) - kept = [] - dropped = collections.Counter() - n = 0 - with opener(a.snp) as f: - for line in f: - c = line.split() - if len(c) < 6: - continue - n += 1 - chrom = eigen_chrom(c[1]) - a1, a2 = c[4].upper(), c[5].upper() - if chrom is None: - dropped["not an autosome"] += 1 - continue - if a1 not in COMPLEMENT or a2 not in COMPLEMENT or a1 == a2: - dropped["alleles not two different bases"] += 1 - continue - hit = lift_point(blocks, f"chr{chrom}", int(float(c[3]))) - if hit is None: - dropped["no single place on GRCh38"] += 1 - continue - q_chrom, q_pos, strand = hit - if q_chrom != f"chr{chrom}": - dropped["moves to another chromosome"] += 1 - continue - if strand == "-": - a1, a2 = COMPLEMENT[a1], COMPLEMENT[a2] - ref = fasta.base(q_chrom, q_pos) - if ref not in (a1, a2): - dropped["neither allele is the GRCh38 base"] += 1 - continue - kept.append((chrom, q_pos, c[0], a1, a2)) - kept.sort() - with open(a.out + ".tmp", "w") as o: - o.write("snp\tchrom\tpos\tallele1\tallele2\n") - for chrom, pos, sid, a1, a2 in kept: - o.write(f"{sid}\tchr{chrom}\t{pos}\t{a1}\t{a2}\n") - os.replace(a.out + ".tmp", a.out) - print(f"Sites of the .snp file: {n:,}; on GRCh38: {len(kept):,}") - for why, k in sorted(dropped.items()): - print(f" dropped, {why}: {k:,}") - if not kept: - raise Fail("no site could be placed on GRCh38: check the chain file and the reference") - return 0 - - -# --- person-geno ------------------------------------------------------------------ - -def read_sites(path): - """sites_grch38.tsv -> {(chrom, pos): [(snp, allele1, allele2)]}.""" - out = collections.defaultdict(list) - with open(path) as f: - next(f, None) - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 5: - out[(c[1], int(c[2]))].append((c[0], c[3], c[4])) - return out - - -def dosage(gt, a1, a2): - """Copies of allele1 in a bcftools %TGT genotype, or None (missing, a - haploid or half call, or an allele that is neither of the two).""" - alleles = gt.replace("|", "/").split("/") - if len(alleles) != 2 or any(x not in (a1, a2) for x in alleles): - return None - return sum(1 for x in alleles if x == a1) - - -def person_dosages(sites, genotype_lines): - """bcftools query lines (CHROM POS REF ALT GT) -> ({snp: dosage}, counts).""" - out = {} - counts = collections.Counter() - for line in genotype_lines: - c = line.rstrip("\n").split("\t") - if len(c) < 5: - continue - here = sites.get((c[0], int(c[1]))) - if not here: - continue - for sid, a1, a2 in here: - d = dosage(c[4].upper(), a1, a2) - if d is None: - counts["other_allele"] += 1 - else: - out[sid] = d - counts["called"] += 1 - return out, counts - - -def cmd_person_geno(a): - sites = read_sites(a.sites) - with open(a.genotypes) as f: - dos, counts = person_dosages(sites, f) - n = used = 0 - with open(a.snp) as snp, open(a.moderns_geno) as geno, open(a.out + ".tmp", "w") as o: - width = None - for sline in snp: - c = sline.split() - if not c: - continue - g = geno.readline().rstrip("\n") - if not g: - raise Fail(f"{a.moderns_geno} has fewer lines than {a.snp}") - if width is None: - width = len(g) - elif len(g) != width: - raise Fail(f"{a.moderns_geno} line {n + 1} has {len(g)} genotypes, not {width}") - d = dos.get(c[0]) - if d is not None: - used += 1 - o.write(g + ("9" if d is None else str(d)) + "\n") - n += 1 - if geno.readline(): - raise Fail(f"{a.moderns_geno} has more lines than {a.snp}") - os.replace(a.out + ".tmp", a.out) - with open(a.stats, "w") as o: - o.write(f"panel_sites\t{n}\nsites_genotyped\t{counts['called'] + counts['other_allele']}\n" - f"other_allele\t{counts['other_allele']}\nsites_used\t{used}\n") - print(f"Sites of the panel: {n:,}; genotyped from the gVCF: {counts['called'] + counts['other_allele']:,}; " - f"used: {used:,} (an allele that is neither of a site's two: {counts['other_allele']:,})") - if used == 0: - raise Fail("none of the panel's sites was genotyped: is the gVCF on GRCh38 with chr names?") - return 0 - - -# --- evec, align ------------------------------------------------------------------ - -def read_evec(path): - """smartpca .evec -> (eigenvalues, {id: (label, [pc...])}).""" - evals, rows = [], {} - with open(path) as f: - for line in f: - c = line.split() - if not c: - continue - if c[0].startswith("#"): - if c[0] == "#eigvals:": - evals = [float(x) for x in c[1:]] - continue - rows[c[0]] = (c[-1], [float(x) for x in c[1:-1]]) - if not evals or not rows: - raise Fail(f"{path} is not a smartpca .evec file (no #eigvals: line or no rows)") - return evals, rows - - -def correlation(x, y): - mx, my = statistics.fmean(x), statistics.fmean(y) - sxy = sum((a - mx) * (b - my) for a, b in zip(x, y)) - sx = math.sqrt(sum((a - mx) ** 2 for a in x)) - sy = math.sqrt(sum((b - my) ** 2 for b in y)) - return sxy / (sx * sy) if sx > 0 and sy > 0 else 0.0 - - -def axis_signs(panel_rows, run_rows, n_pcs): - """The sign that turns each of the run's PCs into the panel's, from the - present-day people both runs used to build the axes. A correlation below - MIN_AXIS_CORRELATION on any PC stops: the two are not the same axes.""" - shared = sorted(k for k, (lab, _) in run_rows.items() - if k in panel_rows and lab not in (ANCIENT_LABEL, PERSON_LABEL) - and panel_rows[k][0] == lab) - if len(shared) < 10: - raise Fail(f"only {len(shared)} present-day people are in both runs; the panel and this run do not match") - signs = [] - for k in range(n_pcs): - r = correlation([panel_rows[i][1][k] for i in shared], [run_rows[i][1][k] for i in shared]) - if abs(r) < MIN_AXIS_CORRELATION: - raise Fail(f"PC{k + 1} of this run correlates {r:.4f} with the panel's over {len(shared)} present-day " - "people: the axes differ. Install the panel again (setup.sh --ancient-panel).") - signs.append(1.0 if r > 0 else -1.0) - return signs - - -def cmd_align(a): - _, panel = read_evec(a.panel_evec) - _, run = read_evec(a.run_evec) - signs = axis_signs(panel, run, a.pcs) - print("Axes agree with the panel's: " + " ".join(f"PC{i + 1}{'+' if s > 0 else '-'}" for i, s in enumerate(signs))) - return 0 - - -# --- report ----------------------------------------------------------------------- - -def year_label(bp): - """A date in years before 1950 -> 'N BCE' or 'N CE', the AADR's own - reading (its 2600-2000 BCE has a mean of 4250 BP): BCE = BP - 1950 and - CE = 1950 - BP, with 1950 BP read as 1 BCE.""" - y = 1950 - int(round(bp)) - return f"{y} CE" if y >= 1 else f"{max(-y, 1)} BCE" - - -def range_label(old_bp, young_bp): - """The span of a population's dates, oldest first, e.g. '5300-4900 BCE', - '200-450 CE', or '300 BCE-200 CE'.""" - a, b = year_label(old_bp), year_label(young_bp) - if a == b: - return a - (na, ea), (nb, eb) = a.split(" "), b.split(" ") - return f"{na}-{nb} {ea}" if ea == eb else f"{a}-{b}" - - -def read_ancients(path): - out = {} - with open(path) as f: - for r in csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE): - out[r["id"]] = r - return out - - -def weights(evals, n): - return [math.sqrt(max(v, 0.0)) for v in evals[:n]] - - -def distance(p, q, w): - return math.sqrt(sum((wi * (a - b)) ** 2 for a, b, wi in zip(p, q, w))) - - -def most_common(values): - vals = [v for v in values if v and v != "."] - if not vals: - return "." - c = collections.Counter(vals).most_common() - top = [v for v, k in c if k == c[0][1]] - return sorted(top)[0] - - -def nearest_populations(person, panel_rows, ancients, evals, n_pcs=DISTANCE_PCS, top=TOP): - """Rank the ancient groups by the weighted distance from the person to the - group's mean position on the first n_pcs PCs.""" - w = weights(evals, n_pcs) - groups = collections.defaultdict(list) - for gid, r in ancients.items(): - if gid in panel_rows and panel_rows[gid][0] == ANCIENT_LABEL: - groups[r["group"]].append((panel_rows[gid][1], r)) - rows = [] - for g, members in groups.items(): - k = len(members[0][0]) - centre = [statistics.fmean(m[0][i] for m in members) for i in range(min(k, n_pcs))] - dates = [to_float(m[1]["date_bp"]) for m in members] - dates = [d for d in dates if d is not None] - rows.append({ - "group": g, - "n": len(members), - "distance": distance(person[:n_pcs], centre, w), - "pc1": centre[0], - "pc2": centre[1] if len(centre) > 1 else 0.0, - "date_bp": statistics.median(dates) if dates else None, - "dates": range_label(max(dates), min(dates)) if dates else ".", - "country": most_common(m[1]["country"] for m in members), - "locality": most_common(m[1]["locality"] for m in members), - }) - rows.sort(key=lambda r: (r["distance"], r["group"])) - for i, r in enumerate(rows, 1): - r["rank"] = i - return rows[:top], len(rows) - - -AGE_BINS = ( # (oldest BP, colour, legend), oldest first; dark-background palette - (math.inf, "#5aa9e6", "before 8000 BCE"), - (9950, "#7fc97f", "8000-4000 BCE"), - (5950, "#f0a35e", "4000-1000 BCE"), - (2950, "#e377c2", "after 1000 BCE"), -) - - -def age_colour(bp): - colour = AGE_BINS[0][1] - for limit, c, _ in AGE_BINS: - if bp is not None and bp < limit: - colour = c - return colour - - -def svg_plot(person, panel_rows, ancients, sample, nearest, width=900, height=720): - """The PC1/PC2 plane: present-day people grey, ancient people coloured by - age, the person a large ring. Dark background, so it reads in a dark - report and on its own.""" - pts = [v[1][:2] for v in panel_rows.values()] + [person[:2]] - xs, ys = [p[0] for p in pts], [p[1] for p in pts] - pad = 0.04 - x0, x1 = min(xs), max(xs) - y0, y1 = min(ys), max(ys) - dx, dy = (x1 - x0) or 1.0, (y1 - y0) or 1.0 - left, right, top, bottom = 60, 20, 50, 110 - pw, ph = width - left - right, height - top - bottom - - def X(v): - return left + ((v - x0) / dx * (1 - 2 * pad) + pad) * pw - - def Y(v): - return top + (1 - ((v - y0) / dy * (1 - 2 * pad) + pad)) * ph - - e = html.escape - out = [f'', - '', - f'', - f'{e(sample)} among ancient West Eurasians ' - '(AADR, projected onto present-day PCs)', - f''] - for label, pcs in panel_rows.values(): - if label != ANCIENT_LABEL: - out.append(f'') - for gid, (label, pcs) in panel_rows.items(): - if label == ANCIENT_LABEL and gid in ancients: - c = age_colour(to_float(ancients[gid]["date_bp"])) - out.append(f'') - for r in nearest[:5]: - out.append(f'{e(r["group"])}') - px, py = X(person[0]), Y(person[1]) - out.append(f'') - out.append(f'') - out.append(f'{e(sample)}') - out.append(f'PC1') - out.append(f'PC2') - lx, ly = left, top + ph + 48 - out.append(f'present day') - lx += 120 - for _, c, name in AGE_BINS: - out.append(f'{e(name)}') - lx += 150 - out.append(f'{e(CAVEAT)}') - out.append("") - return "\n".join(out) + "\n" - - -def cmd_report(a): - evals, panel = read_evec(a.panel_evec) - _, run = read_evec(a.run_evec) - if a.sample not in run or run[a.sample][0] != PERSON_LABEL: - raise Fail(f"{a.run_evec} has no row for {a.sample} labelled {PERSON_LABEL}") - n_pcs = min(DISTANCE_PCS, len(evals), len(run[a.sample][1])) - signs = axis_signs(panel, run, max(2, n_pcs)) - person = [s * v for s, v in zip(signs, run[a.sample][1])] - ancients = read_ancients(a.ancients) - nearest, n_groups = nearest_populations(person, panel, ancients, evals, n_pcs) - stats = {} - if a.stats: - with open(a.stats) as f: - for line in f: - k, _, v = line.rstrip("\n").partition("\t") - stats[k] = v - n_anc = sum(1 for lab, _ in panel.values() if lab == ANCIENT_LABEL) - n_mod = sum(1 for lab, _ in panel.values() if lab not in (ANCIENT_LABEL, PERSON_LABEL)) - with open(a.out_tsv + ".tmp", "w") as o: - o.write("rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n") - for r in nearest: - o.write(f"{r['rank']}\t{clean(r['group'])}\t{r['n']}\t{r['distance']:.5f}\t{r['pc1']:.5f}\t{r['pc2']:.5f}\t" - f"{'.' if r['date_bp'] is None else int(round(r['date_bp']))}\t{r['dates']}\t" - f"{clean(r['country'])}\t{clean(r['locality'])}\n") - os.replace(a.out_tsv + ".tmp", a.out_tsv) - info = [ - ("release", a.release), - ("sites_used", stats.get("sites_used", ".")), - ("panel_sites", stats.get("panel_sites", ".")), - ("present_day_people", n_mod), - ("ancient_people", n_anc), - ("ancient_groups", n_groups), - ("pcs_in_distance", n_pcs), - ("pc1", f"{person[0]:.5f}"), - ("pc2", f"{person[1]:.5f}"), - ] - with open(a.out_info + ".tmp", "w") as o: - o.write("key\tvalue\n") - for k, v in info: - o.write(f"{k}\t{v}\n") - os.replace(a.out_info + ".tmp", a.out_info) - with open(a.out_svg + ".tmp", "w") as o: - o.write(svg_plot(person, panel, ancients, a.sample, nearest)) - os.replace(a.out_svg + ".tmp", a.out_svg) - lines = [f"Ancient populations nearest to {a.sample} (AADR {a.release}, smartpca with lsqproject)", - f"Sites used: {stats.get('sites_used', '.')} of {stats.get('panel_sites', '.')}; axes from {n_mod} " - f"present-day West Eurasians; {n_anc} ancient people in {n_groups} groups projected.", - f"Distance: the first {n_pcs} PCs, each weighted by the square root of its eigenvalue.", ""] - for r in nearest[:10]: - lines.append(f" {r['rank']:>2}. {r['group']:<45} {r['dates']:<22} {r['country']:<20} n={r['n']:<4} " - f"d={r['distance']:.4f}") - lines += ["", CAVEAT, QPADM_NOTE] - with open(a.out_txt, "w") as o: - o.write("\n".join(lines) + "\n") - print("\n".join(lines)) - return 0 - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - sub = p.add_subparsers(dest="cmd", required=True) - s = sub.add_parser("select") - s.add_argument("--anno", required=True) - s.add_argument("--ind", required=True) - s.add_argument("--out-ind", required=True) - s.add_argument("--poplist", required=True) - s.add_argument("--ancients", required=True) - s.add_argument("--min-snps", type=int, default=MIN_ANCIENT_SNPS) - s = sub.add_parser("lift") - s.add_argument("--snp", required=True) - s.add_argument("--chain", required=True) - s.add_argument("--fasta", required=True) - s.add_argument("--out", required=True) - s = sub.add_parser("person-geno") - s.add_argument("--sites", required=True) - s.add_argument("--genotypes", required=True) - s.add_argument("--snp", required=True) - s.add_argument("--moderns-geno", required=True) - s.add_argument("--out", required=True) - s.add_argument("--stats", required=True) - s = sub.add_parser("align") - s.add_argument("--panel-evec", required=True) - s.add_argument("--run-evec", required=True) - s.add_argument("--pcs", type=int, default=DISTANCE_PCS) - s = sub.add_parser("report") - s.add_argument("--panel-evec", required=True) - s.add_argument("--run-evec", required=True) - s.add_argument("--ancients", required=True) - s.add_argument("--sample", required=True) - s.add_argument("--release", required=True) - s.add_argument("--stats") - s.add_argument("--out-tsv", required=True) - s.add_argument("--out-info", required=True) - s.add_argument("--out-txt", required=True) - s.add_argument("--out-svg", required=True) - a = p.parse_args(argv) - try: - return {"select": cmd_select, "lift": cmd_lift, "person-geno": cmd_person_geno, - "align": cmd_align, "report": cmd_report}[a.cmd](a) - except Fail as e: - print(f"ERROR: {e}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/archaic_compare.py b/bin/archaic_compare.py index 88cde4e31..7c6ebdd7d 100644 --- a/bin/archaic_compare.py +++ b/bin/archaic_compare.py @@ -24,8 +24,25 @@ import statistics import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import archaic_report # noqa: E402 (the same reader and cutoffs as step 43) +ARCHAICS = ("Altai", "Vindija", "Denisova") + + +def ibdmix_segments(path, min_lod=4.0, min_length=50000): + """IBDmix's segments for SAMPLE in one of its output files, kept as step 43 + keeps them (og archaic report: LOD 4 and 50 kb or longer), as (start, end).""" + out = [] + with open(path) as f: + head = f.readline().rstrip("\n").split("\t") + if head[:5] != ["ID", "chrom", "start", "end", "slod"]: + sys.exit(f"ERROR: {path} is not IBDmix output (header {' '.join(head[:5])})") + for line in f: + c = line.rstrip("\n").split("\t") + if len(c) < 5 or c[0] != "SAMPLE": + continue + s, e, lod = int(c[2]), int(c[3]), float(c[4]) + if lod >= min_lod and e - s >= min_length: + out.append((s, e)) + return out def norm(c): @@ -170,9 +187,8 @@ def cmd_genomes(a): chroms = list(dict.fromkeys(norm(line.split("\t")[0]) for line in f if line.strip())) per = collections.defaultdict(lambda: collections.defaultdict(list)) for c in chroms: - for n in archaic_report.ARCHAICS: - segs = archaic_report.keep(archaic_report.read_segments(os.path.join(a.ibdmix_dir, f"chr{c}.{n}.txt"))) - per[n][c] += [(s, e) for _, s, e, _ in segs] + for n in ARCHAICS: + per[n][c] += ibdmix_segments(os.path.join(a.ibdmix_dir, f"chr{c}.{n}.txt")) per["Altai+Vindija"] = {c: per["Altai"][c] + per["Vindija"][c] for c in chroms} released = sum(merged_bp(calls[a.sample].get(c, [])) for c in chroms) print(f"| IBDmix output | bp | under the released {a.sample} calls | only here | only released |") diff --git a/bin/archaic_report.py b/bin/archaic_report.py deleted file mode 100755 index 6ada76bd2..000000000 --- a/bin/archaic_report.py +++ /dev/null @@ -1,745 +0,0 @@ -#!/usr/bin/env python3 -"""archaic_report.py: Neanderthal and Denisovan segments in one genome, with IBDmix. - -IBDmix (Chen et al. 2020, Cell 180:677, doi:10.1016/j.cell.2020.01.012) compares -a modern genome with an archaic one site by site and adds up a log-odds score -(LOD) that a stretch of the modern genome shares a recent ancestor with the -archaic genome, against the chance of the same genotypes in the modern -population. It needs no unadmixed reference population, but it does need the -allele frequency of each site in the modern population, and it takes that from -the modern people in its input. With one genome as the only modern input the -frequency is that person's own genotype and no site scores (find_frequency in -IBDmix's src/Genotype_Reader.cc drops every site below its minor allele count), -so `genotypes` adds PSEUDO stand-ins whose genotypes carry the 1000 Genomes -allele count of the chosen population at each site. IBDmix scores each modern -column from the site's frequency and that column's genotype only, so your rows -are the ones it would give you inside a population sample of that frequency; -the stand-ins' own rows are dropped. - -Subcommands: - - table (setup.sh --archaic) one GRCh37 chromosome: the 1000 Genomes - phase 3 biallelic SNVs with their allele frequencies, and the - genotypes of the Altai Neanderthal, the Vindija 33.19 Neanderthal - and the Altai Denisovan at those sites and wherever one of the - three carries a non-reference allele, as a sites-only VCF that - Picard lifts to GRCh38. Each archaic genome is read inside its - own mask (--mask NAME=BED, the MPI-EVA FilterBed of that - genome): a call outside it is kept as masked, code 10 plus its - ALT count. Every call at a site inside an excluded region - (--exclude BED: segmental duplications, the 5 bp around a modern - indel) is masked the same way. That is how Chen et al. filter: - the site stays in IBDmix's input and IBDmix's --mask stops it - scoring, except that a homozygous mismatch there still counts - against a segment. - strict (setup.sh --archaic) the lifted GRCh38 rows of one chromosome with - every row outside the 1000 Genomes strict mask marked masked in - all three archaic columns (the same code 10 to 12), instead of - dropped, so IBDmix still scores a mismatch there. - genotypes (step 43) IBDmix's genotype file of each chromosome: the three - archaic columns, SAMPLE (yours, from the VCF) and the stand-ins; - the mask of each archaic genome (chrN..mask.bed, the - positions where its call is masked); and spans.tsv, the span and - site count of each chromosome. - report (step 43) IBDmix's segments for SAMPLE against each archaic genome, - kept at LOD 4 and 50 kb or longer (the cutoffs of Chen et al.), - merged into Neanderthal and Denisovan segments, with the share of - the analysed span, the ten longest segments and the protein-coding - genes in them. - -Codes: an archaic or modern genotype is the number of ALT alleles (0, 1, 2), or -9 when it is unknown, the coding IBDmix's genotype file uses. In the table an -archaic call outside that genome's mask is 10, 11 or 12 (MASKED plus the ALT -count); `genotypes` writes it to IBDmix as 0, 1 or 2 and lists the position in -that genome's mask file (chrN..mask.bed), which step 43 passes to -IBDmix with -r. - -Used by scripts/setup.sh, scripts/43-archaic.sh and the ARCHAIC_* processes -(bin/ is on the task PATH); tests/test_archaic_report.py checks it. Standard -library only. -""" -import argparse -import bisect -import gzip -import heapq -import os -import re -import sys - -ARCHAICS = ("Altai", "Vindija", "Denisova") -NEANDERTHALS = ("Altai", "Vindija") -POPS = ("ALL", "AFR", "AMR", "EAS", "EUR", "SAS") -TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova"] + [f"AF_{p}" for p in POPS] -MASKED = 10 # table code: MASKED + ALT count, a call outside the genome's mask -PSEUDO = 100 # stand-in modern people: 200 alleles, about one 1000 Genomes population -MIN_LOD = 4.0 # Chen et al. 2020: LOD 4 -MIN_LENGTH = 50000 # and 50 kb -TOP = 10 -SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] -CAVEAT = ("IBDmix finds stretches where your genome matches a Neanderthal or the Denisovan genome more " - "closely than the allele frequencies of a modern population explain. A stretch counts when either " - "of your two copies carries it, and the share is of the analysed span of the autosomes. Some of what " - "it finds in anyone is old ancestry shared with the archaic genomes rather than gene flow from them: " - "Chen et al. 2020 find Neanderthal sequence in every population they studied, Africans included. " - "The numbers move with the cutoffs (LOD 4, 50 kb) and with the frequencies used. An estimate, not " - "a measurement.") - - -def die(msg, code=1): - print(f"ERROR: {msg}", file=sys.stderr) - sys.exit(code) - - -def opener(path): - return gzip.open(path, "rt") if path.endswith(".gz") else open(path) - - -def gt_code(gt): - """'0/1', '1|1', '0/0' -> 1, 2, 0; a missing, partial or multi-allelic call -> None.""" - a = re.split(r"[/|]", gt) - if len(a) != 2 or any(x not in ("0", "1") for x in a): - return None - return int(a[0]) + int(a[1]) - - -# --- table (setup) ------------------------------------------------------------------ - -def read_modern(path): - """1000 Genomes lines: POS REF ALT AF AFR_AF AMR_AF EAS_AF EUR_AF SAS_AF (bcftools query).""" - with opener(path) as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 9: - die(f"{path}: a line has {len(c)} columns, want 9: {line[:80]!r}") - if len(c[1]) != 1 or len(c[2]) != 1: - continue - yield int(c[0]), 0, (c[1], c[2], c[3:9]) - - -def read_archaic(path, idx): - """Archaic lines: POS REF ALT GT (bcftools query).""" - with opener(path) as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 4: - die(f"{path}: a line has {len(c)} columns, want 4: {line[:80]!r}") - yield int(c[0]), 1 + idx, (c[1], c[2], c[3]) - - -def read_mask(path, chrom): - """The intervals of one chromosome of a mask BED, sorted and merged. - A BED is 0-based and half-open, so [start, end) holds the 1-based - positions start + 1 to end. The chromosome is matched with and without - the chr prefix, because the archaic masks name it 22 and others chr22.""" - num = str(chrom)[3:] if str(chrom).startswith("chr") else str(chrom) - want = {num, f"chr{num}"} - iv = [] - with opener(path) as f: - for line in f: - if line.startswith(("#", "track", "browser")): - continue - c = line.split() - if len(c) < 3 or c[0] not in want: - continue - s, e = int(c[1]), int(c[2]) - if e <= s: - continue # an empty or reversed interval holds no position - iv.append((s, e)) - iv.sort() - out = [] - for s, e in iv: - if out and s <= out[-1][1]: - out[-1][1] = max(out[-1][1], e) - else: - out.append([s, e]) - return out - - -class Mask: - """Whether a 1-based position is inside a mask. Positions must come in - ascending order, as they do in the table build: the walk over the - intervals never goes back.""" - - def __init__(self, intervals): - self.iv = intervals - self.i = 0 - - def holds(self, pos): - p = pos - 1 - while self.i < len(self.iv) and self.iv[self.i][1] <= p: - self.i += 1 - return self.i < len(self.iv) and self.iv[self.i][0] <= p - - -def archaic_code(rec, ref, alt): - """The ALT count of one archaic record at a row with this REF and ALT, or 9.""" - if rec is None: - return 9 - aref, aalt, gt = rec - if aref != ref: - return 9 - code = gt_code(gt) - if code is None: - return 9 - if aalt in (".", "<*>") or code == 0: - return 0 if code == 0 else 9 - return code if aalt == alt else 9 - - -def table_row(pos, modern, arch): - """The row of one position, or None. modern: (REF, ALT, AFs) or None; - arch: one (REF, ALT, GT) or None per archaic genome.""" - if modern is not None: - ref, alt, afs = modern - else: - # A site no modern person varies at: the ALT of the first archaic - # genome with a non-reference call, every population frequency 0. - cand = [r for r in arch if r is not None and gt_code(r[2]) not in (None, 0) - and len(r[0]) == 1 and len(r[1]) == 1] - if not cand: - return None - ref, alt, afs = cand[0][0], cand[0][1], ["0"] * len(POPS) - codes = [archaic_code(r, ref, alt) for r in arch] - if all(c == 9 for c in codes): - return None - if modern is None and not any(c in (1, 2) for c in codes): - return None - clean = [] - for v in afs: - try: - x = float(v) - except ValueError: - x = 0.0 - clean.append(f"{min(max(x, 0.0), 1.0):.6g}") - return ref, alt, codes, clean - - -def cmd_table(a): - names = [] - streams = [read_modern(a.modern)] - for i, spec in enumerate(a.archaic): - name, _, path = spec.partition("=") - if name not in ARCHAICS or not path: - die(f"--archaic takes NAME=FILE with NAME one of {', '.join(ARCHAICS)}, got {spec!r}", 2) - names.append(name) - streams.append(read_archaic(path, i)) - if names != list(ARCHAICS): - die(f"--archaic must name {', '.join(ARCHAICS)} in that order, got {', '.join(names)}", 2) - masks = {} - for spec in a.mask: - name, _, path = spec.partition("=") - if name not in ARCHAICS or not path: - die(f"--mask takes NAME=FILE with NAME one of {', '.join(ARCHAICS)}, got {spec!r}", 2) - if name in masks: - die(f"--mask names {name} twice", 2) - iv = read_mask(path, a.chrom) - if not iv: - die(f"{path} holds no interval on chromosome {a.chrom}: a mask that matches nothing " - "would make every site unknown") - masks[name] = Mask(iv) - excluded = [] - for path in a.exclude: - iv = read_mask(path, a.chrom) - if not iv: - die(f"{path} holds no interval on chromosome {a.chrom}: an empty exclusion list is a " - "download that went wrong, not a chromosome without segmental duplications") - excluded.append(Mask(iv)) - chrom = a.chrom if a.chrom.startswith("chr") else f"chr{a.chrom}" - out = open(a.out, "w") - out.write("##fileformat=VCFv4.2\n") - out.write(f"##contig=\n") - for n in ARCHAICS: - out.write(f'##INFO=\n') - for p in POPS: - out.write(f'##INFO=\n') - out.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n") - rows = 0 - cur, modern, arch, last = None, None, [None] * len(ARCHAICS), -1 - - def flush(): - nonlocal rows - if cur is None: - return - # Every Mask is walked for every position, before any early return, - # because the walk only goes forward. - excl = any([m.holds(cur) for m in excluded]) - outside = [excl or (n in masks and not masks[n].holds(cur)) for n in ARCHAICS] - r = table_row(cur, modern, arch) - if r is None: - return - ref, alt, codes, afs = r - # A genome's call outside its mask, or at an excluded site, is kept but - # marked masked (MASKED + code). Step 43 hands IBDmix the call and a - # mask of those positions, as Chen et al. do: IBDmix scores nothing - # there except a homozygous mismatch, which still counts against a - # segment. Dropping the call instead (code 9) let segments run through - # masked sequence nothing could contradict. - codes = [c + MASKED if (out and c != 9) else c for c, out in zip(codes, outside)] - info = ";".join([f"{n}={c}" for n, c in zip(ARCHAICS, codes)] + [f"AF_{p}={v}" for p, v in zip(POPS, afs)]) - out.write(f"{chrom}\t{cur}\t.\t{ref}\t{alt}\t.\t.\t{info}\n") - rows += 1 - - for pos, src, rec in heapq.merge(*streams, key=lambda t: (t[0], t[1])): - if pos < last: - die(f"input not sorted by position at {pos}") - if pos != cur: - flush() - cur, modern, arch = pos, None, [None] * len(ARCHAICS) - last = pos - if src == 0: - # A second modern record at one position (a split multi-allelic - # site) makes the row ambiguous: keep neither. - modern = rec if modern is None else ("N", "N", ["0"] * len(POPS)) - else: - arch[src - 1] = rec - flush() - out.close() - print(f"{chrom}: {rows} rows", file=sys.stderr) - - -def cmd_strict(a): - """The lifted GRCh38 rows of one chromosome (no header, the 13 table - columns), once per position, with the calls of every row outside the 1000 - Genomes strict mask marked masked (MASKED plus the ALT count, 9 stays 9). - The strict mask is the same for the three genomes, so all three are - marked; step 43 then hands IBDmix those positions as its mask.""" - iv = read_mask(a.strict, a.chrom) - if not iv: - die(f"{a.strict} holds no interval on chromosome {a.chrom}: a mask that matches nothing " - "would mark every site masked") - starts = [s for s, _ in iv] - seen = set() - rows = marked = 0 - with open(a.rows) as f, open(a.out, "w") as out: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) != len(TABLE_HEADER): - die(f"{a.rows}: a row has {len(c)} columns, want {len(TABLE_HEADER)}: {line[:80]!r}") - pos = int(c[1]) - if pos in seen: - continue - seen.add(pos) - i = bisect.bisect_right(starts, pos - 1) - 1 - if not (i >= 0 and pos - 1 < iv[i][1]): - hit = False - for k in (4, 5, 6): - if c[k] in ("0", "1", "2"): - c[k] = str(int(c[k]) + MASKED) - hit = True - marked += hit - out.write("\t".join(c) + "\n") - rows += 1 - print(f"{a.chrom}: {rows} rows, {marked} outside the strict mask and marked masked", file=sys.stderr) - - -# setup.sh --archaic stamps its table with the sha256 of everything above this -# section, so an edit below it does not rebuild the table (about 165 GB of downloads). - -# --- genotypes (step 43) ---------------------------------------------------------- - -def pseudo_columns(n): - """The stand-in columns for each ALT allele count 0..2n: as many 2s and - one 1 as the count needs, then 0s.""" - cols = [] - for ac in range(2 * n + 1): - g = ["2"] * (ac // 2) + (["1"] if ac % 2 else []) - g += ["0"] * (n - len(g)) - cols.append("\t".join(g)) - return cols - - -def read_vcf(path): - """(chrom, pos, ref, alt, code) of the biallelic SNVs of a one-sample VCF. - A record whose FILTER is not PASS or '.' gives code 9 (not read), so a - DeepVariant RefCall is neither your allele nor the reference.""" - with opener(path) as f: - for line in f: - if line.startswith("##"): - continue - c = line.rstrip("\n").split("\t") - if line.startswith("#"): - if len(c) != 10: - die(f"{path} holds {max(len(c) - 9, 0)} samples; step 43 reads a one-sample VCF " - "(keep one with bcftools view -s ).") - continue - if len(c) < 10 or len(c[3]) != 1 or c[4] == ".": - continue - if "," in c[4] or len(c[4]) != 1: - yield c[0], int(c[1]), c[3], None, 9 - continue - fmt = c[8].split(":") - if "GT" not in fmt: - continue - vals = c[9].split(":") - gt = vals[fmt.index("GT")] if fmt.index("GT") < len(vals) else "." - code = gt_code(gt) - if code is None or c[6] not in ("PASS", "."): - code = 9 - yield c[0], int(c[1]), c[3], c[4], code - - -CLASH = object() -ARCHAIC_CODES = {str(v) for v in (0, 1, 2, 9, MASKED, MASKED + 1, MASKED + 2)} - - -def pick_call(prev, call): - """The call to keep when a position comes twice, as in a VCF split with - bcftools norm -m-: each is (REF, ALT, code), ALT None for an insertion or a - multi-allelic record. An SNV wins over such a record, in either order. Of - one SNV twice, a usable record wins over code 9, and two different usable - codes are CLASH. Of two different SNVs, the - one that carries its ALT wins over a 0/0 or unusable one (a split 0/1 at - A>G,T is A>G 0/1 and A>T 0/0); both 0/0 is the reference; both carrying - their ALT, or neither readable, is CLASH (unknown, 9).""" - if prev is None: - return call - if prev[1] is None or call[1] is None: - return call if prev[1] is None and call[1] is not None else prev - if call[1] == prev[1]: - if call[2] == 9 or call[2] == prev[2]: - return prev - return call if prev[2] == 9 else CLASH - has_prev, has_call = prev[2] in (1, 2), call[2] in (1, 2) - if has_prev != has_call: - return prev if has_prev else call - if not has_prev and prev[2] == 0 and call[2] == 0: - return (prev[0], None, 0) - return CLASH - - -def read_table(path): - """Rows of one chromosome of the GRCh38 table, with the header checked.""" - with opener(path) as f: - head = f.readline().rstrip("\n").split("\t") - if head != TABLE_HEADER: - die(f"{path} does not start with the table header ({' '.join(TABLE_HEADER)})") - for line in f: - c = line.rstrip("\n").split("\t") - yield c - - -def cmd_genotypes(a): - if a.pop not in POPS: - die(f"--pop must be one of {', '.join(POPS)}, got {a.pop!r}", 2) - af_col = TABLE_HEADER.index(f"AF_{a.pop}") - tables = {} - for name in sorted(os.listdir(a.table_dir)): - m = re.fullmatch(r"(chr(?:[1-9]|1[0-9]|2[0-2]))\.tsv\.gz", name) - if m: - tables[m.group(1)] = os.path.join(a.table_dir, name) - if not tables: - die(f"{a.table_dir} holds no chrN.tsv.gz: install the archaic data with scripts/setup.sh --archaic") - os.makedirs(a.out_dir, exist_ok=True) - pseudo = pseudo_columns(a.pseudo) - two_n = 2 * a.pseudo - head = "\t".join(["chrom", "pos", "ref", "alt", *ARCHAICS, "SAMPLE"] + [f"P{i:03d}" for i in range(1, a.pseudo + 1)]) - with open(os.path.join(a.out_dir, "samples.txt"), "w") as f: - f.write("SAMPLE\n" + "".join(f"P{i:03d}\n" for i in range(1, a.pseudo + 1))) - spans = [] - vcf = read_vcf(a.vcf) - pending = next(vcf, None) - done = set() - while pending is not None: - chrom = pending[0] - if chrom in done: - die(f"{a.vcf} is not sorted: {chrom} comes back after another chromosome") - done.add(chrom) - if chrom not in tables: - while pending is not None and pending[0] == chrom: - pending = next(vcf, None) - continue - # The sample's calls on this chromosome, merged with the table by position. - calls, clash = {}, set() - while pending is not None and pending[0] == chrom: - pos = pending[1] - if pos not in clash: - calls[pos] = pick_call(calls.get(pos), pending[2:]) - if calls[pos] is CLASH: - calls[pos] = (pending[2], None, 9) - clash.add(pos) - pending = next(vcf, None) - n = first = last = dropped = carried = 0 - num = chrom[3:] - path = os.path.join(a.out_dir, f"{chrom}.gt.gz") - # Per archaic genome, the runs of written rows whose call is masked: - # [first, last] positions, closed at the next unmasked row. - runs = {g: [] for g in ARCHAICS} - open_run = {g: None for g in ARCHAICS} - with gzip.open(path, "wt", compresslevel=1) as out: - out.write(head + "\n") - for c in read_table(tables[chrom]): - pos, ref, alt = int(c[1]), c[2], c[3] - arch = [] - for g, v in zip(ARCHAICS, c[4:7]): - if v not in ARCHAIC_CODES: - die(f"{tables[chrom]}: {chrom}:{pos} has {g} code {v!r}; the table holds 0, 1, 2, 9 " - f"or {MASKED} to {MASKED + 2}") - arch.append(int(v)) - call = calls.get(pos) - if call is None: - code = 0 - elif call[0] != ref or (call[1] is not None and call[1] != alt): - # Another allele than the row's: IBDmix's generate_gt drops the line. - dropped += 1 - continue - else: - code = call[2] - if code in (1, 2): - carried += 1 - try: - ac = int(round(float(c[af_col]) * two_n)) - except ValueError: - ac = 0 - ac = min(max(ac, 0), two_n) - for g, v in zip(ARCHAICS, arch): - if v >= MASKED: - if open_run[g] is None: - open_run[g] = [pos, pos] - else: - open_run[g][1] = pos - elif open_run[g] is not None: - runs[g].append(open_run[g]) - open_run[g] = None - gts = "\t".join(str(v - MASKED if v >= MASKED else v) for v in arch) - out.write(f"{num}\t{pos}\t{ref}\t{alt}\t{gts}\t{code}\t{pseudo[ac]}\n") - if n == 0: - first = pos - last = pos - n += 1 - if n == 0: - os.remove(path) - continue - # IBDmix's mask (-r): chromosome as in the genotype file, and a line - # covers the positions start + 1 to end, as in a BED. - for g in ARCHAICS: - if open_run[g] is not None: - runs[g].append(open_run[g]) - with open(os.path.join(a.out_dir, f"{chrom}.{g}.mask.bed"), "w") as f: - f.writelines(f"{num}\t{s0 - 1}\t{e0}\n" for s0, e0 in runs[g]) - spans.append((chrom, first, last, n, carried, dropped)) - with open(os.path.join(a.out_dir, "spans.tsv"), "w") as f: - f.write("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n") - for s in spans: - f.write("\t".join(map(str, s)) + "\n") - if not spans: - die(f"{a.vcf} has no records on a chromosome of the archaic table ({', '.join(sorted(tables))})") - print(f"{len(spans)} chromosomes, {sum(s[3] for s in spans)} sites", file=sys.stderr) - - -# --- report (step 43) ------------------------------------------------------------ - -def read_segments(path, sample="SAMPLE"): - """IBDmix's output rows for SAMPLE: (chrom, start, end, lod).""" - segs = [] - with open(path) as f: - head = f.readline().rstrip("\n").split("\t") - if head[:5] != ["ID", "chrom", "start", "end", "slod"]: - die(f"{path} is not IBDmix output (header {' '.join(head[:5])})") - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 5 or c[0] != sample: - continue - segs.append((c[1], int(c[2]), int(c[3]), float(c[4]))) - return segs - - -def keep(segs, min_lod=MIN_LOD, min_length=MIN_LENGTH): - return [s for s in segs if s[3] >= min_lod and s[2] - s[1] >= min_length] - - -def merge(segs): - """Overlapping or touching segments of one class as one: (chrom, start, end, lod, members).""" - out = [] - for chrom, start, end, lod, who in sorted(segs, key=lambda s: (chrom_key(s[0]), s[1], s[2])): - if out and out[-1][0] == chrom and start <= out[-1][2]: - c, s0, e0, l0, w0 = out[-1] - out[-1] = (c, s0, max(e0, end), max(l0, lod), w0 | who) - else: - out.append((chrom, start, end, lod, set(who))) - return out - - -def chrom_key(c): - c = str(c) - c = c[3:] if c.startswith("chr") else c - return int(c) if c.isdigit() else 99 - - -def overlaps(seg, others): - return [o for o in others if o[0] == seg[0] and o[1] < seg[2] and seg[1] < o[2]] - - -def read_genes(path): - """Protein-coding genes of a GENCODE gene GTF: chrom -> [(start, end, name)].""" - genes = {} - if not path: - return genes - with opener(path) as f: - for line in f: - if line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 9 or c[2] != "gene": - continue - if 'gene_type "protein_coding"' not in c[8]: - continue - m = re.search(r'gene_name "([^"]+)"', c[8]) - if m: - genes.setdefault(c[0], []).append((int(c[3]), int(c[4]), m.group(1))) - return genes - - -def genes_in(genes, chrom, start, end): - """The names of the genes overlapping [start, end), once each, in order. - IBDmix's start is the segment's first site and its end the next site, past - the segment (its README, --inclusive-end); a GTF gene is [s, e], 1-based.""" - name = chrom if str(chrom).startswith("chr") else f"chr{chrom}" - seen = [] - for s, e, n in sorted(genes.get(name, [])): - if s < end and e >= start and n not in seen: - seen.append(n) - return seen - - -def classify(per_archaic): - """Neanderthal: the union of the Altai and Vindija segments. Denisovan: the - Denisova segments that overlap no Neanderthal one (a stretch the Denisova - genome and a Neanderthal both match is counted as Neanderthal, the more - common source outside Oceania). Returns a list of - (chrom, start, end, ancestry, matched, lod).""" - nea = merge([(c, s, e, l, {n}) for n in NEANDERTHALS for c, s, e, l in per_archaic.get(n, [])]) - den = merge([(c, s, e, l, {"Denisova"}) for c, s, e, l in per_archaic.get("Denisova", [])]) - rows = [] - for c, s, e, l, who in nea: - also = {"Denisova"} if overlaps((c, s, e), den) else set() - rows.append((c, s, e, "Neanderthal", sorted(who) + sorted(also), l)) - for c, s, e, l, who in den: - if not overlaps((c, s, e), nea): - rows.append((c, s, e, "Denisovan", ["Denisova"], l)) - rows.sort(key=lambda r: (chrom_key(r[0]), r[1])) - return rows - - -def summarise(rows, spans_bp): - out = {} - for anc in ("Neanderthal", "Denisovan"): - r = [x for x in rows if x[3] == anc] - bp = sum(x[2] - x[1] for x in r) - key = anc.lower() - out[f"{key}_segments"] = len(r) - out[f"{key}_bp"] = bp - out[f"{key}_percent"] = f"{100.0 * bp / spans_bp:.2f}" if spans_bp else "0.00" - out[f"{key}_longest_bp"] = max((x[2] - x[1] for x in r), default=0) - return out - - -def cmd_report(a): - spans = [] - with open(a.spans) as f: - head = f.readline().rstrip("\n").split("\t") - if head[:4] != ["chrom", "first", "last", "sites"]: - die(f"{a.spans} is not the spans table of the genotypes step") - for line in f: - c = line.rstrip("\n").split("\t") - spans.append((c[0], int(c[1]), int(c[2]), int(c[3]))) - if not spans: - die(f"{a.spans} lists no chromosome") - spans_bp = sum(last - first + 1 for _, first, last, _ in spans) - per = {n: [] for n in ARCHAICS} - for chrom, *_ in spans: - for n in ARCHAICS: - p = os.path.join(a.ibdmix_dir, f"{chrom}.{n}.txt") - if not os.path.isfile(p): - die(f"no IBDmix output {p}") - per[n] += [(f"chr{c}" if not str(c).startswith("chr") else c, s, e, l) - for c, s, e, l in keep(read_segments(p))] - rows = classify(per) - genes = read_genes(a.genes) - summary = summarise(rows, spans_bp) - os.makedirs(a.out_dir, exist_ok=True) - seg_path = os.path.join(a.out_dir, f"{a.sample}_archaic_segments.tsv") - with open(seg_path, "w") as f: - f.write("\t".join(SEGMENT_HEADER) + "\n") - for c, s, e, anc, who, lod in rows: - g = genes_in(genes, c, s, e) if a.genes else [] - f.write(f"{c}\t{s}\t{e}\t{e - s}\t{anc}\t{','.join(who)}\t{lod:.1f}\t{','.join(g) if g else '.'}\n") - longest = sorted(rows, key=lambda r: (-(r[2] - r[1]), chrom_key(r[0]), r[1]))[:TOP] - meta = { - "sample": a.sample, - "method": "IBDmix", - "archaic_genomes": "Altai Neanderthal, Vindija 33.19 Neanderthal, Altai Denisovan", - "population": a.pop, - "min_lod": f"{MIN_LOD:g}", - "min_length_bp": MIN_LENGTH, - "analysed_bp": spans_bp, - "analysed_chromosomes": len(spans), - "analysed_sites": sum(s[3] for s in spans), - "genes": "GENCODE protein-coding" if a.genes else "none (no gene annotation installed)", - **summary, - } - with open(os.path.join(a.out_dir, f"{a.sample}_archaic.tsv"), "w") as f: - f.write("key\tvalue\n") - for k, v in meta.items(): - f.write(f"{k}\t{v}\n") - for i, (c, s, e, anc, who, lod) in enumerate(longest, 1): - g = genes_in(genes, c, s, e) if a.genes else [] - f.write(f"longest_{i}\t{c}:{s}-{e} {anc} {(e - s) / 1e6:.2f} Mb " - f"({'/'.join(who)}); genes: {', '.join(g) if g else 'none'}\n") - lines = [f"Archaic ancestry of {a.sample} (IBDmix against the Altai and Vindija Neanderthals and the Altai Denisovan)", - f" Analysed: {len(spans)} autosomes, {spans_bp / 1e6:.1f} Mb, {meta['analysed_sites']} sites; " - f"modern frequencies: 1000 Genomes {a.pop}", - f" Neanderthal: {summary['neanderthal_percent']}% ({summary['neanderthal_bp'] / 1e6:.1f} Mb in " - f"{summary['neanderthal_segments']} segments)", - f" Denisovan: {summary['denisovan_percent']}% ({summary['denisovan_bp'] / 1e6:.1f} Mb in " - f"{summary['denisovan_segments']} segments)"] - if longest: - lines.append(f" The {len(longest)} longest segments:") - for c, s, e, anc, who, lod in longest: - g = genes_in(genes, c, s, e) if a.genes else [] - lines.append(f" {c}:{s}-{e} {(e - s) / 1e6:.2f} Mb {anc} ({'/'.join(who)}, LOD {lod:.0f}) " - f"genes: {', '.join(g) if g else 'none'}") - lines.append(f" {CAVEAT}") - lines.append(f" Every segment: {os.path.basename(seg_path)}") - with open(os.path.join(a.out_dir, f"{a.sample}_archaic.txt"), "w") as f: - f.write("\n".join(lines) + "\n") - print("\n".join(lines)) - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - sub = p.add_subparsers(dest="cmd", required=True) - t = sub.add_parser("table", help="one GRCh37 chromosome of the archaic table (setup.sh --archaic)") - t.add_argument("--chrom", required=True) - t.add_argument("--modern", required=True) - t.add_argument("--archaic", action="append", default=[], help="NAME=FILE, once each for Altai, Vindija, Denisova") - t.add_argument("--mask", action="append", default=[], - help="NAME=BED: read that archaic genome only inside this mask (its MPI-EVA FilterBed)") - t.add_argument("--exclude", action="append", default=[], - help="BED: leave out every site inside these regions (segmental duplications, indel neighbourhoods)") - t.add_argument("--out", required=True) - m = sub.add_parser("strict", help="mark the rows outside the GRCh38 strict mask masked (setup.sh --archaic)") - m.add_argument("--chrom", required=True) - m.add_argument("--strict", required=True, help="BED: the 1000 Genomes GRCh38 strict mask") - m.add_argument("--rows", required=True, help="the lifted rows, tab-separated, no header") - m.add_argument("--out", required=True) - g = sub.add_parser("genotypes", help="IBDmix's genotype files for one sample (step 43)") - g.add_argument("--table-dir", required=True) - g.add_argument("--vcf", required=True) - g.add_argument("--pop", default="ALL") - g.add_argument("--pseudo", type=int, default=PSEUDO) - g.add_argument("--out-dir", required=True) - r = sub.add_parser("report", help="segments, shares and the ten longest (step 43)") - r.add_argument("--sample", required=True) - r.add_argument("--ibdmix-dir", required=True) - r.add_argument("--spans", required=True) - r.add_argument("--genes", default="") - r.add_argument("--pop", default="ALL") - r.add_argument("--out-dir", required=True) - a = p.parse_args(argv) - if a.cmd == "genotypes" and a.pseudo < 1: - die("--pseudo must be 1 or more", 2) - {"table": cmd_table, "strict": cmd_strict, "genotypes": cmd_genotypes, "report": cmd_report}[a.cmd](a) - - -if __name__ == "__main__": - main() diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 054d9a684..5aa4fd6eb 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -2169,7 +2169,7 @@ def sec_bloodtype(d, s): def sec_archaic(d, s): - """Step 43's summary (archaic/_archaic.tsv, bin/archaic_report.py): the + """Step 43's summary (archaic/_archaic.tsv, og archaic report): the Neanderthal and Denisovan shares and segment counts, the analysed span and the ten longest segments.""" p = first_existing(d, [f"archaic/{s}_archaic.tsv"]) diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs index 612a71212..6a3b6fc52 100644 --- a/rust/crates/og-ancestry/src/ancient.rs +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -1529,6 +1529,9 @@ mod tests { assert_eq!(year_label(1950.0), "1 BCE"); assert_eq!(year_label(1949.0), "1 CE"); assert_eq!(year_label(1750.5), "200 CE"); + assert_eq!(year_label(8025.0), "6075 BCE"); + assert_eq!(year_label(450.0), "1500 CE"); + assert_eq!(range_label(3000.0, 3000.0), "1050 BCE"); assert_eq!(range_label(7250.0, 6850.0), "5300-4900 BCE"); assert_eq!(range_label(1750.0, 1500.0), "200-450 CE"); assert_eq!(range_label(2250.0, 1750.0), "300 BCE-200 CE"); diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 37bdf7624..3d1b4e811 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -260,7 +260,6 @@ impl Stream { /// SNVs are skipped. fn next(&mut self) -> Out> { loop { - let path = self.text.path.display().to_string(); let Some(raw) = self.text.raw()? else { return Ok(None); }; @@ -268,10 +267,10 @@ impl Stream { let c: Vec<&str> = line.split('\t').collect(); if self.src == 0 { if c.len() < 9 { + let (n, head) = (c.len(), head80(raw)); return die(format!( - "{path}: a line has {} columns, want 9: {}", - c.len(), - head80(raw) + "{}: a line has {n} columns, want 9: {head}", + self.text.path.display() )); } if c[1].chars().count() != 1 || c[2].chars().count() != 1 { @@ -288,10 +287,10 @@ impl Stream { ))); } if c.len() < 4 { + let (n, head) = (c.len(), head80(raw)); return die(format!( - "{path}: a line has {} columns, want 4: {}", - c.len(), - head80(raw) + "{}: a line has {n} columns, want 4: {head}", + self.text.path.display() )); } let pos = need_int(c[0])?; @@ -732,7 +731,6 @@ struct Vcf { impl Vcf { fn next(&mut self) -> Out> { loop { - let path = self.text.path.display().to_string(); let Some(raw) = self.text.raw()? else { return Ok(None); }; @@ -743,10 +741,11 @@ impl Vcf { let c: Vec<&str> = line.split('\t').collect(); if raw.starts_with('#') { if c.len() != 10 { + let n = c.len().saturating_sub(9); return die(format!( - "{path} holds {} samples; step 43 reads a one-sample VCF \ + "{} holds {n} samples; step 43 reads a one-sample VCF \ (keep one with bcftools view -s ).", - c.len().saturating_sub(9) + self.text.path.display() )); } continue; diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f76068dcf..e2f0a3f24 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -47,11 +47,11 @@ alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-4-bwamem2 scripts/02a-alignment-bwamem2.sh scripts/ci/fake-docker/lib.sh ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/ancestry_fine_report.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/ancient_report.py bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -archaic-ancestry-1 assets/pgs_scores.tsv bin/archaic_report.py bin/collect_summary.py bin/traits_report.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +archaic-ancestry-1 assets/pgs_scores.tsv bin/og bin/collect_summary.py bin/traits_report.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh bash-step-08-hla - bash-step-09-str assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh bash-step-10-telomere scripts/10-telomere-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 49bf9e2a5..645b5a3e4 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -1198,42 +1198,53 @@ def archaic(d, rng): def ancient(d, rng): - """Step 44: bin/ancient_report.py's nearest-group table, info table and - PC plot (its own svg_plot), made-up distances and PCs over a few - AADR-style group labels.""" - import ancient_report - groups = [("England_BellBeaker", "2450-2000 BCE", "United Kingdom", "Amesbury Down"), - ("Germany_CordedWare", "2900-2350 BCE", "Germany", "Esperstedt"), - ("Spain_BellBeaker", "2500-2000 BCE", "Spain", "Camino de las Yeseras"), - ("Czechia_EBA_Unetice", "2200-1700 BCE", "Czech Republic", "Moravia"), - ("Hungary_LBA", "1500-1100 BCE", "Hungary", "Tiszafüred"), - ("Italy_Imperial_Rome", "1-300 CE", "Italy", "Rome"), - ("Sweden_Viking", "800-1050 CE", "Sweden", "Sigtuna"), - ("France_Neolithic", "4500-3000 BCE", "France", "Gurgy"), - ("Turkey_Neolithic", "6500-6000 BCE", "Turkey", "Barcın"), - ("Russia_Yamnaya_Samara", "3300-2700 BCE", "Russia", "Samara")] + """Step 44: what og ancient report writes (the nearest-group table, the + info table, the text and the PC plot) for a made-up panel: present-day + people around the origin, ancient people of a few AADR-style group labels + each around its own point, and the person among them.""" + groups = [("England_BellBeaker", 4200, "United Kingdom", "Amesbury Down"), + ("Germany_CordedWare", 4600, "Germany", "Esperstedt"), + ("Spain_BellBeaker", 4200, "Spain", "Camino de las Yeseras"), + ("Czechia_EBA_Unetice", 3900, "Czech Republic", "Moravia"), + ("Hungary_LBA", 3250, "Hungary", "Tiszafüred"), + ("Italy_Imperial_Rome", 1800, "Italy", "Rome"), + ("Sweden_Viking", 1030, "Sweden", "Sigtuna"), + ("France_Neolithic", 5700, "France", "Gurgy"), + ("Turkey_Neolithic", 8200, "Turkey", "Barcın"), + ("Russia_Yamnaya_Samara", 4950, "Russia", "Samara")] person = (0.01234, -0.00456) - dist, nearest = 0.0, [] - lines = ["rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n"] - for i, (g, dates, country, loc) in enumerate(groups, 1): - dist += rng.uniform(0.002, 0.012) - pc1, pc2 = person[0] + rng.uniform(-dist, dist), person[1] + rng.uniform(-dist, dist) - nearest.append({"group": g, "pc1": pc1, "pc2": pc2}) - lines.append(f"{i}\t{g}\t{rng.randint(3, 40)}\t{dist:.5f}\t{pc1:.5f}\t" - f"{pc2:.5f}\t{rng.randint(1000, 8500)}\t{dates}\t{country}\t{loc}\n") - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient.tsv"), "".join(lines)) - panel = {f"m{i}": ("present", (rng.gauss(0.0, 0.03), rng.gauss(0.0, 0.025))) for i in range(300)} - ancients = {} - for i in range(200): - panel[f"a{i}"] = (ancient_report.ANCIENT_LABEL, (rng.gauss(0.005, 0.035), rng.gauss(0.0, 0.03))) - ancients[f"a{i}"] = {"date_bp": str(rng.randint(500, 9000))} - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient_pca.svg"), - ancient_report.svg_plot(person, panel, ancients, SAMPLE, nearest)) - info = [("release", "v62.0"), ("sites_used", rng.randint(520_000, 560_000)), ("panel_sites", 597_573), - ("present_day_people", 819), ("ancient_people", 4_210), ("ancient_groups", 612), - ("pcs_in_distance", 4), ("pc1", f"{person[0]:.5f}"), ("pc2", f"{person[1]:.5f}")] - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient_info.tsv"), - "key\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in info)) + evals = "#eigvals: 9.000 4.000 2.000 1.000\n" + moderns = [f"m{i} " + " ".join(f"{rng.gauss(0.0, sd):.6f}" for sd in (0.03, 0.025, 0.02, 0.015)) + + f" {('French', 'Sardinian', 'Basque')[i % 3]}\n" for i in range(300)] + ancients, anc_rows = [], ["id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n"] + for i, (g, bp, country, loc) in enumerate(groups): + spread = 0.004 + 0.006 * i + cx, cy = person[0] + rng.uniform(-spread, spread), person[1] + rng.uniform(-spread, spread) + for j in range(rng.randint(3, 12)): + aid = f"a{i}_{j}" + ancients.append(f"{aid} {cx + rng.gauss(0, 0.004):.6f} {cy + rng.gauss(0, 0.004):.6f} 0.000000 0.000000 Ancient\n") + date = bp + rng.randint(-150, 150) + anc_rows.append(f"{aid}\t{g}\t{date}\t50\t.\t{loc}\t{country}\t50\t10\t500000\n") + with tempfile.TemporaryDirectory() as work: + paths = {n: os.path.join(work, n) for n in ("panel.evec", "run.evec", "ancients.tsv", "stats.tsv")} + put(paths["panel.evec"], evals + "".join(moderns + ancients)) + put(paths["run.evec"], evals + "".join(moderns) + + f"{SAMPLE} {person[0]:.6f} {person[1]:.6f} 0.000000 0.000000 Person\n") + put(paths["ancients.tsv"], "".join(anc_rows)) + put(paths["stats.tsv"], f"panel_sites\t597573\nsites_genotyped\t560000\nother_allele\t120\n" + f"sites_used\t{rng.randint(520_000, 560_000)}\n") + out = os.path.join(d, "ancient") + os.makedirs(out, exist_ok=True) + r = subprocess.run([og_binary(), "ancient", "report", "--panel-evec", paths["panel.evec"], + "--run-evec", paths["run.evec"], "--ancients", paths["ancients.tsv"], + "--sample", SAMPLE, "--release", "v62.0", "--stats", paths["stats.tsv"], + "--out-tsv", os.path.join(out, f"{SAMPLE}_ancient.tsv"), + "--out-info", os.path.join(out, f"{SAMPLE}_ancient_info.tsv"), + "--out-txt", os.path.join(out, f"{SAMPLE}_ancient.txt"), + "--out-svg", os.path.join(out, f"{SAMPLE}_ancient_pca.svg")], + stdout=subprocess.DEVNULL) + if r.returncode: + sys.exit("ERROR: og ancient report failed on the made-up panel") def exomiser(d): diff --git a/tests/e2e/ancient-similarity-1.sh b/tests/e2e/ancient-similarity-1.sh index c1b0e8e7f..21941d932 100755 --- a/tests/e2e/ancient-similarity-1.sh +++ b/tests/e2e/ancient-similarity-1.sh @@ -46,7 +46,7 @@ for e in ANNO:anno IND:ind SNP:snp; do curl -fsSL --retry 3 -o "${REAL}/aadr.${e#*:}" "https://dataverse.harvard.edu/api/access/datafile/${!id_var}" check_eq "AADR ${AADR_VERSION} .${e#*:}: the md5 versions.env pins" "$(md5sum "${REAL}/aadr.${e#*:}" | cut -d' ' -f1)" "${!md5_var}" done -python3 "${REPO}/bin/ancient_report.py" select --anno "${REAL}/aadr.anno" --ind "${REAL}/aadr.ind" \ +"${REPO}/bin/og" ancient select --anno "${REAL}/aadr.anno" --ind "${REAL}/aadr.ind" \ --out-ind "${REAL}/panel.ind" --poplist "${REAL}/poplist.txt" --ancients "${REAL}/ancients.tsv" | tee "${CASE_TMP}/select.log" check_ge "real AADR: present-day reference people" "$(awk '$3 != "Ignore" && $3 != "Ancient"' "${REAL}/panel.ind" | wc -l | tr -d ' ')" 800 check_ge "real AADR: present-day groups" "$(wc -l < "${REAL}/poplist.txt" | tr -d ' ')" 35 diff --git a/tests/test_ancient_report.py b/tests/test_ancient_report.py index f91218a6d..b0e7efee9 100644 --- a/tests/test_ancient_report.py +++ b/tests/test_ancient_report.py @@ -1,34 +1,35 @@ #!/usr/bin/env python3 -"""bin/ancient_report.py (step 44) on synthetic files. +"""og ancient (step 44) on synthetic files, run as the pipeline runs it. Checks: - 1. dates: years before 1950 become BCE and CE the way the AADR writes - them (4250 BP is the middle of its 2600-2000 BCE; 1950 BP is 1 BCE, - 1949 BP is 1 CE), and a population's span is written oldest first, with - one era when both ends share it; - 2. ranking: on a synthetic .evec the nearest ancient group is the one whose + 1. ranking: on a synthetic .evec the nearest ancient group is the one whose mean position is closest on the weighted PCs, a group far on PC1 but with a small eigenvalue on PC3 is ranked by the weighted distance, and - each row carries its dates, country and member count; - 3. axes: a run whose PC2 has the opposite sign is turned back before the + each row carries its dates (oldest first, AADR style BCE), country and + member count; + 2. axes: a run whose PC2 has the opposite sign is turned back before the person is placed, and a run whose present-day coordinates differ stops with the reason instead of placing the person on other axes; - 4. liftover: a site inside a block maps with the block's offset, a site on - a minus-strand chain maps to the mirrored position with its alleles - complemented, a site in a gap or covered by two chains is dropped; - 5. genotypes: a bcftools genotype gives the copies of the .snp file's first - allele, phased or not; a half call, an indel and an allele that is - neither of the two are missing; person-geno appends one column, 9 where - the person has no genotype; - 6. select: the present-day reference people come from MODERN_GROUPS, the + 3. liftover: lift keeps a site inside a block, a site on a minus-strand + chain with its alleles complemented, and drops a site in a gap, one + whose alleles miss the GRCh38 base, chrX and a monomorphic marker; + 4. genotypes: person-geno appends one column, 9 where the person has no + usable genotype, and stops on a genotype file shorter than the .snp file; + 5. select: the present-day reference people come from MODERN_GROUPS, the ancient ones need a date, a passing assessment, enough sites and a place in West Eurasia, one genetic id per individual; - 7. the report command writes the table, the summary, the text and an SVG + 6. the report command writes the table, the summary, the text and an SVG with the person marked; - 8. bin/render_report.py shows them: an HTML card with the table and the + 7. bin/render_report.py shows them: an HTML card with the table and the plot, and a section of the text report. -Run: python3 tests/test_ancient_report.py +The Rust unit tests (rust/crates/og-ancestry, og-intervals chain) check the +date labels, the dosages and the single-position lift in detail, and +tests/parity checks each command against the Python it replaced. Without an +og binary the test is skipped with a note; with OG_REQUIRED set the skip is a +failure. + +Run: OG=bin/og python3 tests/test_ancient_report.py """ import os import shutil @@ -37,11 +38,13 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import ancient_report as ar # noqa: E402 - -SCRIPT = os.path.join(REPO, "bin", "ancient_report.py") +OG = os.environ.get("OG", "") FAILS = 0 +CAVEAT = "Closeness on a PCA is genetic similarity, not descent" + +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) def check(desc, ok, detail=""): @@ -57,17 +60,11 @@ def write(path, text): return path -# --- 1. dates --------------------------------------------------------------------- -check("4250 BP is 2300 BCE, the AADR's midpoint of 2600-2000 BCE", ar.year_label(4250) == "2300 BCE", ar.year_label(4250)) -check("8025 BP is 6075 BCE", ar.year_label(8025) == "6075 BCE", ar.year_label(8025)) -check("1950 BP is 1 BCE", ar.year_label(1950) == "1 BCE", ar.year_label(1950)) -check("1949 BP is 1 CE", ar.year_label(1949) == "1 CE", ar.year_label(1949)) -check("450 BP is 1500 CE", ar.year_label(450) == "1500 CE", ar.year_label(450)) -check("a span in one era is written once", ar.range_label(7250, 6850) == "5300-4900 BCE", ar.range_label(7250, 6850)) -check("a span across the eras names both", ar.range_label(2250, 1750) == "300 BCE-200 CE", ar.range_label(2250, 1750)) -check("a span of one date is that date", ar.range_label(3000, 3000) == "1050 BCE", ar.range_label(3000, 3000)) +def og(*args): + return subprocess.run([OG, "ancient", *args], capture_output=True, text=True) + -# --- 2. ranking ------------------------------------------------------------------- +# --- 1. ranking ------------------------------------------------------------------- # Present-day people M0..M19 build the axes; three ancient groups. EVALS = [9.0, 4.0, 0.01, 0.01] @@ -94,36 +91,62 @@ def evec(rows, evals=EVALS): "B1": {"id": "B1", "group": "Anatolia_N", "date_bp": "8200", "country": "Turkey", "locality": "Barcin"}, "C1": {"id": "C1", "group": "Odd_C", "date_bp": "3000", "country": "Italy", "locality": "Somewhere"}, } -person = [0.08, 0.09, 0.0, 0.0] -rows, n = ar.nearest_populations(person, PANEL, ANCIENTS, EVALS, 4) -check("three ancient groups ranked", n == 3 and len(rows) == 3, (n, rows)) -order = [r["group"] for r in rows] -check("the nearest group is first (Steppe_EBA)", order[0] == "Steppe_EBA", order) -check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1] == "Odd_C", order) -check("the farthest group is last (Anatolia_N)", order[-1] == "Anatolia_N", order) -st = rows[0] -check("the row has the member count, dates and place", st["n"] == 2 and st["dates"] == "2850-2650 BCE" - and st["country"] == "Russia" and st["locality"] == "Samara", st) -check("ranks are 1..n", [r["rank"] for r in rows] == [1, 2, 3], rows) -unweighted = ar.nearest_populations(person, PANEL, ANCIENTS, [1.0, 1.0, 1.0, 1.0], 4)[0] -check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", - [r["group"] for r in unweighted] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], [r["group"] for r in unweighted]) +# The person's run: the present-day people with PC2 flipped, as smartpca may +# write it, and the person. +RUN = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} +RUN["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) -# --- 3. axes ---------------------------------------------------------------------- -run = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} -run["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) -signs = ar.axis_signs(PANEL, run, 2) -check("a flipped PC2 is found and turned back", signs == [1.0, -1.0], signs) -bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} -try: - ar.axis_signs(PANEL, bad, 2) - check("other axes stop the report", False, "no error") -except ar.Fail as e: - check("other axes stop the report, naming the PC", "PC1" in str(e), str(e)) - -# --- 4. liftover -------------------------------------------------------------------- tmp = tempfile.mkdtemp() try: + stats = write(os.path.join(tmp, "stats.tsv"), "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n") + at = os.path.join(tmp, "anc.tsv") + with open(at, "w") as f: + f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") + for r in ANCIENTS.values(): + f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") + outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} + + def report(panel_evec, run_evec, with_stats=True): + return og("report", "--panel-evec", panel_evec, "--run-evec", run_evec, "--ancients", at, + "--sample", "S", "--release", "test", *(["--stats", stats] if with_stats else []), + "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], + "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]) + + def table(): + with open(outs["ancient.tsv"]) as f: + head, *rows = [line.split("\t") for line in f.read().splitlines()] + return [dict(zip(head, r)) for r in rows] + + pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) + re_ = write(os.path.join(tmp, "run.evec"), evec(RUN)) + p = report(pe, re_) + check("report exits 0", p.returncode == 0, p.stderr) + rows = table() if p.returncode == 0 else [] + order = [r["group"] for r in rows] + check("three ancient groups ranked", len(rows) == 3, rows) + check("the nearest group is first (Steppe_EBA), from the flipped run turned back", + order[:1] == ["Steppe_EBA"], order) + check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1:2] == ["Odd_C"], order) + check("the farthest group is last (Anatolia_N)", order[-1:] == ["Anatolia_N"], order) + st = rows[0] if rows else {} + check("the row has the member count, dates and place", st.get("n") == "2" and st.get("dates") == "2850-2650 BCE" + and st.get("country") == "Russia" and st.get("locality") == "Samara", st) + check("ranks are 1..n", [r["rank"] for r in rows] == ["1", "2", "3"], rows) + flat_pe = write(os.path.join(tmp, "flat_panel.evec"), evec(PANEL, [1.0, 1.0, 1.0, 1.0])) + flat_re = write(os.path.join(tmp, "flat_run.evec"), evec(RUN, [1.0, 1.0, 1.0, 1.0])) + p = report(flat_pe, flat_re) + check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", + p.returncode == 0 and [r["group"] for r in table()] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], p.stderr) + + # --- 2. axes ---------------------------------------------------------------------- + p = og("align", "--panel-evec", pe, "--run-evec", re_, "--pcs", "2") + check("align: a flipped PC2 is found", p.returncode == 0 and "PC1+ PC2-" in p.stdout, (p.stdout, p.stderr)) + bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} + bad_re = write(os.path.join(tmp, "bad.evec"), evec(bad)) + p = og("align", "--panel-evec", pe, "--run-evec", bad_re, "--pcs", "2") + check("align: other axes stop, naming the PC", p.returncode == 1 and "PC1" in p.stderr, (p.returncode, p.stderr)) + + # --- 3. liftover -------------------------------------------------------------------- chain = write(os.path.join(tmp, "test.chain"), "\n".join([ # chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then 1150-1200 -> 2150-2200 "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", @@ -134,19 +157,6 @@ def evec(rows, evals=EVALS): # chr3: two chains over 100-200 "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", "100", "", "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", "100", ""]) + "\n") - ix = ar.index_blocks(ar.read_chain(chain)) - check("a site in the first block moves by its offset", ar.lift_point(ix, "chr1", 1051) == ("chr1", 2051, "+"), - ar.lift_point(ix, "chr1", 1051)) - check("a site in the second block moves by its offset", ar.lift_point(ix, "chr1", 1160) == ("chr1", 2160, "+"), - ar.lift_point(ix, "chr1", 1160)) - check("a site in the gap is dropped", ar.lift_point(ix, "chr1", 1120) is None, ar.lift_point(ix, "chr1", 1120)) - # 0-based 510 -> reverse 310 -> forward 10000 - 1 - 310 = 9689 (0-based) = 9690 1-based - check("a minus-strand site is mirrored", ar.lift_point(ix, "chr2", 511) == ("chr2", 9690, "-"), - ar.lift_point(ix, "chr2", 511)) - check("a site two chains cover is dropped", ar.lift_point(ix, "chr3", 171) is None, ar.lift_point(ix, "chr3", 171)) - check("a site one of them covers maps", ar.lift_point(ix, "chr3", 121) == ("chr3", 121, "+"), - ar.lift_point(ix, "chr3", 121)) - # The lift command against a small FASTA. seq1 = ["N"] * 3000 seq1[2050] = "G" # 1-based 2051 seq1[2159] = "T" # 1-based 2160 @@ -174,38 +184,32 @@ def evec(rows, evals=EVALS): "rs7 1 0.0 1051 G X", # monomorphic marker: dropped ]) + "\n") out = os.path.join(tmp, "sites.tsv") - rc = ar.main(["lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out]) + p = og("lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out) got = open(out).read().splitlines() if os.path.exists(out) else [] check("lift keeps rs1, rs2 and the complemented rs4, in GRCh38 order", - rc == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", - "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (rc, got)) + p.returncode == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", + "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (p.returncode, got, p.stderr)) - # --- 5. genotypes ----------------------------------------------------------------- - check("A/A is two copies of allele1 A", ar.dosage("A/A", "A", "G") == 2) - check("A|G is one copy", ar.dosage("A|G", "A", "G") == 1) - check("G/G is none", ar.dosage("G/G", "A", "G") == 0) - check("a half call is missing", ar.dosage("A", "A", "G") is None) - check("an allele outside the pair is missing", ar.dosage("A/T", "A", "G") is None) - check("an indel is missing", ar.dosage("A/AT", "A", "G") is None) + # --- 4. genotypes ----------------------------------------------------------------- msnp = write(os.path.join(tmp, "moderns.snp"), "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n") mgeno = write(os.path.join(tmp, "moderns.geno"), "012\n220\n999\n101\n") gts = write(os.path.join(tmp, "gts.tsv"), "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n") merged = os.path.join(tmp, "merged.geno") - stats = os.path.join(tmp, "stats.tsv") - rc = ar.main(["person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, - "--out", merged, "--stats", stats]) + pstats = os.path.join(tmp, "pstats.tsv") + p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, + "--out", merged, "--stats", pstats) got = open(merged).read().splitlines() if os.path.exists(merged) else [] check("person-geno appends one column: 1, 0, 9 for X, 9 for an allele outside the pair", - rc == 0 and got == ["0121", "2200", "9999", "1019"], (rc, got)) - st = dict(l.split("\t") for l in open(stats).read().splitlines()) if os.path.exists(stats) else {} + p.returncode == 0 and got == ["0121", "2200", "9999", "1019"], (p.returncode, got, p.stderr)) + st = dict(line.split("\t") for line in open(pstats).read().splitlines()) if os.path.exists(pstats) else {} check("its counts", st == {"panel_sites": "4", "sites_genotyped": "3", "other_allele": "1", "sites_used": "2"}, st) short = write(os.path.join(tmp, "short.geno"), "012\n220\n") - rc = ar.main(["person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, - "--out", merged, "--stats", stats]) - check("a genotype file shorter than the .snp file stops", rc == 1, rc) + p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, + "--out", merged, "--stats", pstats) + check("a genotype file shorter than the .snp file stops", p.returncode == 1, (p.returncode, p.stderr)) - # --- 6. select ------------------------------------------------------------------------ + # --- 5. select ------------------------------------------------------------------------ head = ["Genetic ID (long text)", "Persistent Genetic ID", "Individual ID", '"Date mean in BP in years before 1950"', "Date standard deviation in BP [x]", "Full Date One of two formats", "Group ID", "Locality", "Political Entity", "Latitude", "Longitude", "Data type", @@ -231,43 +235,32 @@ def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): anno = write(os.path.join(tmp, "x.anno"), "\n".join("\t".join(r) for r in [head] + rows_) + "\n") ind = write(os.path.join(tmp, "x.ind"), "".join(f"{r[0]} U {r[6]}\n" for r in rows_)) oind, plist, anc = (os.path.join(tmp, n) for n in ("panel.ind", "poplist.txt", "ancients.tsv")) - rc = ar.main(["select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc]) - labels = dict(l.split("\t")[0::2] for l in open(oind).read().splitlines()) if rc == 0 else {} + p = og("select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc) + rc = p.returncode + labels = dict(line.split("\t")[0::2] for line in open(oind).read().splitlines()) if rc == 0 else {} check("select: the 60 French shotgun genomes are the reference, one id per person (F0.DG)", - rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", labels) + rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", + (labels, p.stderr)) check("select: Yakut and a Human Origins genome are ignored", labels.get("Y1.DG") == "Ignore" and labels.get("H1.HO") == "Ignore", labels) check("select: only A1 is projected", [k for k, v in labels.items() if v == "Ancient"] == ["A1.AG"], labels) check("select: the population list", open(plist).read() == "French\n" if rc == 0 else False) check("select: ancients.tsv holds A1 with its group", - rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), open(anc).read() if rc == 0 else "") + rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), + open(anc).read() if rc == 0 else "") - # --- 7. the report command ---------------------------------------------------------- - pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) - re_ = write(os.path.join(tmp, "run.evec"), evec(run)) - at = os.path.join(tmp, "anc.tsv") - with open(at, "w") as f: - f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for r in ANCIENTS.values(): - f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") - outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} - p = subprocess.run([sys.executable, SCRIPT, "report", "--panel-evec", pe, "--run-evec", re_, "--ancients", at, - "--sample", "S", "--release", "test", "--stats", stats, - "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], - "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]], capture_output=True, text=True) + # --- 6. the report command ---------------------------------------------------------- + p = report(pe, re_) check("report exits 0", p.returncode == 0, p.stderr) - tsv = open(outs["ancient.tsv"]).read().splitlines() if p.returncode == 0 else [] - check("report: the table's first row is Steppe_EBA, from the flipped run turned back", - len(tsv) == 4 and tsv[1].split("\t")[1] == "Steppe_EBA", tsv) - info = dict(l.split("\t") for l in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} + info = dict(line.split("\t") for line in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} check("report: the person's PC2 in the panel's sign", info.get("pc2") == "0.09000", info) check("report: the sites used", info.get("sites_used") == "2", info) svg = open(outs["pca.svg"]).read() if p.returncode == 0 else "" check("report: the SVG is an SVG with a dark background and the person's label", svg.startswith("S" in svg and svg.rstrip().endswith(""), svg[:200]) txt = open(outs["ancient.txt"]).read() if p.returncode == 0 else "" - check("report: the text carries the caveat and says qpAdm is not run", ar.CAVEAT in txt and "qpAdm" in txt, txt) - # --- 8. both reports read the outputs ---------------------------------------------- + check("report: the text carries the caveat and says qpAdm is not run", CAVEAT in txt and "qpAdm" in txt, txt) + # --- 7. both reports read the outputs ---------------------------------------------- sd = os.path.join(tmp, "S") os.makedirs(os.path.join(sd, "ancient")) for k, f in outs.items(): @@ -284,10 +277,7 @@ def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): check("the text report has the section with the nearest group and the caveat", "Ancient Populations (AADR PCA, step 44)" in text and "1. Steppe_EBA (" in text and "not descent" in text, text[-600:]) - p = subprocess.run([sys.executable, SCRIPT, "report", "--panel-evec", pe, "--run-evec", pe, "--ancients", at, - "--sample", "S", "--release", "test", "--out-tsv", outs["ancient.tsv"], "--out-info", - outs["info.tsv"], "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]], - capture_output=True, text=True) + p = report(pe, pe, with_stats=False) check("report: a run without the person stops", p.returncode == 1 and "no row for S" in p.stderr, p.stderr) finally: shutil.rmtree(tmp) diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index a54d508e9..844658cd4 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""bin/archaic_report.py (step 43) on a synthetic archaic panel, with the real IBDmix image. +"""og archaic (step 43) on a synthetic archaic panel, with the real IBDmix image. Checks: 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic @@ -37,7 +37,12 @@ Set ARCHAIC_TEST_NO_DOCKER=1 to skip part 4 on a machine without docker; CI never sets it. -Run: python3 tests/test_archaic_report.py +The Rust unit tests (rust/crates/og-ancestry) check pick_call's symmetry and +which genes touch a segment's ends, and tests/parity checks each command +against the Python it replaced. Without an og binary the test is skipped with +a note; with OG_REQUIRED set the skip is a failure. + +Run: OG=bin/og python3 tests/test_archaic_report.py """ import gzip import os @@ -49,10 +54,17 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import archaic_report as ar # noqa: E402 - -SCRIPT = os.path.join(REPO, "bin", "archaic_report.py") +OG = os.environ.get("OG", "") FAILS = 0 +TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", + "AF_ALL", "AF_AFR", "AF_AMR", "AF_EAS", "AF_EUR", "AF_SAS"] +SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] +MASKED = 10 +ARCHAICS = ("Altai", "Vindija", "Denisova") + +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) def check(desc, ok, detail=""): @@ -68,7 +80,7 @@ def write(path, text): def run(*args): - return subprocess.run([sys.executable, SCRIPT, *args], capture_output=True, text=True) + return subprocess.run([OG, "archaic", *args], capture_output=True, text=True) def image(var): @@ -166,25 +178,30 @@ def table_info(out, extra=()): r.returncode == 1 and "no interval" in r.stderr, r.stderr) # --- 1b. strict: rows outside the GRCh38 strict mask are marked, not dropped ------- -# Lifted rows (13 columns, no header). Strict mask 1-based 101-300 on chr22: -# 100 is outside, 150 and 300 inside, 301 outside. A duplicate position keeps -# the first row. Codes 0, 1, 2 outside become 10, 11, 12; 9 stays 9; a code -# already masked is not marked twice. +# Lifted rows (13 columns, no header), sorted by position as Picard writes +# them. Strict mask 1-based 101-300 on chr22: 100 is outside, 150 and 300 +# inside, 301 outside. A duplicate position keeps the first row. Codes 0, 1, 2 +# outside become 10, 11, 12; 9 stays 9; a code already masked is not marked +# twice. rows = ["chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", + "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0", - "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0"] + "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0"] write(f"{TMP}/lifted.tsv", "\n".join(rows) + "\n") write(f"{TMP}/strict.bed", "chr22\t100\t300\n") r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked.tsv") check("strict exits 0", r.returncode == 0, r.stderr) got = [l.rstrip("\n").split("\t") for l in open(f"{TMP}/marked.tsv")] by = {int(c[1]): c[4:7] for c in got} -check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 301, 300], [c[1] for c in got]) +check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 300, 301], [c[1] for c in got]) check("outside the strict mask 2, 0 and unknown become 12, 10 and 9", by[100] == ["12", "10", "9"], by.get(100)) check("inside the strict mask the calls are untouched (last base 300 too)", by[150] == ["1", "1", "1"] and by[300] == ["0", "0", "0"], (by.get(150), by.get(300))) check("a masked call stays as it is, and the first of a duplicate wins", by[301] == ["10", "10", "12"], by.get(301)) +write(f"{TMP}/unsorted.tsv", "\n".join([rows[0], rows[3], rows[2]]) + "\n") +r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/unsorted.tsv", "--out", f"{TMP}/marked3.tsv") +check("rows out of position order stop the step (a duplicate is found only next to its twin)", + r.returncode == 2 and "sorted" in r.stderr and not os.path.exists(f"{TMP}/marked3.tsv"), (r.returncode, r.stderr)) write(f"{TMP}/strict_other.bed", "chr21\t0\t10\n") r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict_other.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked2.tsv") check("a strict mask with no interval on the chromosome stops the step", r.returncode != 0 and "no interval" in r.stderr, r.stderr) @@ -192,7 +209,7 @@ def table_info(out, extra=()): # --- 2. genotypes ------------------------------------------------------------------ tdir = f"{TMP}/table" os.makedirs(tdir) -hdr = "\t".join(ar.TABLE_HEADER) + "\n" +hdr = "\t".join(TABLE_HEADER) + "\n" rows = ["chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", @@ -267,12 +284,6 @@ def table_info(out, extra=()): check("a 0/0 half after the carried SNV keeps the SNV call", code3(6000) == "1", g3.get(6000)) check("a usable record of the same SNV wins over a filtered one", code3(7000) == "2", g3.get(7000)) check("two 0/0 halves are the reference (0)", code3(8000) == "0", g3.get(8000)) -pc_code = lambda r: 9 if r is ar.CLASH else r[2] -check("pick_call gives the same code in either order", - all(pc_code(ar.pick_call(x, y)) == pc_code(ar.pick_call(y, x)) - for x, y in [(("A", "G", 1), ("A", None, 9)), (("A", "G", 1), ("A", "T", 0)), (("A", "G", 9), ("A", "G", 2)), - (("A", "G", 0), ("A", "T", 0)), (("A", "G", 1), ("A", "T", 2)), (("A", "G", 9), ("A", "T", 0)), - (("A", "G", 1), ("A", "G", 2))])) # Masked calls: IBDmix gets the call itself and a mask per genome of the # positions where it is masked, the runs of written rows closed by the next @@ -291,7 +302,7 @@ def table_info(out, extra=()): check("IBDmix gets each masked call as its ALT count (12, 10, 11 are 2, 0, 1)", gm.get(100) == ["2", "0", "9"] and gm.get(200) == ["0", "1", "0"] and gm.get(600) == ["1", "0", "0"], gm) masks = {g: open(f"{TMP}/gtm/chr22.{g}.mask.bed").read() if os.path.exists(f"{TMP}/gtm/chr22.{g}.mask.bed") else None - for g in ar.ARCHAICS} + for g in ARCHAICS} check("Altai's mask: 100-200, then 400-600 (the left-out row 500 does not close it)", masks["Altai"] == "22\t99\t200\n22\t399\t600\n", masks["Altai"]) check("Vindija's mask: 200 alone", masks["Vindija"] == "22\t199\t200\n", masks["Vindija"]) @@ -321,7 +332,7 @@ def table_info(out, extra=()): "--out-dir", f"{TMP}/out") check("report exits 0", r.returncode == 0, r.stderr) segs = [l.split("\t") for l in open(f"{TMP}/out/S1_archaic_segments.tsv").read().splitlines()] -check("segments header", segs[0] == ar.SEGMENT_HEADER, segs[0]) +check("segments header", segs[0] == SEGMENT_HEADER, segs[0]) body = segs[1:] check("two segments: the merged Neanderthal one and the Denisovan one", len(body) == 2, body) check("Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge into Neanderthal 1.0-1.5 Mb", @@ -367,9 +378,6 @@ def table_info(out, extra=()): check("a row cut after its key is unreadable, not a crash", False) except cs.Unreadable: check("a row cut after its key is unreadable, not a crash", True) -genes = {"chr22": [(90, 100, "ENDS_AT_START"), (100, 120, "INSIDE"), (200, 210, "STARTS_AT_END"), (50, 99, "BEFORE")]} -check("genes: a segment [100, 200) takes a gene ending at its first site, not one starting at its end", - ar.genes_in(genes, "22", 100, 200) == ["ENDS_AT_START", "INSIDE"], ar.genes_in(genes, "22", 100, 200)) check("no file: the section is missing", cs.sec_archaic(f"{TMP}/nowhere", "S1") == (None, {})) # --- 4. the real IBDmix ------------------------------------------------------------- @@ -411,14 +419,14 @@ def synthetic(d, plant, mid=None): if mid and MID[0] <= pos < MID[1]: if kind == 0: g = [False, False] - ta, tv = (al + ar.MASKED, vi + ar.MASKED) if mid == "masked" else (9, 9) + ta, tv = (al + MASKED, vi + MASKED) if mid == "masked" else (9, 9) rows.append(f"chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t" + "\t".join([f"{f:.4f}"] * 6)) if plant and DEN[0] <= pos < DEN[1]: g[0] = de == 2 if any(g): calls.append(f"chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{'1/1' if all(g) else '0/1'}") with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(ar.TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") + f.write("\t".join(TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") write(f"{d}/s.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + "\n".join(calls) + "\n") From 796f4cd6681d4a72aa0a6c4393002e95352a576d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:24:21 +0100 Subject: [PATCH 04/19] docs(ancestry): steps 43 and 44 run og archaic and og ancient Bead: pgp-bs9u.21 --- CHANGELOG.md | 8 ++++++++ docs/43-archaic.md | 6 +++--- docs/44-ancient.md | 2 +- docs/rust.md | 6 +++--- rust/README.md | 2 +- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f5f4625..7dfccbfb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 43's genotype files for 3 million table rows take 6 s and 159 MB instead of 12 s and 213 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: + - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. + - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. + - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. + - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. + - Malformed input stops with one `ERROR:` line, not a Python traceback. + - `sites_genotyped` in step 44's stats still counts a position twice when the AADR holds two SNPs there, as the Python did; the genotypes themselves are right. + - Step 37's own Y tools now run in the Rust `og` binary: `og y tree-positions` and `og y tree-place` (the placement on the current YFull YTree), `og y private-snps` (the private SNP list) and `og y yleaf-positions` (Yleaf's marker positions). They replace `bin/y_tree_place.py` and `bin/y_private_snps.py`, which are gone, and the `positions` command of `bin/yleaf_run.py`, which keeps only `predict`. og is static, so each part still runs in the Yleaf image. The three tables are the same, byte for byte, on the 43 parity cases, except where a line below says otherwise. On the pinned YTree 14.06.0 with a synthetic pileup of 362,055 positions, `og y tree-place` takes 0.8 s and 99 MB where the Python took 3.6 s and 389 MB; on a synthetic chrY VCF of 20,000 records, `og y private-snps` takes 0.3 s and 24 MB where the Python took 1.1 s and 330 MB. What changes on purpose: - `og y tree-place` reads the pileup by contig and position, and only its `chrY` (or `Y`) lines. The Python keyed it by position only, so a line on another contig could overwrite a chrY count. Lines on another contig are now counted at the end of the step line, and a pileup with both `chrY` and `Y` lines stops with exit 1. Step 37 makes a chrY-only pileup, so its results do not change. - `og y private-snps` refuses a callable mask with more than one FASTA record (exit 1). The Python joined the records into one sequence, which shifts every position after the first. The pinned mask has one record. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index 25e33f93b..3e706e1ba 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -13,7 +13,7 @@ Most people outside Africa carry about 2% Neanderthal DNA, and people with ances ## Tool - [IBDmix](https://github.com/PrincetonUniversity/IBDmix) 1.0.1 (Chen et al. 2020, Cell 180:677, doi:10.1016/j.cell.2020.01.012) -- `bin/archaic_report.py` (standard library Python): IBDmix's genotype files, then the segments and the summary +- `og archaic` (OwnGenome's Rust binary, `docs/rust.md`): the table at setup, IBDmix's genotype files, then the segments and the summary IBDmix needs no unadmixed reference population. It scores each site from the modern allele frequency and the two genotypes, so it finds archaic segments in Africans too, where methods that use Africans as the unadmixed reference cannot. @@ -57,13 +57,13 @@ Chen et al. do not score every site. Each archaic genome comes with its own mask - a genome's call outside that genome's FilterBed is kept and marked masked in the table (10, 11 or 12: 10 plus its ALT count); - every call at a site inside a segmental duplication (UCSC's GRCh37 track), or inside the window of a 1000 Genomes indel, is marked masked the same way. The window is 5 bp on each side of the indel, reaching past the longer of its REF and ALT, as the published script writes it; - step 43 writes each masked call to IBDmix as its ALT count, and the masked positions to a mask file per chromosome and archaic genome, which it passes to IBDmix with `-r`; -- every call at a site outside the 1000 Genomes GRCh38 strict mask is marked masked the same way, in all three archaic columns. `setup.sh --archaic` does that after the liftover (`archaic_report.py strict`), because the mask is on GRCh38 and the table is built on GRCh37. +- every call at a site outside the 1000 Genomes GRCh38 strict mask is marked masked the same way, in all three archaic columns. `setup.sh --archaic` does that after the liftover (`og archaic strict`), because the mask is on GRCh38 and the table is built on GRCh37. The first version of the masks dropped the masked calls instead (code 9, or no row), and a second one still dropped every row outside the strict mask. IBDmix then had nothing to score at those sites, the mismatches included, so a segment could run through masked sequence that contradicts it. That is why HG002's share stayed above the published figure after the masks; see Runtime below. One of their exclusions is not applied here: the CpG mask, which needs a GRCh37 reference and the chimpanzee, orangutan and macaque SNP sets, none of which this pipeline installs. What that leaves out is written down in the Notes below. -The filters cut hard: on chr22 the strict mask covers 25.2 Mb of 50.8, the Vindija FilterBed 19.6 Mb, and segmental duplications take out 4.3 Mb. Without them the step called sequence from positions the archaic genomes cannot be read at, and HG002's Neanderthal total was higher still: the filters took it from 115.2 Mb to 107.5 Mb, which is still 2.1 times the published median (see the same-person section under Runtime). The filters are part of the data version: a change to a pin or to a mask means `snpad2016-1kgp3-v3` and a rebuild. `setup.sh --archaic` also rebuilds a table whose build code changed (its stamp holds the sha256 of the table half of `bin/archaic_report.py`), so a table built before the masked codes, or before the strict-mask rows were kept, is rebuilt on the next `setup.sh --archaic`. Until then step 43 reads it as before: a table with no masked code gives empty masks. +The filters cut hard: on chr22 the strict mask covers 25.2 Mb of 50.8, the Vindija FilterBed 19.6 Mb, and segmental duplications take out 4.3 Mb. Without them the step called sequence from positions the archaic genomes cannot be read at, and HG002's Neanderthal total was higher still: the filters took it from 115.2 Mb to 107.5 Mb, which is still 2.1 times the published median (see the same-person section under Runtime). The filters are part of the data version: a change to a pin or to a mask means `snpad2016-1kgp3-v3` and a rebuild. `setup.sh --archaic` also rebuilds a table whose format changed (its stamp holds the table format id of `og archaic`, `ARCHAIC_TABLE_FORMAT` in `setup.sh`), so a table built before the masked codes, or before the strict-mask rows were kept, is rebuilt on the next `setup.sh --archaic`. A table the last Python build code wrote has the same bytes, so its stamp is taken as current once and rewritten, and no install downloads the archaic genomes again for the move to Rust. Until then step 43 reads it as before: a table with no masked code gives empty masks. The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf`. A site that does not lift, or lifts to another chromosome, is dropped. The archaic files are downloaded one chromosome at a time (150 GB in all, at most 13.8 GB at once, for chromosome 2) and deleted once the chromosome is built, so an interrupted install resumes at the chromosome it stopped in. `ARCHAIC_REGIONS` (a GRCh37 BED, chromosomes named 1 to 22) builds only those regions. diff --git a/docs/44-ancient.md b/docs/44-ancient.md index 785913ca9..ec0c31253 100644 --- a/docs/44-ancient.md +++ b/docs/44-ancient.md @@ -14,7 +14,7 @@ Most people who look at their ancestry want to know where they fall among ancien - **smartpca** and **convertf** (EIGENSOFT 9.0.0): the PCA, and the conversion of the present-day people to EIGENSTRAT text. 9.0.0 is the first release that reads the AADR's TGENO genotype format. - **bcftools** (`convert --gvcf2vcf`, `view`, `query`) reads your genotypes at the panel's sites from the gVCF. -- **`bin/ancient_report.py`** (Python standard library): chooses the panel's people, lifts the AADR sites to GRCh38, adds your column to the present-day genotypes, aligns the axes and writes the report and the plot. +- **`og ancient`** (OwnGenome's Rust binary, `docs/rust.md`): chooses the panel's people, lifts the AADR sites to GRCh38, adds your column to the present-day genotypes, aligns the axes and writes the report and the plot. ## Docker Image diff --git a/docs/rust.md b/docs/rust.md index 241f92311..5c60ea02c 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -2,7 +2,7 @@ This page is for contributors. OwnGenome's own tools are moving from Python to Rust, one tool per pull request, into a single binary, `og`. This page says how the Rust code is laid out, how the binary reaches the pipeline, how it stays byte-for-byte compatible with the Python it replaces, and what CI checks. -Steps 04, 05, 07 and 25 call `og` (the filters of `og sv` and `og pgx`, and `og prs format`), step 06 and its Nextflow module `CLINVAR_SCREEN` call `og clinvar`, step 33 calls it (`og qc sex-loss`, the mosaic loss of Y or X), steps 23 and 31 and their Nextflow modules call it (`og vcf csq-rank` and `og table constraint-join`), step 27 and `CPIC_LOOKUP` call it (`og pgx cpic-report`), and so do steps 09b, 21 and 32: the CYP2D6 depth check (`og pgx cyp2d6-bed`, `cyp2d6-measure`, `cyp2d6-check`) and the RFC1 motif count (`og str motif`) replaced their Python. Steps 06, 17 and 31 and their Nextflow modules run the reads at the variant in `og`: the cDNA read check and scan (`og cdna check`, `og cdna scan`) and the phase of the compound-het pairs (`og comphet phase`); step 32 and its module read the BAM's contig names with `og bam contigs`. `scripts/setup.sh` calls it for the STRchive catalog, Exomiser's zips, the genetic map and the masked reference. Step 26b calls `og ancestry-fine`. Step 40 calls `og consumer`. Step 38 and the Nextflow family workflow call `og family`. So a run that reaches those steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull. The pieces below are the base every port builds on. +Steps 04, 05, 07 and 25 call `og` (the filters of `og sv` and `og pgx`, and `og prs format`), step 06 and its Nextflow module `CLINVAR_SCREEN` call `og clinvar`, step 33 calls it (`og qc sex-loss`, the mosaic loss of Y or X), steps 23 and 31 and their Nextflow modules call it (`og vcf csq-rank` and `og table constraint-join`), step 27 and `CPIC_LOOKUP` call it (`og pgx cpic-report`), and so do steps 09b, 21 and 32: the CYP2D6 depth check (`og pgx cyp2d6-bed`, `cyp2d6-measure`, `cyp2d6-check`) and the RFC1 motif count (`og str motif`) replaced their Python. Steps 06, 17 and 31 and their Nextflow modules run the reads at the variant in `og`: the cDNA read check and scan (`og cdna check`, `og cdna scan`) and the phase of the compound-het pairs (`og comphet phase`); step 32 and its module read the BAM's contig names with `og bam contigs`. `scripts/setup.sh` calls it for the STRchive catalog, Exomiser's zips, the genetic map and the masked reference. Step 26b calls `og ancestry-fine`, step 43 and `setup.sh --archaic` call `og archaic`, and step 44 and `setup.sh --ancient-panel` call `og ancient`. Step 40 calls `og consumer`. Step 38 and the Nextflow family workflow call `og family`. So a run that reaches those steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull. The pieces below are the base every port builds on. ## Layout @@ -26,7 +26,7 @@ rust/ crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read crates/og-bloodtype/ step 41, blood groups: og bloodtype report, check-markers, systems - crates/og-ancestry/ step 26b, fine-grained ancestry: og ancestry-fine + crates/og-ancestry/ steps 26b, 43 and 44: og ancestry-fine, og archaic, og ancient crates/og-reports/ the opt-in step reports of steps 45, 47, 48 and 50 to 53, and step 37's Y tree tools crates/og-lookup/ the genotype lookup of scripts/owngenome-lookup.sh: og lookup query, resolve, report crates/og-fresh/ og dev freshness: the monthly freshness issue's upstream lookups, typed answers @@ -36,7 +36,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. diff --git a/rust/README.md b/rust/README.md index 5071aad12..a6a412a2a 100644 --- a/rust/README.md +++ b/rust/README.md @@ -18,7 +18,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-lookup` | the genotype lookup of `scripts/owngenome-lookup.sh`: `og lookup query`, `og lookup resolve`, `og lookup report` | | `crates/og-filters` | the VCF filters of steps 04, 05 and 07: `og sv svlen-fix`, `og sv inv-filter`, `og pgx refblock-fill`, `og pgx mito-merge` | | `crates/og-setup` | what `scripts/setup.sh` runs once: `og str catalog` (the STRchive catalog of steps 09 and 9b), `og setup unzip` (Exomiser's zips), `og setup genetic-map` (the genetic map of steps 26b and 51) | -| `crates/og-ancestry` | step 26b, fine-grained ancestry: `og ancestry-fine panel-map`, `check-map`, `order`, `sample-bed`, `chroms`, `report` | +| `crates/og-ancestry` | step 26b, fine-grained ancestry: `og ancestry-fine panel-map`, `check-map`, `order`, `sample-bed`, `chroms`, `report`; step 43's `og archaic table`, `strict`, `genotypes`, `report`; step 44's `og ancient select`, `lift`, `person-geno`, `align`, `report` | | `crates/og-steps` | step helpers, among them `og pgx outside-calls`, `og pgx hla-concordance` (steps 08, 36), `og pgx pypgx-summary` (step 32) and `og paraphase check-bam`, `og paraphase table` (step 35b) | | `crates/og-fresh` | `og dev freshness`: the upstream lookups of the monthly freshness issue, each answer typed, HTTP through the host's `curl` | | `crates/og-recheck` | `og recheck`: which of a finished sample's results a newer ClinVar or catalog would change, `--rerun`, `--affected`, `--launch-dir` | From 7cb54dc8ef4641e4d472ea7142a1c85cac92484f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:28:18 +0100 Subject: [PATCH 05/19] test(ancestry): a mask holds the 1-based bases of its BED intervals only Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 3d1b4e811..5e6a01b60 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -1609,6 +1609,20 @@ mod tests { } } + #[test] + fn a_mask_holds_its_bed_bases_only() { + // BED 99-100 and 200-300 are the 1-based bases 100 and 201 to 300. + let mut m = Mask { + iv: vec![(99, 100), (200, 300)], + i: 0, + }; + let got: Vec = [99, 100, 101, 200, 201, 300, 301] + .into_iter() + .filter(|&p| m.holds(p)) + .collect(); + assert_eq!(got, [100, 201, 300]); + } + #[test] fn gt_codes() { assert_eq!(gt_code("0/1"), Some(1)); From 3b492b9a30db6e823bae87486040d9c0462f94e6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:28:18 +0100 Subject: [PATCH 06/19] test(ancestry): RED CONTROL, plant a 1-based mask and a chain block that holds its end Reverted in the next commit. Guard must go red: cargo test in og-ancestry and og-intervals, the archaic-table and ancient-lift parity cases, and tests/test_archaic_report.py. Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 2 +- rust/crates/og-intervals/src/chain.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 5e6a01b60..ff03f7791 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -224,7 +224,7 @@ struct Mask { impl Mask { fn holds(&mut self, pos: i64) -> bool { - let p = pos - 1; + let p = pos; while self.i < self.iv.len() && self.iv[self.i].1 <= p { self.i += 1; } diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs index 559a4d71e..160aaf502 100644 --- a/rust/crates/og-intervals/src/chain.rs +++ b/rust/crates/og-intervals/src/chain.rs @@ -240,10 +240,10 @@ impl Chain { // may still reach past p. let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); let mut first: Option<&Block> = None; - while j > 0 && i64::from(src.reach[j - 1]) > p { + while j > 0 && i64::from(src.reach[j - 1]) >= p { j -= 1; let b = &src.blocks[j]; - if i64::from(b.end) > p { + if i64::from(b.end) >= p { match first { None => first = Some(b), Some(f) if f.chain != b.chain => return None, From bb71d5692e77523235e27243e6453b9d4f3e4cf2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:38:15 +0100 Subject: [PATCH 07/19] test(ancestry): revert the red control of 3b492b9a Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 2 +- rust/crates/og-intervals/src/chain.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index ff03f7791..5e6a01b60 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -224,7 +224,7 @@ struct Mask { impl Mask { fn holds(&mut self, pos: i64) -> bool { - let p = pos; + let p = pos - 1; while self.i < self.iv.len() && self.iv[self.i].1 <= p { self.i += 1; } diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs index 160aaf502..559a4d71e 100644 --- a/rust/crates/og-intervals/src/chain.rs +++ b/rust/crates/og-intervals/src/chain.rs @@ -240,10 +240,10 @@ impl Chain { // may still reach past p. let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); let mut first: Option<&Block> = None; - while j > 0 && i64::from(src.reach[j - 1]) >= p { + while j > 0 && i64::from(src.reach[j - 1]) > p { j -= 1; let b = &src.blocks[j]; - if i64::from(b.end) >= p { + if i64::from(b.end) > p { match first { None => first = Some(b), Some(f) if f.chain != b.chain => return None, From 4ab9d0754bc8c11eff60d6230b4798544a6ac6d1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:41:17 +0100 Subject: [PATCH 08/19] fix(ancestry): join genotype calls in one position window --- .github/workflows/guard.yml | 18 ++ rust/crates/og-ancestry/src/archaic.rs | 263 ++++++++++++++++--------- tests/parity/cases.tsv | 2 +- tests/test_archaic_report.py | 31 +++ 4 files changed, 222 insertions(+), 92 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index e22eba988..0f2d5f70a 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -266,6 +266,24 @@ jobs: grep -E 'planted_failure.*FAILED' "$log" || { cat "$log"; echo "ERROR: the run failed, but not on the planted test"; exit 1; } echo "Control: the planted test failed, as it must." + - name: Control (archaic duplicate window must fail when equal positions are split) + run: | + source=crates/og-ancestry/src/archaic.rs + saved=$(mktemp "${RUNNER_TEMP}/archaic-window.XXXXXX") + cp "$source" "$saved" + trap 'cp -f "$saved" "$source"; rm -f "$saved"' EXIT + sed 's/r.pos == kept.pos/r.pos < kept.pos/' "$saved" > "$source" + grep -q 'r.pos < kept.pos' "$source" + log="${RUNNER_TEMP}/archaic-window-red.log" + if cargo test --locked --no-fail-fast -p og-ancestry duplicate_window_keeps_only_one_record_of_lookahead > "$log" 2>&1; then + cat "$log" + echo "ERROR: splitting a duplicate-position window passed" + exit 1 + fi + cat "$log" + grep -E 'duplicate_window_keeps_only_one_record_of_lookahead.*FAILED' "$log" + echo "Control: splitting the window failed its regression test." + - name: cargo test run: cargo test --workspace --locked --no-fail-fast diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 5e6a01b60..4468fc815 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -25,7 +25,7 @@ //! Each command writes what `bin/archaic_report.py` wrote (checked by //! `tests/parity`). Deliberate changes: every output is written under a //! temporary name and renamed into place; a gzip input is told by its first -//! bytes, not its name; `strict` reads its rows in position order and stops +//! bytes, not its name; `strict` and `genotypes` read in position order and stop //! (exit 2) on a row that goes back, instead of keeping every position of a //! chromosome in memory; and malformed input is an `ERROR:` line, never a //! traceback. @@ -726,9 +726,17 @@ struct VcfRec { /// the reference. struct Vcf { text: Text, + previous: Option<(String, i64)>, } impl Vcf { + fn new(text: Text) -> Self { + Self { + text, + previous: None, + } + } + fn next(&mut self) -> Out> { loop { let Some(raw) = self.text.raw()? else { @@ -750,13 +758,28 @@ impl Vcf { } continue; } - if c.len() < 10 || c[3].chars().count() != 1 || c[4] == "." { + if c.len() < 10 { + continue; + } + let pos = need_int(c[1])?; + if let Some((chrom, prev)) = &self.previous + && chrom == c[0] + && pos < *prev + { + let message = format!("{}:{pos} comes after {prev}", c[0]); + return usage(format!( + "{} is not sorted: {message}", + self.text.path.display() + )); + } + self.previous = Some((c[0].to_owned(), pos)); + if c[3].chars().count() != 1 || c[4] == "." { continue; } if c[4].contains(',') || c[4].chars().count() != 1 { return Ok(Some(VcfRec { chrom: c[0].to_owned(), - pos: need_int(c[1])?, + pos, call: Sample { r#ref: c[3].to_owned(), alt: None, @@ -775,7 +798,7 @@ impl Vcf { } return Ok(Some(VcfRec { chrom: c[0].to_owned(), - pos: need_int(c[1])?, + pos, call: Sample { r#ref: c[3].to_owned(), alt: Some(c[4].to_owned()), @@ -786,39 +809,48 @@ impl Vcf { } } -/// The sample's calls on one chromosome, one per position after -/// [`pick_call`], sorted by position: a compact array, not a map. -fn settle(mut recs: Vec<(i64, Sample)>) -> Vec<(i64, Sample)> { - // A stable sort keeps the records of one position in file order. - recs.sort_by_key(|(p, _)| *p); - let mut out: Vec<(i64, Sample)> = Vec::new(); - let mut clashed = false; - for (pos, call) in recs { - match out.last_mut() { - Some((p, kept)) if *p == pos => { - if clashed { - continue; - } - let r#ref = call.r#ref.clone(); - match pick_call(Some(kept.clone()), call) { - Picked::Call(c) => *kept = c, - Picked::Clash => { - *kept = Sample { - r#ref, - alt: None, - code: 9, +/// Reduce one duplicate-position window in file order. Only the reduced +/// call and one record of lookahead survive, even for very long windows. +struct Calls { + vcf: Vcf, + pending: Option, +} + +impl Calls { + fn next(&mut self) -> Out> { + let first = match self.pending.take() { + Some(r) => Some(r), + None => self.vcf.next()?, + }; + let Some(mut kept) = first else { + return Ok(None); + }; + let mut clashed = false; + loop { + match self.vcf.next()? { + Some(r) if r.chrom == kept.chrom && r.pos == kept.pos => { + if !clashed { + let r#ref = r.call.r#ref.clone(); + kept.call = match pick_call(Some(kept.call), r.call) { + Picked::Call(c) => c, + Picked::Clash => { + clashed = true; + Sample { + r#ref, + alt: None, + code: 9, + } + } }; - clashed = true; } } - } - _ => { - clashed = false; - out.push((pos, call)); + other => { + self.pending = other; + return Ok(Some(kept)); + } } } } - out } /// `chrN.tsv.gz` with N from 1 to 22: the chromosome's name. @@ -893,13 +925,14 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; let vcf_path = a.req("--vcf"); - let mut vcf = Vcf { - text: Text::open(vcf_path)?, + let mut calls = Calls { + vcf: Vcf::new(Text::open(vcf_path)?), + pending: None, }; let mut spans: Vec = Vec::new(); let mut done: Vec = Vec::new(); - let mut pending = vcf.next()?; - while let Some(first) = pending.take() { + let mut pending = calls.next()?; + while let Some(first) = &pending { let chrom = first.chrom.clone(); if done.contains(&chrom) { return die(format!( @@ -908,33 +941,24 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { } done.push(chrom.clone()); let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); - // The sample's calls on this chromosome; on a chromosome the table - // does not hold they are read past. - let mut recs = Vec::new(); - if table.is_some() { - recs.push((first.pos, first.call)); - } - loop { - match vcf.next()? { - Some(r) if r.chrom == chrom => { - if table.is_some() { - recs.push((r.pos, r.call)); - } - } - other => { - pending = other; - break; - } + if let Some(table) = table { + if let Some(span) = write_chrom( + &chrom, + table, + &mut calls, + &mut pending, + &head, + &pseudo, + af_col, + two_n, + out_dir, + )? { + spans.push(span); + } + } else { + while pending.as_ref().is_some_and(|r| r.chrom == chrom) { + pending = calls.next()?; } - } - let Some(table) = table else { - continue; - }; - let calls = settle(recs); - if let Some(span) = write_chrom( - &chrom, table, &calls, &head, &pseudo, af_col, two_n, out_dir, - )? { - spans.push(span); } } let mut text = @@ -967,7 +991,8 @@ type Span = (String, i64, i64, u64, u64, u64); fn write_chrom( chrom: &str, table: &str, - calls: &[(i64, Sample)], + calls: &mut Calls, + pending: &mut Option, head: &str, pseudo: &[String], af_col: usize, @@ -991,8 +1016,14 @@ fn write_chrom( let (mut n, mut first, mut last, mut dropped, mut carried) = (0u64, 0i64, 0i64, 0u64, 0u64); // Per archaic genome, the runs of written rows whose call is masked: // [first, last] positions, closed at the next unmasked row. - let mut runs: [Vec<(i64, i64)>; 3] = [Vec::new(), Vec::new(), Vec::new()]; + let mask_paths = + ARCHAICS.map(|name| Path::new(out_dir).join(format!("{chrom}.{name}.mask.bed"))); + let mut masks: Vec = mask_paths + .iter() + .map(|p| AtomicFile::create(p).map_err(|e| cannot_write(p, &e))) + .collect::>()?; let mut open_run: [Option<(i64, i64)>; 3] = [None; 3]; + let mut previous = None; let mut row = String::new(); while let Some(line) = t.next()? { let c: Vec<&str> = line.split('\t').collect(); @@ -1004,6 +1035,13 @@ fn write_chrom( )); } let pos = need_int(c[1])?; + if previous.is_some_and(|p| pos < p) { + return usage(format!( + "{table} is not sorted: {chrom}:{pos} comes after {}", + previous.unwrap() + )); + } + previous = Some(pos); let (r, alt) = (c[2], c[3]); let mut arch = [0u8; 3]; for k in 0..3 { @@ -1020,10 +1058,19 @@ fn write_chrom( } }; } - let code = match calls.binary_search_by_key(&pos, |(p, _)| *p) { - Err(_) => 0, - Ok(i) => { - let call = &calls[i].1; + while pending + .as_ref() + .is_some_and(|r| r.chrom == chrom && r.pos < pos) + { + *pending = calls.next()?; + } + let code = match pending + .as_ref() + .filter(|r| r.chrom == chrom && r.pos == pos) + { + None => 0, + Some(record) => { + let call = &record.call; if call.r#ref != r || call.alt.as_ref().is_some_and(|x| x != alt) { // Another allele than the row's: IBDmix's generate_gt // drops the line. @@ -1052,7 +1099,8 @@ fn write_chrom( Some(run) => run.1 = pos, } } else if let Some(run) = open_run[k].take() { - runs[k].push(run); + writeln!(masks[k], "{num}\t{}\t{}", run.0 - 1, run.1) + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } } row.clear(); @@ -1069,6 +1117,11 @@ fn write_chrom( last = pos; n += 1; } + // Read the tail too: an unsorted or unreadable record past the last + // table site must fail before committing this chromosome's outputs. + while pending.as_ref().is_some_and(|r| r.chrom == chrom) { + *pending = calls.next()?; + } if n == 0 { drop(gz); match std::fs::remove_file(&path) { @@ -1082,18 +1135,13 @@ fn write_chrom( gz.finish().and_then(AtomicFile::commit).map_err(werr)?; // IBDmix's mask (-r): the chromosome as in the genotype file, and a // line covers the positions start + 1 to end, as in a BED. - for k in 0..3 { - if let Some(run) = open_run[k] { - runs[k].push(run); + for (k, mut mask) in masks.into_iter().enumerate() { + if let Some((s, e)) = open_run[k] { + writeln!(mask, "{num}\t{}\t{e}", s - 1) + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } - let text: String = runs[k] - .iter() - .map(|(s, e)| format!("{num}\t{}\t{e}\n", s - 1)) - .collect(); - put( - &Path::new(out_dir).join(format!("{chrom}.{}.mask.bed", ARCHAICS[k])), - text.as_bytes(), - )?; + mask.commit() + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) } @@ -1155,7 +1203,7 @@ type Merged = (String, i64, i64, f64, Vec<&'static str>); /// Overlapping or touching segments of one class as one. fn merge(mut segs: Vec<(Seg, &'static str)>) -> Vec { - segs.sort_by(|(a, _), (b, _)| (chrom_key(&a.0), a.1, a.2).cmp(&(chrom_key(&b.0), b.1, b.2))); + segs.sort_by_key(|(a, _)| (chrom_key(&a.0), a.1, a.2)); let mut out: Vec = Vec::new(); for ((chrom, start, end, lod), who) in segs { match out.last_mut() { @@ -1205,7 +1253,7 @@ fn classify(per: &[Vec; 3]) -> Vec { rows.push((c, s, e, "Denisovan", vec!["Denisova"], l)); } } - rows.sort_by(|a, b| (chrom_key(&a.0), a.1).cmp(&(chrom_key(&b.0), b.1))); + rows.sort_by_key(|a| (chrom_key(&a.0), a.1)); rows } @@ -1659,16 +1707,49 @@ mod tests { #[test] fn a_clash_ends_its_position() { - let recs = vec![ - (4000, s("T", Some("C"), 1)), - (4000, s("T", Some("G"), 1)), - (4000, s("T", Some("C"), 1)), - (10, s("A", Some("G"), 2)), - ]; - assert_eq!( - settle(recs), - vec![(10, s("A", Some("G"), 2)), (4000, s("T", None, 9))] - ); + let mut calls = sample_calls(&[ + (10, "A", "G", "1/1"), + (4000, "T", "C", "0/1"), + (4000, "T", "G", "0/1"), + (4000, "T", "C", "0/1"), + ]); + assert_eq!(calls.next().unwrap().unwrap().call, s("A", Some("G"), 2)); + assert_eq!(calls.next().unwrap().unwrap().call, s("T", None, 9)); + assert!(calls.next().unwrap().is_none()); + } + + fn sample_calls(rows: &[(i64, &str, &str, &str)]) -> Calls { + let mut text = String::new(); + for (p, r, a, gt) in rows { + text += &format!("chr22\t{p}\t.\t{r}\t{a}\t.\tPASS\t.\tGT\t{gt}\n"); + } + Calls { + vcf: Vcf::new(Text { + path: "sample.vcf".into(), + lines: Lines::new(Box::new(io::Cursor::new(text.into_bytes())), Utf8::Strict), + line: String::new(), + }), + pending: None, + } + } + + #[test] + fn duplicate_window_keeps_only_one_record_of_lookahead() { + let mut rows = vec![(100, "A", "AT", "0/1"); 10_000]; + rows.push((100, "A", "G", "1/1")); + rows.push((200, "C", "T", "0/1")); + rows.push((150, "G", "A", "0/1")); // must not be read in the first window + let mut calls = sample_calls(&rows); + let first = calls.next().unwrap().unwrap(); + assert_eq!(first.pos, 100); + assert_eq!(first.call, s("A", Some("G"), 2)); + assert_eq!(calls.pending.as_ref().unwrap().pos, 200); + let error = calls + .next() + .err() + .expect("the second window sees the unsorted tail"); + assert_eq!(error.0, 2); + assert!(error.1.contains("not sorted")); } #[test] diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index dd63836b6..aae29404c 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -963,7 +963,7 @@ archaic-table empty-exclude gen:archaic-inputs.sh --chrom 22 --modern modern.tsv archaic-strict plain gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows.tsv --out s.tsv s.tsv bytes PYTHON_IMAGE archaic-strict unsorted gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows-unsorted.tsv --out s.tsv - bytes PYTHON_IMAGE archaic-strict empty-mask gen:archaic-inputs.sh --chrom 22 --strict empty.bed --rows rows.tsv --out s.tsv - bytes PYTHON_IMAGE -archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr22.Altai.mask.bed *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr21.Altai.mask.bed,gt/chr21.Vindija.mask.bed,gt/chr21.Denisova.mask.bed,gt/chr22.Altai.mask.bed,gt/chr22.Vindija.mask.bed,gt/chr22.Denisova.mask.bed *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes default gen:archaic-inputs.sh --table-dir . --vcf s.vcf.gz --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes afr-one gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop AFR --pseudo 1 --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes two-samples gen:archaic-inputs.sh --table-dir . --vcf two.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index 844658cd4..cffe87e77 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -315,6 +315,37 @@ def table_info(out, extra=()): r = run("genotypes", "--table-dir", bdir, "--vcf", f"{TMP}/m.vcf", "--out-dir", f"{TMP}/gtb") check("a code the table never holds (7) stops the step", r.returncode == 1 and "code '7'" in r.stderr, r.stderr) +# A streaming join must refuse backwards positions, including records past +# the table's end and records whose genotype it would otherwise ignore. +vcf_header = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" +for label, records in ( + ("within-table", [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")]), + ("past-table", [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")]), + ("ignored-call", [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")])): + bad_vcf = f"{TMP}/{label}.vcf" + write(bad_vcf, vcf_header + "".join( + f"chr22\t{p}\t.\t{ref}\t{alt}\t.\tPASS\t.\t{fmt}\t0/1\n" + for p, ref, alt, fmt in records)) + out = f"{TMP}/order-{label}" + os.makedirs(out) + protected = ["chr22.gt.gz", *(f"chr22.{g}.mask.bed" for g in ARCHAICS)] + for name in protected: + write(f"{out}/{name}", "previous output\n") + r = run("genotypes", "--table-dir", tdir, "--vcf", bad_vcf, "--out-dir", out) + check(f"unsorted VCF {label} exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) + check(f"unsorted VCF {label} keeps prior genotype and masks", + all(open(f"{out}/{name}", "rb").read() == b"previous output\n" for name in protected)) + +unordered_table = f"{TMP}/table_unordered" +os.makedirs(unordered_table) +with gzip.open(f"{unordered_table}/chr22.tsv.gz", "wt") as f: + f.write(hdr + "\n".join([rows[1], rows[0], *rows[2:]]) + "\n") +r = run("genotypes", "--table-dir", unordered_table, "--vcf", f"{TMP}/s.vcf", + "--out-dir", f"{TMP}/gt_unordered") +check("an unsorted table exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) +check("an unsorted table leaves no partial genotype or mask", + not any(name.endswith((".gt.gz", ".mask.bed")) for name in os.listdir(f"{TMP}/gt_unordered"))) + # --- 3. report --------------------------------------------------------------------- idir = f"{TMP}/ibd" os.makedirs(idir) From b101631c6db40d1fe2318d33815b72a70dc0f33c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:44:17 +0100 Subject: [PATCH 09/19] docs(ancestry): record sorted-input contract and bounded memory --- CHANGELOG.md | 3 ++- docs/43-archaic.md | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08831d0a4..bfa29e96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 43's genotype files for 3 million table rows take 6 s and 159 MB instead of 12 s and 213 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. + - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index 3e706e1ba..d9689a32c 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -72,6 +72,12 @@ The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf` - `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A site with no record is read as the reference allele, a record whose FILTER is not PASS as unknown, and a site where you carry another ALT than the table's is left out. - The GENCODE gene lines `setup.sh` installs, for the genes of each segment (without them the segments are listed without genes) +`og archaic genotypes` reads the VCF and table together, keeping one +duplicate-position window. Positions within each chromosome must be in +ascending order in both inputs. A position that goes backwards stops the +command with exit 2. Mask runs are written as they close. Duplicate calls, +clashes, masked sites and the output columns keep their previous meaning. + ## Command ```bash From c47cd892115bdc1c0c55897d72bce3845b86004d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:17:11 +0100 Subject: [PATCH 10/19] feat(summary): whole summary.json and its post pass in og summary collect Without --only, og summary collect reads all three groups over one walk of the folder and runs the post pass: the readings that join sections, the cyp2d6 block, not_run, not_assessed, the database re-check and new_since. A part of the post pass that fails makes only its own section unreadable; the summary is still written. Bead: pgp-bs9u.44 --- rust/Cargo.lock | 1 + rust/crates/og-summary/Cargo.toml | 1 + rust/crates/og-summary/src/collect.rs | 257 ++++++++----- rust/crates/og-summary/src/collect/group_b.rs | 108 +++--- rust/crates/og-summary/src/collect/group_c.rs | 46 ++- .../og-summary/src/collect/group_c/repeats.rs | 15 +- rust/crates/og-summary/src/collect/model.rs | 14 + rust/crates/og-summary/src/collect/post.rs | 339 +++++++++++++++++ .../og-summary/src/collect/post/tests.rs | 355 ++++++++++++++++++ rust/crates/og-summary/src/collect/tests.rs | 12 +- 10 files changed, 974 insertions(+), 174 deletions(-) create mode 100644 rust/crates/og-summary/src/collect/post.rs create mode 100644 rust/crates/og-summary/src/collect/post/tests.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c5d9ae54a..a99006d13 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -846,6 +846,7 @@ dependencies = [ "og-clinvar", "og-cn", "og-io", + "og-runlog", "og-steps", "pycompat", ] diff --git a/rust/crates/og-summary/Cargo.toml b/rust/crates/og-summary/Cargo.toml index 325b1f475..dedbfb695 100644 --- a/rust/crates/og-summary/Cargo.toml +++ b/rust/crates/og-summary/Cargo.toml @@ -15,6 +15,7 @@ og-steps.workspace = true og-ancestry.workspace = true og-chip.workspace = true og-cn.workspace = true +og-runlog.workspace = true indexmap.workspace = true memchr.workspace = true diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index e351fd308..13b22eb76 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -14,6 +14,10 @@ //! ancestry, structural variants, repeats, telomeres, runs of //! homozygosity, the haplogroups, the opt-in step tables, chrM, traits with //! the APOL1 G2 deletion, and the couple screen ([`group_c`]). +//! Without `--only` it reads all three groups and runs the post pass +//! ([`post`]): the readings that join sections, the steps not run, what no +//! step assessed, the last database re-check and what changed since the +//! previous run. That is the whole summary.json. //! //! How it reads a folder: one walk of the folder ([`scan::FolderIndex`]) //! instead of a `glob` per candidate path; each input read once (the @@ -32,6 +36,7 @@ pub mod group_b; pub mod group_c; pub mod model; pub mod pgx; +pub mod post; pub mod scan; pub mod sections; pub mod text; @@ -47,17 +52,17 @@ use std::io::Write; use std::path::Path; const USAGE: &str = - "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only {a,b,c} - --out OUT [--declared-sex DECLARED_SEX] [--assets ASSETS] + "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] + [--declared-sex DECLARED_SEX] [--assets ASSETS] [--only {a,b,c}] "; const OPTS: &[Opt] = &[ Opt("--sample", Kind::Value, true), Opt("--sample-dir", Kind::Value, true), Opt("--declared-sex", Kind::Value, false), - Opt("--out", Kind::Value, true), + Opt("--out", Kind::Value, false), Opt("--assets", Kind::Value, false), - Opt("--only", Kind::Choice(&["a", "b", "c"]), true), + Opt("--only", Kind::Choice(&["a", "b", "c"]), false), ]; /// What one collect reads. @@ -70,7 +75,7 @@ pub struct Input { pub declared_sex: Option, /// The repository's `assets/` folder. pub assets: String, - /// The group of sections to read: `a`, `b` or `c`. + /// The group of sections to read: `a`, `b`, `c`, or all of them. pub group: Group, /// The folder of the two tables group c reads from `bin/`. pub tables: String, @@ -82,6 +87,8 @@ pub enum Group { A, B, C, + /// Every section and the post pass: the whole summary.json. + All, } /// `og summary collect`: exit 0 when summary.json is written (whatever @@ -102,28 +109,39 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { declared_sex: a.get("--declared-sex").map(str::to_owned), assets: a.get("--assets").map_or_else(default_assets, str::to_owned), tables: group_c::util::default_tables(), - group: match a.text("--only") { - "b" => Group::B, - "c" => Group::C, - _ => Group::A, + group: match a.get("--only") { + Some("a") => Group::A, + Some("b") => Group::B, + Some("c") => Group::C, + _ => Group::All, }, }; // A part of the sections must never replace the sample's complete - // summary.json, so there is no default path while `--only` exists. - let target = a.text("--out").to_owned(); - let read = match a.text("--only") { - "b" => group_b::collect, - _ => collect, - }; - let summary = match read(&input) { - Ok(s) => s, - Err(e) => return crate::finish(Err(e), err), + // summary.json, so a part is written only where `--out` says. + let target = match a.get("--out") { + Some(t) => t.to_owned(), + None if input.group == Group::All => crate::join(&input.sample_dir, "summary.json"), + None => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og summary collect: error: --only needs --out: a part of the sections never replaces summary.json" + ); + return 2; + } }; // The sections' own data is freed before the text is made, and the // tree before the file is written: a large ClinVar list is in memory in // one form at a time. - let value = summary.to_json(); - drop(summary); + let read = match input.group { + Group::All => post::collect(&input), + Group::B => group_b::collect(&input).map(|s| s.to_json()), + Group::A | Group::C => collect(&input).map(|s| s.to_json()), + }; + let value = match read { + Ok(v) => v, + Err(e) => return crate::finish(Err(e), err), + }; let body = match dumps( &value, &Dump { @@ -161,47 +179,83 @@ fn default_assets() -> String { .to_string() } -/// Reads the folder into the summary of group a. An error is a run status -/// or a manifest that is there and cannot be read: without it a result of -/// an earlier run would be called current, so no summary is written. +/// What every group reads first: one walk of the folder, the run status +/// and the manifest. +pub struct Folder { + pub index: FolderIndex, + pub run: Option, + pub manifest: Option, +} + +impl Folder { + /// Walks the folder and reads its run status and manifest. An error is + /// a run status or a manifest that is there and cannot be read: without + /// it a result of an earlier run would be called current, so no summary + /// is written. + pub fn open(input: &Input) -> Result { + let dir = Path::new(&input.sample_dir); + let index = FolderIndex::walk(dir); + let run = + read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); + let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); + Ok(Folder { + index, + run, + manifest, + }) + } +} + +/// Reads the folder into the summary of group a or group c. pub fn collect(input: &Input) -> Result { - let dir = Path::new(&input.sample_dir); - let index = FolderIndex::walk(dir); - let run = - read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); - let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); - let s = pycompat::mtime::escape(&input.sample); - let find = |pats: &[String]| index.first(pats).map(str::to_owned); - let file = |rel: String| -> (String, String) { (crate::join(&input.sample_dir, &rel), rel) }; - if input.group == Group::C { + let folder = Folder::open(input)?; + let (sections, declared_sex) = if input.group == Group::C { // The declared sex the chrY filter of CNVpytor reads comes from // step 33's table when the run status and manifest have none. - let qc = find(&[format!("qc/{s}_sample_qc.tsv")]).map(file); - let qc_declared = qc.and_then(|(p, _)| { + let s = pycompat::mtime::escape(&input.sample); + let qc = folder + .index + .first(&[format!("qc/{s}_sample_qc.tsv")]) + .map(|rel| crate::join(&input.sample_dir, rel)); + let qc_declared = qc.and_then(|p| { let rows = sections::read_tsv(&p).ok()?; SampleQc::read(&p, &rows) .ok()? .declared_sex() .map(str::to_owned) }); - let declared_sex = declared( + let declared_sex = declared(input, &folder, qc_declared.as_deref()); + let mut sections = read_c(input, &folder); + post_c( input, - run.as_ref(), - manifest.as_ref(), - qc_declared.as_deref(), + &folder.index, + &mut sections, + &group_c::Others::default(), + declared_sex.as_deref(), ); - let sections = read_group_c(input, &index, run.as_ref(), declared_sex.as_deref()); - return Ok(Summary { - sample: input.sample.clone(), - generated_utc: time::utc_minute(now()).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6: None, - }); - } + (sections, declared_sex) + } else { + let sections = read_a(input, &folder); + let declared_sex = declared(input, &folder, qc_declared(§ions)); + (sections, declared_sex) + }; + Ok(Summary { + sample: input.sample.clone(), + generated_utc: time::utc_minute(now()).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6: None, + }) +} +/// Reads the sections of group a, in [`GROUP_A`] order: the sample VCF on +/// its own thread while the other six are read. +pub fn read_a(input: &Input, folder: &Folder) -> Vec
{ + let s = pycompat::mtime::escape(&input.sample); + let find = |pats: &[String]| folder.index.first(pats).map(str::to_owned); + let file = |rel: String| -> (String, String) { (crate::join(&input.sample_dir, &rel), rel) }; let variants_at = find(&[format!("vcf/{s}.vcf.gz")]); let (variants, rest) = std::thread::scope(|scope| { let v = scope.spawn(|| { @@ -233,29 +287,23 @@ pub fn collect(input: &Input) -> Result { read_count, variants, ]; - let sections: Vec
= GROUP_A + GROUP_A .iter() .zip(reads) - .map(|(spec, r): (&Spec, _)| Section::new(*spec, r, run.as_ref())) - .collect(); + .map(|(spec, r): (&Spec, _)| Section::new(*spec, r, folder.run.as_ref())) + .collect() +} - let qc_declared = sections +/// The declared sex step 33's table records, from its section (read when +/// the section is ok or stale). +pub fn qc_declared(sections: &[Section]) -> Option<&str> { + sections .iter() .find(|s| s.spec.key == "sample_qc") .and_then(|s| match &s.data { Some(Data::SampleQc(q)) => q.declared_sex(), _ => None, - }); - let declared_sex = declared(input, run.as_ref(), manifest.as_ref(), qc_declared); - Ok(Summary { - sample: input.sample.clone(), - generated_utc: time::utc_minute(now()).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6: None, - }) + }) } fn now() -> f64 { @@ -266,16 +314,14 @@ fn now() -> f64 { /// The declared sex: the command line's, the run status's, the /// manifest's, then step 33's; the first that is not empty. -fn declared( - input: &Input, - run: Option<&RunStatus>, - manifest: Option<&Manifest>, - qc: Option<&str>, -) -> Option { +pub fn declared(input: &Input, folder: &Folder, qc: Option<&str>) -> Option { [ input.declared_sex.as_deref(), - run.and_then(|r| r.declared_sex.as_deref()), - manifest.and_then(|m| m.value("run", "declared_sex")), + folder.run.as_ref().and_then(|r| r.declared_sex.as_deref()), + folder + .manifest + .as_ref() + .and_then(|m| m.value("run", "declared_sex")), qc, ] .into_iter() @@ -284,39 +330,56 @@ fn declared( .map(str::to_owned) } -/// Reads the sections of group c, then runs the parts of the post pass -/// that change them. -fn read_group_c( +/// Reads the sections of group c, in [`group_c::GROUP_C`] order. +pub fn read_c(input: &Input, folder: &Folder) -> Vec
{ + let c = ctx_c(input, &folder.index); + group_c::read_all(&c) + .into_iter() + .zip(group_c::GROUP_C.iter()) + .map(|(r, spec)| { + let r = r.and_then(|found| match found { + None => Ok(None), + Some((f, v)) => with_mtime(&f.path, f.rel, Data::Json(v)), + }); + Section::new(*spec, r, folder.run.as_ref()) + }) + .collect() +} + +/// The parts of the post pass that change group c's sections: the repeat +/// readings, Locityper's primary calls and CNVpytor's chrY filter. A part +/// that fails makes its own section unreadable, with a note that names the +/// file and the reading; the other sections and the summary stay. +pub fn post_c( input: &Input, index: &FolderIndex, - run: Option<&RunStatus>, + sections: &mut [Section], + others: &group_c::Others, declared_sex: Option<&str>, -) -> Vec
{ - let c = group_c::Ctx { +) { + let c = ctx_c(input, index); + let failed = { + let mut view: Vec<(Spec, model::State, Option<&mut Data>)> = sections + .iter_mut() + .map(|s| (s.spec, s.state, s.data.as_mut())) + .collect(); + group_c::post(&c, &mut view, others, declared_sex) + }; + for (key, why) in failed { + if let Some(s) = sections.iter_mut().find(|s| s.spec.key == key) { + s.make_unreadable(&why); + } + } +} + +fn ctx_c<'a>(input: &'a Input, index: &'a FolderIndex) -> group_c::Ctx<'a> { + group_c::Ctx { sample: &input.sample, dir: &input.sample_dir, assets: &input.assets, tables: &input.tables, index, - }; - let reads = group_c::read_all(&c); - let mut sections: Vec
= group_c::GROUP_C - .iter() - .zip(reads) - .map(|(spec, r)| { - let r = r.and_then(|found| match found { - None => Ok(None), - Some((f, v)) => with_mtime(&f.path, f.rel, Data::Json(v)), - }); - Section::new(*spec, r, run) - }) - .collect(); - let mut view: Vec<(Spec, model::State, Option<&mut Data>)> = sections - .iter_mut() - .map(|s| (s.spec, s.state, s.data.as_mut())) - .collect(); - group_c::post(&c, &mut view, &group_c::Others::default(), declared_sex); - sections + } } /// What one reader gave: the file (relative), its mtime and its data. diff --git a/rust/crates/og-summary/src/collect/group_b.rs b/rust/crates/og-summary/src/collect/group_b.rs index cb0ba8a1e..394178cf5 100644 --- a/rust/crates/og-summary/src/collect/group_b.rs +++ b/rust/crates/og-summary/src/collect/group_b.rs @@ -14,13 +14,12 @@ use super::carrier::Carrier; use super::clinical::{Clinical, Slivar}; use super::clinvar::{ClinVar, Floor, HitsVcf}; use super::cpsr::{self, Cpsr}; -use super::model::{Manifest, RunStatus, Section, Spec, Summary}; +use super::model::{Section, Spec, Summary}; use super::pgx::{Cpic, Cyp2d6, Cyrius, PharmCat, Pypgx}; use super::scan::FolderIndex; use super::sections::{Data, SampleQc}; use super::two_hits::{self, DeletionInputs}; -use super::{Input, Read, read_at, read_optional, with_mtime}; -use std::path::Path; +use super::{Folder, Input, Read, read_at, with_mtime}; /// The sections `og summary collect --only b` reads, in summary.json's order. pub const GROUP_B: [Spec; 9] = [ @@ -74,12 +73,42 @@ pub const GROUP_B: [Spec; 9] = [ /// Reads the folder into the summary of group b. An error is a run status /// or a manifest that is there and cannot be read. pub fn collect(input: &Input) -> Result { - let dir = Path::new(&input.sample_dir); + let folder = Folder::open(input)?; + let sections = read(input, &folder); + let cyp2d6 = cyp2d6(input, &folder.index, §ions); + let s = pycompat::mtime::escape(&input.sample); + // The run block is group a's: the declared sex from the command line, + // the run status, the manifest or step 33's table, the first that has one. + let qc_declared = folder + .index + .first(&[format!("qc/{s}_sample_qc.tsv")]) + .and_then(|rel| { + let p = crate::join(&input.sample_dir, rel); + let rows = super::sections::read_tsv(&p).ok()?; + SampleQc::read(&p, &rows) + .ok()? + .declared_sex() + .map(str::to_owned) + }); + let declared_sex = super::declared(input, &folder, qc_declared.as_deref()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + Ok(Summary { + sample: input.sample.clone(), + generated_utc: super::time::utc_minute(now).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6, + }) +} + +/// Reads the sections of group b, in [`GROUP_B`] order. +pub fn read(input: &Input, folder: &Folder) -> Vec
{ let d = input.sample_dir.as_str(); - let index = FolderIndex::walk(dir); - let run = - read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); - let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); + let index = &folder.index; let s = pycompat::mtime::escape(&input.sample); let raw = input.sample.as_str(); let file = |rel: &str| -> (String, String) { (crate::join(d, rel), rel.to_owned()) }; @@ -122,7 +151,7 @@ pub fn collect(input: &Input) -> Result { )?; with_mtime(&p, rel, Data::Clinical(data)) }); - let rest = read_rest(input, &s, &index, &find, &hits, &deletions); + let rest = read_rest(input, &s, index, &find, &hits, &deletions); let join = |h: std::thread::ScopedJoinHandle<'_, Read>| { h.join() .unwrap_or_else(|_| Err("internal error: the section reader stopped".into())) @@ -133,55 +162,26 @@ pub fn collect(input: &Input) -> Result { let reads = [ clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical, slivar, ]; - let sections: Vec
= GROUP_B + GROUP_B .iter() .zip(reads) - .map(|(spec, r)| Section::new(*spec, r, run.as_ref())) - .collect(); + .map(|(spec, r)| Section::new(*spec, r, folder.run.as_ref())) + .collect() +} + +/// The `cyp2d6` block: CPIC, pypgx and Cyrius side by side, with step 36's +/// row. `None` when `sections` lack one of the three. +pub fn cyp2d6(input: &Input, index: &FolderIndex, sections: &[Section]) -> Option { + let s = pycompat::mtime::escape(&input.sample); + let d = input.sample_dir.as_str(); let sec = |k: &str| sections.iter().find(|x| x.spec.key == k); - let cyp2d6 = match (sec("cpic"), sec("pypgx"), sec("cyrius")) { - (Some(c), Some(p), Some(y)) => Some(Cyp2d6::new( - c, - p, - y, - one(format!("pgx_consensus/{s}_pgx_consensus.tsv")), - d, - )), + let consensus = index + .first(&[format!("pgx_consensus/{s}_pgx_consensus.tsv")]) + .map(|rel| (crate::join(d, rel), rel.to_owned())); + match (sec("cpic"), sec("pypgx"), sec("cyrius")) { + (Some(c), Some(p), Some(y)) => Some(Cyp2d6::new(c, p, y, consensus, d)), _ => None, - }; - - // The run block is group a's: the declared sex from the command line, - // the run status, the manifest or step 33's table, the first that has one. - let qc_declared = one(format!("qc/{s}_sample_qc.tsv")).and_then(|(p, _)| { - let rows = super::sections::read_tsv(&p).ok()?; - SampleQc::read(&p, &rows) - .ok()? - .declared_sex() - .map(str::to_owned) - }); - let declared_sex = [ - input.declared_sex.clone(), - run.as_ref().and_then(|r| r.declared_sex.clone()), - manifest - .as_ref() - .and_then(|m| m.value("run", "declared_sex").map(str::to_owned)), - qc_declared, - ] - .into_iter() - .flatten() - .find(|v| !v.is_empty()); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0.0, |d| d.as_secs_f64()); - Ok(Summary { - sample: input.sample.clone(), - generated_utc: super::time::utc_minute(now).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6, - }) + } } /// Finds a section's file: the full path and the path in the folder. diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 15ab27782..3450f6b5d 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -9,7 +9,8 @@ //! readings with the traits and CPIC states, Locityper's primary calls, and //! the chrY filter of CNVpytor's count for a declared female. What they //! need from group b (the CPIC state, pypgx and Cyrius) comes in -//! [`Others`]; `--only c` reads none of it, so it is empty there. +//! [`Others`]; `--only c` reads none of it, so it is empty there. A part +//! that fails makes only its own section unreadable. mod chrm; mod cn; @@ -236,13 +237,15 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { /// The post pass over the read sections (`sections` in [`GROUP_C`] /// order, each with its state and data): the repeat readings with the /// traits and CPIC states, each Locityper locus's primary call, and the -/// chrY filter of CNVpytor's count. +/// chrY filter of CNVpytor's count. Each part runs on its own: one that +/// fails (an error or a panic) is returned as its section's key and why, +/// and the others still run. pub fn post( c: &Ctx<'_>, sections: &mut [(Spec, State, Option<&mut Data>)], others: &Others, declared_sex: Option<&str>, -) { +) -> Vec<(&'static str, String)> { let state = |key: &str, secs: &[(Spec, State, Option<&mut Data>)]| { secs.iter() .find(|(s, ..)| s.key == key) @@ -257,19 +260,42 @@ pub fn post( .and_then(|x| x.2.as_deref()) .and_then(json) .cloned(); + let mut failed = Vec::new(); for (spec, st, data) in sections.iter_mut() { let live = matches!(st, State::Ok | State::Stale); let Some(Data::Json(v)) = data.as_deref_mut() else { continue; }; - match spec.key { - "expansions" if live => repeats::post(c, v, traits_state, cpic_state), - "locityper" if live => { - tables::locityper_post(v, hla_state, hla_data.as_ref(), others); + let part = || -> Result<(), String> { + match spec.key { + "expansions" if live => repeats::post(c, v, traits_state, cpic_state) + .map_err(|e| format!("the repeat readings could not be made: {e}")), + "locityper" if live => { + tables::locityper_post(v, hla_state, hla_data.as_ref(), others); + Ok(()) + } + "cnvpytor" => { + sv::cnvpytor_sex_filter(v, declared_sex); + Ok(()) + } + _ => Ok(()), } - "cnvpytor" => sv::cnvpytor_sex_filter(v, declared_sex), - _ => {} - } + }; + failed.extend(contain(spec.key, part)); + } + failed +} + +/// Runs one part of the post pass: `None` when it went through, else its +/// section's key and why it failed, for an error and for a panic alike. +pub fn contain( + key: &'static str, + part: impl FnOnce() -> Result<(), String>, +) -> Option<(&'static str, String)> { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { + Ok(Ok(())) => None, + Ok(Err(e)) => Some((key, e)), + Err(_) => Some((key, format!("internal error: the post pass of {key} stopped"))), } } diff --git a/rust/crates/og-summary/src/collect/group_c/repeats.rs b/rust/crates/og-summary/src/collect/group_c/repeats.rs index 9d9610fd0..3081543d3 100644 --- a/rust/crates/og-summary/src/collect/group_c/repeats.rs +++ b/rust/crates/og-summary/src/collect/group_c/repeats.rs @@ -910,9 +910,10 @@ fn readings( } /// The post pass: the readings again, with the traits and CPIC states, over -/// the loci as the section lists them. -pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) { - let Value::Object(m) = v else { return }; +/// the loci as the section lists them. An error is a reading that could +/// not be made; the section's data is then left as it was. +pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) -> Result<(), String> { + let Value::Object(m) = v else { return Ok(()) }; let loci: Vec<(String, Obj)> = match m.get("loci") { Some(Value::Array(xs)) => xs .iter() @@ -924,10 +925,10 @@ pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) { _ => None, }) .collect(), - _ => return, + _ => return Ok(()), }; let by = by_locus(loci.iter().map(|(k, o)| (k, o))); - if let Ok(r) = readings(c, &by, traits, cpic) { - m.insert("readings".into(), r); - } + let r = readings(c, &by, traits, cpic)?; + m.insert("readings".into(), r); + Ok(()) } diff --git a/rust/crates/og-summary/src/collect/model.rs b/rust/crates/og-summary/src/collect/model.rs index c6d6e22a3..2d6e7c994 100644 --- a/rust/crates/og-summary/src/collect/model.rs +++ b/rust/crates/og-summary/src/collect/model.rs @@ -178,6 +178,20 @@ impl Section { s } + /// Marks a section whose reading failed after its file was read (a + /// part of the post pass): `unreadable`, as a reader's error leaves it, + /// with a note that names the file the section came from. + pub fn make_unreadable(&mut self, why: &str) { + self.note = Some(match self.source.as_deref() { + Some(src) => format!("{}: {why}", crate::basename(src)), + None => why.to_owned(), + }); + self.state = State::Unreadable; + self.source = None; + self.file_date = None; + self.data = None; + } + pub fn to_json(&self) -> Value { let mut m = IndexMap::new(); m.insert("title".into(), Value::Str(self.spec.title.into())); diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs new file mode 100644 index 000000000..42b9fe1ad --- /dev/null +++ b/rust/crates/og-summary/src/collect/post.rs @@ -0,0 +1,339 @@ +//! The whole summary.json: every section of groups a, b and c, read over +//! one walk of the folder, then the post pass. +//! +//! The post pass is what joins sections after each is read: +//! +//! - the repeat readings with the traits and CPIC states, Locityper's +//! primary calls with HLA, pypgx and Cyrius, and CNVpytor's chrY filter +//! with the declared sex ([`super::post_c`]); +//! - the `cyp2d6` block over CPIC, pypgx and Cyrius; +//! - `not_run`: the steps whose section has no file; +//! - `not_assessed`: what no step looked at, with the lines that depend on +//! the polygenic scores, HBA1/HBA2, mobile elements, VNtyper and MoChA; +//! - `recheck`: the last database re-check of the sample; +//! - `new_since`: what changed since the previous run kept in `history/`. +//! +//! Each part that changes a section runs on its own: when it fails, only +//! its section becomes `unreadable`, with a note naming the file, and the +//! summary is still written. The Python collector ran this pass outside +//! its per-section error handling, so one failure there lost summary.json +//! and both reports. A previous run that cannot be read leaves `new_since` +//! out, as before; the comparison itself has no error to hide. + +use super::group_c::Others; +use super::model::{Section, State, Summary}; +use super::sections::Data; +use super::{Folder, Input, declared, group_b, post_c, qc_declared, read_a, read_c, time}; +use og_runlog::{delta, files}; +use pycompat::json::Value; +use pycompat::value::truthy; +use std::path::Path; + +/// summary.json's sections, in its order. +pub const ORDER: [&str; 48] = [ + "coverage", + "callability", + "sex_check", + "sample_qc", + "mosaic_sex_loss", + "read_count", + "variants", + "clinvar", + "couple", + "carrier", + "pharmcat", + "cpic", + "pypgx", + "cyrius", + "parascopy", + "paralogs", + "hla", + "prs", + "ancestry_fine", + "manta", + "delly", + "cnvpytor", + "sv_consensus", + "annotsv", + "expansions", + "telomere", + "roh", + "haplogroup", + "y_haplogroup", + "y_tree", + "y_str", + "consumer_export", + "bloodtype", + "lpa", + "vntyper", + "locityper", + "mito", + "traits", + "archaic", + "ancient", + "viral", + "exomiser", + "chip", + "mei", + "mosaic", + "cpsr", + "clinical", + "slivar", +]; + +/// Sections that are no step of their own, so never in `not_run`: the read +/// count is written by alignment, which a run from a BAM or a VCF never +/// does; the couple screen and the carrier list come with other steps. +const NOT_A_STEP: [&str; 3] = ["read_count", "couple", "carrier"]; + +/// What no step assesses. +const NOT_ASSESSED: [&str; 5] = [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", +]; + +/// Added unless step 25 ran with an ancestry reference panel. +const PRS_NOT_ADJUSTED: &str = + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums"; + +/// Added unless step 35's depth count (short reads) or step 35b +/// (Paraphase, PacBio HiFi, both genes) made a call. +const HBA_NOT_ASSESSED: &str = "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count \ +(short reads) nor step 35b (Paraphase, PacBio HiFi) made a call"; + +/// Added when step 50 (opt-in, MEGAnE) did not run. A failed step 50 is +/// listed under not run, and an unreadable table under its section. +const MEI_NOT_ASSESSED: &str = "Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): \ +not looked for (step 50, MEGAnE, is opt-in)"; + +/// Added unless step 52 (VNtyper) gave a positive, possible or negative +/// reading: no other step sees a frameshift inside the MUC1 VNTR. +const VNTYPER_NOT_ASSESSED: &str = "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; \ +no standard caller sees them"; + +/// Added unless step 51 (MoChA) assessed the sample. +const MCA_NOT_ASSESSED: &str = "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share \ +of the blood cells): step 51 (MoChA) did not assess this sample"; + +/// Reads the whole summary of a sample folder. The three groups are read +/// at once over one walk of the folder. An error is a run status or a +/// manifest that is there and cannot be read. +pub fn collect(input: &Input) -> Result { + let folder = Folder::open(input)?; + let (a, b, mut c) = std::thread::scope(|scope| { + let a = scope.spawn(|| read_a(input, &folder)); + let b = scope.spawn(|| group_b::read(input, &folder)); + let c = read_c(input, &folder); + let stopped = |specs: &[super::model::Spec]| -> Vec
{ + specs + .iter() + .map(|s| { + Section::new( + *s, + Err("internal error: the section reader stopped".into()), + folder.run.as_ref(), + ) + }) + .collect() + }; + let a = a + .join() + .unwrap_or_else(|_| stopped(&super::model::GROUP_A)); + let b = b.join().unwrap_or_else(|_| stopped(&group_b::GROUP_B)); + (a, b, c) + }); + let declared_sex = declared(input, &folder, qc_declared(&a)); + post_c( + input, + &folder.index, + &mut c, + &others(&b), + declared_sex.as_deref(), + ); + let cyp2d6 = group_b::cyp2d6(input, &folder.index, &b); + let sections = in_order(a.into_iter().chain(b).chain(c).collect()); + let not_run = not_run(§ions); + let not_assessed = not_assessed(§ions); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + let summary = Summary { + sample: input.sample.clone(), + generated_utc: time::utc_minute(now).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6, + }; + let mut v = summary.to_json(); + drop(summary); + let Value::Object(m) = &mut v else { + return Err("internal error: the summary is not an object".into()); + }; + m.insert( + "not_run".into(), + Value::Array(not_run.into_iter().map(Value::Str).collect()), + ); + m.insert( + "not_assessed".into(), + Value::Array( + not_assessed + .into_iter() + .map(|x| Value::Str(x.to_owned())) + .collect(), + ), + ); + let dir = Path::new(&input.sample_dir); + if let Some(r) = delta::read_recheck(&dir.join("recheck").join("state.json")) { + m.insert("recheck".into(), Value::Object(r)); + } + if let Some(n) = new_since(dir, &v) { + if let Value::Object(m) = &mut v { + m.insert("new_since".into(), n); + } + } + Ok(v) +} + +/// What the post pass of group c needs from group b: the CPIC state, +/// pypgx's CYP2D6 call and Cyrius's genotype and Filter. +fn others(b: &[Section]) -> Others { + let sec = |k: &str| b.iter().find(|s| s.spec.key == k); + let live = |s: &Section| matches!(s.state, State::Ok | State::Stale); + Others { + cpic: sec("cpic").map(|s| s.state), + traits: None, + pypgx: sec("pypgx") + .filter(|s| live(s)) + .and_then(|s| match &s.data { + Some(Data::Pypgx(p)) => Some((s.state, p.cyp2d6.clone())), + _ => None, + }), + cyrius: sec("cyrius") + .filter(|s| live(s)) + .and_then(|s| match &s.data { + Some(Data::Cyrius(c)) => Some((s.state, Some(c.genotype.clone()), c.filter.clone())), + _ => None, + }), + } +} + +/// The sections in summary.json's order ([`ORDER`]). +fn in_order(mut sections: Vec
) -> Vec
{ + sections.sort_by_key(|s| ORDER.iter().position(|k| *k == s.spec.key)); + sections +} + +/// The steps whose section has no file, with " (step N failed)" for a step +/// that failed in the last run. +fn not_run(sections: &[Section]) -> Vec { + sections + .iter() + .filter(|s| matches!(s.state, State::Missing | State::Failed)) + .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) + .map(|s| { + if s.state == State::Failed { + format!("{} (step {} failed)", s.spec.title, s.spec.step) + } else { + s.spec.title.to_owned() + } + }) + .collect() +} + +/// What no step assessed for this sample. +fn not_assessed(sections: &[Section]) -> Vec<&'static str> { + let sec = |k: &str| sections.iter().find(|s| s.spec.key == k); + let state = |k: &str| sec(k).map_or(State::Missing, |s| s.state); + // A section's data is there when it is ok or stale; else it reads {}. + let field = |k: &str, key: &str| -> Option<&Value> { + match sec(k).and_then(|s| s.data.as_ref()) { + Some(Data::Json(Value::Object(m))) => m.get(key), + _ => None, + } + }; + let mut out = NOT_ASSESSED.to_vec(); + if !field("prs", "adjusted").is_some_and(truthy) { + out.push(PRS_NOT_ADJUSTED); + } + // Step 35b on a region subset (--paraphase_regions) may leave hba out. + let hba_genes: Vec<&str> = match field("paralogs", "genes") { + Some(Value::Array(genes)) => genes + .iter() + .filter(|g| is(at(Some(g), "region"), "hba") && is(at(Some(g), "status"), "called")) + .filter_map(|g| match at(Some(g), "gene") { + Some(Value::Str(x)) => Some(x.as_str()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + let paraphase_hba = ["HBA1", "HBA2"].iter().all(|g| hba_genes.contains(g)); + if !(paraphase_hba || is(at(field("parascopy", "hba"), "status"), "called")) { + out.push(HBA_NOT_ASSESSED); + } + if state("mei") == State::Missing { + out.push(MEI_NOT_ASSESSED); + } + let vntyper_read = state("vntyper") == State::Ok + && ["positive", "possible", "negative"] + .iter() + .any(|w| is(field("vntyper", "status"), w)); + if !vntyper_read { + out.push(VNTYPER_NOT_ASSESSED); + } + if !is(field("mosaic", "status"), "assessed") { + out.push(MCA_NOT_ASSESSED); + } + out +} + +/// True when `v` is the string `word`. +fn is(v: Option<&Value>, word: &str) -> bool { + matches!(v, Some(Value::Str(x)) if x == word) +} + +/// `v[key]` when `v` is an object that has it. +fn at<'a>(v: Option<&'a Value>, key: &str) -> Option<&'a Value> { + match v { + Some(Value::Object(m)) => m.get(key), + _ => None, + } +} + +/// `{previous_utc, sections}`: what changed since the newest run kept in +/// `history/` that holds a summary.json (generate-report.sh keeps one per +/// run), or `None` without one. A previous run whose summary or manifest +/// cannot be read, or whose summary is not shaped as a summary, counts as +/// none. +fn new_since(dir: &Path, cur: &Value) -> Option { + let prev_dir = delta::newest_history(dir)?; + let bytes = std::fs::read(prev_dir.join("summary.json")).ok()?; + let prev = pycompat::json::loads(std::str::from_utf8(&bytes).ok()?).ok()?; + let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; + let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; + let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; + let previous = match &prev { + Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), + _ => Value::Null, + }; + let mut out = indexmap::IndexMap::new(); + out.insert("previous_utc".to_owned(), previous); + out.insert( + "sections".to_owned(), + Value::Array( + delta::changed_sections(&changes) + .into_iter() + .map(Value::Str) + .collect(), + ), + ); + Some(Value::Object(out)) +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs new file mode 100644 index 000000000..e33709b92 --- /dev/null +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -0,0 +1,355 @@ +use super::super::group_c::{self, GROUP_C}; +use super::super::model::{GROUP_A, Spec}; +use super::super::{Group, Input, group_b}; +use super::*; +use crate::testutil::{Dir, run}; +use pycompat::json::{Value, loads}; + +fn input(d: &Dir) -> Input { + Input { + sample: "S".into(), + sample_dir: d.path("S"), + declared_sex: None, + assets: d.path("assets"), + group: Group::All, + tables: d.path("tables"), + } +} + +fn summary(d: &Dir) -> Value { + std::fs::create_dir_all(d.path("S")).unwrap(); + collect(&input(d)).unwrap() +} + +fn at<'a>(v: &'a Value, path: &[&str]) -> &'a Value { + path.iter().fold(v, |v, k| match v { + Value::Object(m) => m.get(*k).unwrap_or_else(|| panic!("no {k} in {path:?}")), + Value::Array(a) => &a[k.parse::().unwrap()], + _ => panic!("{path:?}"), + }) +} + +fn has(v: &Value, key: &str) -> bool { + matches!(v, Value::Object(m) if m.contains_key(key)) +} + +fn texts(v: &Value) -> Vec { + match v { + Value::Array(a) => a + .iter() + .map(|x| match x { + Value::Str(s) => s.clone(), + _ => panic!("{x:?}"), + }) + .collect(), + _ => panic!("{v:?}"), + } +} + +fn word(v: &Value) -> &str { + match v { + Value::Str(s) => s, + _ => panic!("{v:?}"), + } +} + +#[test] +fn the_order_holds_every_section_once() { + let mut keys: Vec<&str> = GROUP_A + .iter() + .chain(group_b::GROUP_B.iter()) + .chain(GROUP_C.iter()) + .map(|s: &Spec| s.key) + .collect(); + assert_eq!(keys.len(), ORDER.len()); + keys.sort_unstable(); + let mut order = ORDER.to_vec(); + order.sort_unstable(); + assert_eq!(keys, order); +} + +#[test] +fn an_empty_folder_lists_every_step_and_every_line() { + let d = Dir::new("post-empty"); + let v = summary(&d); + let Value::Object(m) = &v else { panic!() }; + let keys: Vec<&str> = m.keys().map(String::as_str).collect(); + assert_eq!( + keys, + [ + "schema_version", + "sample", + "generated_utc", + "run", + "databases", + "manifest", + "sections", + "cyp2d6", + "not_run", + "not_assessed" + ] + ); + let Value::Object(sections) = at(&v, &["sections"]) else { + panic!() + }; + assert_eq!(sections.keys().map(String::as_str).collect::>(), ORDER); + let not_run = texts(at(&v, &["not_run"])); + assert_eq!(not_run.len(), 45); + assert_eq!(not_run[0], "Coverage (mosdepth)"); + assert!(!not_run.contains(&"Reads kept by alignment".to_owned())); + assert!(!not_run.contains(&"Carrier list".to_owned())); + assert!(!not_run.contains(&"Couple carrier screen".to_owned())); + let lines = texts(at(&v, &["not_assessed"])); + let mut expected: Vec<&str> = NOT_ASSESSED.to_vec(); + expected.extend([ + PRS_NOT_ADJUSTED, + HBA_NOT_ASSESSED, + MEI_NOT_ASSESSED, + VNTYPER_NOT_ASSESSED, + MCA_NOT_ASSESSED, + ]); + assert_eq!(lines, expected); +} + +#[test] +fn a_failed_step_says_so_under_not_run() { + let d = Dir::new("post-failed"); + d.put( + "S/logs/run_status.tsv", + "meta\tstarted_epoch\t1577836800\nstep\t06\tfailed (exit 1)\nstep\t02\tfailed (exit 1)\n", + ); + let v = summary(&d); + let not_run = texts(at(&v, &["not_run"])); + assert!(not_run.contains(&"ClinVar screen (step 06 failed)".to_owned())); + // The read count is no step of its own, failed or not; the carrier + // list shares step 06 with ClinVar. + assert!(!not_run.iter().any(|x| x.starts_with("Reads kept"))); + assert!(!not_run.iter().any(|x| x.starts_with("Carrier list"))); + assert_eq!(word(at(&v, &["sections", "carrier", "state"])), "failed"); +} + +#[test] +fn the_lines_go_when_their_step_assessed_the_sample() { + let d = Dir::new("post-assessed"); + d.put( + "S/vntyper/S_vntyper.tsv", + "key\tvalue\nstatus\tnegative\n", + ); + d.put( + "S/paralogs/S_hba_copy_number.tsv", + "metric\tvalue\nstatus\tcalled\ncopies\t4\nreading\tfour\n", + ); + let v = summary(&d); + let lines = texts(at(&v, &["not_assessed"])); + let vntyper = at(&v, &["sections", "vntyper"]); + if word(at(vntyper, &["state"])) == "ok" && word(at(vntyper, &["data", "status"])) == "negative" { + assert!(!lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); + } else { + assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); + } + let parascopy = at(&v, &["sections", "parascopy"]); + if word(at(parascopy, &["state"])) == "ok" + && word(at(parascopy, &["data", "hba", "status"])) == "called" + { + assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); + } +} + +#[test] +fn a_stale_vntyper_reading_is_not_assessed() { + // Only an ok section counts for VNtyper; any other state leaves the line. + let mk = |state: State, status: &str| { + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "vntyper").unwrap(), + Ok(None), + None, + ); + s.state = state; + s.data = Some(Data::Json( + loads(&format!(r#"{{"status": "{status}"}}"#)).unwrap(), + )); + s + }; + let lines = not_assessed(&[mk(State::Ok, "positive")]); + assert!(!lines.contains(&VNTYPER_NOT_ASSESSED)); + let lines = not_assessed(&[mk(State::Stale, "positive")]); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED)); + let lines = not_assessed(&[mk(State::Ok, "failed")]); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED)); +} + +#[test] +fn paraphase_needs_both_hba_genes_called() { + let paralogs = |genes: &str| { + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "paralogs").unwrap(), + Ok(None), + None, + ); + s.state = State::Ok; + s.data = Some(Data::Json( + loads(&format!(r#"{{"genes": {genes}}}"#)).unwrap(), + )); + s + }; + let both = r#"[{"region": "hba", "gene": "HBA1", "status": "called"}, + {"region": "hba", "gene": "HBA2", "status": "called"}]"#; + let one = r#"[{"region": "hba", "gene": "HBA1", "status": "called"}, + {"region": "hba", "gene": "HBA2", "status": "no call"}]"#; + // A row without a region, the shape that stopped the Python collector. + let odd = r#"[{"gene": "HBA1", "status": "called"}, 7]"#; + assert!(!not_assessed(&[paralogs(both)]).contains(&HBA_NOT_ASSESSED)); + assert!(not_assessed(&[paralogs(one)]).contains(&HBA_NOT_ASSESSED)); + assert!(not_assessed(&[paralogs(odd)]).contains(&HBA_NOT_ASSESSED)); +} + +#[test] +fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { + // An error and a panic alike come back as the section's key and why. + assert_eq!(group_c::contain("cnvpytor", || Ok(())), None); + assert_eq!( + group_c::contain("expansions", || Err("no table".into())), + Some(("expansions", "no table".to_owned())) + ); + let panicked = group_c::contain("locityper", || panic!("planted")); + assert_eq!( + panicked, + Some(( + "locityper", + "internal error: the post pass of locityper stopped".to_owned() + )) + ); + // The section is then unreadable, with no data, and names its file. + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "locityper").unwrap(), + Ok(Some(( + "locityper/S_locityper.tsv".into(), + 1_600_000_000.0, + Data::Json(Value::Object(indexmap::IndexMap::new())), + ))), + None, + ); + s.make_unreadable("internal error: the post pass of locityper stopped"); + let j = s.to_json(); + assert_eq!(word(at(&j, &["state"])), "unreadable"); + assert_eq!( + word(at(&j, &["note"])), + "S_locityper.tsv: internal error: the post pass of locityper stopped" + ); + assert_eq!(at(&j, &["source"]), &Value::Null); + assert_eq!(at(&j, &["data"]), &Value::Object(indexmap::IndexMap::new())); +} + +#[test] +fn an_infinite_telomere_read_length_keeps_the_summary() { + // The Python collector stopped on this table with an OverflowError and + // wrote no summary.json; here only the telomere section is unreadable. + let d = Dir::new("post-inf"); + d.put( + "S/telomere/S/S/S_summary.tsv", + "PID\tsample\ttotal_reads\tread_lengths\tintratel_reads\ttel_content\nS\ttumor\t1000\tinf\t40\t1.5\n", + ); + d.put( + "S/mosdepth/S.mosdepth.summary.txt", + "chrom\tlength\tbases\tmean\tmin\tmax\ntotal\t100\t3000\t30\t0\t60\n", + ); + let v = summary(&d); + assert_eq!(word(at(&v, &["sections", "telomere", "state"])), "unreadable"); + assert_eq!(word(at(&v, &["sections", "coverage", "state"])), "ok"); +} + +#[test] +fn a_declared_female_drops_cnvpytor_chry_calls_across_groups() { + // The declared sex comes from step 33's table (group a) and changes + // CNVpytor's count (group c). + let d = Dir::new("post-sex"); + d.put( + "S/qc/S_sample_qc.tsv", + "key\tvalue\nsample\tS\ndeclared_sex\tfemale\ninferred_sex\tfemale\nsex_check\tok\n", + ); + d.put( + "S/cnvpytor/S_cnvs.txt", + "deletion\tchrY:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n\ +deletion\tchr2:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n", + ); + let v = summary(&d); + assert_eq!(word(at(&v, &["run", "declared_sex"])), "female"); + let cnv = at(&v, &["sections", "cnvpytor"]); + if word(at(cnv, &["state"])) == "ok" { + assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); + assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); + } +} + +const PREV: &str = r#"{"schema_version": 2, "generated_utc": "2020-01-01 00:00 UTC", + "sections": {"coverage": {"state": "ok", "data": {}}, "clinvar": {"state": "missing", "data": {}}}}"#; + +#[test] +fn new_since_compares_with_the_newest_kept_run() { + let d = Dir::new("post-history"); + d.put("S/history/20200101T000000/summary.json", PREV); + // A newer folder without a summary.json is a half-copied one: passed over. + d.put("S/history/20990101T000000/run_manifest.tsv", "run\tx\ty\n"); + let v = summary(&d); + assert_eq!( + at(&v, &["new_since", "previous_utc"]), + &Value::Str("2020-01-01 00:00 UTC".into()) + ); + assert!(texts(at(&v, &["new_since", "sections"])).contains(&"coverage".to_owned())); +} + +#[test] +fn a_previous_run_that_cannot_be_read_leaves_new_since_out() { + for body in ["{not json", "[]", r#"{"sections": 7}"#] { + let d = Dir::new("post-history-bad"); + d.put("S/history/20200101T000000/summary.json", body); + let v = summary(&d); + assert!(!has(&v, "new_since"), "{body}"); + } +} + +#[test] +fn the_recheck_block_comes_from_its_state_file() { + let d = Dir::new("post-recheck"); + d.put( + "S/recheck/state.json", + r#"{"checked_utc": "2026-01-02 03:04:05 UTC", "items": [{"item": "clinvar", "class": "newer", "x": 1}, 5], + "affected_steps": ["06"]}"#, + ); + let v = summary(&d); + assert_eq!( + word(at(&v, &["recheck", "checked_utc"])), + "2026-01-02 03:04:05 UTC" + ); + assert_eq!(word(at(&v, &["recheck", "items", "0", "item"])), "clinvar"); + // A check older than the run's manifest is of an earlier run. + d.put( + "S/run_manifest.tsv", + "run\twritten_utc\t2027-01-01 00:00:00 UTC\n", + ); + assert!(!has(&summary(&d), "recheck")); +} + +#[test] +fn without_only_the_command_writes_summary_json_in_the_folder() { + let d = Dir::new("post-cli"); + std::fs::create_dir_all(d.path("S")).unwrap(); + let dir = d.path("S"); + let assets = d.path("assets"); + let (code, out, err) = run( + crate::collect::cli, + &["--sample", "S", "--sample-dir", &dir, "--assets", &assets], + ); + assert_eq!((code, err.as_str()), (0, "")); + let target = format!("{dir}/summary.json"); + assert_eq!(out, format!("Summary written: {target}\n")); + let v = loads(&d.read("S/summary.json")).unwrap(); + assert!(has(&v, "not_assessed")); + // A part of the sections never takes summary.json's place. + let (code, _, err) = run( + crate::collect::cli, + &["--sample", "S", "--sample-dir", &dir, "--only", "a"], + ); + assert_eq!(code, 2); + assert!(err.contains("--only needs --out"), "{err}"); +} diff --git a/rust/crates/og-summary/src/collect/tests.rs b/rust/crates/og-summary/src/collect/tests.rs index 75031f745..4f622c77f 100644 --- a/rust/crates/og-summary/src/collect/tests.rs +++ b/rust/crates/og-summary/src/collect/tests.rs @@ -448,22 +448,22 @@ fn the_command_line() { let text = d.read("out.json"); assert!(text.starts_with("{\n \"schema_version\": 2,\n \"sample\": \"S\",\n")); assert!(text.ends_with("}\n")); - // --only is required until every section is ported. + // Without --only it writes every section and the post pass. let (code, _, err) = run( cli, &["--sample", "S", "--sample-dir", &d.path("S"), "--out", &out], ); - assert_eq!(code, 2); - assert!(err.contains("required: --only"), "{err}"); - // So is --out: a part of the sections must not land on the sample's - // complete summary.json. + assert_eq!((code, err.as_str()), (0, "")); + assert!(d.read("out.json").contains("\n \"not_assessed\": [\n")); + // --only needs --out: a part of the sections must not land on the + // sample's complete summary.json. std::fs::write(d.path("S/summary.json"), "complete\n").unwrap(); let (code, _, err) = run( cli, &["--sample", "S", "--sample-dir", &d.path("S"), "--only", "a"], ); assert_eq!(code, 2); - assert!(err.contains("required: --out"), "{err}"); + assert!(err.contains("--only needs --out"), "{err}"); assert_eq!(d.read("S/summary.json"), "complete\n"); let (code, _, err) = run( cli, From 5e22114a3ad3a25009b7e76bf3211631d567d375 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:34:15 +0100 Subject: [PATCH 11/19] test(parity): compare the whole summary.json with the Python collector Nine cases run og summary collect without --only against bin/collect_summary.py on folders that join the three groups' fixtures, with failed steps, an empty folder, kept runs in history/ and a database re-check. A telomere table with an infinite read length stopped the Python with an uncaught OverflowError; og keeps the summary with that section unreadable (the all-inf golden). Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect.rs | 3 +- rust/crates/og-summary/src/collect/group_c.rs | 5 +- rust/crates/og-summary/src/collect/post.rs | 16 +- .../og-summary/src/collect/post/tests.rs | 56 +- tests/parity/cases.tsv | 9 + tests/parity/gen/summary-collect-full.sh | 107 +++ .../summary-collect/all-inf/all-inf.json | 820 ++++++++++++++++++ .../golden/summary-collect/all-inf/exit | 1 + .../golden/summary-collect/all-inf/stderr | 0 .../golden/summary-collect/all-inf/stdout | 1 + 10 files changed, 981 insertions(+), 37 deletions(-) create mode 100644 tests/parity/gen/summary-collect-full.sh create mode 100644 tests/parity/golden/summary-collect/all-inf/all-inf.json create mode 100644 tests/parity/golden/summary-collect/all-inf/exit create mode 100644 tests/parity/golden/summary-collect/all-inf/stderr create mode 100644 tests/parity/golden/summary-collect/all-inf/stdout diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 13b22eb76..4bbc9529f 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -51,8 +51,7 @@ use sections::{Callability, Coverage, Data, Mosaic, ReadCount, SampleQc, SexChec use std::io::Write; use std::path::Path; -const USAGE: &str = - "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] +const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] [--declared-sex DECLARED_SEX] [--assets ASSETS] [--only {a,b,c}] "; diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 3450f6b5d..29f66b870 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -295,7 +295,10 @@ pub fn contain( match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), - Err(_) => Some((key, format!("internal error: the post pass of {key} stopped"))), + Err(_) => Some(( + key, + format!("internal error: the post pass of {key} stopped"), + )), } } diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 42b9fe1ad..54c5c36da 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -139,9 +139,7 @@ pub fn collect(input: &Input) -> Result { }) .collect() }; - let a = a - .join() - .unwrap_or_else(|_| stopped(&super::model::GROUP_A)); + let a = a.join().unwrap_or_else(|_| stopped(&super::model::GROUP_A)); let b = b.join().unwrap_or_else(|_| stopped(&group_b::GROUP_B)); (a, b, c) }); @@ -191,10 +189,10 @@ pub fn collect(input: &Input) -> Result { if let Some(r) = delta::read_recheck(&dir.join("recheck").join("state.json")) { m.insert("recheck".into(), Value::Object(r)); } - if let Some(n) = new_since(dir, &v) { - if let Value::Object(m) = &mut v { - m.insert("new_since".into(), n); - } + if let Some(n) = new_since(dir, &v) + && let Value::Object(m) = &mut v + { + m.insert("new_since".into(), n); } Ok(v) } @@ -216,7 +214,9 @@ fn others(b: &[Section]) -> Others { cyrius: sec("cyrius") .filter(|s| live(s)) .and_then(|s| match &s.data { - Some(Data::Cyrius(c)) => Some((s.state, Some(c.genotype.clone()), c.filter.clone())), + Some(Data::Cyrius(c)) => { + Some((s.state, Some(c.genotype.clone()), c.filter.clone())) + } _ => None, }), } diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs index e33709b92..bcf38afd5 100644 --- a/rust/crates/og-summary/src/collect/post/tests.rs +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -92,7 +92,10 @@ fn an_empty_folder_lists_every_step_and_every_line() { let Value::Object(sections) = at(&v, &["sections"]) else { panic!() }; - assert_eq!(sections.keys().map(String::as_str).collect::>(), ORDER); + assert_eq!( + sections.keys().map(String::as_str).collect::>(), + ORDER + ); let not_run = texts(at(&v, &["not_run"])); assert_eq!(not_run.len(), 45); assert_eq!(not_run[0], "Coverage (mosdepth)"); @@ -129,30 +132,17 @@ fn a_failed_step_says_so_under_not_run() { } #[test] -fn the_lines_go_when_their_step_assessed_the_sample() { +fn step_35s_hba_count_assesses_hba() { let d = Dir::new("post-assessed"); - d.put( - "S/vntyper/S_vntyper.tsv", - "key\tvalue\nstatus\tnegative\n", - ); d.put( "S/paralogs/S_hba_copy_number.tsv", "metric\tvalue\nstatus\tcalled\ncopies\t4\nreading\tfour\n", ); let v = summary(&d); + assert_eq!(word(at(&v, &["sections", "parascopy", "state"])), "ok"); let lines = texts(at(&v, &["not_assessed"])); - let vntyper = at(&v, &["sections", "vntyper"]); - if word(at(vntyper, &["state"])) == "ok" && word(at(vntyper, &["data", "status"])) == "negative" { - assert!(!lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); - } else { - assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); - } - let parascopy = at(&v, &["sections", "parascopy"]); - if word(at(parascopy, &["state"])) == "ok" - && word(at(parascopy, &["data", "hba", "status"])) == "called" - { - assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); - } + assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); } #[test] @@ -160,7 +150,11 @@ fn a_stale_vntyper_reading_is_not_assessed() { // Only an ok section counts for VNtyper; any other state leaves the line. let mk = |state: State, status: &str| { let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "vntyper").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "vntyper") + .unwrap(), Ok(None), None, ); @@ -182,7 +176,11 @@ fn a_stale_vntyper_reading_is_not_assessed() { fn paraphase_needs_both_hba_genes_called() { let paralogs = |genes: &str| { let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "paralogs").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "paralogs") + .unwrap(), Ok(None), None, ); @@ -221,7 +219,11 @@ fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { ); // The section is then unreadable, with no data, and names its file. let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "locityper").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "locityper") + .unwrap(), Ok(Some(( "locityper/S_locityper.tsv".into(), 1_600_000_000.0, @@ -254,7 +256,10 @@ fn an_infinite_telomere_read_length_keeps_the_summary() { "chrom\tlength\tbases\tmean\tmin\tmax\ntotal\t100\t3000\t30\t0\t60\n", ); let v = summary(&d); - assert_eq!(word(at(&v, &["sections", "telomere", "state"])), "unreadable"); + assert_eq!( + word(at(&v, &["sections", "telomere", "state"])), + "unreadable" + ); assert_eq!(word(at(&v, &["sections", "coverage", "state"])), "ok"); } @@ -275,10 +280,9 @@ deletion\tchr2:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n" let v = summary(&d); assert_eq!(word(at(&v, &["run", "declared_sex"])), "female"); let cnv = at(&v, &["sections", "cnvpytor"]); - if word(at(cnv, &["state"])) == "ok" { - assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); - assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); - } + assert_eq!(word(at(cnv, &["state"])), "ok"); + assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); + assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); } const PREV: &str = r#"{"schema_version": 2, "generated_utc": "2020-01-01 00:00 UTC", diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 7c21916ad..14edaa05e 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -870,6 +870,15 @@ summary-collect cb-notbi gen:summary-clinical.sh,summary-collect/oracle.sh,summa summary-collect cb-notgz gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-notgz --assets assets --out cb-notgz.json --only b cb-notgz.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-empty gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-empty --assets assets --out cb-empty.json --only b cb-empty.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-no-assets gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets nowhere --out cb-no-assets.json --only b cb-no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect all gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir all --assets assets --out all.json all.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-nf gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir nf --assets assets --out all-nf.json all-nf.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-bad gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir bad --assets assets --out all-bad.json all-bad.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-failed gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir failed --assets assets --out all-failed.json all-failed.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-empty gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir empty --assets assets --out all-empty.json all-empty.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-history gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history --assets assets --out all-history.json all-history.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-history-bad gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history-bad --assets assets --out all-history-bad.json all-history-bad.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-inf gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir inf --assets assets --out all-inf.json all-inf.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-default-out gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history-bad --assets assets history-bad/summary.json json:mask=generated_utc PYTHON_IMAGE y-tree-positions small gen:y-inputs.sh --tree-dir t pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions big gen:y-inputs.sh --tree-dir ./big/ pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions odd gen:y-inputs.sh --tree-dir odd pos.txt pos.txt bytes PYTHON_IMAGE diff --git a/tests/parity/gen/summary-collect-full.sh b/tests/parity/gen/summary-collect-full.sh new file mode 100644 index 000000000..94d4db167 --- /dev/null +++ b/tests/parity/gen/summary-collect-full.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# summary-collect-full.sh: writes the sample folders of the whole-file +# summary-collect parity cases (og summary collect without --only) in the +# current directory, the same every time. It runs the generators of the +# three groups (summary-collect.sh, summary-clinical.sh and +# summary-collect-c.sh) and joins their folders, so one folder holds every +# section and the post pass reads across the groups: the repeat readings +# with the CPIC state, Locityper's primary calls with pypgx and Cyrius, +# CNVpytor's chrY filter with step 33's declared sex, the cyp2d6 block, +# not_run and not_assessed. On top: a run with failed steps, an empty +# folder, previous runs kept in history/ (one half-copied, one whose +# summary is no JSON) with a database re-check, and a telomere table whose +# infinite read length stopped the Python collector (a golden: og keeps +# the summary and marks that one section unreadable). +# py/ holds the modules the oracle imports at the oracle's commit, with the +# real run_delta.py for recheck and new_since. The case stages +# cb_vep.vcf.gz and its index beside the folders. +# scripts/ci/oracle-diff.sh runs it. +set -euo pipefail +export LC_ALL=C TZ=UTC +here=$(cd "$(dirname "$0")" && pwd) +root=$(cd "${here}/../../.." && pwd) +ORACLE_SHA=98aa9252ff77825a8a797d388b449931fa9a234e + +put() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" > "$1"; at "$NEW" "$1"; } +at() { touch -t "$1" "${@:2}"; } # [[CC]YY]MMDDhhmm[.SS], in UTC +NEW=202103040506.07 # after the run of the three groups' folders + +for g in a b c; do mkdir -p "parts/$g"; done +(cd parts/a && bash "${here}/summary-collect.sh") +(cd parts/b && bash "${here}/summary-clinical.sh") +(cd parts/c && bash "${here}/summary-collect-c.sh") + +# merge DEST SRC...: the folders' files, the later folder's file winning; +# times and symlinks kept. +merge() { + local dest=$1 src + shift + mkdir -p "$dest" + for src in "$@"; do cp -pPR "parts/${src}/." "$dest"/; done +} + +mkdir -p assets py pymods +for g in a b c; do cp -pR "parts/${g}/assets/." assets/; done +cp -pR parts/b/pymods/. pymods/ +cp -pR parts/c/py/. py/ +git -C "$root" cat-file -e "${ORACLE_SHA}^{commit}" 2> /dev/null \ + || git -C "$root" fetch -q --depth=1 origin "$ORACLE_SHA" +git -C "$root" show "${ORACLE_SHA}:bin/run_delta.py" > py/run_delta.py + +# all: every section of the three groups, the bash names. +merge all a/full b/cb-full c/full +# nf: the Nextflow names, steps failed and from an earlier run. +merge nf a/nf b/cb-nf c/nf +# bad: an unreadable file in sections of each group. +merge bad a/bad-a b/cb-bad c/bad1 +# failed: steps that failed and wrote nothing, under not_run. +merge failed a/failed b/cb-failed +mkdir -p empty + +# history: all, with three earlier runs kept. The newest has no summary +# (a half-copied folder) and is passed over; the next is compared: its +# ClinVar, CPIC, pypgx and PRS results and its section states differ, and +# so do the inputs of its manifest. A database re-check newer than the +# run's manifest. +merge history a/full b/cb-full c/full +put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" +put history/history/20200101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2019-12-01 00:00 UTC", "sections": {}}' +put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2021-01-01 00:00 UTC", + "sections": { + "coverage": {"title": "Coverage (mosdepth)", "state": "missing", "data": {}}, + "clinvar": {"title": "ClinVar screen", "state": "ok", "data": {"hits": [ + {"chrom": "chr1", "pos": "45331556", "ref": "C", "alt": "T", "gene": "MUTYH", "significance": "Likely_pathogenic", "stars": 1, "review_status": "x", "genotype": "1/1"}, + {"chrom": "chr9", "pos": "1", "ref": "A", "alt": "G", "gene": "X", "significance": "Pathogenic", "stars": 2}]}}, + "cpic": {"title": "CPIC", "state": "ok", "data": {"genes": [{"gene": "CYP2D6", "diplotype": "*1/*1", "phenotype": "Normal"}]}}, + "pypgx": {"title": "pypgx", "state": "ok", "data": {"cyp2d6": "*1/*1"}}, + "prs": {"title": "PRS", "state": "ok", "data": {"scores": [{"pgs_id": "PGS000001", "condition": "x", "percentile": 10}]}}, + "mei": {"title": "Mobile element insertions (MEGAnE)", "state": "ok", "data": {}}}, + "cyp2d6": {"consensus": {"result": "*1/*1"}}}\n' +put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" +mkdir -p history/history/20220101T000000 +put history/history/20220101T000000/run_manifest.tsv "run\twritten_utc\t2022-01-01 00:00:00 UTC\n" +put history/recheck/state.json '{"checked_utc": "2021-03-05 00:00:00 UTC", "items": [ + {"item": "clinvar", "label": "ClinVar", "class": "newer-release", "ran_with": "2021-03-01", "upstream": "2021-03-04", "action": "rerun step 06", "extra": 1}, + {"item": "vep", "label": "VEP", "class": "no-newer-release"}, 7], + "affected_steps": ["06"], "rerun": {"command": "run-all.sh"}}\n' + +# history-bad: the only kept run has a summary that is no JSON, so nothing +# is compared; a re-check older than the run's manifest is of an earlier +# run and left out. +mkdir -p history-bad +put history-bad/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\n" +put history-bad/history/20210101T000000/summary.json '{"schema_version": 2,' +put history-bad/recheck/state.json '{"checked_utc": "2021-03-01 00:00:00 UTC", "items": []}\n' + +# inf: a telomere table whose read length is infinite, beside the +# coverage, step 33 and repeat sections. The Python collector stopped with +# an uncaught OverflowError and wrote no summary; og writes it with the +# telomere section unreadable (a golden). +mkdir -p inf +for f in mosdepth/S.mosdepth.summary.txt qc/S_sample_qc.tsv logs/run_status.tsv; do + mkdir -p "inf/$(dirname "$f")" && cp -p "parts/a/full/$f" "inf/$f" +done +cp -pR parts/c/full/expansion_hunter inf/ +put inf/telomere/S/S/S_summary.tsv "PID\tsample\ttotal_reads\tread_lengths\tintratel_reads\ttel_content\nS\ttumor\t1000\tinf\t40\t1.5\n" + +rm -rf parts diff --git a/tests/parity/golden/summary-collect/all-inf/all-inf.json b/tests/parity/golden/summary-collect/all-inf/all-inf.json new file mode 100644 index 000000000..99b10ea8d --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/all-inf.json @@ -0,0 +1,820 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-13 23:22 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). Not compared with PharmCAT (no unambiguous UGT1A1 call from step 27)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "OverflowError: cannot round", + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Variant calling", + "ClinVar screen", + "PharmCAT", + "CPIC drug-gene recommendations", + "pypgx", + "Cyrius (CYP2D6)", + "SMN1/SMN2 copy number (Parascopy, short reads)", + "Paralog genes (Paraphase, PacBio HiFi)", + "HLA typing (T1K)", + "Polygenic risk scores", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "Structural variants (Manta)", + "Structural variants (Delly)", + "Copy number (CNVpytor)", + "SV consensus merge", + "Structural variants, class 4 and 5 (AnnotSV)", + "Runs of homozygosity", + "Mitochondrial haplogroup", + "Y-chromosome haplogroup (Yleaf)", + "Y placement on the current public tree", + "Y-STR markers (HipSTR)", + "Raw-data export (23andMe v5 layout)", + "Blood groups (predicted)", + "Lp(a): LPA KIV-2 repeat count", + "MUC1 VNTR frameshifts (VNtyper)", + "Locityper second opinion at hard loci", + "Mitochondrial variants", + "Traits panel", + "Archaic ancestry (IBDmix)", + "Ancient populations (AADR PCA)", + "Viral DNA in the unmapped reads", + "Symptom-driven ranking (Exomiser)", + "Clonal haematopoiesis (CHIP)", + "Mobile element insertions (MEGAnE)", + "Mosaic chromosomal alterations (MoChA)", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): not looked for (step 50, MEGAnE, is opt-in)", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/all-inf/exit b/tests/parity/golden/summary-collect/all-inf/exit new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/exit @@ -0,0 +1 @@ +0 diff --git a/tests/parity/golden/summary-collect/all-inf/stderr b/tests/parity/golden/summary-collect/all-inf/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/summary-collect/all-inf/stdout b/tests/parity/golden/summary-collect/all-inf/stdout new file mode 100644 index 000000000..437515d43 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/stdout @@ -0,0 +1 @@ +Summary written: all-inf.json From 88dd793198d73798bb50013332594a1bc61a1031 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:04:00 +0100 Subject: [PATCH 12/19] docs(summary): og summary collect writes the whole summary.json Bead: pgp-bs9u.44 --- CHANGELOG.md | 3 +++ docs/rust.md | 2 +- rust/README.md | 2 +- rust/og/src/main.rs | 2 +- tests/parity/inputs/summary-collect/oracle.sh | 4 ++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7489faaa..0a14652ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The Rust `og` binary now writes the whole summary.json: `og summary collect` without `--only` reads all 48 sections over one walk of the sample folder, then runs the post pass. The post pass joins sections: the repeat readings with the traits and CPIC states, Locityper's primary calls with HLA, pypgx and Cyrius, and CNVpytor's chrY filter with the declared sex. It also builds the CYP2D6 block and writes `not_run`, `not_assessed`, `recheck` and `new_since`. Without `--out` it writes `summary.json` in the sample folder, as the Python did; `--only` still needs `--out`. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The whole file is the same as the Python's on 8 of 9 new parity cases, compared as JSON with key order and only `generated_utc` masked; the ninth holds the deliberate fix below. On a synthetic folder with every section it takes 0.01 to 0.03 s and 15 MB, where the Python took 0.15 to 0.21 s and 42 to 44 MB. With a 1,000,000-record VCF it takes 0.44 to 0.62 s and 17 to 20 MB, where the Python took 1.5 s and 42 to 44 MB. What differs from the Python: + - A part of the post pass that fails makes only its own section unreadable, with a note that names the file. The Python ran the post pass outside its per-section error handling, so one failure there lost summary.json and both reports. + - A section reader that stops on an error the Python did not catch keeps the summary too. A telomere table with an infinite read length stopped the Python with an OverflowError and no summary.json; og marks only the telomere section unreadable. - Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error with the path instead of a traceback. Both tables are staged before either is published. A staging failure or a directory destination leaves both previous tables intact. Each file is then replaced by a separate atomic rename. The pair is not a transaction: a failure of the second rename can leave the first table updated. Catalogs and output remain plain TSV; quotes never hide a tab or bypass validation. - The unused Python copy-number engine is removed. Its historical catalog tests materialise a pinned test oracle and keep testing the Rust commands. diff --git a/docs/rust.md b/docs/rust.md index 57471737d..528d0a63e 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -21,7 +21,7 @@ rust/ crates/og-qc/ og qc sex-loss: mosaic loss of Y or X for step 33 crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) - crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect --only a|b|c, QC, clinical and the remaining summary sections (no caller yet); og roh summary (step 11) + crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass, or one group with --only a|b|c (no caller yet); og roh summary (step 11) crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read diff --git a/rust/README.md b/rust/README.md index 449ce5867..14724865c 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 6b4833c7b..82e29d218 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -50,7 +50,7 @@ const USAGE: &str = "usage: og [args] og prs format [args] og prs table [args] og mito annotate|deletions|copies [args] - og summary collect --only {a,b,c} [args] + og summary collect [--only {a,b,c}] [args] og roh summary --sample SAMPLE --roh ROH [--out OUT] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] diff --git a/tests/parity/inputs/summary-collect/oracle.sh b/tests/parity/inputs/summary-collect/oracle.sh index 3e33b6992..7add40e40 100644 --- a/tests/parity/inputs/summary-collect/oracle.sh +++ b/tests/parity/inputs/summary-collect/oracle.sh @@ -6,8 +6,8 @@ # --only is og's alone (the Python writes every section, and the case # compares the group's subtrees). The modules collect_summary imports are # stubbed, except those the group b and c generators stage in pymods/ -# and py/. No other module is reached for these sample folders, except -# run_delta's two readers of history/ and recheck/, which find nothing here. +# and py/. run_delta is stubbed to find no history/ and no recheck/, unless +# py/ brings the real one (the whole-file cases, summary-collect-full.sh). prog=$1 shift d=$(mktemp -d) From 888cd36fcdb34eeba513bb77fbec943e763997e6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:05:22 +0100 Subject: [PATCH 13/19] test(summary): plant four post-pass faults (red control, reverted next) not_run keeps the sections that are no step, a stale VNtyper reading counts as assessed, new_since takes the current run's date, and a panic in a post-pass part is no longer contained. The next commit removes them. Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect/group_c.rs | 3 ++- rust/crates/og-summary/src/collect/post.rs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 29f66b870..dcc7380fd 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -292,7 +292,8 @@ pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { + // Planted fault (red control): the panic is no longer caught. + match Ok::<_, ()>(part()) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), Err(_) => Some(( diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 54c5c36da..faec5a064 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -234,7 +234,7 @@ fn not_run(sections: &[Section]) -> Vec { sections .iter() .filter(|s| matches!(s.state, State::Missing | State::Failed)) - .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) + .filter(|s| !NOT_A_STEP[..0].contains(&s.spec.key)) // planted fault (red control) .map(|s| { if s.state == State::Failed { format!("{} (step {} failed)", s.spec.title, s.spec.step) @@ -279,7 +279,7 @@ fn not_assessed(sections: &[Section]) -> Vec<&'static str> { if state("mei") == State::Missing { out.push(MEI_NOT_ASSESSED); } - let vntyper_read = state("vntyper") == State::Ok + let vntyper_read = matches!(state("vntyper"), State::Ok | State::Stale) && ["positive", "possible", "negative"] .iter() .any(|w| is(field("vntyper", "status"), w)); @@ -317,7 +317,7 @@ fn new_since(dir: &Path, cur: &Value) -> Option { let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; - let previous = match &prev { + let previous = match cur { Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), _ => Value::Null, }; From af005e271db960a44135b5208b8a587b5496e515 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:05:55 +0100 Subject: [PATCH 14/19] test(summary): remove the planted post-pass faults Reverts 888cd36f, the red control of this PR. Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect/group_c.rs | 3 +-- rust/crates/og-summary/src/collect/post.rs | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index dcc7380fd..29f66b870 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -292,8 +292,7 @@ pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - // Planted fault (red control): the panic is no longer caught. - match Ok::<_, ()>(part()) { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), Err(_) => Some(( diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index faec5a064..54c5c36da 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -234,7 +234,7 @@ fn not_run(sections: &[Section]) -> Vec { sections .iter() .filter(|s| matches!(s.state, State::Missing | State::Failed)) - .filter(|s| !NOT_A_STEP[..0].contains(&s.spec.key)) // planted fault (red control) + .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) .map(|s| { if s.state == State::Failed { format!("{} (step {} failed)", s.spec.title, s.spec.step) @@ -279,7 +279,7 @@ fn not_assessed(sections: &[Section]) -> Vec<&'static str> { if state("mei") == State::Missing { out.push(MEI_NOT_ASSESSED); } - let vntyper_read = matches!(state("vntyper"), State::Ok | State::Stale) + let vntyper_read = state("vntyper") == State::Ok && ["positive", "possible", "negative"] .iter() .any(|w| is(field("vntyper", "status"), w)); @@ -317,7 +317,7 @@ fn new_since(dir: &Path, cur: &Value) -> Option { let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; - let previous = match cur { + let previous = match &prev { Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), _ => Value::Null, }; From 7b0a8e735331d3e7e8411ec670bcb9918b38c079 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:23:59 +0100 Subject: [PATCH 15/19] test(parity): name no image in the whole-summary fixture manifests The images check reads any image name in a tracked file as one that bypasses versions.env; the fixture's DEEPVARIANT_IMAGE rows now hold plain version text. Bead: pgp-bs9u.44 --- tests/parity/gen/summary-collect-full.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parity/gen/summary-collect-full.sh b/tests/parity/gen/summary-collect-full.sh index 94d4db167..accdcdf73 100644 --- a/tests/parity/gen/summary-collect-full.sh +++ b/tests/parity/gen/summary-collect-full.sh @@ -64,7 +64,7 @@ mkdir -p empty # so do the inputs of its manifest. A database re-check newer than the # run's manifest. merge history a/full b/cb-full c/full -put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" +put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tv1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" put history/history/20200101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2019-12-01 00:00 UTC", "sections": {}}' put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2021-01-01 00:00 UTC", "sections": { @@ -77,7 +77,7 @@ put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample" "prs": {"title": "PRS", "state": "ok", "data": {"scores": [{"pgs_id": "PGS000001", "condition": "x", "percentile": 10}]}}, "mei": {"title": "Mobile element insertions (MEGAnE)", "state": "ok", "data": {}}}, "cyp2d6": {"consensus": {"result": "*1/*1"}}}\n' -put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" +put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tv1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" mkdir -p history/history/20220101T000000 put history/history/20220101T000000/run_manifest.tsv "run\twritten_utc\t2022-01-01 00:00:00 UTC\n" put history/recheck/state.json '{"checked_utc": "2021-03-05 00:00:00 UTC", "items": [ From 8dd88d6bcc29808a3454e7743d514a8c4965e530 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:46:19 +0100 Subject: [PATCH 16/19] docs(ancestry): give the archaic table's measured cost on the real table Chromosomes 1 to 6 and 21 of the installed table, rebuilt by og from the same public inputs, equal the Python-built table after decompression. The synthetic table figure understated mask memory, so the changelog now gives the measured chromosome 1 cost. --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d74fe2a71..30fe87bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On the real archaic table, `og archaic table` merges chromosome 1 (4.1 million sites) in 18 s and 190 MB where the Python took 76 s and 1.25 GB; bcftools and Picard still take most of each chromosome's setup. On a synthetic input of real size, step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: + - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes (chromosomes 1 to 6 and 21 of the real table, rebuilt from the same public files, are the same after decompression), so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. From a3f08e08baa9a849eac0643faa8ea92f9e1e91f0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:36:49 +0100 Subject: [PATCH 17/19] fix(summary): claim only error containment in the post pass The release profile aborts on a panic, so catch_unwind in contain() caught nothing in the shipped og. contain() now maps the error a part returns, and the docs say the parts read with defaults instead of promising to catch a panic. --- rust/crates/og-summary/src/collect/group_c.rs | 17 ++++++----------- rust/crates/og-summary/src/collect/post.rs | 8 +++++--- .../crates/og-summary/src/collect/post/tests.rs | 17 +++-------------- rust/crates/og-summary/src/lib.rs | 4 ++-- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 29f66b870..d9c8c9506 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -238,8 +238,10 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { /// order, each with its state and data): the repeat readings with the /// traits and CPIC states, each Locityper locus's primary call, and the /// chrY filter of CNVpytor's count. Each part runs on its own: one that -/// fails (an error or a panic) is returned as its section's key and why, -/// and the others still run. +/// returns an error is returned as its section's key and why, and the +/// others still run. The parts read with defaults, so a row of an +/// unexpected shape is no error. A panic is not caught: the release binary +/// aborts on one (`panic = "abort"`). pub fn post( c: &Ctx<'_>, sections: &mut [(Spec, State, Option<&mut Data>)], @@ -287,19 +289,12 @@ pub fn post( } /// Runs one part of the post pass: `None` when it went through, else its -/// section's key and why it failed, for an error and for a panic alike. +/// section's key and the error it returned. pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { - Ok(Ok(())) => None, - Ok(Err(e)) => Some((key, e)), - Err(_) => Some(( - key, - format!("internal error: the post pass of {key} stopped"), - )), - } + part().err().map(|e| (key, e)) } fn json(d: &Data) -> Option<&Value> { diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 54c5c36da..6065d5402 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -13,9 +13,11 @@ //! - `recheck`: the last database re-check of the sample; //! - `new_since`: what changed since the previous run kept in `history/`. //! -//! Each part that changes a section runs on its own: when it fails, only -//! its section becomes `unreadable`, with a note naming the file, and the -//! summary is still written. The Python collector ran this pass outside +//! Each part that changes a section runs on its own: when it returns an +//! error, only its section becomes `unreadable`, with a note naming the +//! file, and the summary is still written. The parts read with defaults, so +//! a row of an unexpected shape is no error; a panic is not caught, as the +//! release binary aborts on one. The Python collector ran this pass outside //! its per-section error handling, so one failure there lost summary.json //! and both reports. A previous run that cannot be read leaves `new_since` //! out, as before; the comparison itself has no error to hide. diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs index bcf38afd5..34d18cc61 100644 --- a/rust/crates/og-summary/src/collect/post/tests.rs +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -203,20 +203,12 @@ fn paraphase_needs_both_hba_genes_called() { #[test] fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { - // An error and a panic alike come back as the section's key and why. + // An error comes back as the section's key and why. assert_eq!(group_c::contain("cnvpytor", || Ok(())), None); assert_eq!( group_c::contain("expansions", || Err("no table".into())), Some(("expansions", "no table".to_owned())) ); - let panicked = group_c::contain("locityper", || panic!("planted")); - assert_eq!( - panicked, - Some(( - "locityper", - "internal error: the post pass of locityper stopped".to_owned() - )) - ); // The section is then unreadable, with no data, and names its file. let mut s = Section::new( GROUP_C @@ -231,13 +223,10 @@ fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { ))), None, ); - s.make_unreadable("internal error: the post pass of locityper stopped"); + s.make_unreadable("no table"); let j = s.to_json(); assert_eq!(word(at(&j, &["state"])), "unreadable"); - assert_eq!( - word(at(&j, &["note"])), - "S_locityper.tsv: internal error: the post pass of locityper stopped" - ); + assert_eq!(word(at(&j, &["note"])), "S_locityper.tsv: no table"); assert_eq!(at(&j, &["source"]), &Value::Null); assert_eq!(at(&j, &["data"]), &Value::Object(indexmap::IndexMap::new())); } diff --git a/rust/crates/og-summary/src/lib.rs b/rust/crates/og-summary/src/lib.rs index ac4040411..4aa53eeca 100644 --- a/rust/crates/og-summary/src/lib.rs +++ b/rust/crates/og-summary/src/lib.rs @@ -7,8 +7,8 @@ //! table from pgsc_calc's output (`prs/_prs_summary.tsv`, //! `ancestry/_ancestry.tsv`). //! - [`mito`]: `og mito annotate|deletions|copies`, step 20's chrM tables. -//! - [`collect`]: `og summary collect`, summary.json, ported one group of -//! sections at a time beside the Python collector (`--only a`). +//! - [`collect`]: `og summary collect`, the whole summary.json, or one +//! group with `--only a|b|c`, beside the Python collector (no caller yet). //! //! Each tool exits 0 when it wrote its table, 1 with `ERROR: ` //! when an input cannot be read or the table cannot be written, and 2 with From fe1542c5908c452e42f8cc9d7a3ab197820a662a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:23:42 +0100 Subject: [PATCH 18/19] fix(ancestry): keep the previous archaic outputs when a run stops, and refuse bad report inputs og archaic genotypes now stages samples.txt, every genotype file and every mask, and publishes them only when the whole run has passed. A chromosome with no row deletes its old genotype file only on success. The VCF order check compares each record with the last record handed to the join, so a skipped record of another chromosome cannot hide an unsorted return. og ancient report stops on a non-finite eigenvalue or person PC, and on an ancient row with fewer PCs than the distance uses. The parity goldens of the four failing genotypes cases now expect the previous outputs to stay (no new file). --- CHANGELOG.md | 5 +- rust/crates/og-ancestry/src/ancient.rs | 120 +++++++++- rust/crates/og-ancestry/src/archaic.rs | 209 +++++++++++++++--- .../bad-code/gt/samples.txt.absent | 0 .../comes-back/gt/chr21.Altai.mask.bed.absent | 0 .../gt/chr21.Denisova.mask.bed.absent | 0 .../gt/chr21.Vindija.mask.bed.absent | 0 .../comes-back/gt/chr21.gt.gz.absent | 0 .../comes-back/gt/chr22.Altai.mask.bed.absent | 0 .../gt/chr22.Denisova.mask.bed.absent | 0 .../gt/chr22.Vindija.mask.bed.absent | 0 .../comes-back/gt/chr22.gt.gz.absent | 0 .../comes-back/gt/samples.txt.absent | 0 .../no-table-chrom/gt/samples.txt.absent | 0 .../no-table-chrom/gt/spans.tsv.absent | 0 .../two-samples/gt/samples.txt.absent | 0 16 files changed, 302 insertions(+), 32 deletions(-) create mode 100644 tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent create mode 100644 tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a01e5d8..e30598b54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,12 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On the real archaic table, `og archaic table` merges chromosome 1 (4.1 million sites) in 18 s and 190 MB where the Python took 76 s and 1.25 GB; bcftools and Picard still take most of each chromosome's setup. On a synthetic input of real size, step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes (chromosomes 1 to 6 and 21 of the real table, rebuilt from the same public files, are the same after decompression), so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. - - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. + - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2, also when a skipped record (an indel, ALT `.`) of another chromosome sits between the two. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. + - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. `og archaic genotypes` stages `samples.txt`, every genotype file and every mask, and publishes them only when the whole run has passed: a run that stops leaves the previous set as it was, where the Python had already rewritten `samples.txt`, earlier chromosomes and, for a chromosome with no row, deleted the old genotype file. The renames are one at a time, so the set is not one transaction. - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. - Malformed input stops with one `ERROR:` line, not a Python traceback. + - `og ancient report` stops when an eigenvalue or the person's PCs in the distance are not finite (NaN or infinity), and when an ancient group's row holds fewer PCs than the distance uses. The Python wrote NaN distances, or ranked that group on fewer PCs. - `sites_genotyped` in step 44's stats still counts a position twice when the AADR holds two SNPs there, as the Python did; the genotypes themselves are right. - CI evidence rejects FASTA offset overflow and offsets outside the file before reading or publishing. Guard tests this in both debug and release builds. Every new dev verb fails on stdout or stderr write or flush errors. A truth VCF header after the first data record fails without replacing the destination. Additional deliberate refusals are documented and tested: zero-byte gzip inputs, zero-width unused FAI rows, non-ASCII or out-of-range annotation limits, non-writable staging parents, and gzip FASTA files with long FNAME headers. Publication replaces a file and leaves other hard links on the old inode. diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs index 6a3b6fc52..3693dec13 100644 --- a/rust/crates/og-ancestry/src/ancient.rs +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -1063,9 +1063,10 @@ fn nearest_populations( let mut rows = Vec::with_capacity(order.len()); for g in order { let members = &groups[g]; - let k = members[0].0.len().min(n_pcs); - let mut centre = Vec::with_capacity(k); - for i in 0..k { + // Every member must hold all n_pcs PCs: a shorter row would rank + // its group on fewer PCs than the person's distance claims. + let mut centre = Vec::with_capacity(n_pcs); + for i in 0..n_pcs { let mut xs = Vec::with_capacity(members.len()); for (pcs, _) in members { xs.push(*pcs.get(i).ok_or_else(|| { @@ -1348,6 +1349,19 @@ fn cmd_report(a: &Args, out: &mut dyn Write) -> Result<()> { let n_pcs = (DISTANCE_PCS as usize) .min(panel.evals.len()) .min(pcs.len()); + // NaN or infinity in the eigenvalues or the person's PCs would make + // every distance and plotted point meaningless: stop instead. + let not_finite = |path: &str, what: &str| { + Fail(format!( + "{path}: {what} holds a value that is not a finite number" + )) + }; + if panel.evals.iter().take(n_pcs).any(|v| !v.is_finite()) { + return Err(not_finite(pp, "the #eigvals: line")); + } + if pcs.iter().take(n_pcs.max(2)).any(|v| !v.is_finite()) { + return Err(not_finite(rp, &format!("the row of {sample}"))); + } let signs = axis_signs(&panel, pp, &run, rp, n_pcs.max(2))?; let person: Vec = signs.iter().zip(pcs).map(|(s, v)| s * v).collect(); let ancients = read_ancients(a.req("--ancients"))?; @@ -1569,4 +1583,104 @@ mod tests { assert_eq!(clean(" a\t b "), "a b"); assert_eq!(clean(" \t "), "."); } + + fn evec(evals: &[f64], rows: &[(&str, &str, &[f64])]) -> Evec { + let mut e = Evec { + evals: evals.to_vec(), + rows: Vec::new(), + index: HashMap::new(), + }; + for (id, label, pcs) in rows { + e.index.insert((*id).to_owned(), e.rows.len()); + e.rows + .push(((*id).to_owned(), (*label).to_owned(), pcs.to_vec())); + } + e + } + + fn ancient(group: &str) -> AncientRow { + AncientRow { + group: group.to_owned(), + date_bp: None, + country: None, + locality: None, + } + } + + #[test] + fn every_ancient_member_carries_the_distance_pcs() { + // A group whose rows hold 2 PCs must not be ranked on 2 of 4. + let panel = evec( + &[1.0; 4], + &[ + ("a1", ANCIENT_LABEL, &[0.0, 0.0]), + ("a2", ANCIENT_LABEL, &[0.0, 0.0, 9.0, 0.0]), + ], + ); + let ancients = vec![ + ("a1".to_owned(), ancient("Short")), + ("a2".to_owned(), ancient("Full")), + ]; + let Err(Fail(msg)) = nearest_populations(&[0.0, 0.0, 10.0, 0.0], &panel, &ancients, 4) + else { + panic!("a group short of PCs was ranked"); + }; + assert!(msg.contains("ancient group Short has no PC3"), "{msg}"); + } + + #[test] + fn a_person_or_eigenvalue_that_is_not_finite_stops_the_report() { + let d = std::env::temp_dir().join(format!("og-ancient-finite-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + let modern: String = (1..=12) + .map(|i| format!("M{i} {}.0 {}.5 {} {} Pop\n", i, 13 - i, i % 3, i % 5)) + .collect(); + let path = |n: &str| d.join(n).to_str().unwrap().to_owned(); + for (person, evals) in [ + ("S1 nan 0.1 0.2 0.3 Person\n", "4 3 2 1"), + ("S1 0.1 0.2 0.3 0.4 Person\n", "4 inf 2 1"), + ] { + std::fs::write( + d.join("panel.evec"), + format!("#eigvals: {evals}\n{modern}A1 1 2 3 4 Ancient\n"), + ) + .unwrap(); + std::fs::write( + d.join("run.evec"), + format!("#eigvals: 4 3 2 1\n{modern}{person}"), + ) + .unwrap(); + std::fs::write(d.join("anc.tsv"), "id\tgroup\nA1\tG\n").unwrap(); + let args: Vec = [ + "report", + "--panel-evec", + &path("panel.evec"), + "--run-evec", + &path("run.evec"), + "--ancients", + &path("anc.tsv"), + "--sample", + "S1", + "--release", + "v1", + "--out-tsv", + &path("o.tsv"), + "--out-info", + &path("o.info"), + "--out-txt", + &path("o.txt"), + "--out-svg", + &path("o.svg"), + ] + .map(str::to_owned) + .to_vec(); + let mut err = Vec::new(); + assert_eq!(cli(&args, &mut Vec::new(), &mut err), 1, "{person}{evals}"); + let err = String::from_utf8(err).unwrap(); + assert!(err.contains("not a finite number"), "{err}"); + assert!(!d.join("o.tsv").exists()); + } + let _ = std::fs::remove_dir_all(&d); + } } diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 4468fc815..6f668b698 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -723,7 +723,9 @@ struct VcfRec { /// The biallelic SNVs (and the other records, as code 9) of a one-sample /// VCF, in file order. A record whose FILTER is not PASS or `.` gives /// code 9 (not read), so a DeepVariant RefCall is neither your allele nor -/// the reference. +/// the reference. Every record is checked against the last record handed +/// out (`previous`), so a record that is skipped cannot hide a chromosome +/// that comes back at a lower position. struct Vcf { text: Text, previous: Option<(String, i64)>, @@ -772,11 +774,11 @@ impl Vcf { self.text.path.display() )); } - self.previous = Some((c[0].to_owned(), pos)); if c[3].chars().count() != 1 || c[4] == "." { continue; } if c[4].contains(',') || c[4].chars().count() != 1 { + self.previous = Some((c[0].to_owned(), pos)); return Ok(Some(VcfRec { chrom: c[0].to_owned(), pos, @@ -796,6 +798,7 @@ impl Vcf { if c[6] != "PASS" && c[6] != "." { code = 9; } + self.previous = Some((c[0].to_owned(), pos)); return Ok(Some(VcfRec { chrom: c[0].to_owned(), pos, @@ -922,7 +925,16 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { samples += &format!("{id}\n"); } head.push('\n'); - put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; + // Every output is staged and published only when the whole run has + // passed, so a run that stops leaves the previous set as it was. + let samples_path = Path::new(out_dir).join("samples.txt"); + let mut samples_file = + AtomicFile::create(&samples_path).map_err(|e| cannot_write(&samples_path, &e))?; + samples_file + .write_all(samples.as_bytes()) + .map_err(|e| cannot_write(&samples_path, &e))?; + let mut staged: Vec<(PathBuf, AtomicFile)> = vec![(samples_path, samples_file)]; + let mut stale: Vec = Vec::new(); let vcf_path = a.req("--vcf"); let mut calls = Calls { @@ -942,7 +954,7 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { done.push(chrom.clone()); let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); if let Some(table) = table { - if let Some(span) = write_chrom( + match write_chrom( &chrom, table, &mut calls, @@ -953,7 +965,11 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { two_n, out_dir, )? { - spans.push(span); + Written::Rows(span, files) => { + spans.push(span); + staged.extend(files); + } + Written::Empty(path) => stale.push(path), } } else { while pending.as_ref().is_some_and(|r| r.chrom == chrom) { @@ -961,12 +977,6 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { } } } - let mut text = - String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); - for (c, f, l, n, carried, dropped) in &spans { - text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); - } - put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; if spans.is_empty() { let names: Vec<&str> = tables.iter().map(|(c, _)| c.as_str()).collect(); let mut sorted = names.clone(); @@ -976,6 +986,25 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { sorted.join(", ") )); } + // Published one rename at a time: the set is not one transaction, and + // a rename that fails part way leaves part of the new set in place. + for (path, file) in staged { + file.commit().map_err(|e| cannot_write(&path, &e))?; + } + for path in stale { + match std::fs::remove_file(&path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => { + return Err(Stop::from(cannot_write(&path, &e))); + } + _ => {} + } + } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; let sites: u64 = spans.iter().map(|s| s.3).sum(); let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); Ok(()) @@ -985,8 +1014,18 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { /// position, sites, sites with your ALT, sites with another allele. type Span = (String, i64, i64, u64, u64, u64); +/// What one chromosome leaves to publish when the whole run has passed. +enum Written { + /// Rows were written: the span row, and the staged genotype file and + /// masks with their final paths. + Rows(Span, Vec<(PathBuf, AtomicFile)>), + /// No row was written: a genotype file of this name from an earlier + /// run is stale and goes. + Empty(PathBuf), +} + /// IBDmix's genotype file of one chromosome, and each archaic genome's -/// mask; the chromosome's span row, or `None` when no row was written. +/// mask, staged under temporary names. #[allow(clippy::too_many_arguments)] fn write_chrom( chrom: &str, @@ -998,7 +1037,7 @@ fn write_chrom( af_col: usize, two_n: f64, out_dir: &str, -) -> Out> { +) -> Out { let num = &chrom[3..]; let path = Path::new(out_dir).join(format!("{chrom}.gt.gz")); let file = AtomicFile::create(&path).map_err(|e| cannot_write(&path, &e))?; @@ -1124,26 +1163,22 @@ fn write_chrom( } if n == 0 { drop(gz); - match std::fs::remove_file(&path) { - Err(e) if e.kind() != io::ErrorKind::NotFound => { - return Err(Stop::from(cannot_write(&path, &e))); - } - _ => {} - } - return Ok(None); + return Ok(Written::Empty(path)); } - gz.finish().and_then(AtomicFile::commit).map_err(werr)?; + let gz = gz.finish().map_err(werr)?; + let mut files = vec![(path, gz)]; // IBDmix's mask (-r): the chromosome as in the genotype file, and a // line covers the positions start + 1 to end, as in a BED. - for (k, mut mask) in masks.into_iter().enumerate() { + for ((k, mut mask), mask_path) in masks.into_iter().enumerate().zip(mask_paths) { if let Some((s, e)) = open_run[k] { - writeln!(mask, "{num}\t{}\t{e}", s - 1) - .map_err(|e| cannot_write(&mask_paths[k], &e))?; + writeln!(mask, "{num}\t{}\t{e}", s - 1).map_err(|e| cannot_write(&mask_path, &e))?; } - mask.commit() - .map_err(|e| cannot_write(&mask_paths[k], &e))?; + files.push((mask_path, mask)); } - Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) + Ok(Written::Rows( + (chrom.to_owned(), first, last, n, carried, dropped), + files, + )) } // --- report (step 43) ------------------------------------------------------------- @@ -1752,6 +1787,126 @@ mod tests { assert!(error.1.contains("not sorted")); } + #[test] + fn an_ignored_record_cannot_hide_a_chromosome_return() { + // chr22 comes back after an ignored chrX record (ALT "."), at a + // lower position: the join would write 0 at chr22:1000 and lose + // the call, so it must stop as unsorted. + let text = "chr22\t3000\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\n\ + chrX\t1\t.\tA\t.\t.\tPASS\t.\tGT\t0/0\n\ + chr22\t1000\t.\tC\tT\t.\tPASS\t.\tGT\t1/1\n"; + let mut vcf = Vcf::new(Text { + path: "sample.vcf".into(), + lines: Lines::new( + Box::new(io::Cursor::new(text.as_bytes().to_vec())), + Utf8::Strict, + ), + line: String::new(), + }); + assert_eq!(vcf.next().unwrap().unwrap().pos, 3000); + let error = vcf.next().err().expect("chr22 goes back to 1000"); + assert_eq!(error.0, 2); + assert!( + error.1.contains("chr22:1000 comes after 3000"), + "{}", + error.1 + ); + } + + /// A scratch folder of its own under the system temp folder. + fn scratch(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-archaic-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(d.join("table")).unwrap(); + d + } + + fn gzip_to(path: &Path, text: &str) { + let mut gz = GzEncoder::new(std::fs::File::create(path).unwrap(), Compression::new(1)); + gz.write_all(text.as_bytes()).unwrap(); + gz.finish().unwrap(); + } + + /// `og archaic genotypes` on the folder's table into `gt`, with a VCF + /// of these lines: the exit code. + fn genotypes(d: &Path, vcf_lines: &str, pseudo: &str) -> u8 { + let vcf = d.join("s.vcf"); + std::fs::write( + &vcf, + format!("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n{vcf_lines}"), + ) + .unwrap(); + let args: Vec = [ + "genotypes", + "--table-dir", + d.join("table").to_str().unwrap(), + "--vcf", + vcf.to_str().unwrap(), + "--pseudo", + pseudo, + "--out-dir", + d.join("gt").to_str().unwrap(), + ] + .map(str::to_owned) + .to_vec(); + cli(&args, &mut Vec::new(), &mut Vec::new()) + } + + fn snapshot(d: &Path) -> Vec<(String, Vec)> { + let mut files: Vec<(String, Vec)> = std::fs::read_dir(d.join("gt")) + .unwrap() + .map(|e| { + let e = e.unwrap(); + ( + e.file_name().into_string().unwrap(), + std::fs::read(e.path()).unwrap(), + ) + }) + .collect(); + files.sort(); + files + } + + #[test] + fn a_failed_run_leaves_every_previous_output() { + let d = scratch("failed-run"); + let head = TABLE_HEADER.join("\t"); + let row = |c: &str, p: i64, r: &str, a: &str| { + format!("{c}\t{p}\t{r}\t{a}\t0\t1\t12\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n") + }; + gzip_to( + &d.join("table/chr21.tsv.gz"), + &format!( + "{head}\n{}{}", + row("chr21", 100, "A", "G"), + row("chr21", 200, "C", "T") + ), + ); + gzip_to( + &d.join("table/chr22.tsv.gz"), + &format!("{head}\n{}", row("chr22", 100, "G", "A")), + ); + let good = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t100\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n"; + assert_eq!(genotypes(&d, good, "2"), 0); + let before = snapshot(&d); + assert!(before.iter().any(|(n, _)| n == "chr22.gt.gz")); + // chr21 is written, then chr22 is found unsorted: nothing changes, + // not samples.txt for the new --pseudo nor chr21's files. + let unsorted = "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t300\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t400\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t100\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n"; + assert_eq!(genotypes(&d, unsorted, "3"), 2); + assert_eq!(snapshot(&d), before); + // No row written on any chromosome (chr22's only site holds + // another allele): the run fails and chr22's file stays. + let other = "chr22\t100\t.\tG\tC\t.\tPASS\t.\tGT\t1/1\n"; + assert_ne!(genotypes(&d, other, "3"), 0); + assert_eq!(snapshot(&d), before); + let _ = std::fs::remove_dir_all(&d); + } + #[test] fn pseudo_columns_carry_the_count() { let p = pseudo_columns(2); diff --git a/tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent b/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb From fa7a6571f57ecb4e8ecc43c93a5b59221e47216a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:41:42 +0100 Subject: [PATCH 19/19] fix(ancestry): remove a stale genotype file only after every new archaic output is in place spans.tsv is now staged with the other outputs, and a chromosome's old genotype file is removed last. A failed publication keeps it. --- rust/crates/og-ancestry/src/archaic.rs | 47 ++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 6f668b698..0dbd7db0c 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -986,8 +986,21 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { sorted.join(", ") )); } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + let spans_path = Path::new(out_dir).join("spans.tsv"); + let mut spans_file = + AtomicFile::create(&spans_path).map_err(|e| cannot_write(&spans_path, &e))?; + spans_file + .write_all(text.as_bytes()) + .map_err(|e| cannot_write(&spans_path, &e))?; + staged.push((spans_path, spans_file)); // Published one rename at a time: the set is not one transaction, and // a rename that fails part way leaves part of the new set in place. + // Stale genotype files go last, once every new file is in place. for (path, file) in staged { file.commit().map_err(|e| cannot_write(&path, &e))?; } @@ -999,12 +1012,6 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { _ => {} } } - let mut text = - String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); - for (c, f, l, n, carried, dropped) in &spans { - text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); - } - put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; let sites: u64 = spans.iter().map(|s| s.3).sum(); let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); Ok(()) @@ -1907,6 +1914,34 @@ mod tests { let _ = std::fs::remove_dir_all(&d); } + #[test] + fn a_failed_publication_keeps_a_stale_genotype_file() { + let d = scratch("failed-publication"); + let head = TABLE_HEADER.join("\t"); + let row = + |c: &str, p: i64| format!("{c}\t{p}\tA\tG\t0\t1\t2\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n"); + gzip_to( + &d.join("table/chr21.tsv.gz"), + &format!("{head}\n{}", row("chr21", 100)), + ); + gzip_to( + &d.join("table/chr22.tsv.gz"), + &format!("{head}\n{}", row("chr22", 100)), + ); + let both = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\n"; + assert_eq!(genotypes(&d, both, "2"), 0); + // spans.tsv cannot be replaced: chr22 now has no row, and its old + // genotype file must outlive the failed publication. + std::fs::remove_file(d.join("gt/spans.tsv")).unwrap(); + std::fs::create_dir_all(d.join("gt/spans.tsv/full")).unwrap(); + let chr21_only = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t1/1\nchr22\t100\t.\tA\tC\t.\tPASS\t.\tGT\t1/1\n"; + assert_ne!(genotypes(&d, chr21_only, "2"), 0); + assert!(d.join("gt/chr22.gt.gz").exists()); + let _ = std::fs::remove_dir_all(&d); + } + #[test] fn pseudo_columns_carry_the_count() { let p = pseudo_columns(2);