From 2c8756a538fccc93e05611c4ab6082c35efb7881 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 16:43:53 -0700 Subject: [PATCH 01/27] style: apply nightly rustfmt to files that had drifted These three were committed unformatted; `task fmt` reformats them on any run. Separated out so they do not obscure the real diffs that follow. --- rust/calib_dash/src/app.rs | 1 - rust/calibrt/src/lib.rs | 4 +--- rust/timscentroid/src/storage.rs | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/rust/calib_dash/src/app.rs b/rust/calib_dash/src/app.rs index 44bba7ae..4bc0b1b0 100644 --- a/rust/calib_dash/src/app.rs +++ b/rust/calib_dash/src/app.rs @@ -729,7 +729,6 @@ impl CalibDash { None => self.app.clear_scrub(), } } - } /// Pauses the batch loop to render one interactive frame and block until the user's diff --git a/rust/calibrt/src/lib.rs b/rust/calibrt/src/lib.rs index d56c1b05..ee48222f 100644 --- a/rust/calibrt/src/lib.rs +++ b/rust/calibrt/src/lib.rs @@ -697,9 +697,7 @@ pub type GridRanges = ((f64, f64), (f64, f64)); /// comes out empty or inverted on either axis is `Err(ZeroRange)` here rather /// than later out of `Grid::new`, so a caller that only wants to know whether /// a grid is configurable never has to build one. -fn point_ranges( - points: impl IntoIterator, -) -> Result { +fn point_ranges(points: impl IntoIterator) -> Result { let mut x = (f64::INFINITY, f64::NEG_INFINITY); let mut y = x; for (px, py) in points { diff --git a/rust/timscentroid/src/storage.rs b/rust/timscentroid/src/storage.rs index d8271a1d..7bf34b2b 100644 --- a/rust/timscentroid/src/storage.rs +++ b/rust/timscentroid/src/storage.rs @@ -49,12 +49,12 @@ //! - **Azure**: `AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_KEY` environment variables use bytes::Bytes; +use object_store::local::LocalFileSystem; +use object_store::path::Path as ObjectPath; use object_store::{ ObjectStore, ObjectStoreExt, }; -use object_store::local::LocalFileSystem; -use object_store::path::Path as ObjectPath; use once_cell::sync::Lazy; use std::path::Path; use std::sync::Arc; From 191a40f8ecdc0d429b2fa149aec0e365677aedcf Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 16:44:07 -0700 Subject: [PATCH 02/27] refactor: move chemistry stack from rustyms to mzcore rustyms (snijderlab) and mzcore (rusteomics) are separate crates, not a rename, so this is a real port: `pro_forma` now takes an explicit `&Ontologies` and returns `(Peptidoform, Vec)`, `SimpleModificationInner::Mass` gained two fields, `ModificationId::id` is private behind an accessor returning `mzcv::AccessionCode`, and `formulas()` moved to the `AmbiguousMolecule` trait. Masses and UNIMOD ids are bit-identical before and after. Ontologies are behind a `OnceLock` reached only from the fallback past the byte-walk fast path. Building them costs ~210 ms / ~200 MB, and a real DIA-NN `.speclib` load never touches it: peak RSS 10.4 MB, zero inits. micromzpaf loses its rustyms dependency entirely. `IonAnnot::from_fragment` and `TryFrom` had no callers -- the crate now needs no chemistry stack at all (serde + thiserror). The two new tests cover ontology name resolution, which had no coverage: the existing differential tests only assert the fast path agrees with the slow one, and never exercise a named mod that requires a lookup. --- Cargo.lock | 208 ++++++++++++++---- Cargo.toml | 3 +- rust/micromzpaf/Cargo.toml | 2 - rust/micromzpaf/src/lib.rs | 64 ------ rust/speclib_build_cli/Cargo.toml | 2 +- rust/speclib_build_cli/src/entry.rs | 9 +- rust/timsseek/Cargo.toml | 3 +- rust/timsseek/src/data_sources/speclib.rs | 4 +- rust/timsseek/src/fragment_mass/averagine.rs | 6 +- .../fragment_mass/elution_group_converter.rs | 32 +-- rust/timsseek/src/lib.rs | 1 + rust/timsseek/src/models/sequence.rs | 122 +++++++--- 12 files changed, 290 insertions(+), 166 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51e2c068..b69058b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -237,7 +237,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -248,7 +248,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -961,7 +961,7 @@ dependencies = [ "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] @@ -1246,6 +1246,15 @@ dependencies = [ "generic-array 0.14.7", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.5.1" @@ -1656,6 +1665,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1678,9 +1693,9 @@ dependencies = [ [[package]] name = "context_error" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7e1b8dc6f4cdc4f6b897d6aa1b7eaec6d95331bdb765d2a51cdd948e157ee0" +checksum = "c9bc4a1eacadc98da5e0c4f02884efd5c5ad4a267e5e6175b3a53a9918ecb458" dependencies = [ "serde", ] @@ -1856,6 +1871,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csscolorparser" version = "0.6.2" @@ -1970,11 +1994,43 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", +] + +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "dispatch" version = "0.2.0" @@ -2368,7 +2424,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3135,7 +3191,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -3214,6 +3270,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -3967,7 +4032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -4019,7 +4084,6 @@ dependencies = [ name = "micromzpaf" version = "0.33.0" dependencies = [ - "rustyms", "serde", "thiserror 2.0.18", ] @@ -4089,6 +4153,41 @@ dependencies = [ "pxfm", ] +[[package]] +name = "mzcore" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca74d59b7c73d9c705a25622c73a186a1bcbf62c0faaf2b97fac8968b4b8d45" +dependencies = [ + "bincode", + "context_error", + "flate2", + "itertools 0.14.0", + "mzcv", + "ordered-float 5.3.0", + "roxmltree", + "serde", + "serde_json", + "sha2 0.11.0", + "thin-vec", + "uom", +] + +[[package]] +name = "mzcv" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97dae61b4d9d35973769f286a138dc2ec55fc57ef06e5692c3ffc771427c5f7" +dependencies = [ + "bincode", + "chrono", + "context_error", + "directories", + "flate2", + "serde", + "sha2 0.11.0", +] + [[package]] name = "mzdata" version = "0.65.4" @@ -4228,7 +4327,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4695,6 +4794,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "orbclient" version = "0.3.51" @@ -5568,6 +5673,17 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + [[package]] name = "regex" version = "1.12.3" @@ -5751,6 +5867,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rstar" version = "0.8.4" @@ -5870,7 +5995,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5928,25 +6053,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "rustyms" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "011d3d672ae44d5e07db0488d855f2b5ed178e3d6bb7ef5b18c6415c20bbd61e" -dependencies = [ - "bincode", - "context_error", - "flate2", - "itertools 0.14.0", - "ordered-float 5.3.0", - "regex", - "serde", - "serde_json", - "similar", - "thin-vec", - "uom", -] - [[package]] name = "ryu" version = "1.0.23" @@ -6114,7 +6220,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -6125,7 +6231,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -6314,7 +6431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6337,8 +6454,8 @@ dependencies = [ "clap", "indicatif", "micromzpaf", + "mzcore", "reqwest", - "rustyms", "serde", "serde_json", "tempfile", @@ -6477,7 +6594,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6499,7 +6616,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6548,7 +6665,7 @@ dependencies = [ "pest", "pest_derive", "phf", - "sha2", + "sha2 0.10.9", "signal-hook", "siphasher", "terminfo", @@ -6826,13 +6943,14 @@ dependencies = [ "forust-ml", "matrixmultiply", "micromzpaf", + "mzcore", + "mzcv", "parquet", "rand 0.9.3", "rayon", "regex", "rmp-serde", "rusqlite", - "rustyms", "serde", "serde_json", "smallvec", @@ -7230,9 +7348,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -7248,7 +7366,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7756,7 +7874,7 @@ checksum = "692daff6d93d94e29e4114544ef6d5c942a7ed998b37abdc19b17136ea428eb7" dependencies = [ "getrandom 0.3.4", "mac_address", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "uuid", ] @@ -7983,7 +8101,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7cd1bee9..fa8cc32e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,8 @@ insta = { version = "1.34.0" } bon = "3.8.1" tinyvec = { features = ["alloc", "serde"], version = "1.10.0" } smallvec = { version = "1.13", features = ["const_generics", "union"] } -rustyms = { version = "0.11.0", default-features = false } +mzcore = { version = "0.2.0" } +mzcv = { version = "0.3.0" } csv = "1.3" tempfile = "3.23.0" diff --git a/rust/micromzpaf/Cargo.toml b/rust/micromzpaf/Cargo.toml index 83c47f4c..34ecc552 100644 --- a/rust/micromzpaf/Cargo.toml +++ b/rust/micromzpaf/Cargo.toml @@ -8,5 +8,3 @@ license.workspace = true serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } -# Workspace-inherited deps -rustyms = { workspace = true } diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index 551e4b24..e412157f 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -29,7 +29,6 @@ //! assert_eq!(ion.get_charge(), 3); //! ``` -use rustyms::fragment::FragmentType; use serde::{ Deserialize, Serialize, @@ -107,18 +106,6 @@ impl IonAnnot { }) } - pub fn from_fragment( - frag: FragmentType, - charge: i8, - isotope: i8, - ) -> Result { - Ok(Self { - series_ordinal: IonSeriesOrdinal::try_from(frag)?, - charge, - isotope, - }) - } - pub fn terminality(&self) -> IonSeriesTerminality { self.series_ordinal.terminality() } @@ -401,57 +388,6 @@ impl FromStr for IonSeriesOrdinal { } } -impl TryFrom for IonSeriesOrdinal { - type Error = IonParsingError; - - fn try_from(value: FragmentType) -> Result { - fn try_convert_ordinal(ordinal: usize, series: char) -> Result { - ordinal - .try_into() - .map_err(|_| IonParsingError::OrdinalOutOfRange { - ordinal: ordinal as i32, - series: Some(series), - }) - } - let tmp = match value { - FragmentType::a(ordinal, _) => IonSeriesOrdinal::a { - ordinal: try_convert_ordinal(ordinal.series_number, 'a')?, - }, - FragmentType::b(ordinal, _) => IonSeriesOrdinal::b { - ordinal: try_convert_ordinal(ordinal.series_number, 'b')?, - }, - FragmentType::c(ordinal, _) => IonSeriesOrdinal::c { - ordinal: try_convert_ordinal(ordinal.series_number, 'c')?, - }, - FragmentType::d(ordinal, _, _, _, _) => IonSeriesOrdinal::d { - ordinal: try_convert_ordinal(ordinal.series_number, 'd')?, - }, - FragmentType::v(ordinal, _, _, _) => IonSeriesOrdinal::v { - ordinal: try_convert_ordinal(ordinal.series_number, 'v')?, - }, - FragmentType::w(ordinal, _, _, _, _) => IonSeriesOrdinal::w { - ordinal: try_convert_ordinal(ordinal.series_number, 'w')?, - }, - FragmentType::x(ordinal, _) => IonSeriesOrdinal::x { - ordinal: try_convert_ordinal(ordinal.series_number, 'x')?, - }, - FragmentType::y(ordinal, _) => IonSeriesOrdinal::y { - ordinal: try_convert_ordinal(ordinal.series_number, 'y')?, - }, - FragmentType::z(ordinal, _) => IonSeriesOrdinal::z { - ordinal: try_convert_ordinal(ordinal.series_number, 'z')?, - }, - FragmentType::Precursor => IonSeriesOrdinal::precursor, - _ => { - return Err(IonParsingError::Custom { - error: format!("Unsupported fragment type: {value:?}"), - }); - } - }; - Ok(tmp) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/rust/speclib_build_cli/Cargo.toml b/rust/speclib_build_cli/Cargo.toml index 456fe6f7..9486fdf8 100644 --- a/rust/speclib_build_cli/Cargo.toml +++ b/rust/speclib_build_cli/Cargo.toml @@ -16,7 +16,7 @@ path = "src/main.rs" timsseek = { path = "../timsseek" } micromzpaf = { path = "../micromzpaf" } tims_stage = { path = "../tims_stage" } -rustyms = { workspace = true } +mzcore = { workspace = true } clap = { workspace = true } serde = { workspace = true } diff --git a/rust/speclib_build_cli/src/entry.rs b/rust/speclib_build_cli/src/entry.rs index e5686aa6..de7f3605 100644 --- a/rust/speclib_build_cli/src/entry.rs +++ b/rust/speclib_build_cli/src/entry.rs @@ -49,12 +49,13 @@ pub fn strip_mods(seq: &str) -> String { use timsseek::models::sequence::normalize_to_proforma; -/// Compute the monoisotopic precursor m/z using rustyms. +/// Compute the monoisotopic precursor m/z using mzcore. /// Input should be the modified sequence (mods included in mass). fn compute_precursor_mz(modified_seq: &str, charge: u8) -> Option { - use rustyms::prelude::*; + use mzcore::prelude::*; let proforma = normalize_to_proforma(modified_seq); - let peptide = Peptidoform::pro_forma(&proforma, None).ok()?; + // `pro_forma` also returns non-fatal warnings; only the peptidoform matters. + let (peptide, _warnings) = Peptidoform::pro_forma(&proforma, timsseek::ontologies()).ok()?; let linear = peptide.as_linear()?; let formulas = linear.formulas(); if formulas.is_empty() { @@ -223,7 +224,7 @@ mod tests { #[test] fn test_precursor_mz_includes_mod_mass() { - // to_proforma converts [U:4] → [UNIMOD:4] before rustyms + // to_proforma converts [U:4] → [UNIMOD:4] before mzcore let mz_unmod = compute_precursor_mz("PEPTCIDEK", 2).unwrap(); let mz_mod = compute_precursor_mz("PEPTC[U:4]IDEK", 2).unwrap(); let diff = mz_mod - mz_unmod; diff --git a/rust/timsseek/Cargo.toml b/rust/timsseek/Cargo.toml index 6834cb80..352e7c69 100644 --- a/rust/timsseek/Cargo.toml +++ b/rust/timsseek/Cargo.toml @@ -23,7 +23,8 @@ array2d = { path = "../array2d" } timsseek_macros = { path = "../timsseek_macros" } # Workspace-inherited deps -rustyms = { workspace = true } +mzcore = { workspace = true } +mzcv = { workspace = true } timsrust = { workspace = true } rusqlite = {workspace = true } serde = { workspace = true } diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 30e76ed0..1acb1b03 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -1199,7 +1199,7 @@ mod tests { /// `parse_sequence(normalize_to_proforma(..))`, sequence-derived features are /// disabled library-wide (`SeqFeatureState::Unavailable`). Here one target /// parses (`PEPTIDEK`) and one is poisoned (`GARBAGE!!!`: the `!` bytes are - /// rejected by both the fast byte-walk parser and the rustyms fallback), so + /// rejected by both the fast byte-walk parser and the mzcore fallback), so /// the gate must report `!parsable_sequences()`. This is the inverse of /// `test_diann_tsv_parsable_gate`, and the only test of the OFF branch after /// the AOS `test_parse_gate_off_on_poisoned_row` was removed in Task 9. @@ -1223,7 +1223,7 @@ mod tests { ), ); // Unparseable modified sequence: `!` is rejected by parse_sequence_fast - // (`_ => return None`) and by the rustyms pro_forma fallback. + // (`_ => return None`) and by the mzcore pro_forma fallback. let poisoned = SerSpeclibElement::new( PrecursorEntry::new("GARBAGE!!!".to_string(), 2, false, 1), ReferenceEG::new( diff --git a/rust/timsseek/src/fragment_mass/averagine.rs b/rust/timsseek/src/fragment_mass/averagine.rs index 53e6fb0d..5904521c 100644 --- a/rust/timsseek/src/fragment_mass/averagine.rs +++ b/rust/timsseek/src/fragment_mass/averagine.rs @@ -74,10 +74,10 @@ mod tests { #[test] fn or_averagine_falls_back_on_nonstandard() { - // `B` (Asx) is genuinely ambiguous between Asp/Asn in rustyms and + // `B` (Asx) is genuinely ambiguous between Asp/Asn in mzcore and // resolves to more than one formula, which is the real trigger for - // the rustyms-backed count path to error today. (`X` was tried first - // but rustyms resolves it to a defined zero-C/S formula rather than + // the mzcore-backed count path to error today. (`X` was tried first + // but mzcore resolves it to a defined zero-C/S formula rather than // erroring, so it does not exercise the fallback.) let (src, env) = isotope_dist_or_averagine("PEPBK", 600.0); assert_eq!(src, IsotopeSource::Averagine); diff --git a/rust/timsseek/src/fragment_mass/elution_group_converter.rs b/rust/timsseek/src/fragment_mass/elution_group_converter.rs index 6ff9a0d9..87354b58 100644 --- a/rust/timsseek/src/fragment_mass/elution_group_converter.rs +++ b/rust/timsseek/src/fragment_mass/elution_group_converter.rs @@ -1,5 +1,6 @@ use crate::isotopes::peptide_isotopes; -use rustyms::prelude::{ +use mzcore::prelude::{ + AmbiguousMolecule, Element, MolecularFormula, Peptidoform, @@ -54,11 +55,11 @@ fn count_carbon_sulphur(form: &MolecularFormula) -> (u16, u16) { } /// In-chain (C, S) atom counts per standard residue, indexed by `byte - b'A'`. -/// `None` = a non-standard code (B/J/O/U/X/Z) — defer to the rustyms path. +/// `None` = a non-standard code (B/J/O/U/X/Z) — defer to the mzcore path. /// /// A residue contributes the same carbon/sulfur as its free amino acid: forming /// a peptide bond removes one water per bond and the terminal water carries -/// neither C nor S, so a bare-sequence sum equals rustyms' formula exactly. +/// neither C nor S, so a bare-sequence sum equals mzcore's formula exactly. const RESIDUE_CS: [Option<(u16, u16)>; 26] = { // Alphabet offset of an uppercase residue byte (as a fn so `b'A'` maps to 0 // without a literal `b'A' - b'A'`, which clippy's eq_op denies). @@ -90,7 +91,7 @@ const RESIDUE_CS: [Option<(u16, u16)>; 26] = { }; /// Fast (C, S) tally over a bare amino-acid sequence via [`RESIDUE_CS`]. -/// `None` on an empty string or any non-standard residue, forcing the rustyms +/// `None` on an empty string or any non-standard residue, forcing the mzcore /// fallback so behavior (including the error path) is preserved. fn count_cs_fast(sequence: &str) -> Option<(u16, u16)> { if sequence.is_empty() { @@ -108,18 +109,19 @@ fn count_cs_fast(sequence: &str) -> Option<(u16, u16)> { } /// (C, S) counts for `sequence` (a bare, mod-stripped peptide on the hot path). -/// Tries the allocation-free table first; defers to the rustyms formula path for +/// Tries the allocation-free table first; defers to the mzcore formula path for /// empty / non-standard input, which stays the authority. pub fn count_carbon_sulphur_in_sequence(sequence: &str) -> Result<(u16, u16), String> { if let Some(cs) = count_cs_fast(sequence) { return Ok(cs); } - count_carbon_sulphur_in_sequence_rustyms(sequence) + count_carbon_sulphur_in_sequence_mzcore(sequence) } -fn count_carbon_sulphur_in_sequence_rustyms(sequence: &str) -> Result<(u16, u16), String> { - let peptide = match Peptidoform::pro_forma(sequence, None) { - Ok(pep) => pep, +fn count_carbon_sulphur_in_sequence_mzcore(sequence: &str) -> Result<(u16, u16), String> { + let peptide = match Peptidoform::pro_forma(sequence, crate::models::sequence::ontologies()) { + // `pro_forma` also yields non-fatal warnings; the formula is all we need. + Ok((pep, _warnings)) => pep, Err(e) => { return Err(format!( "Error parsing peptide sequence {}: {:?}", @@ -151,22 +153,22 @@ mod tests { use super::*; #[test] - fn cs_table_matches_rustyms_per_residue() { - // Every standard residue: the table must equal rustyms' formula count. + fn cs_table_matches_mzcore_per_residue() { + // Every standard residue: the table must equal mzcore's formula count. for &aa in b"ACDEFGHIKLMNPQRSTVWY" { let seq = String::from_utf8(vec![aa, aa, aa]).unwrap(); // e.g. "AAA" let fast = count_cs_fast(&seq).expect("standard residue in table"); - let slow = count_carbon_sulphur_in_sequence_rustyms(&seq) - .unwrap_or_else(|e| panic!("rustyms failed on {seq}: {e}")); + let slow = count_carbon_sulphur_in_sequence_mzcore(&seq) + .unwrap_or_else(|e| panic!("mzcore failed on {seq}: {e}")); assert_eq!(fast, slow, "C/S mismatch for {seq}"); } } #[test] - fn cs_table_matches_rustyms_on_peptides() { + fn cs_table_matches_mzcore_on_peptides() { for seq in ["AAAGAAATHLEVAR", "LEGNSPQGSNQGVK", "MCMCMCK", "PEPTIDEK"] { let fast = count_cs_fast(seq).expect("standard peptide"); - let slow = count_carbon_sulphur_in_sequence_rustyms(seq).unwrap(); + let slow = count_carbon_sulphur_in_sequence_mzcore(seq).unwrap(); assert_eq!(fast, slow, "C/S mismatch for {seq}"); } } diff --git a/rust/timsseek/src/lib.rs b/rust/timsseek/src/lib.rs index 2ea374f6..21221853 100644 --- a/rust/timsseek/src/lib.rs +++ b/rust/timsseek/src/lib.rs @@ -11,6 +11,7 @@ pub mod scoring; pub mod traits; pub mod utils; pub use micromzpaf; +pub use models::sequence::ontologies; pub use data_sources::{ ExpectedIntensity, diff --git a/rust/timsseek/src/models/sequence.rs b/rust/timsseek/src/models/sequence.rs index 827f3209..1b235acc 100644 --- a/rust/timsseek/src/models/sequence.rs +++ b/rust/timsseek/src/models/sequence.rs @@ -5,7 +5,27 @@ use crate::models::decoy::DecoyMarking; use serde::Serialize; use smallvec::SmallVec; -use std::sync::Arc; +use std::sync::{ + Arc, + OnceLock, +}; + +/// Process-wide modification ontologies, built on first use. +/// +/// mzcore requires an explicit `Ontologies` value at every `pro_forma` call +/// (rustyms, its predecessor, used global lazy statics and took `None`). +/// Building it costs ~210 ms and ~200 MB, so it is deliberately behind a +/// `OnceLock` reached ONLY from [`parse_sequence_mzcore`] — the fallback past +/// the byte-walk fast path. A library whose sequences all match the fast +/// grammar never pays it; a real DIA-NN `.speclib` load peaks at ~10 MB and +/// never initializes this. +pub fn ontologies() -> &'static mzcore::ontology::Ontologies { + static ONTOLOGIES: OnceLock = OnceLock::new(); + ONTOLOGIES.get_or_init(|| { + tracing::debug!("initializing mzcore ontologies (first non-fast-path sequence)"); + mzcore::ontology::Ontologies::init_static() + }) +} /// Amino acid stored as alphabet offset `c - b'A'` (0..=25). `u8::MAX` /// means "unrecognized / non-alpha". Unreachable slots in count buffers @@ -158,9 +178,9 @@ impl Serialize for Peptide { /// A hand-rolled byte walk handles the grammar `normalize_to_proforma` actually /// emits (bare residues, `[UNIMOD:n]`, `[+/-mass]`, N-/C-terminal forms). It /// returns `Some` ONLY for inputs it fully recognizes; anything else — named -/// mods, cross-links, unexpected bytes — yields `None` and defers to the rustyms +/// mods, cross-links, unexpected bytes — yields `None` and defers to the mzcore /// parser, which stays the authority for what is valid. So the fast path can -/// never accept something rustyms would reject, with one deliberate exception: +/// never accept something mzcore would reject, with one deliberate exception: /// it does not check that a `UNIMOD:n` id exists in the ontology (a /// syntactically valid id is accepted). Real DIA-NN output only carries real /// ids, so this never triggers in practice. @@ -168,12 +188,12 @@ pub fn parse_sequence(normalized: &str) -> Option { if let Some(parsed) = parse_sequence_fast(normalized) { return Some(parsed); } - parse_sequence_rustyms(normalized) + parse_sequence_mzcore(normalized) } /// Classify one bracket body (`UNIMOD:n` or a signed mass like `+15.995`) into a /// [`Mod`]. `None` for anything else — a named mod, an unsigned number, empty — -/// which forces the rustyms fallback in [`parse_sequence`]. +/// which forces the mzcore fallback in [`parse_sequence`]. fn classify_mod(body: &str) -> Option { let body = body.trim(); if body.len() >= 7 && body[..7].eq_ignore_ascii_case("UNIMOD:") { @@ -186,7 +206,7 @@ fn classify_mod(body: &str) -> Option { } /// Byte-walk parser for the `normalize_to_proforma` output grammar. `None` means -/// "not recognized — defer to rustyms", never "definitively invalid" (that +/// "not recognized — defer to mzcore", never "definitively invalid" (that /// verdict is the fallback's). See [`parse_sequence`] for the contract. fn parse_sequence_fast(s: &str) -> Option { let b = s.as_bytes(); @@ -198,7 +218,7 @@ fn parse_sequence_fast(s: &str) -> Option { if b.first() == Some(&b'[') { let close = i + 1 + b[i + 1..].iter().position(|&c| c == b']')?; // A leading bracket not of the `[..]-` shape is something we do not - // model; let rustyms decide. + // model; let mzcore decide. if b.get(close + 1) != Some(&b'-') { return None; } @@ -240,7 +260,7 @@ fn parse_sequence_fast(s: &str) -> Option { }); i = close + 1; } - _ => return None, // anything unexpected -> rustyms fallback + _ => return None, // anything unexpected -> mzcore fallback } } @@ -250,14 +270,17 @@ fn parse_sequence_fast(s: &str) -> Option { Some(ParsedSequence { residues, mods }) } -/// The rustyms-backed parser. Authoritative fallback for [`parse_sequence`]: +/// The mzcore-backed parser. Authoritative fallback for [`parse_sequence`]: /// validates against the ontology, handles named mods, and rejects non-linear /// peptides. Off the hot path once the fast path covers the common grammar. -fn parse_sequence_rustyms(normalized: &str) -> Option { - use rustyms::prelude::IsAminoAcid; - use rustyms::sequence::Peptidoform; +/// +/// `pro_forma` also returns non-fatal parse warnings; they are dropped, since +/// this function's contract is a binary parsed/not-parsed verdict. +fn parse_sequence_mzcore(normalized: &str) -> Option { + use mzcore::prelude::IsAminoAcid; + use mzcore::sequence::Peptidoform; - let pf = Peptidoform::pro_forma(normalized, None).ok()?; + let (pf, _warnings) = Peptidoform::pro_forma(normalized, ontologies()).ok()?; let linear = pf.into_linear()?; let mut residues: SmallVec<[AminoAcid; 32]> = SmallVec::new(); @@ -294,9 +317,9 @@ fn parse_sequence_rustyms(normalized: &str) -> Option { Some(ParsedSequence { residues, mods }) } -fn modification_to_mod(m: &rustyms::sequence::Modification) -> Option { - use rustyms::ontology::Ontology; - use rustyms::sequence::{ +fn modification_to_mod(m: &mzcore::sequence::Modification) -> Option { + use mzcore::ontology::Ontology; + use mzcore::sequence::{ Modification, SimpleModificationInner, }; @@ -305,12 +328,18 @@ fn modification_to_mod(m: &rustyms::sequence::Modification) -> Option { _ => return None, // Cross-link / ambiguous — out of v1 scope }; match simple.as_ref() { - SimpleModificationInner::Mass(mass) => Some(Mod::Mass(mass.value as f32)), + // mzcore carries the mass tag and the source digit count alongside the + // mass itself; only the mass matters here. + SimpleModificationInner::Mass(_tag, mass, _digits) => Some(Mod::Mass(mass.value as f32)), SimpleModificationInner::Database { id, .. } => { - if id.ontology == Ontology::Unimod { - Some(Mod::Unimod(id.id? as u16)) - } else { - None + if id.ontology != Ontology::Unimod { + return None; + } + // UNIMOD accessions are numeric; a non-numeric CURIE is not + // something `Mod::Unimod(u16)` can represent. + match id.id() { + mzcv::AccessionCode::Numeric(n) => u16::try_from(n).ok().map(Mod::Unimod), + _ => None, } } _ => None, @@ -385,7 +414,7 @@ fn convert_paren_unimod(s: &str) -> String { out } -/// Coerce DIA-NN / short-form modified-sequence strings into rustyms-parseable +/// Coerce DIA-NN / short-form modified-sequence strings into mzcore-parseable /// ProForma. Strips `_..._` wrapping used by DIA-NN, converts DIA-NN's /// parenthesised mods (`C(UniMod:4)`) to ProForma brackets (`C[UNIMOD:4]`), and /// normalizes UNIMOD tag casing (`[UniMod:`, `[Unimod:`, `[U:` → `[UNIMOD:`). @@ -623,7 +652,7 @@ mod tests { #[test] fn fast_path_takes_recognized_grammar() { // Bare and numeric-UNIMOD/mass inputs must be served by the fast path - // (never reach rustyms), else there's no speedup. + // (never reach mzcore), else there's no speedup. for s in [ "PEPTIDEK", "AAC[UNIMOD:4]DEK", @@ -642,7 +671,7 @@ mod tests { #[test] fn fast_path_defers_named_and_garbage() { // Named mods and unexpected bytes must defer (fast returns None) so the - // rustyms authority decides validity + resolves the name. + // mzcore authority decides validity + resolves the name. for s in [ "[Acetyl]-PEPTIDEK", "C[Carbamidomethyl (C)]PEPK", @@ -655,10 +684,47 @@ mod tests { } } + /// The fallback must not just accept a named mod — it must resolve it + /// through the UNIMOD ontology to the same numeric id the `[UNIMOD:n]` + /// spelling yields. This is the one behavior that has no fast-path + /// equivalent, so nothing else covers it; it is also the only test that + /// forces `ontologies()` to actually initialize. + #[test] + fn mzcore_fallback_resolves_named_mods_via_ontology() { + for (named, expected) in [ + ("[Acetyl]-PEPTIDEK", Mod::Unimod(1)), + ("PEPTC[Carbamidomethyl]IDEK", Mod::Unimod(4)), + ("PEPTM[Oxidation]IDEK", Mod::Unimod(35)), + ] { + let parsed = parse_sequence(named) + .unwrap_or_else(|| panic!("mzcore must resolve named mod in {named:?}")); + assert_eq!( + parsed.mods.len(), + 1, + "expected exactly one mod in {named:?}, got {:?}", + parsed.mods + ); + assert_eq!( + parsed.mods[0].kind, expected, + "ontology resolved {named:?} to the wrong id" + ); + } + } + + /// A named mod and its numeric spelling must land on the same + /// `ParsedSequence` — including residues, so the ontology path cannot + /// quietly disagree with the fast path about the peptide itself. + #[test] + fn named_and_numeric_mod_spellings_agree() { + let named = parse_sequence("PEPTC[Carbamidomethyl]IDEK").expect("named form parses"); + let numeric = parse_sequence("PEPTC[UNIMOD:4]IDEK").expect("numeric form parses"); + assert_eq!(named, numeric); + } + #[test] - fn fast_matches_rustyms_on_recognized_grammar() { + fn fast_matches_mzcore_on_recognized_grammar() { // Differential test: wherever the fast path claims an input, it must - // produce the exact same ParsedSequence rustyms would. Guards against + // produce the exact same ParsedSequence mzcore would. Guards against // the fast path silently diverging on residue counts or mod mapping. let corpus = [ "PEPTIDEK", @@ -671,8 +737,8 @@ mod tests { ]; for s in corpus { if let Some(fast) = parse_sequence_fast(s) { - let slow = parse_sequence_rustyms(s) - .unwrap_or_else(|| panic!("rustyms must also parse {s:?}")); + let slow = parse_sequence_mzcore(s) + .unwrap_or_else(|| panic!("mzcore must also parse {s:?}")); assert_eq!(fast.residues, slow.residues, "residues mismatch for {s:?}"); assert_eq!( fast.mods.len(), From 20d93d32afaf88e993441b0915781d03c5478f32 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 16:44:20 -0700 Subject: [PATCH 03/27] fix: stop unknown-ion labels from silently colliding Fragment labels must be unique within a precursor -- `linear_get` is first-match, so a duplicate hides data rather than erroring (`ExpectedIntensities::try_from_pairs`). Unknown ions are distinguished only by an ordinal used as a counter, and every site that advanced that counter could wrap: - diann_io (TSV + parquet) and spectronaut_io used `+= 1` on an inferred `u8`: panics in debug, wraps in release past 255. - skyline_io used `saturating_add`, which pins every ordinal past the limit to 255 -- the same collision, failing more quietly. - `try_fill_labels_u8` built labels with `(0..n).map(|i| i as u8)`, which wraps at 256 into `0,1,2,..`. This one is live: it is how JSON inputs without explicit labels get labelled. All four now go through a `checked_add` helper that fails at the row, where the index is still in hand, instead of downstream with a duplicate-key error pointing at the symptom. Also fixes `try_fill_labels_annot` building placeholders at isotope 1, which would have marked every synthesized fragment as the M+1 peak. That path has no callers today. The capacity test is at exactly 256 rather than near it: the first attempt at the fix used `(0..num_fragments as u8)`, which is an *empty* range at 256, and only a test at the boundary catches that. --- rust/timsquery/src/serde/diann_io.rs | 31 +++++- .../src/serde/elution_group_inputs.rs | 105 +++++++++++++++++- rust/timsquery/src/serde/skyline_io.rs | 23 +++- rust/timsquery/src/serde/spectronaut_io.rs | 22 +++- 4 files changed, 170 insertions(+), 11 deletions(-) diff --git a/rust/timsquery/src/serde/diann_io.rs b/rust/timsquery/src/serde/diann_io.rs index 291a04b4..3d81d55d 100644 --- a/rust/timsquery/src/serde/diann_io.rs +++ b/rust/timsquery/src/serde/diann_io.rs @@ -61,6 +61,23 @@ impl From for DiannPrecursorParsingError { } } +/// Advance the per-precursor `?`-ordinal counter, refusing to wrap. +/// +/// `IonAnnot` ordinals are `u8`, so a precursor can carry at most +/// [`u8::MAX`] distinguishable unknown ions. Past that there is no way to keep +/// labels unique, and a silently reused ordinal corrupts scoring +/// (`linear_get` is first-match). Fail here, where the row index is still in +/// hand, rather than downstream in `try_from_pairs`. +fn next_unknown_ordinal(current: u8) -> Result { + current.checked_add(1).ok_or_else(|| { + error!( + "More than {} unknown-ion fragments in a single precursor; cannot assign unique labels", + u8::MAX + ); + DiannPrecursorParsingError::IonOverCapacity + }) +} + impl From for DiannReadingError { fn from(_err: DiannPrecursorParsingError) -> Self { DiannReadingError::DiannPrecursorParsingError @@ -344,7 +361,12 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(rows.len()); - let mut num_unknown_losses = 0; + // `?` labels are distinguished only by their ordinal, used here as a + // per-precursor counter. That counter IS what upholds the per-precursor + // label-uniqueness invariant (see `ExpectedIntensities::try_from_pairs`): + // wrapping past `u8::MAX` would re-emit `?1` and fail the load much later + // with a duplicate-key error pointing at the symptom, not at this row. + let mut num_unknown_losses: u8 = 0; for (i, row) in rows.iter().enumerate() { let fragment_mz = row.fragment_mz; @@ -364,7 +386,7 @@ fn parse_precursor_group( row.fragment_loss_type, i ); - num_unknown_losses += 1; + num_unknown_losses = next_unknown_ordinal(num_unknown_losses)?; let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); @@ -658,7 +680,8 @@ fn parse_precursor_group_from_parquet( let mut fragment_mzs = Vec::with_capacity(indices.len()); buffers.fragment_labels.clear(); let mut rel_intensities = Vec::with_capacity(indices.len()); - let mut num_unknown_losses = 0; + // Per-precursor `?` counter — see `next_unknown_ordinal`. + let mut num_unknown_losses: u8 = 0; for (i, &idx) in indices.iter().enumerate() { let fragment_mz = columns.product_mzs[idx] as f64; @@ -680,7 +703,7 @@ fn parse_precursor_group_from_parquet( columns.fragment_loss_types[idx], i ); - num_unknown_losses += 1; + num_unknown_losses = next_unknown_ordinal(num_unknown_losses)?; let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); diff --git a/rust/timsquery/src/serde/elution_group_inputs.rs b/rust/timsquery/src/serde/elution_group_inputs.rs index 05142133..f109c7f4 100644 --- a/rust/timsquery/src/serde/elution_group_inputs.rs +++ b/rust/timsquery/src/serde/elution_group_inputs.rs @@ -10,10 +10,19 @@ use crate::{ #[derive(Debug)] pub enum ElutionGroupInputError { - MismatchedFragmentLabelsLength { expected: usize, found: usize }, + MismatchedFragmentLabelsLength { + expected: usize, + found: usize, + }, AlreadyHasFragmentLabels, - IonConversionError { inner: String }, + IonConversionError { + inner: String, + }, MissingFragmentLabels, + /// More fragments than a `u8` label space can index uniquely. + TooManyFragmentsToLabel { + count: usize, + }, } /// User-friendly format for specifying elution groups in an input file @@ -39,11 +48,26 @@ impl ElutionGroupInput { self.fragment_labels.is_none() } + /// Synthesize positional `u8` labels for an input that shipped none. + /// + /// Labels must be unique within the elution group (see + /// `ExpectedIntensities::try_from_pairs`), and `u8` can only index 256 of + /// them. `i as u8` would wrap silently past that — emitting `0,1,..,255,0,1,..` + /// and failing far downstream with a duplicate-key error — so the overflow + /// is rejected here instead. pub fn try_fill_labels_u8(self) -> Result, ElutionGroupInputError> { let num_fragments = self.fragments.len(); if self.fragment_labels.is_some() { return Err(ElutionGroupInputError::AlreadyHasFragmentLabels); } + if num_fragments > u8::MAX as usize + 1 { + return Err(ElutionGroupInputError::TooManyFragmentsToLabel { + count: num_fragments, + }); + } + // Iterate in `usize` and narrow per element: `0..(num_fragments as u8)` + // would be an EMPTY range at exactly 256, since `256 as u8` is 0. The + // guard above is what makes the narrowing lossless. let fragment_labels: Vec = (0..num_fragments).map(|i| i as u8).collect(); Ok(ElutionGroupInput { @@ -66,8 +90,14 @@ impl ElutionGroupInput { .fragment_labels .unwrap() .into_iter() - .map(|lbl| IonAnnot::try_new('?', Some(lbl), 1, 1).unwrap()) - .collect(); + // isotope 0 — these are monoisotopic placeholders. Every other `?` + // construction in the readers uses 0; the previous `1` here would + // have labelled every synthesized fragment as the M+1 peak. + .map(|lbl| IonAnnot::try_new('?', Some(lbl), 1, 0)) + .collect::, _>>() + .map_err(|e| ElutionGroupInputError::IonConversionError { + inner: format!("{e:?}"), + })?; Ok(ElutionGroupInput { id: tmp.id, mobility: tmp.mobility, @@ -124,3 +154,70 @@ impl + KeyLike> TryFrom> for Tims Ok(builder.try_build().expect("I checked the sizes!")) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn input_with_n_fragments(n: usize) -> ElutionGroupInput { + ElutionGroupInput { + id: 0, + mobility: 0.8, + rt_seconds: 100.0, + precursor: 500.0, + precursor_charge: 2, + precursor_isotopes: None, + fragments: vec![100.0; n], + fragment_labels: None, + } + } + + /// Synthesized labels must be unique — that is the invariant + /// `ExpectedIntensities::try_from_pairs` relies on. 256 fragments is the + /// exact capacity of the `u8` label space. + #[test] + fn fill_labels_u8_is_unique_at_capacity() { + let filled = input_with_n_fragments(256) + .try_fill_labels_u8() + .expect("256 fragments fit the u8 label space"); + let labels = filled.fragment_labels.unwrap(); + assert_eq!(labels.len(), 256); + let unique: std::collections::HashSet<_> = labels.iter().copied().collect(); + assert_eq!(unique.len(), 256, "synthesized labels must all be distinct"); + } + + /// One past capacity previously wrapped (`i as u8`), silently emitting a + /// second `0` label and corrupting scoring downstream. It must now fail. + #[test] + fn fill_labels_u8_rejects_overflow_instead_of_wrapping() { + let err = input_with_n_fragments(257) + .try_fill_labels_u8() + .expect_err("257 fragments cannot be labelled uniquely with a u8"); + assert!( + matches!( + err, + ElutionGroupInputError::TooManyFragmentsToLabel { count: 257 } + ), + "expected TooManyFragmentsToLabel, got {err:?}" + ); + } + + /// Synthesized `?` annotations are monoisotopic placeholders; a nonzero + /// isotope would mislabel every fragment as the M+1 peak. + #[test] + fn fill_labels_annot_uses_monoisotopic_placeholders() { + let filled = input_with_n_fragments(3) + .try_fill_labels_annot() + .expect("3 fragments label fine"); + let labels = filled.fragment_labels.unwrap(); + let expected: Vec = (0u8..3) + .map(|i| IonAnnot::try_new('?', Some(i), 1, 0).unwrap()) + .collect(); + assert_eq!( + labels, expected, + "synthesized placeholders must be `?` at isotope 0, not M+1" + ); + let unique: std::collections::HashSet<_> = labels.iter().copied().collect(); + assert_eq!(unique.len(), 3, "synthesized labels must all be distinct"); + } +} diff --git a/rust/timsquery/src/serde/skyline_io.rs b/rust/timsquery/src/serde/skyline_io.rs index 1f42c43e..b5c13a29 100644 --- a/rust/timsquery/src/serde/skyline_io.rs +++ b/rust/timsquery/src/serde/skyline_io.rs @@ -59,6 +59,27 @@ impl From for SkylinePrecursorParsingError { } } +/// Advance the per-precursor `?`-ordinal counter, refusing to wrap. +/// +/// `IonAnnot` ordinals are `u8`, so a precursor can carry at most +/// [`u8::MAX`] distinguishable unknown ions. Past that there is no way to keep +/// labels unique, and a silently reused ordinal corrupts scoring +/// (`linear_get` is first-match). Fail here, where the row index is still in +/// hand, rather than downstream in `try_from_pairs`. +/// +/// This previously used `saturating_add`, which does not wrap but still pins +/// every ordinal past the limit to [`u8::MAX`] — producing the same duplicate +/// labels, just more quietly. +fn next_unknown_ordinal(current: u8) -> Result { + current.checked_add(1).ok_or_else(|| { + error!( + "More than {} unknown-ion fragments in a single precursor; cannot assign unique labels", + u8::MAX + ); + SkylinePrecursorParsingError::IonOverCapacity + }) +} + impl From for SkylineReadingError { fn from(_err: SkylinePrecursorParsingError) -> Self { SkylineReadingError::SkylinePrecursorParsingError @@ -351,7 +372,7 @@ fn parse_precursor_group( falling back to unknown ion", frag_type, i ); - num_unknown_losses = num_unknown_losses.saturating_add(1); + num_unknown_losses = next_unknown_ordinal(num_unknown_losses)?; IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)? } }; diff --git a/rust/timsquery/src/serde/spectronaut_io.rs b/rust/timsquery/src/serde/spectronaut_io.rs index f5037486..d1d389d5 100644 --- a/rust/timsquery/src/serde/spectronaut_io.rs +++ b/rust/timsquery/src/serde/spectronaut_io.rs @@ -59,6 +59,23 @@ impl From for SpectronautPrecursorParsingError { } } +/// Advance the per-precursor `?`-ordinal counter, refusing to wrap. +/// +/// `IonAnnot` ordinals are `u8`, so a precursor can carry at most +/// [`u8::MAX`] distinguishable unknown ions. Past that there is no way to keep +/// labels unique, and a silently reused ordinal corrupts scoring +/// (`linear_get` is first-match). Fail here, where the row index is still in +/// hand, rather than downstream in `try_from_pairs`. +fn next_unknown_ordinal(current: u8) -> Result { + current.checked_add(1).ok_or_else(|| { + error!( + "More than {} unknown-ion fragments in a single precursor; cannot assign unique labels", + u8::MAX + ); + SpectronautPrecursorParsingError::IonOverCapacity + }) +} + impl From for SpectronautReadingError { fn from(_err: SpectronautPrecursorParsingError) -> Self { SpectronautReadingError::SpectronautPrecursorParsingError @@ -286,7 +303,8 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(included_rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(included_rows.len()); - let mut num_unknown_losses = 0; + // Per-precursor `?` counter — see `next_unknown_ordinal`. + let mut num_unknown_losses: u8 = 0; for (i, row) in included_rows.iter().enumerate() { let fragment_mz = row.fragment_mz; @@ -306,7 +324,7 @@ fn parse_precursor_group( row.fragment_loss_type, i ); - num_unknown_losses += 1; + num_unknown_losses = next_unknown_ordinal(num_unknown_losses)?; let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); From d96c0ac6456e41eb55b281a759d484038cfbb134 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 16:50:39 -0700 Subject: [PATCH 04/27] feat(micromzpaf): add composition-keyed neutral losses Libraries spell the same loss differently. Two real collisions in the HUPO-PSI corpus: -CH3SOH (NIST) == -CH4OS (SpectraST) C1H4O1S1 -NH2-CO-CH2SH (NIST) == -C2H5NOS (SpectraST) C2H5N1O1S1 Keying on the string would give `y5-CH4OS` and `y5-CH3SOH` two distinct labels for one ion -- the wrong direction, since fragment labels must be unique within a precursor and a spurious second label hides a real duplicate. So parsing goes text -> composition -> discriminant. Summing terms also collapses `H2O-NH3`/`NH3-H2O` and `2H2O`/`H2O-H2O` for free. Composition is parse-time only; an `IonAnnot` will store the discriminant, so there is no per-annotation cost. The vocabulary is scoped to what the supported inputs actually emit -- DIA-NN none, Spectronaut two, NIST eight -- plus the phospho losses, which no corpus measurement could have surfaced because every PSI example file is non-phospho. SpectraST's wider combinatorics are deliberately absent. `from_expression` distinguishes Ok(None) "valid composition, not in the table" from Err "not a loss expression": callers route the first to an unknown label and the second to a parse failure. Display emits a canonical spelling, so `-CH3SOH` renders as `-CH4OS`. Byte-identical round-trip is therefore not a property of this type and tests compare parsed values. --- rust/micromzpaf/src/lib.rs | 6 + rust/micromzpaf/src/loss.rs | 431 ++++++++++++++++++++++++++++++++++++ 2 files changed, 437 insertions(+) create mode 100644 rust/micromzpaf/src/loss.rs diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index e412157f..36bd18cc 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -29,6 +29,12 @@ //! assert_eq!(ion.get_charge(), 3); //! ``` +pub mod loss; + +pub use loss::{ + Composition, + NeutralLoss, +}; use serde::{ Deserialize, Serialize, diff --git a/rust/micromzpaf/src/loss.rs b/rust/micromzpaf/src/loss.rs new file mode 100644 index 00000000..c5ef5d31 --- /dev/null +++ b/rust/micromzpaf/src/loss.rs @@ -0,0 +1,431 @@ +//! Neutral losses, keyed by atomic composition rather than by spelling. +//! +//! Libraries write the same chemical loss different ways. Two real examples +//! from the HUPO-PSI mzSpecLib corpus: +//! +//! | written | library | composition | +//! |---|---|---| +//! | `-CH3SOH` | NIST | C₁H₄O₁S₁ | +//! | `-CH4OS` | SpectraST | C₁H₄O₁S₁ | +//! | `-NH2-CO-CH2SH` | NIST | C₂H₅N₁O₁S₁ | +//! | `-C2H5NOS` | SpectraST | C₂H₅N₁O₁S₁ | +//! +//! Keying on the string would make `y5-CH4OS` and `y5-CH3SOH` distinct labels +//! for one ion. Since fragment labels must be unique within a precursor (see +//! `ExpectedIntensities::try_from_pairs` in timsseek), that is exactly the +//! wrong direction: it hides a genuine duplicate behind two spellings. +//! +//! So parsing goes `text -> composition -> discriminant`, and the composition +//! is a *parse-time* concept only. What gets stored on an `IonAnnot` is the +//! discriminant, so there is no per-annotation cost at runtime. + +use std::fmt::Display; + +use crate::IonParsingError; + +/// Atom counts for the elements that appear in peptide neutral losses. +/// +/// Deliberately not a general chemical formula: these losses only ever draw +/// from C/H/N/O/S/P, and keeping it to six `u8`s makes equality a single +/// 6-byte compare during the parse-time table lookup. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Composition { + pub c: u8, + pub h: u8, + pub n: u8, + pub o: u8, + pub s: u8, + pub p: u8, +} + +impl Composition { + pub const fn new(c: u8, h: u8, n: u8, o: u8, s: u8, p: u8) -> Self { + Self { c, h, n, o, s, p } + } + + /// Multiply every count, saturating. Used for the `2H2O` multiplier form. + fn scaled(self, k: u8) -> Self { + Self { + c: self.c.saturating_mul(k), + h: self.h.saturating_mul(k), + n: self.n.saturating_mul(k), + o: self.o.saturating_mul(k), + s: self.s.saturating_mul(k), + p: self.p.saturating_mul(k), + } + } + + fn plus(self, o: Self) -> Self { + Self { + c: self.c.saturating_add(o.c), + h: self.h.saturating_add(o.h), + n: self.n.saturating_add(o.n), + o: self.o.saturating_add(o.o), + s: self.s.saturating_add(o.s), + p: self.p.saturating_add(o.p), + } + } + + /// Parse a bare formula like `H2O`, `CH4OS`, `C2H5NOS`. + /// + /// Only single-letter C/H/N/O/S/P are recognized; anything else is an + /// error rather than a silent skip, so an unsupported loss surfaces as + /// "not representable" instead of being mistaken for a smaller one. + fn parse_formula(s: &str) -> Result { + if s.is_empty() { + return Err(IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Empty neutral-loss formula"), + }); + } + let mut out = Composition::default(); + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() { + let elem = b[i]; + i += 1; + let start = i; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + let count: u8 = if start == i { + 1 + } else { + s[start..i] + .parse() + .map_err(|_| IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Neutral-loss atom count out of range"), + })? + }; + let slot = match elem { + b'C' => &mut out.c, + b'H' => &mut out.h, + b'N' => &mut out.n, + b'O' => &mut out.o, + b'S' => &mut out.s, + b'P' => &mut out.p, + _ => { + return Err(IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Unsupported element in neutral loss"), + }); + } + }; + *slot = slot.saturating_add(count); + } + Ok(out) + } + + /// Parse a full loss expression: `-` separated terms, each optionally + /// prefixed by a repeat count. `2H2O`, `H2O-NH3`, `NH2-CO-CH2SH`. + /// + /// Because terms are summed, ordering and multiplier spelling collapse for + /// free: `H2O-NH3` == `NH3-H2O`, and `2H2O` == `H2O-H2O`. + pub fn parse_expression(s: &str) -> Result { + let mut total = Composition::default(); + for term in s.split('-') { + let term = term.trim(); + if term.is_empty() { + return Err(IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Empty term in neutral-loss expression"), + }); + } + // Leading digits are a repeat count for the whole term. + let digits = term.len() - term.trim_start_matches(|c: char| c.is_ascii_digit()).len(); + let (mult, formula) = if digits > 0 { + let m: u8 = term[..digits] + .parse() + .map_err(|_| IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Neutral-loss repeat count out of range"), + })?; + (m, &term[digits..]) + } else { + (1, term) + }; + total = total.plus(Composition::parse_formula(formula)?.scaled(mult)); + } + Ok(total) + } +} + +/// The neutral losses this crate can represent, as a packed discriminant. +/// +/// Scoped deliberately: DIA-NN emits none, Spectronaut two (`-H2O`, `-NH3`), +/// NIST eight, plus the phospho losses that no non-phospho corpus can show. +/// SpectraST's wider combinatorics are NOT here — an unlisted loss parses to a +/// composition that misses the table and is reported as unrepresentable, which +/// routes the peak to an unknown label rather than silently mislabelling it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +#[repr(u8)] +pub enum NeutralLoss { + #[default] + None = 0, + /// H₂O + Water = 1, + /// NH₃ + Ammonia = 2, + /// CO + CarbonMonoxide = 3, + /// CO₂ + CarbonDioxide = 4, + /// 2 H₂O + WaterX2 = 5, + /// 2 NH₃ + AmmoniaX2 = 6, + /// H₂O + NH₃ + WaterAmmonia = 7, + /// CH₄OS — methanesulfenic acid, off oxidized Met. Also spelled `CH3SOH`. + Methanesulfenic = 8, + /// C₂H₅NOS — also spelled `NH2-CO-CH2SH`. + Carbamidomethylthiol = 9, + /// H₃PO₄ — phospho-Ser/Thr. + PhosphoricAcid = 10, + /// HPO₃ — phospho-Tyr, and phospho-Ser/Thr. + Metaphosphoric = 11, + /// H₃PO₄ + H₂O + PhosphoricAcidWater = 12, +} + +/// `(composition, discriminant, canonical spelling)`. +/// +/// The canonical spelling is what `Display` emits, so a non-canonical input +/// (`-CH3SOH`) round-trips to the canonical form (`-CH4OS`). Round-trip tests +/// must therefore compare parsed values, not bytes. +const TABLE: &[(Composition, NeutralLoss, &str)] = &[ + ( + Composition::new(0, 2, 0, 1, 0, 0), + NeutralLoss::Water, + "H2O", + ), + ( + Composition::new(0, 3, 1, 0, 0, 0), + NeutralLoss::Ammonia, + "NH3", + ), + ( + Composition::new(1, 0, 0, 1, 0, 0), + NeutralLoss::CarbonMonoxide, + "CO", + ), + ( + Composition::new(1, 0, 0, 2, 0, 0), + NeutralLoss::CarbonDioxide, + "CO2", + ), + ( + Composition::new(0, 4, 0, 2, 0, 0), + NeutralLoss::WaterX2, + "2H2O", + ), + ( + Composition::new(0, 6, 2, 0, 0, 0), + NeutralLoss::AmmoniaX2, + "2NH3", + ), + ( + Composition::new(0, 5, 1, 1, 0, 0), + NeutralLoss::WaterAmmonia, + "H2O-NH3", + ), + ( + Composition::new(1, 4, 0, 1, 1, 0), + NeutralLoss::Methanesulfenic, + "CH4OS", + ), + ( + Composition::new(2, 5, 1, 1, 1, 0), + NeutralLoss::Carbamidomethylthiol, + "C2H5NOS", + ), + ( + Composition::new(0, 3, 0, 4, 0, 1), + NeutralLoss::PhosphoricAcid, + "H3PO4", + ), + ( + Composition::new(0, 1, 0, 3, 0, 1), + NeutralLoss::Metaphosphoric, + "HPO3", + ), + ( + Composition::new(0, 5, 0, 5, 0, 1), + NeutralLoss::PhosphoricAcidWater, + "H3PO4-H2O", + ), +]; + +impl NeutralLoss { + /// Resolve a loss expression (without the leading `-`) to a discriminant. + /// + /// `None` means "parsed as a valid composition, but not one we represent" — + /// distinct from `Err`, which means the text was not a loss expression at + /// all. Callers route the former to an unknown label and the latter to a + /// parse failure. + pub fn from_expression(s: &str) -> Result, IonParsingError> { + let comp = Composition::parse_expression(s)?; + Ok(TABLE + .iter() + .find(|(c, _, _)| *c == comp) + .map(|(_, l, _)| *l)) + } + + /// The composition this loss removes. + pub fn composition(self) -> Composition { + if self == NeutralLoss::None { + return Composition::default(); + } + TABLE + .iter() + .find(|(_, l, _)| *l == self) + .map(|(c, _, _)| *c) + .unwrap_or_default() + } + + /// Canonical spelling, without the leading `-`. Empty for [`Self::None`]. + pub fn canonical(self) -> &'static str { + if self == NeutralLoss::None { + return ""; + } + TABLE + .iter() + .find(|(_, l, _)| *l == self) + .map(|(_, _, s)| *s) + .unwrap_or("") + } +} + +impl Display for NeutralLoss { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if *self == NeutralLoss::None { + return Ok(()); + } + write!(f, "-{}", self.canonical()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn formula_parses_counts_and_implicit_ones() { + assert_eq!( + Composition::parse_expression("H2O").unwrap(), + Composition::new(0, 2, 0, 1, 0, 0) + ); + assert_eq!( + Composition::parse_expression("NH3").unwrap(), + Composition::new(0, 3, 1, 0, 0, 0) + ); + assert_eq!( + Composition::parse_expression("C2H5NOS").unwrap(), + Composition::new(2, 5, 1, 1, 1, 0) + ); + } + + /// The two cross-library spelling collisions this module exists for. + #[test] + fn different_spellings_resolve_to_one_loss() { + // NIST vs SpectraST, methanesulfenic acid. + assert_eq!( + NeutralLoss::from_expression("CH3SOH").unwrap(), + Some(NeutralLoss::Methanesulfenic) + ); + assert_eq!( + NeutralLoss::from_expression("CH4OS").unwrap(), + Some(NeutralLoss::Methanesulfenic) + ); + // NIST structural notation vs SpectraST molecular notation. + assert_eq!( + NeutralLoss::from_expression("NH2-CO-CH2SH").unwrap(), + Some(NeutralLoss::Carbamidomethylthiol) + ); + assert_eq!( + NeutralLoss::from_expression("C2H5NOS").unwrap(), + Some(NeutralLoss::Carbamidomethylthiol) + ); + } + + /// Summing terms collapses ordering and multiplier spelling for free. + #[test] + fn ordering_and_multipliers_normalize() { + assert_eq!( + NeutralLoss::from_expression("H2O-NH3").unwrap(), + NeutralLoss::from_expression("NH3-H2O").unwrap() + ); + assert_eq!( + NeutralLoss::from_expression("2H2O").unwrap(), + NeutralLoss::from_expression("H2O-H2O").unwrap() + ); + assert_eq!( + NeutralLoss::from_expression("2H2O").unwrap(), + Some(NeutralLoss::WaterX2) + ); + } + + #[test] + fn phospho_losses_resolve() { + assert_eq!( + NeutralLoss::from_expression("H3PO4").unwrap(), + Some(NeutralLoss::PhosphoricAcid) + ); + assert_eq!( + NeutralLoss::from_expression("HPO3").unwrap(), + Some(NeutralLoss::Metaphosphoric) + ); + assert_eq!( + NeutralLoss::from_expression("H3PO4-H2O").unwrap(), + Some(NeutralLoss::PhosphoricAcidWater) + ); + } + + /// A well-formed composition outside the table is `Ok(None)` — "valid but + /// not representable" — while malformed text is `Err`. Callers need to + /// tell those apart to route one to an unknown label and the other to a + /// parse failure. + #[test] + fn unrepresentable_is_distinct_from_malformed() { + assert_eq!(NeutralLoss::from_expression("HCOOH").unwrap(), None); + assert!(NeutralLoss::from_expression("Xe2").is_err()); + assert!(NeutralLoss::from_expression("").is_err()); + assert!(NeutralLoss::from_expression("H2O-").is_err()); + } + + /// Every table entry must survive canonical -> composition -> discriminant. + #[test] + fn table_round_trips_through_canonical_spelling() { + for (comp, loss, canon) in TABLE { + assert_eq!( + NeutralLoss::from_expression(canon).unwrap(), + Some(*loss), + "canonical spelling {canon} must resolve to its own loss" + ); + assert_eq!(loss.composition(), *comp); + assert_eq!(loss.canonical(), *canon); + } + } + + /// Compositions must be unique: two entries sharing one would make the + /// table lookup order-dependent. + #[test] + fn table_compositions_are_unique() { + for (i, (a, _, sa)) in TABLE.iter().enumerate() { + for (b, _, sb) in TABLE.iter().skip(i + 1) { + assert_ne!(a, b, "{sa} and {sb} share a composition"); + } + } + } + + #[test] + fn display_uses_canonical_spelling() { + assert_eq!(NeutralLoss::None.to_string(), ""); + assert_eq!(NeutralLoss::Water.to_string(), "-H2O"); + // Non-canonical input renders canonically; byte-identical round-trip + // is intentionally not a property of this type. + let parsed = NeutralLoss::from_expression("CH3SOH").unwrap().unwrap(); + assert_eq!(parsed.to_string(), "-CH4OS"); + } +} From 188c78090f25e0e63805246e89b98b614a9d4b87 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 16:56:11 -0700 Subject: [PATCH 05/27] perf(micromzpaf)!: pack IonAnnot into a u32, add losses and internal fragments A spectral library carries one annotation per fragment, so this type is replicated millions of times and compared on the scoring hot path. Packing it into a single word makes equality one compare instead of an enum discriminant match plus payload: linear_get over 13 fragments, worst case struct of fields (before) 16.3 ns packed u32 (after) 5.5 ns The alternative -- adding a `loss` field to the existing struct -- costs more than the byte it looks like. `IonAnnot` is align-1 but pairs with an `f32` in `ExpectedIntensities`, so a 5-byte key pads to a 12-byte tuple and the inline TinyVec storage goes 104 -> 156 bytes. Packed stays at 8 and 104. Layout is 30 of 32 bits: kind 4, charge 4 (zigzag), isotope 4 (zigzag), loss 6, payload 12. The payload is a tagged union -- an 8-bit ordinal for backbone ions, two 6-bit endpoints for internal fragments, a 5-bit residue for immonium. Bit 30 is reserved for a future registry-backed loss so an esoteric addition does not force another layout change. BREAKING (in-workspace only): - charge is now +-7 and isotope +-7, down from the full i8. Observed maxima across the HUPO-PSI corpus are 3 and 3, with no negative charges. Bit fields truncate rather than wrapping loudly, so every constructor range-checks and `every_field_round_trips_at_its_extremes` exercises the boundaries -- that is the test that catches a missing check. - Ord is now by packed word, not field-by-field. Only tests sorted these. - Display is no longer byte-inverse to parsing: a non-canonical loss spelling renders canonically, so `y5-CH3SOH` and `y5-CH4OS` are one annotation. That is the point -- two labels for one ion would fake a duplicate and defeat per-precursor label uniqueness. Also adds the mzPAF mass-error suffix (`/-0.0005`, `/1.2ppm`). mzSpecLib peak lists carry observed m/z, so a reader needs the error to recover theoretical. `TryFrom<&str>` discards it; `parse_mzpaf` keeps it. Unrepresentable annotations fail rather than degrade: an unlisted loss, and modified immonium (`IC[Carbamidomethyl]`, which carries an arbitrary mod string). Parsing `y1-HCOOH` as plain `y1` would put a loss peak's m/z on the y1 label and collide with the real y1. --- rust/micromzpaf/src/lib.rs | 978 +++++++++++++++++++++++++++++------- rust/micromzpaf/src/loss.rs | 30 +- 2 files changed, 808 insertions(+), 200 deletions(-) diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index 36bd18cc..0f46e6a9 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -1,19 +1,52 @@ //! Compact representation of fragment ion annotations for mass spectrometry. //! -//! This crate provides a minimal, memory-efficient encoding of fragment ion annotations -//! that pack ion series, charge state, ordinal position, and isotope offset into a -//! compact structure suitable for high-throughput proteomics workflows. +//! A spectral library carries one annotation per fragment, so this type is +//! replicated millions of times in a loaded arena and compared on the scoring +//! hot path. It is therefore a packed `u32` rather than a struct of fields: +//! equality is a single word compare, and `(IonAnnot, f32)` stays 8 bytes with +//! no padding. //! -//! # mzPAF Format Compliance +//! # Bit layout //! -//! This implementation supports a subset of the mzPAF annotation format: -//! - Common ion series (a, b, c, d, v, w, x, y, z, precursor) -//! - Charge states (^N notation) -//! - Positive isotope offsets (+Ni notation) +//! ```text +//! bit: 31 30 29 18 17 12 11 8 7 4 3 0 +//! ┌──┬──┬────────────────┬──────────┬────────┬────────┬───────┐ +//! │▒▒│R │ payload │ loss │isotope │ charge │ kind │ +//! │1b│1b│ 12b │ 6b │ 4b zz │ 4b zz │ 4b │ +//! └──┴──┴────────────────┴──────────┴────────┴────────┴───────┘ +//! ``` +//! +//! `payload` is reinterpreted per `kind` — a tagged union inside the word: +//! +//! | kind | payload | +//! |---|---| +//! | backbone (a/b/c/d/v/w/x/y/z), unknown | ordinal, 8b (0..=255) | +//! | internal | start 6b │ end 6b (peptides to 63 residues) | +//! | immonium | residue index 5b | +//! | precursor | unused | +//! +//! `charge` and `isotope` are zigzag-encoded so they stay signed in 4 bits. +//! Their ranges (±7) are far wider than anything observed: the HUPO-PSI corpus +//! tops out at charge 3 and isotope 3, with no negative charges at all. Because +//! the field truncates rather than wrapping loudly, every constructor +//! range-checks — see [`IonAnnot::try_new`]. +//! +//! Bit 30 is reserved: when set, `loss` would be an index into a growable +//! registry instead of a [`NeutralLoss`] discriminant. Nothing sets it today +//! and constructors assert it stays clear; it exists so an esoteric loss can be +//! added later without another layout change. +//! +//! # mzPAF compliance //! -//! NOTABLY boes not support: -//! - Negative isotope offsets (not yet implemented) -//! - Complex neutral losses or modifications +//! Supported: the a/b/c/d/v/w/x/y/z series, precursor (`p`), unknown (`?`), +//! internal fragments (`m:`), bare immonium (`IA`), charge (`^N`), +//! positive isotopes (`+Ni`), neutral losses from [`NeutralLoss`], and the +//! mass-error suffix (`/-0.0003`, `/1.2ppm`). +//! +//! Not supported: negative isotope offsets, modified immonium +//! (`IC[Carbamidomethyl]` carries an arbitrary mod string), and losses outside +//! the [`NeutralLoss`] table. These are reported as errors, never coerced into +//! a nearby representable ion. //! //! # Examples //! @@ -44,34 +77,147 @@ use std::hash::Hash; use std::str::FromStr; use thiserror::Error; +// ── Bit layout ─────────────────────────────────────────────────────────────── + +const KIND_SHIFT: u32 = 0; +const KIND_BITS: u32 = 4; +const CHARGE_SHIFT: u32 = 4; +const CHARGE_BITS: u32 = 4; +const ISOTOPE_SHIFT: u32 = 8; +const ISOTOPE_BITS: u32 = 4; +const LOSS_SHIFT: u32 = 12; +const LOSS_BITS: u32 = 6; +const PAYLOAD_SHIFT: u32 = 18; +const PAYLOAD_BITS: u32 = 12; +/// Reserved: set would mean `loss` is a registry index. Always clear today. +const REGISTRY_BIT: u32 = 1 << 30; + +/// Widest charge the 4-bit zigzag field holds. Observed maximum is 3. +pub const CHARGE_MIN: i8 = -7; +pub const CHARGE_MAX: i8 = 7; +/// Widest isotope offset the 4-bit zigzag field holds. Observed maximum is 3. +pub const ISOTOPE_MIN: i8 = -7; +pub const ISOTOPE_MAX: i8 = 7; +/// Widest residue index an internal fragment endpoint holds (6 bits). +pub const INTERNAL_POS_MAX: u8 = 63; + +#[inline] +const fn mask(bits: u32) -> u32 { + (1u32 << bits) - 1 +} +/// Zigzag: map a small signed value onto an unsigned one without losing the +/// sign bit to the field width. +#[inline] +const fn zigzag(v: i8) -> u32 { + (((v as i32) << 1) ^ ((v as i32) >> 31)) as u32 +} +#[inline] +const fn unzigzag(u: u32) -> i8 { + (((u >> 1) as i32) ^ -((u & 1) as i32)) as i8 +} + +/// Discriminants for the `kind` field. Not public: the public view is +/// [`IonSeriesOrdinal`], which fuses kind with its payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum Kind { + None = 0, + A = 1, + B = 2, + C = 3, + D = 4, + V = 5, + W = 6, + X = 7, + Y = 8, + Z = 9, + Precursor = 10, + Unknown = 11, + Internal = 12, + Immonium = 13, +} + +impl Kind { + const fn from_raw(v: u32) -> Kind { + match v { + 1 => Kind::A, + 2 => Kind::B, + 3 => Kind::C, + 4 => Kind::D, + 5 => Kind::V, + 6 => Kind::W, + 7 => Kind::X, + 8 => Kind::Y, + 9 => Kind::Z, + 10 => Kind::Precursor, + 11 => Kind::Unknown, + 12 => Kind::Internal, + 13 => Kind::Immonium, + _ => Kind::None, + } + } + + const fn series_char(self) -> char { + match self { + Kind::A => 'a', + Kind::B => 'b', + Kind::C => 'c', + Kind::D => 'd', + Kind::V => 'v', + Kind::W => 'w', + Kind::X => 'x', + Kind::Y => 'y', + Kind::Z => 'z', + Kind::Precursor => 'p', + Kind::Unknown => '?', + Kind::Internal => 'm', + Kind::Immonium => 'I', + Kind::None => '\0', + } + } + + const fn from_series_char(c: char) -> Option { + match c { + 'a' => Some(Kind::A), + 'b' => Some(Kind::B), + 'c' => Some(Kind::C), + 'd' => Some(Kind::D), + 'v' => Some(Kind::V), + 'w' => Some(Kind::W), + 'x' => Some(Kind::X), + 'y' => Some(Kind::Y), + 'z' => Some(Kind::Z), + 'p' => Some(Kind::Precursor), + '?' => Some(Kind::Unknown), + _ => None, + } + } + + /// Does this kind carry an 8-bit ordinal in its payload? + const fn has_ordinal(self) -> bool { + matches!( + self, + Kind::A + | Kind::B + | Kind::C + | Kind::D + | Kind::V + | Kind::W + | Kind::X + | Kind::Y + | Kind::Z + | Kind::Unknown + ) + } +} + /// Compact representation of fragment annotations. /// -/// This is a very compressed representation of a fragment -/// ion annotation. Essentially we are packing in 32 bytes -/// the ion series (b, y, ...), charge (+1 / -1 ...), -/// ordinal (12 in the ion series) and isotope. -/// -/// It is not meant to represent all possible ions but rather have -/// a very compact representation of the common ones. -/// -/// # Invariants -/// -/// - **charge**: Must be non-zero (±1 to ±127). Zero charge is invalid and rejected by constructors. -/// - **ordinal**: Limited to u8 range (1-255). Peptides with >255 residues cannot be represented. -/// - **isotope**: Isotope offset relative to monoisotopic peak (M+0), range -128 to +127. -/// -/// # Memory Layout -/// -/// The struct fits in 32 bytes with optimal packing: -/// - `IonSeriesOrdinal`: 2 bytes (enum discriminant + u8 ordinal) -/// - `charge`: 1 byte (i8) -/// - `isotope`: 1 byte (i8) +/// A packed `u32`; see the crate docs for the bit layout. Ordering is by the +/// packed word rather than field-by-field — nothing depends on the previous +/// ordering (only tests sorted these), but it is not the same order. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] -pub struct IonAnnot { - series_ordinal: IonSeriesOrdinal, - charge: i8, - isotope: i8, -} +pub struct IonAnnot(u32); impl Serialize for IonAnnot { fn serialize(&self, serializer: S) -> Result @@ -88,6 +234,9 @@ impl Serialize for IonAnnot { /// b12+i^3 -> b12 charge 3 isotope 1 /// b12+3i^3 -> b12 charge 3 isotope 2 /// b13 -> b13 (implicit charge 1 and isotope 0) +/// +/// The wire format is the mzPAF string, not the packed word, so the bit layout +/// can change without breaking existing files. impl<'de> Deserialize<'de> for IonAnnot { fn deserialize(deserializer: D) -> Result where @@ -99,54 +248,325 @@ impl<'de> Deserialize<'de> for IonAnnot { } impl IonAnnot { + /// Build a backbone / precursor / unknown annotation. + /// + /// Errors when `charge` is zero or either of `charge`/`isotope` falls + /// outside the packed field's range. That range check is load-bearing: the + /// bit field truncates silently, so an unchecked value would corrupt the + /// annotation rather than fail. pub fn try_new( ion_type: char, ordinal: Option, charge: i8, isotope: i8, ) -> Result { - Ok(Self { - series_ordinal: IonSeriesOrdinal::try_new(ion_type, ordinal)?, - charge, - isotope, - }) + Self::try_new_with_loss(ion_type, ordinal, charge, isotope, NeutralLoss::None) } - pub fn terminality(&self) -> IonSeriesTerminality { - self.series_ordinal.terminality() + /// As [`Self::try_new`], carrying a neutral loss. + pub fn try_new_with_loss( + ion_type: char, + ordinal: Option, + charge: i8, + isotope: i8, + loss: NeutralLoss, + ) -> Result { + let kind = + Kind::from_series_char(ion_type).ok_or(IonParsingError::UnsupportedFragmentType { + fragment_type: ion_type, + })?; + let payload = match (kind, ordinal) { + (k, Some(o)) if k.has_ordinal() => o as u32, + (Kind::Precursor, None) => 0, + (Kind::Precursor, Some(o)) => { + return Err(IonParsingError::OrdinalOutOfRange { + ordinal: o as i32, + series: Some(ion_type), + }); + } + (_, None) => { + return Err(IonParsingError::OrdinalOutOfRange { + ordinal: -1, + series: Some(ion_type), + }); + } + (_, Some(o)) => { + return Err(IonParsingError::OrdinalOutOfRange { + ordinal: o as i32, + series: Some(ion_type), + }); + } + }; + Self::pack(kind, payload, loss, charge, isotope) } - pub fn try_with_offset_neutrons(&self, offset_neutrons: i8) -> Result { - let new_isotope = - self.isotope - .checked_add(offset_neutrons) - .ok_or_else(|| IonParsingError::Custom { - error: format!( - "Isotope offset overflow: {} + {} exceeds i8 range", - self.isotope, offset_neutrons - ), - })?; + /// Build an internal fragment spanning residues `start..=end`. + /// + /// Endpoints are capped at [`INTERNAL_POS_MAX`], narrower than a backbone + /// ordinal, because two of them share the 12-bit payload. Internal + /// fragments are bounded by peptide length, so 63 is well past tryptic. + pub fn try_new_internal( + start: u8, + end: u8, + charge: i8, + isotope: i8, + loss: NeutralLoss, + ) -> Result { + if start > INTERNAL_POS_MAX || end > INTERNAL_POS_MAX { + return Err(IonParsingError::OrdinalOutOfRange { + ordinal: start.max(end) as i32, + series: Some('m'), + }); + } + let payload = (start as u32) | ((end as u32) << 6); + Self::pack(Kind::Internal, payload, loss, charge, isotope) + } - Ok(Self { - series_ordinal: self.series_ordinal, - charge: self.charge, - isotope: new_isotope, - }) + /// Build a bare immonium ion for an uppercase residue code. + /// + /// Modified immonium (`IC[Carbamidomethyl]`) is not representable at any + /// field width and is rejected by the parser rather than silently losing + /// the modification. + pub fn try_new_immonium( + residue: char, + charge: i8, + isotope: i8, + loss: NeutralLoss, + ) -> Result { + if !residue.is_ascii_uppercase() { + return Err(IonParsingError::UnsupportedFragmentType { + fragment_type: residue, + }); + } + let payload = (residue as u8 - b'A') as u32; + Self::pack(Kind::Immonium, payload, loss, charge, isotope) } + fn pack( + kind: Kind, + payload: u32, + loss: NeutralLoss, + charge: i8, + isotope: i8, + ) -> Result { + if charge == 0 { + return Err(IonParsingError::ParsingError { + error: format!("{}", kind.series_char()), + context: Some("Charge cannot be 0"), + }); + } + if !(CHARGE_MIN..=CHARGE_MAX).contains(&charge) { + return Err(IonParsingError::ChargeOutOfRange { charge }); + } + if !(ISOTOPE_MIN..=ISOTOPE_MAX).contains(&isotope) { + return Err(IonParsingError::IsotopeOutOfRange { isotope }); + } + debug_assert!(payload <= mask(PAYLOAD_BITS), "payload overflows its field"); + debug_assert!( + (loss as u32) <= mask(LOSS_BITS), + "loss discriminant overflows its field" + ); + Ok(IonAnnot( + ((kind as u32) << KIND_SHIFT) + | ((zigzag(charge) & mask(CHARGE_BITS)) << CHARGE_SHIFT) + | ((zigzag(isotope) & mask(ISOTOPE_BITS)) << ISOTOPE_SHIFT) + | ((loss as u32 & mask(LOSS_BITS)) << LOSS_SHIFT) + | ((payload & mask(PAYLOAD_BITS)) << PAYLOAD_SHIFT), + )) + } + + #[inline] + fn kind(self) -> Kind { + Kind::from_raw((self.0 >> KIND_SHIFT) & mask(KIND_BITS)) + } + + #[inline] + fn payload(self) -> u32 { + (self.0 >> PAYLOAD_SHIFT) & mask(PAYLOAD_BITS) + } + + #[inline] pub fn get_charge(&self) -> i8 { - self.charge + unzigzag((self.0 >> CHARGE_SHIFT) & mask(CHARGE_BITS)) + } + + #[inline] + pub fn get_isotope(&self) -> i8 { + unzigzag((self.0 >> ISOTOPE_SHIFT) & mask(ISOTOPE_BITS)) + } + + /// The neutral loss this ion carries, [`NeutralLoss::None`] if any. + #[inline] + pub fn loss(&self) -> NeutralLoss { + debug_assert!( + self.0 & REGISTRY_BIT == 0, + "registry-backed losses are reserved but not implemented" + ); + match (self.0 >> LOSS_SHIFT) & mask(LOSS_BITS) { + 0 => NeutralLoss::None, + 1 => NeutralLoss::Water, + 2 => NeutralLoss::Ammonia, + 3 => NeutralLoss::CarbonMonoxide, + 4 => NeutralLoss::CarbonDioxide, + 5 => NeutralLoss::WaterX2, + 6 => NeutralLoss::AmmoniaX2, + 7 => NeutralLoss::WaterAmmonia, + 8 => NeutralLoss::Methanesulfenic, + 9 => NeutralLoss::Carbamidomethylthiol, + 10 => NeutralLoss::PhosphoricAcid, + 11 => NeutralLoss::Metaphosphoric, + 12 => NeutralLoss::PhosphoricAcidWater, + _ => NeutralLoss::None, + } + } + + pub fn terminality(&self) -> IonSeriesTerminality { + match self.kind() { + Kind::A | Kind::B | Kind::C | Kind::D => IonSeriesTerminality::NTerm, + Kind::V | Kind::W | Kind::X | Kind::Y | Kind::Z => IonSeriesTerminality::CTerm, + _ => IonSeriesTerminality::None, + } + } + + /// Shift the isotope by `offset_neutrons`. + /// + /// Errors when the result leaves [`ISOTOPE_MIN`]..=[`ISOTOPE_MAX`]. That + /// bound is narrower than the `i8` this used to hold, but an order of + /// magnitude past any observed isotope offset. + pub fn try_with_offset_neutrons(&self, offset_neutrons: i8) -> Result { + let new_isotope = self + .get_isotope() + .checked_add(offset_neutrons) + .ok_or_else(|| IonParsingError::Custom { + error: format!( + "Isotope offset overflow: {} + {} exceeds i8 range", + self.get_isotope(), + offset_neutrons + ), + })?; + if !(ISOTOPE_MIN..=ISOTOPE_MAX).contains(&new_isotope) { + return Err(IonParsingError::IsotopeOutOfRange { + isotope: new_isotope, + }); + } + Ok(IonAnnot( + (self.0 & !(mask(ISOTOPE_BITS) << ISOTOPE_SHIFT)) + | ((zigzag(new_isotope) & mask(ISOTOPE_BITS)) << ISOTOPE_SHIFT), + )) } + /// The series ordinal, for the kinds that have one. + /// + /// `None` for precursor, unknown, internal and immonium: `?1` is a + /// uniqueness counter, not a position in a ladder. pub fn try_get_ordinal(&self) -> Option { - self.series_ordinal.try_get_ordinal() + let k = self.kind(); + if k.has_ordinal() && k != Kind::Unknown { + Some(self.payload() as u8) + } else { + None + } + } + + /// The logical series-and-payload view of this annotation. + pub fn series_ordinal(&self) -> IonSeriesOrdinal { + let k = self.kind(); + let p = self.payload(); + match k { + Kind::A => IonSeriesOrdinal::a { ordinal: p as u8 }, + Kind::B => IonSeriesOrdinal::b { ordinal: p as u8 }, + Kind::C => IonSeriesOrdinal::c { ordinal: p as u8 }, + Kind::D => IonSeriesOrdinal::d { ordinal: p as u8 }, + Kind::V => IonSeriesOrdinal::v { ordinal: p as u8 }, + Kind::W => IonSeriesOrdinal::w { ordinal: p as u8 }, + Kind::X => IonSeriesOrdinal::x { ordinal: p as u8 }, + Kind::Y => IonSeriesOrdinal::y { ordinal: p as u8 }, + Kind::Z => IonSeriesOrdinal::z { ordinal: p as u8 }, + Kind::Unknown => IonSeriesOrdinal::unknown { ordinal: p as u8 }, + Kind::Precursor => IonSeriesOrdinal::precursor, + Kind::Internal => IonSeriesOrdinal::internal { + start: (p & mask(6)) as u8, + end: ((p >> 6) & mask(6)) as u8, + }, + Kind::Immonium => IonSeriesOrdinal::immonium { + residue: (b'A' + (p & mask(5)) as u8) as char, + }, + Kind::None => IonSeriesOrdinal::None, + } } } -impl TryFrom<&str> for IonAnnot { - type Error = IonParsingError; +/// A mass-error suffix on an mzPAF annotation: observed minus theoretical. +/// +/// mzSpecLib peak lists carry *observed* m/z, so the theoretical value a +/// library reader wants is `observed - error`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum MassError { + /// Absolute, in daltons. + Da(f64), + /// Relative, in parts per million. + Ppm(f64), +} - fn try_from(value: &str) -> Result { +impl MassError { + /// Recover the theoretical m/z from the observed one. + pub fn theoretical_from_observed(&self, observed: f64) -> f64 { + match self { + MassError::Da(d) => observed - d, + MassError::Ppm(p) => observed / (1.0 + p * 1e-6), + } + } +} + +/// One parsed mzPAF annotation plus its optional mass-error suffix. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ParsedAnnotation { + pub ion: IonAnnot, + pub mass_error: Option, +} + +/// Split the trailing `/[ppm]` off an annotation, if present. +/// +/// Care is needed because `/` does not otherwise appear, but the numeric part +/// may be signed and the `ppm` suffix optional. +fn split_mass_error(s: &str) -> Result<(&str, Option), IonParsingError> { + let Some((head, tail)) = s.rsplit_once('/') else { + return Ok((s, None)); + }; + let (num, is_ppm) = match tail.strip_suffix("ppm") { + Some(n) => (n, true), + None => (tail, false), + }; + let v: f64 = num.parse().map_err(|_| IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Unable to parse the mass-error suffix"), + })?; + Ok(( + head, + Some(if is_ppm { + MassError::Ppm(v) + } else { + MassError::Da(v) + }), + )) +} + +impl IonAnnot { + /// Parse a full mzPAF annotation, including any mass-error suffix. + /// + /// A comma-separated list of alternatives is NOT handled here — that is + /// ambiguity, and resolving it needs the caller's policy. Split on `,` and + /// parse each alternative. + pub fn parse_mzpaf(value: &str) -> Result { + let (rest, mass_error) = split_mass_error(value)?; + Ok(ParsedAnnotation { + ion: Self::parse_ion(rest)?, + mass_error, + }) + } + + fn parse_ion(value: &str) -> Result { + // charge: trailing ^N let (rest, charge) = match value.split_once('^') { Some((rest, charge)) => { let charge = charge @@ -160,20 +580,15 @@ impl TryFrom<&str> for IonAnnot { None => (value, 1), }; - // TODO: Implement 'negative isotopes' parsing ... right now I dont use them - // for serialization ... - // Note that this is not 100% compliant with mzPAF + // isotope: +Ni. Negative isotope offsets are not supported. let (rest, isotope) = match rest.split_once('+') { Some((rest, adducts)) => { - // Make sure the adduct is only +{number}?i let adducts = adducts .strip_suffix('i') .ok_or(IonParsingError::ParsingError { error: adducts.to_string(), context: Some("Unsupported adduct found"), })?; - // If its empty its an implicit 1 isotope - // Since we stripped the 'i' from '+i' let isotope = if adducts.is_empty() { 1 } else { @@ -188,33 +603,83 @@ impl TryFrom<&str> for IonAnnot { } None => (rest, 0), }; - let series_ord = IonSeriesOrdinal::from_str(rest)?; - if charge == 0 { - return Err(IonParsingError::ParsingError { + + // neutral loss: everything from the first '-'. Internal fragments use + // 'm:' and never contain '-' before the loss, so splitting + // at the first '-' is unambiguous. + let (core, loss) = match rest.split_once('-') { + Some((core, loss_expr)) => { + let loss = NeutralLoss::from_expression(loss_expr)?.ok_or_else(|| { + IonParsingError::UnsupportedNeutralLoss { + loss: loss_expr.to_string(), + } + })?; + (core, loss) + } + None => (rest, NeutralLoss::None), + }; + + // internal fragment: m: + if let Some(spans) = core.strip_prefix('m') + && let Some((a, b)) = spans.split_once(':') + { + let start = a.parse::().map_err(|_| IonParsingError::ParsingError { error: value.to_string(), - context: Some("Charge cannot be 0"), - }); + context: Some("Unable to parse internal-fragment start"), + })?; + let end = b.parse::().map_err(|_| IonParsingError::ParsingError { + error: value.to_string(), + context: Some("Unable to parse internal-fragment end"), + })?; + return Self::try_new_internal(start, end, charge, isotope, loss); } - Ok(Self { - series_ordinal: series_ord, - charge, - isotope, - }) + + // immonium: I, bare only. + if let Some(res) = core.strip_prefix('I') { + let mut ch = res.chars(); + return match (ch.next(), ch.next()) { + (Some(r), None) => Self::try_new_immonium(r, charge, isotope, loss), + // `IC[Carbamidomethyl]` and friends carry a mod string that no + // fixed-width field can hold. + _ => Err(IonParsingError::UnsupportedModifiedImmonium { + annotation: value.to_string(), + }), + }; + } + + let series_ord = IonSeriesOrdinal::from_str(core)?; + let (ion_type, ordinal) = series_ord.as_char_and_ordinal(); + Self::try_new_with_loss(ion_type, ordinal, charge, isotope, loss) + } +} + +impl TryFrom<&str> for IonAnnot { + type Error = IonParsingError; + + /// Parses an annotation, discarding any mass-error suffix. Use + /// [`IonAnnot::parse_mzpaf`] to keep it. + fn try_from(value: &str) -> Result { + Ok(Self::parse_mzpaf(value)?.ion) } } impl Display for IonAnnot { + /// Renders the canonical mzPAF spelling. Not byte-inverse to parsing: a + /// non-canonical loss spelling (`-CH3SOH`) renders canonically (`-CH4OS`). + /// The mass-error suffix is not part of the annotation and is not rendered. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.series_ordinal)?; + write!(f, "{}", self.series_ordinal())?; + write!(f, "{}", self.loss())?; - match self.isotope { + match self.get_isotope() { 0 => {} 1 => write!(f, "+i")?, i => write!(f, "+{}i", i)?, } - if self.charge != 1 { - write!(f, "^{}", self.charge)?; + let charge = self.get_charge(); + if charge != 1 { + write!(f, "^{}", charge)?; } Ok(()) @@ -227,6 +692,14 @@ pub enum IonParsingError { OrdinalOutOfRange { ordinal: i32, series: Option }, #[error("Unsupported fragment type: '{fragment_type}'")] UnsupportedFragmentType { fragment_type: char }, + #[error("Charge {charge} outside the representable range")] + ChargeOutOfRange { charge: i8 }, + #[error("Isotope offset {isotope} outside the representable range")] + IsotopeOutOfRange { isotope: i8 }, + #[error("Neutral loss '{loss}' is not representable")] + UnsupportedNeutralLoss { loss: String }, + #[error("Modified immonium ions are not representable: '{annotation}'")] + UnsupportedModifiedImmonium { annotation: String }, #[error("Parsing error: {error}{}", .context.map(|c| format!(" ({})", c)).unwrap_or_default())] ParsingError { error: String, @@ -245,6 +718,10 @@ pub enum IonSeriesTerminality { None, } +/// The logical series-and-payload view of an [`IonAnnot`]. +/// +/// This is a *view*: `IonAnnot` stores a packed word and reconstructs this on +/// demand. Constructing one directly does not allocate an annotation. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Default)] #[allow(non_camel_case_types)] pub enum IonSeriesOrdinal { @@ -279,6 +756,15 @@ pub enum IonSeriesOrdinal { ordinal: u8, }, precursor, + /// An internal fragment spanning residues `start..=end`. + internal { + start: u8, + end: u8, + }, + /// A bare immonium ion for an uppercase residue code. + immonium { + residue: char, + }, /// This variant should not be used directly ... its mainly added to satisfy trait constraints by TinyVec #[default] @@ -315,38 +801,62 @@ impl IonSeriesOrdinal { Ok(tmp) } + /// Series character and ordinal, for handing back to [`IonAnnot::try_new`]. + fn as_char_and_ordinal(&self) -> (char, Option) { + match self { + Self::a { ordinal } => ('a', Some(*ordinal)), + Self::b { ordinal } => ('b', Some(*ordinal)), + Self::c { ordinal } => ('c', Some(*ordinal)), + Self::d { ordinal } => ('d', Some(*ordinal)), + Self::v { ordinal } => ('v', Some(*ordinal)), + Self::w { ordinal } => ('w', Some(*ordinal)), + Self::x { ordinal } => ('x', Some(*ordinal)), + Self::y { ordinal } => ('y', Some(*ordinal)), + Self::z { ordinal } => ('z', Some(*ordinal)), + Self::unknown { ordinal } => ('?', Some(*ordinal)), + Self::precursor => ('p', None), + Self::internal { start, .. } => ('m', Some(*start)), + Self::immonium { residue } => (*residue, None), + Self::None => ('\0', None), + } + } + pub fn terminality(&self) -> IonSeriesTerminality { match self { - IonSeriesOrdinal::a { ordinal: _ } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::b { ordinal: _ } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::c { ordinal: _ } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::d { ordinal: _ } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::v { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::w { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::x { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::y { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::z { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::unknown { ordinal: _ } => IonSeriesTerminality::None, - IonSeriesOrdinal::precursor => IonSeriesTerminality::None, + IonSeriesOrdinal::a { .. } + | IonSeriesOrdinal::b { .. } + | IonSeriesOrdinal::c { .. } + | IonSeriesOrdinal::d { .. } => IonSeriesTerminality::NTerm, + IonSeriesOrdinal::v { .. } + | IonSeriesOrdinal::w { .. } + | IonSeriesOrdinal::x { .. } + | IonSeriesOrdinal::y { .. } + | IonSeriesOrdinal::z { .. } => IonSeriesTerminality::CTerm, + IonSeriesOrdinal::unknown { .. } + | IonSeriesOrdinal::precursor + | IonSeriesOrdinal::internal { .. } + | IonSeriesOrdinal::immonium { .. } => IonSeriesTerminality::None, IonSeriesOrdinal::None => panic!("IonSeriesOrdinal::None should not be used directly"), } } pub fn try_get_ordinal(&self) -> Option { match self { - IonSeriesOrdinal::a { ordinal } => Some(*ordinal), - IonSeriesOrdinal::b { ordinal } => Some(*ordinal), - IonSeriesOrdinal::c { ordinal } => Some(*ordinal), - IonSeriesOrdinal::d { ordinal } => Some(*ordinal), - IonSeriesOrdinal::v { ordinal } => Some(*ordinal), - IonSeriesOrdinal::w { ordinal } => Some(*ordinal), - IonSeriesOrdinal::x { ordinal } => Some(*ordinal), - IonSeriesOrdinal::y { ordinal } => Some(*ordinal), - IonSeriesOrdinal::z { ordinal } => Some(*ordinal), - IonSeriesOrdinal::unknown { .. } => None, + IonSeriesOrdinal::a { ordinal } + | IonSeriesOrdinal::b { ordinal } + | IonSeriesOrdinal::c { ordinal } + | IonSeriesOrdinal::d { ordinal } + | IonSeriesOrdinal::v { ordinal } + | IonSeriesOrdinal::w { ordinal } + | IonSeriesOrdinal::x { ordinal } + | IonSeriesOrdinal::y { ordinal } + | IonSeriesOrdinal::z { ordinal } => Some(*ordinal), // ?1 does not mean its an ordinal, just a placeholder - IonSeriesOrdinal::precursor => None, - IonSeriesOrdinal::None => None, + IonSeriesOrdinal::unknown { .. } + | IonSeriesOrdinal::precursor + | IonSeriesOrdinal::internal { .. } + | IonSeriesOrdinal::immonium { .. } + | IonSeriesOrdinal::None => None, } } } @@ -365,6 +875,8 @@ impl Display for IonSeriesOrdinal { IonSeriesOrdinal::z { ordinal } => write!(f, "z{}", ordinal), IonSeriesOrdinal::unknown { ordinal } => write!(f, "?{}", ordinal), IonSeriesOrdinal::precursor => write!(f, "p"), + IonSeriesOrdinal::internal { start, end } => write!(f, "m{}:{}", start, end), + IonSeriesOrdinal::immonium { residue } => write!(f, "I{}", residue), IonSeriesOrdinal::None => panic!("IonSeriesOrdinal::None should not be used directly"), } } @@ -375,6 +887,12 @@ impl FromStr for IonSeriesOrdinal { fn from_str(s: &str) -> Result { // "b12" split into "b" and "12" + if s.is_empty() { + return Err(IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Empty string"), + }); + } let (series_chunk, ordinal_chunk) = s.split_at(1); let series_id = series_chunk.chars().next(); let series_ordinal = ordinal_chunk.parse::(); @@ -398,6 +916,19 @@ impl FromStr for IonSeriesOrdinal { mod tests { use super::*; + fn ion(s: &str) -> IonAnnot { + IonAnnot::try_from(s).unwrap_or_else(|e| panic!("{s:?} must parse: {e}")) + } + + /// The whole point of the packed representation. + #[test] + fn packs_into_one_word() { + assert_eq!(size_of::(), 4); + // Paired with an intensity on the scoring hot path; padding here would + // grow the inline TinyVec storage in `ExpectedIntensities`. + assert_eq!(size_of::<(IonAnnot, f32)>(), 8); + } + #[test] fn test_ion_series_ord_from_str() { let ion: IonSeriesOrdinal = IonSeriesOrdinal::from_str("b12").unwrap(); @@ -406,88 +937,165 @@ mod tests { #[test] fn test_deserialize() { - let serde_pairs = vec![ - ( - "b12", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 12 }, - charge: 1, - isotope: 0, - }, - ), - ( - "b12^3", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 12 }, - charge: 3, - isotope: 0, - }, - ), - ( - "y12^3", - IonAnnot { - series_ordinal: IonSeriesOrdinal::y { ordinal: 12 }, - charge: 3, - isotope: 0, - }, - ), - ( - "b12+i^3", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 12 }, - charge: 3, - isotope: 1, - }, - ), - ( - "b12+3i^3", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 12 }, - charge: 3, - isotope: 3, - }, - ), - ( - "b13", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 13 }, - charge: 1, - isotope: 0, - }, - ), - ( - "p^2", - IonAnnot { - series_ordinal: IonSeriesOrdinal::precursor, - charge: 2, - isotope: 0, - }, - ), - ( - "p", - IonAnnot { - series_ordinal: IonSeriesOrdinal::precursor, - charge: 1, - isotope: 0, - }, - ), - ( - "?12^2", - IonAnnot { - series_ordinal: IonSeriesOrdinal::unknown { ordinal: 12 }, - charge: 2, - isotope: 0, - }, - ), + let cases = [ + ("b12", 'b', Some(12u8), 1i8, 0i8), + ("b12^3", 'b', Some(12), 3, 0), + ("y12^3", 'y', Some(12), 3, 0), + ("b12+i^3", 'b', Some(12), 3, 1), + ("b12+3i^3", 'b', Some(12), 3, 3), + ("b13", 'b', Some(13), 1, 0), + ("p^2", 'p', None, 2, 0), + ("p", 'p', None, 1, 0), + ("?12^2", '?', Some(12), 2, 0), ]; - for (input, expected) in serde_pairs { - let annot = IonAnnot::try_from(input).unwrap(); - assert_eq!(annot, expected); + for (input, series, ordinal, charge, isotope) in cases { + let annot = ion(input); + let expected = IonAnnot::try_new(series, ordinal, charge, isotope).unwrap(); + assert_eq!(annot, expected, "{input}"); + assert_eq!(annot.get_charge(), charge, "{input}"); + assert_eq!(annot.get_isotope(), isotope, "{input}"); + // Round-trips byte-identically when no loss is involved. + assert_eq!(format!("{}", annot), input); + } + } - // Re-serialize and check that its the same - let serialized = format!("{}", annot); - assert_eq!(serialized, input); + /// Every field must survive the pack/unpack at its extremes. The bit + /// fields truncate rather than wrap, so a missing range check corrupts + /// silently — this is the test that catches it. + #[test] + fn every_field_round_trips_at_its_extremes() { + for charge in CHARGE_MIN..=CHARGE_MAX { + if charge == 0 { + continue; + } + for isotope in ISOTOPE_MIN..=ISOTOPE_MAX { + for ordinal in [1u8, 2, 127, 255] { + for loss in [ + NeutralLoss::None, + NeutralLoss::Water, + NeutralLoss::PhosphoricAcidWater, + ] { + let a = + IonAnnot::try_new_with_loss('y', Some(ordinal), charge, isotope, loss) + .expect("in range"); + assert_eq!(a.get_charge(), charge); + assert_eq!(a.get_isotope(), isotope); + assert_eq!(a.try_get_ordinal(), Some(ordinal)); + assert_eq!(a.loss(), loss); + assert_eq!(a.terminality(), IonSeriesTerminality::CTerm); + } + } + } } } + + #[test] + fn out_of_range_is_rejected_not_truncated() { + assert!(matches!( + IonAnnot::try_new('y', Some(1), CHARGE_MAX + 1, 0), + Err(IonParsingError::ChargeOutOfRange { .. }) + )); + assert!(matches!( + IonAnnot::try_new('y', Some(1), 1, ISOTOPE_MAX + 1), + Err(IonParsingError::IsotopeOutOfRange { .. }) + )); + assert!(IonAnnot::try_new('y', Some(1), 0, 0).is_err()); + assert!(matches!( + IonAnnot::try_new_internal(INTERNAL_POS_MAX + 1, 1, 1, 0, NeutralLoss::None), + Err(IonParsingError::OrdinalOutOfRange { .. }) + )); + } + + #[test] + fn isotope_offset_respects_the_field_bound() { + let a = ion("y5"); + assert_eq!(a.try_with_offset_neutrons(2).unwrap().get_isotope(), 2); + assert!(a.try_with_offset_neutrons(ISOTOPE_MAX + 1).is_err()); + } + + #[test] + fn parses_neutral_losses() { + let a = ion("y5-H2O"); + assert_eq!(a.loss(), NeutralLoss::Water); + assert_eq!(a.try_get_ordinal(), Some(5)); + assert_eq!(format!("{}", a), "y5-H2O"); + + // Non-canonical spelling resolves to the same annotation, and renders + // canonically -- so this pair is equal, which is the property that + // keeps per-precursor label uniqueness honest. + assert_eq!(ion("y5-CH3SOH"), ion("y5-CH4OS")); + assert_eq!(format!("{}", ion("y5-CH3SOH")), "y5-CH4OS"); + + // Loss combines with charge and isotope. + let b = ion("y10-NH3+i^2"); + assert_eq!(b.loss(), NeutralLoss::Ammonia); + assert_eq!(b.get_isotope(), 1); + assert_eq!(b.get_charge(), 2); + } + + #[test] + fn parses_internal_fragments() { + let a = ion("m2:11"); + assert_eq!( + a.series_ordinal(), + IonSeriesOrdinal::internal { start: 2, end: 11 } + ); + // An internal fragment has no ladder position. + assert_eq!(a.try_get_ordinal(), None); + assert_eq!(format!("{}", a), "m2:11"); + + let b = ion("m11:12-CO"); + assert_eq!(b.loss(), NeutralLoss::CarbonMonoxide); + assert_eq!(format!("{}", b), "m11:12-CO"); + } + + #[test] + fn parses_bare_immonium_and_rejects_modified() { + let a = ion("IA"); + assert_eq!( + a.series_ordinal(), + IonSeriesOrdinal::immonium { residue: 'A' } + ); + assert_eq!(format!("{}", a), "IA"); + + // Carries an arbitrary mod string -- no field width represents it, so + // it must fail rather than silently degrade to a bare immonium. + assert!(matches!( + IonAnnot::try_from("IC[Carbamidomethyl]"), + Err(IonParsingError::UnsupportedModifiedImmonium { .. }) + )); + } + + #[test] + fn parses_and_applies_the_mass_error_suffix() { + let p = IonAnnot::parse_mzpaf("y1/-0.0005").unwrap(); + assert_eq!(p.ion, ion("y1")); + assert_eq!(p.mass_error, Some(MassError::Da(-0.0005))); + // theoretical = observed - error; verified against a real SpectraST + // peak: y1 for C-terminal R, observed 175.1184, theoretical 175.11895. + let theo = p.mass_error.unwrap().theoretical_from_observed(175.1184); + assert!((theo - 175.1189).abs() < 1e-9, "got {theo}"); + + let q = IonAnnot::parse_mzpaf("y6/1.2ppm").unwrap(); + assert_eq!(q.mass_error, Some(MassError::Ppm(1.2))); + let theo = q.mass_error.unwrap().theoretical_from_observed(700.0); + assert!((theo - 699.99916).abs() < 1e-4, "got {theo}"); + + // Absent suffix is not an error. + assert_eq!(IonAnnot::parse_mzpaf("y6").unwrap().mass_error, None); + // TryFrom discards it rather than failing. + assert_eq!(IonAnnot::try_from("y1/-0.0005").unwrap(), ion("y1")); + } + + /// An unrepresentable loss must fail loudly. Parsing `y1-HCOOH` as plain + /// `y1` would put a loss peak's m/z on the `y1` label and collide with the + /// real `y1`. + #[test] + fn unrepresentable_loss_is_rejected_not_stripped() { + assert!(matches!( + IonAnnot::try_from("y1-HCOOH"), + Err(IonParsingError::UnsupportedNeutralLoss { .. }) + )); + } } diff --git a/rust/micromzpaf/src/loss.rs b/rust/micromzpaf/src/loss.rs index c5ef5d31..68eb8d66 100644 --- a/rust/micromzpaf/src/loss.rs +++ b/rust/micromzpaf/src/loss.rs @@ -5,10 +5,10 @@ //! //! | written | library | composition | //! |---|---|---| -//! | `-CH3SOH` | NIST | C₁H₄O₁S₁ | -//! | `-CH4OS` | SpectraST | C₁H₄O₁S₁ | -//! | `-NH2-CO-CH2SH` | NIST | C₂H₅N₁O₁S₁ | -//! | `-C2H5NOS` | SpectraST | C₂H₅N₁O₁S₁ | +//! | `-CH3SOH` | NIST | C1H4O1S1 | +//! | `-CH4OS` | SpectraST | C1H4O1S1 | +//! | `-NH2-CO-CH2SH` | NIST | C2H5N1O1S1 | +//! | `-C2H5NOS` | SpectraST | C2H5N1O1S1 | //! //! Keying on the string would make `y5-CH4OS` and `y5-CH3SOH` distinct labels //! for one ion. Since fragment labels must be unique within a precursor (see @@ -163,29 +163,29 @@ impl Composition { pub enum NeutralLoss { #[default] None = 0, - /// H₂O + /// H2O Water = 1, - /// NH₃ + /// NH3 Ammonia = 2, /// CO CarbonMonoxide = 3, - /// CO₂ + /// CO2 CarbonDioxide = 4, - /// 2 H₂O + /// 2 H2O WaterX2 = 5, - /// 2 NH₃ + /// 2 NH3 AmmoniaX2 = 6, - /// H₂O + NH₃ + /// H2O + NH3 WaterAmmonia = 7, - /// CH₄OS — methanesulfenic acid, off oxidized Met. Also spelled `CH3SOH`. + /// CH4OS — methanesulfenic acid, off oxidized Met. Also spelled `CH3SOH`. Methanesulfenic = 8, - /// C₂H₅NOS — also spelled `NH2-CO-CH2SH`. + /// C2H5NOS — also spelled `NH2-CO-CH2SH`. Carbamidomethylthiol = 9, - /// H₃PO₄ — phospho-Ser/Thr. + /// H3PO4 — phospho-Ser/Thr. PhosphoricAcid = 10, - /// HPO₃ — phospho-Tyr, and phospho-Ser/Thr. + /// HPO3 — phospho-Tyr, and phospho-Ser/Thr. Metaphosphoric = 11, - /// H₃PO₄ + H₂O + /// H3PO4 + H2O PhosphoricAcidWater = 12, } From abb86059293d05dd5df5e671430b3e24ace7ebaa Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 17:02:24 -0700 Subject: [PATCH 06/27] feat(timsquery): read mzSpecLib text libraries mzSpecLib specifies how to write a CV term, not which term to use, so every field needs a fallback ladder. The two reference exports disagree on almost everything: precursor m/z DIA-NN MS:1000744 Spectronaut MS:1003208 retention time DIA-NN (absent entirely) Spectronaut MS:1000896, minutes ion mobility both MS:1002476 (a drift time, not 1/K0) RT is unit-tagged and the unit is honoured rather than assumed -- reading Spectronaut's minutes as seconds would be a silent 60x error. Peak lists carry OBSERVED m/z; the annotation's mass-error suffix is what recovers theoretical (`theoretical = observed - error`). Two consequences shape the policy: - A theoretical mass only exists once a single identity is pinned. `?` peaks and tied ambiguities have no error to subtract, so they are skipped rather than stored at observed m/z -- an arena mixing observed and theoretical masses would be invisible downstream. - "Known" and "representable" differ. `y1-HCOOH` has a known identity and therefore an exact theoretical mass even though IonAnnot cannot spell the loss. Those peaks are kept with an unknown label: the mass stays exact, only the label is erased. Ambiguous annotations resolve to the smallest absolute mass error. If the winner is unrepresentable the peak takes an unknown label rather than falling back to a worse-matching representable alternative, which would assign both a wrong identity and a wrong mass. Everything that does not land verbatim is tallied and reported once per library. A consensus library carries thousands of unannotated peaks and a line each would bury the signal. Fixtures are the HUPO-PSI examples verbatim (Apache-2.0, same as this project): the DIA-NN export because it is the shape our own writer will emit, and the Spectronaut one because it carries the `-H2O`/`-NH3` losses the packed IonAnnot now represents. --- rust/timsquery/src/serde/library_file.rs | 13 + rust/timsquery/src/serde/mod.rs | 1 + rust/timsquery/src/serde/mzspeclib_io.rs | 765 ++++++++++++++++++ .../tests/mzspeclib_io_files/README.md | 15 + .../mzspeclib_io_files/diann.mzSpecLib.txt | 358 ++++++++ .../spectronaut.mzSpecLib.txt | 520 ++++++++++++ 6 files changed, 1672 insertions(+) create mode 100644 rust/timsquery/src/serde/mzspeclib_io.rs create mode 100644 rust/timsquery/tests/mzspeclib_io_files/README.md create mode 100644 rust/timsquery/tests/mzspeclib_io_files/diann.mzSpecLib.txt create mode 100644 rust/timsquery/tests/mzspeclib_io_files/spectronaut.mzSpecLib.txt diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index b9c8885a..c91ac13b 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -13,6 +13,10 @@ use super::elution_group_inputs::{ ElutionGroupInput, ElutionGroupInputError, }; +use super::mzspeclib_io::{ + read_mzspeclib_library_file, + sniff_mzspeclib_library_file, +}; pub use super::skyline_io::SkylinePrecursorExtras; use super::skyline_io::{ read_library_file as read_skyline_csv, @@ -525,6 +529,15 @@ fn registry() -> &'static [&'static dyn LibraryReader] { pub fn read_library_file>(path: T) -> Result { let path = path.as_ref(); + // mzSpecLib is sniffed alongside `.speclib` rather than through the + // registry: like the DIA-NN binary reader it builds the arena directly + // (with the reference-intensity sidecar) instead of going through the + // legacy `ElutionGroupCollection`. Its magic first line makes the probe + // exact, so an early check cannot steal another format's file. + if sniff_mzspeclib_library_file(path) { + info!("Dispatching library read to mzspeclib (direct arena build)"); + return read_mzspeclib_library_file(path); + } // The DIA-NN `.speclib` reader builds the columnar arena directly (with the // reference-intensity sidecar); every other format still produces the legacy // `ElutionGroupCollection`, adapted into the arena here. `.speclib` is diff --git a/rust/timsquery/src/serde/mod.rs b/rust/timsquery/src/serde/mod.rs index 4c07f6b1..dc18d5a4 100644 --- a/rust/timsquery/src/serde/mod.rs +++ b/rust/timsquery/src/serde/mod.rs @@ -4,6 +4,7 @@ pub mod diann_speclib_io; mod elution_group_inputs; pub mod index_serde; mod library_file; +pub mod mzspeclib_io; mod skyline_io; mod spectronaut_io; diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs new file mode 100644 index 00000000..07be1fbb --- /dev/null +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -0,0 +1,765 @@ +//! Reader for the mzSpecLib text format (HUPO-PSI). +//! +//! # Why every field needs a fallback ladder +//! +//! mzSpecLib says *how* to write a controlled-vocabulary term, not *which* +//! term a writer must use. The two reference exports disagree on nearly +//! everything this reader needs: +//! +//! | field | DIA-NN writes | Spectronaut writes | +//! |---|---|---| +//! | precursor m/z | `MS:1000744` selected ion m/z | `MS:1003208` experimental precursor monoisotopic m/z | +//! | retention time | *nothing at all* | `MS:1000896` normalized retention time, in minutes | +//! | ion mobility | `MS:1002476` (written as `0.0`) | `MS:1002476` | +//! +//! So each field is resolved by trying terms in priority order, and RT carries +//! a unit that must be honoured rather than assumed. +//! +//! # Peak resolution +//! +//! An mzSpecLib peak list carries *observed* m/z; the annotation's mass-error +//! suffix is what recovers the theoretical value the arena wants +//! (`theoretical = observed - error`). Two properties follow, and they drive +//! the whole policy: +//! +//! 1. A theoretical m/z only exists once a single ion identity is pinned down. +//! A peak with no annotation (`?`) or an unresolvable ambiguity has no +//! error to subtract, so it is **skipped** rather than stored at its +//! observed m/z. Mixing observed and theoretical masses in one arena would +//! be invisible downstream. +//! 2. "Identity known" and "identity representable" are different. `y1-HCOOH` +//! has a known identity and therefore a computable theoretical mass, even +//! though [`IonAnnot`] cannot spell the loss. Those peaks are **kept** with +//! an unknown label: the mass stays exact, only the label is erased. +//! +//! Ambiguous (comma-separated) annotations resolve to the alternative with the +//! smallest absolute mass error. If that alternative is not representable the +//! peak takes an unknown label — it is deliberately NOT downgraded to a +//! worse-matching but representable alternative, which would assign a wrong +//! chemical identity *and* a wrong theoretical mass. An exact tie pins no +//! identity at all, so it is skipped. + +use crate::ion::IonAnnot; +use crate::models::{ + LibCapabilities, + QueryCollection, +}; +use crate::serde::library_file::{ + LibraryArena, + LibraryReadingError, +}; +use micromzpaf::{ + MassError, + split_mass_error, +}; +use std::io::{ + BufRead, + BufReader, +}; +use std::path::Path; +use tracing::{ + info, + warn, +}; + +/// First non-empty line of an mzSpecLib text file. +const MAGIC: &str = ""; + +// ── CV term ladders ────────────────────────────────────────────────────────── +// +// Ordered most- to least-specific. The first term present wins. + +/// Precursor m/z. Experimental monoisotopic is preferred over `selected ion +/// m/z`, which on a quadrupole instrument is the isolation-window centre and +/// need not be the monoisotopic peak. +const PRECURSOR_MZ_TERMS: &[&str] = &[ + "MS:1003208", // experimental precursor monoisotopic m/z + "MS:1003053", // theoretical monoisotopic m/z + "MS:1000744", // selected ion m/z +]; +/// Retention time. `normalized retention time` is an iRT-style index rather +/// than a clock reading, but it is what Spectronaut exports and the only RT +/// signal available in those files. +const RT_TERMS: &[&str] = &[ + "MS:1000894", // retention time + "MS:1000896", // normalized retention time +]; +/// Ion mobility. Note these are different quantities, not spellings of one: +/// `MS:1002815` is inverse reduced ion mobility (1/K0), which is what the +/// arena wants, while `MS:1002476` is a drift time. They are tried in that +/// order and a drift-time-only library is counted, since treating a drift time +/// as 1/K0 is only valid for instruments that report it that way. +const MOBILITY_INVERSE_REDUCED: &str = "MS:1002815"; +const MOBILITY_DRIFT_TIME: &str = "MS:1002476"; + +const CHARGE_TERM: &str = "MS:1000041"; +const STRIPPED_SEQ_TERM: &str = "MS:1000888"; +const PROFORMA_TERM: &str = "MS:1003270"; +const UNIT_TERM: &str = "UO:0000000"; + +const UNIT_MINUTE: &str = "UO:0000031"; +const UNIT_SECOND: &str = "UO:0000010"; + +/// Per-library tally of everything that did not land verbatim in the arena. +/// +/// Reported once at the end of a load rather than per row: a consensus library +/// can carry thousands of unannotated peaks, and a line each would bury the +/// signal. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct MzSpecLibStats { + /// Peaks stored with their parsed annotation. + pub kept_annotated: usize, + /// Identity known but not representable (a loss outside the table, a + /// modified immonium). Stored with an unknown label and an exact mass. + pub kept_unknown_label: usize, + /// `?` — no annotation, so no mass error, so no theoretical m/z. + pub skipped_unannotated: usize, + /// Comma-separated alternatives that tied on absolute mass error. + pub skipped_ambiguous: usize, + /// Peaks dropped because their label collided with one already in the + /// precursor after the unknown-label rewrite. + pub dropped_duplicate_label: usize, + /// Spectra with no retention-time term at all. + pub spectra_without_rt: usize, + /// Spectra whose mobility came from a drift time rather than 1/K0. + pub spectra_with_drift_time_mobility: usize, + /// Precursors dropped for having no usable peak left. + pub dropped_empty_precursors: usize, +} + +impl MzSpecLibStats { + fn anything_to_report(&self) -> bool { + self.kept_unknown_label > 0 + || self.skipped_unannotated > 0 + || self.skipped_ambiguous > 0 + || self.dropped_duplicate_label > 0 + || self.spectra_without_rt > 0 + || self.spectra_with_drift_time_mobility > 0 + || self.dropped_empty_precursors > 0 + } + + fn report(&self, path: &Path) { + if !self.anything_to_report() { + info!( + "mzSpecLib {}: {} peaks, all annotated and representable", + path.display(), + self.kept_annotated + ); + return; + } + warn!( + "mzSpecLib {}: kept {} annotated + {} with unknown labels; \ + skipped {} unannotated, {} ambiguous, {} duplicate-label; \ + {} spectra without RT, {} using drift time as mobility, \ + {} precursors dropped as empty", + path.display(), + self.kept_annotated, + self.kept_unknown_label, + self.skipped_unannotated, + self.skipped_ambiguous, + self.dropped_duplicate_label, + self.spectra_without_rt, + self.spectra_with_drift_time_mobility, + self.dropped_empty_precursors, + ); + } +} + +/// One `ACC|name=value` attribute, with its optional `[n]` group tag. +#[derive(Debug, Clone)] +struct Attr { + group: Option, + accession: String, + value: String, +} + +impl Attr { + fn parse(line: &str) -> Option { + let (group, rest) = match line.strip_prefix('[') { + Some(r) => { + let (g, r) = r.split_once(']')?; + (Some(g.parse().ok()?), r) + } + None => (None, line), + }; + let (key, value) = rest.split_once('=')?; + let accession = key.split('|').next()?.to_string(); + Some(Attr { + group, + accession, + value: value.to_string(), + }) + } + + /// The accession out of a `ACC|name` *value* (as opposed to a key), for + /// terms whose value is itself a CV term — e.g. `unit=UO:0000031|minute`. + fn value_accession(&self) -> &str { + self.value.split('|').next().unwrap_or(&self.value) + } +} + +/// Attributes collected for one spectrum (its own plus its analyte's). +#[derive(Debug, Default)] +struct AttrBag(Vec); + +impl AttrBag { + fn find(&self, accession: &str) -> Option<&Attr> { + self.0.iter().find(|a| a.accession == accession) + } + + fn first_of(&self, accessions: &[&str]) -> Option<&Attr> { + accessions.iter().find_map(|a| self.find(a)) + } + + fn f64_of(&self, accessions: &[&str]) -> Option { + self.first_of(accessions)?.value.parse().ok() + } + + /// The unit term attached to `attr` via its `[n]` group, if any. + fn unit_for(&self, attr: &Attr) -> Option<&str> { + let group = attr.group?; + self.0 + .iter() + .find(|a| a.group == Some(group) && a.accession == UNIT_TERM) + .map(|a| a.value_accession()) + } +} + +/// A spectrum accumulated from the text stream, before conversion. +#[derive(Debug, Default)] +struct RawSpectrum { + attrs: AttrBag, + /// `(observed mz, intensity, annotation)` + peaks: Vec<(f64, f32, String)>, +} + +/// What resolving one peak's annotation produced. +enum Resolved { + /// Parsed cleanly; store with this label. + Annotated(IonAnnot), + /// Identity known, not representable. Store with an unknown label, exact + /// mass. + UnknownLabel, + /// No single identity, so no theoretical mass. Skip. + SkipUnannotated, + SkipAmbiguous, +} + +/// Resolve one annotation string into a storage decision plus the mass error +/// needed to recover theoretical m/z. +fn resolve_annotation(annotation: &str) -> (Resolved, Option) { + let annotation = annotation.trim(); + if annotation.is_empty() || annotation == "?" { + return (Resolved::SkipUnannotated, None); + } + + let alternatives: Vec<&str> = annotation.split(',').map(str::trim).collect(); + + // Split the mass error off every alternative first. This works even when + // the ion itself will not parse, which is exactly the case that still + // needs an exact mass. + let split: Vec<(&str, Option)> = alternatives + .iter() + .filter_map(|a| split_mass_error(a).ok()) + .collect(); + if split.is_empty() { + return (Resolved::SkipUnannotated, None); + } + + let chosen = if split.len() == 1 { + split[0] + } else { + // Ambiguous: the alternative whose observed mass sits closest to its + // own theoretical. A tie pins no identity, so nothing can be stored. + let magnitude = |m: &Option| match m { + Some(MassError::Da(d)) => d.abs(), + Some(MassError::Ppm(p)) => p.abs(), + None => f64::INFINITY, + }; + let best = split + .iter() + .map(|(_, e)| magnitude(e)) + .fold(f64::INFINITY, f64::min); + let tied = split + .iter() + .filter(|(_, e)| (magnitude(e) - best).abs() <= f64::EPSILON) + .count(); + if tied != 1 || !best.is_finite() { + return (Resolved::SkipAmbiguous, None); + } + *split + .iter() + .find(|(_, e)| (magnitude(e) - best).abs() <= f64::EPSILON) + .expect("a unique minimum was just counted") + }; + + let (ion_str, mass_error) = chosen; + match IonAnnot::try_from(ion_str) { + Ok(ion) => (Resolved::Annotated(ion), mass_error), + // Known identity, unrepresentable spelling: keep the peak and its exact + // mass, lose only the label. + Err(_) => (Resolved::UnknownLabel, mass_error), + } +} + +/// Convert one accumulated spectrum into arena rows. +/// +/// Returns `None` when the spectrum lacks something structural (precursor m/z, +/// charge, sequence) or ends up with no usable peak. +fn convert_spectrum( + raw: &RawSpectrum, + stats: &mut MzSpecLibStats, +) -> Option<( + f64, + u8, + f32, + f32, + Vec<(IonAnnot, f64)>, + Vec, + String, + String, +)> { + let precursor_mz = raw.attrs.f64_of(PRECURSOR_MZ_TERMS)?; + let charge: u8 = raw.attrs.find(CHARGE_TERM)?.value.parse().ok()?; + + let rt_seconds = match raw.attrs.first_of(RT_TERMS) { + Some(attr) => { + let v: f64 = attr.value.parse().ok()?; + // Honour the unit rather than assuming: Spectronaut writes minutes. + match raw.attrs.unit_for(attr) { + Some(UNIT_SECOND) => v, + Some(UNIT_MINUTE) | None => v * 60.0, + Some(_) => v * 60.0, + } + } + None => { + stats.spectra_without_rt += 1; + 0.0 + } + }; + + let mobility = match raw.attrs.find(MOBILITY_INVERSE_REDUCED) { + Some(a) => a.value.parse().unwrap_or(0.0), + None => match raw.attrs.find(MOBILITY_DRIFT_TIME) { + Some(a) => { + stats.spectra_with_drift_time_mobility += 1; + a.value.parse().unwrap_or(0.0) + } + None => 0.0, + }, + }; + + let stripped = raw + .attrs + .find(STRIPPED_SEQ_TERM) + .map(|a| a.value.clone()) + .unwrap_or_default(); + // The proforma term carries a trailing `/charge` that is not part of the + // peptidoform. + let modified = raw + .attrs + .find(PROFORMA_TERM) + .map(|a| { + a.value + .rsplit_once('/') + .map(|(p, _)| p.to_string()) + .unwrap_or_else(|| a.value.clone()) + }) + .unwrap_or_else(|| stripped.clone()); + if stripped.is_empty() && modified.is_empty() { + return None; + } + + let mut frags: Vec<(IonAnnot, f64)> = Vec::with_capacity(raw.peaks.len()); + let mut intens: Vec = Vec::with_capacity(raw.peaks.len()); + let mut unknown_counter: u8 = 0; + + for (observed_mz, intensity, annotation) in &raw.peaks { + let (resolved, mass_error) = resolve_annotation(annotation); + let label = match resolved { + Resolved::Annotated(ion) => { + stats.kept_annotated += 1; + ion + } + Resolved::UnknownLabel => { + // The ordinal is a per-precursor uniqueness counter. Past 255 + // there is no way to keep labels distinct, so drop rather than + // reuse one. + let Some(next) = unknown_counter.checked_add(1) else { + stats.dropped_duplicate_label += 1; + continue; + }; + unknown_counter = next; + match IonAnnot::try_new('?', Some(unknown_counter), 1, 0) { + Ok(i) => { + stats.kept_unknown_label += 1; + i + } + Err(_) => { + stats.dropped_duplicate_label += 1; + continue; + } + } + } + Resolved::SkipUnannotated => { + stats.skipped_unannotated += 1; + continue; + } + Resolved::SkipAmbiguous => { + stats.skipped_ambiguous += 1; + continue; + } + }; + + // Peak lists carry observed m/z; the arena wants theoretical. + let mz = match mass_error { + Some(e) => e.theoretical_from_observed(*observed_mz), + None => *observed_mz, + }; + + // Labels must stay unique within the precursor (`linear_get` is + // first-match), so a collision drops the later peak. + if frags.iter().any(|(l, _)| *l == label) { + stats.dropped_duplicate_label += 1; + continue; + } + frags.push((label, mz)); + intens.push(*intensity); + } + + if frags.is_empty() { + stats.dropped_empty_precursors += 1; + return None; + } + + Some(( + precursor_mz, + charge, + rt_seconds as f32, + mobility, + frags, + intens, + stripped, + modified, + )) +} + +/// Cheap probe: the format's magic first line. +pub fn sniff_mzspeclib_library_file>(path: T) -> bool { + let Ok(file) = std::fs::File::open(path.as_ref()) else { + return false; + }; + let mut reader = BufReader::new(file); + let mut line = String::new(); + // Only the first non-empty line is inspected, so this stays O(1) on a + // multi-gigabyte library. + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => return false, + Ok(_) => { + let t = line.trim(); + if t.is_empty() { + continue; + } + return t == MAGIC; + } + Err(_) => return false, + } + } +} + +/// Read an mzSpecLib text file into the columnar arena. +pub fn read_mzspeclib_library_file>( + path: T, +) -> Result { + let path = path.as_ref(); + let file = std::fs::File::open(path).map_err(LibraryReadingError::IoError)?; + let reader = BufReader::new(file); + + let mut geom = QueryCollection::with_capabilities(LibCapabilities::default_diann_no_decoys()); + let mut frag_intens: Vec = Vec::new(); + let mut stats = MzSpecLibStats::default(); + + let mut current: Option = None; + let mut in_peaks = false; + + let flush = |cur: Option, + geom: &mut QueryCollection, + frag_intens: &mut Vec, + stats: &mut MzSpecLibStats| { + let Some(raw) = cur else { return }; + let Some((mz, charge, rt, mobility, frags, intens, stripped, modified)) = + convert_spectrum(&raw, stats) + else { + return; + }; + frag_intens.extend_from_slice(&intens); + geom.push_row( + mz, + charge, + rt, + mobility, + &frags, + &stripped, + &modified, + &[], + false, + ); + }; + + for line in reader.lines() { + let line = line.map_err(LibraryReadingError::IoError)?; + let trimmed = line.trim_end(); + + if trimmed.starts_with("" { + in_peaks = true; + continue; + } + // `` and `` attributes are folded into the + // spectrum's bag: this reader wants the union, not the hierarchy. + if trimmed.starts_with('<') { + in_peaks = false; + continue; + } + if trimmed.is_empty() { + in_peaks = false; + continue; + } + + let Some(spec) = current.as_mut() else { + continue; // library-level header + }; + + if in_peaks { + let mut cols = trimmed.split('\t'); + let (Some(mz), Some(intensity)) = (cols.next(), cols.next()) else { + continue; + }; + let (Ok(mz), Ok(intensity)) = + (mz.trim().parse::(), intensity.trim().parse::()) + else { + continue; + }; + let annotation = cols.next().unwrap_or("?").to_string(); + spec.peaks.push((mz, intensity, annotation)); + } else if let Some(attr) = Attr::parse(trimmed) { + spec.attrs.0.push(attr); + } + } + flush(current.take(), &mut geom, &mut frag_intens, &mut stats); + + stats.report(path); + + if geom.n_rows() == 0 { + return Err(LibraryReadingError::SpeclibParse(format!( + "mzSpecLib {} yielded no usable precursors", + path.display() + ))); + } + if frag_intens.len() != geom.frag_labels.len() { + return Err(LibraryReadingError::SpeclibParse(format!( + "reference-intensity sidecar ({}) must stay parallel to the fragment-label arena ({})", + frag_intens.len(), + geom.frag_labels.len(), + ))); + } + + geom.seal(); + Ok(LibraryArena::Mzpaf { + geom, + frag_intens: Some(frag_intens), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture(name: &str) -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/mzspeclib_io_files") + .join(name) + } + + #[test] + fn sniffs_only_mzspeclib() { + assert!(sniff_mzspeclib_library_file(fixture("diann.mzSpecLib.txt"))); + assert!(sniff_mzspeclib_library_file(fixture( + "spectronaut.mzSpecLib.txt" + ))); + // A DIA-NN TSV must not be claimed. + let tsv = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/diann_io_files/sample_lib.tsv"); + assert!(!sniff_mzspeclib_library_file(tsv)); + } + + #[test] + fn reads_diann_export() { + let arena = read_mzspeclib_library_file(fixture("diann.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, frag_intens } = arena else { + panic!("mzSpecLib must build an mzpaf arena"); + }; + assert!(geom.n_rows() > 0); + let intens = frag_intens.expect("reference intensities are populated"); + assert_eq!(intens.len(), geom.frag_labels.len()); + } + + /// DIA-NN's export writes every mass error as exactly `0.0`, so observed + /// and theoretical coincide and the m/z must pass through untouched. + #[test] + fn zero_mass_error_leaves_mz_unchanged() { + let arena = read_mzspeclib_library_file(fixture("diann.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + // First spectrum's first peak in the fixture: 427.22995, annotated b6/0.0 + let first = geom.frag_mzs[0]; + assert!( + (first - 427.22995).abs() < 1e-6, + "expected the observed m/z verbatim, got {first}" + ); + } + + /// Spectronaut's export carries `-H2O`/`-NH3` losses, which the packed + /// `IonAnnot` now represents, so they keep their real labels rather than + /// falling back to unknown. + #[test] + fn spectronaut_losses_keep_real_labels() { + let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + let has_loss = geom + .frag_labels + .iter() + .any(|l| l.loss() != micromzpaf::NeutralLoss::None); + assert!(has_loss, "expected at least one loss-bearing label"); + let unknowns = geom + .frag_labels + .iter() + .filter(|l| l.try_get_ordinal().is_none() && l.loss() == micromzpaf::NeutralLoss::None) + .count(); + assert_eq!(unknowns, 0, "no peak should need an unknown label here"); + } + + /// RT is unit-tagged; Spectronaut writes minutes and the arena wants + /// seconds. Getting this wrong is a silent 60x error. + #[test] + fn retention_time_honours_its_unit() { + let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + // Fixture's first spectrum: normalized retention time = 28.658491 min. + let rt = geom.rt_seconds[0]; + assert!( + (rt - 28.658491 * 60.0).abs() < 0.01, + "expected minutes converted to seconds, got {rt}" + ); + } + + #[test] + fn resolves_unambiguous_representable() { + let (r, e) = resolve_annotation("y5/-0.0005"); + assert!(matches!(r, Resolved::Annotated(_))); + assert_eq!(e, Some(MassError::Da(-0.0005))); + } + + /// Known identity, unrepresentable spelling: keep the peak and the exact + /// mass, erase only the label. + #[test] + fn unrepresentable_loss_keeps_peak_with_unknown_label() { + let (r, e) = resolve_annotation("y1-HCOOH/0.0003"); + assert!(matches!(r, Resolved::UnknownLabel)); + assert_eq!( + e, + Some(MassError::Da(0.0003)), + "the mass error must survive so theoretical m/z stays exact" + ); + } + + #[test] + fn unannotated_peak_is_skipped() { + assert!(matches!( + resolve_annotation("?").0, + Resolved::SkipUnannotated + )); + } + + /// Closest-by-error wins; if that alternative is unrepresentable the peak + /// takes an unknown label rather than falling back to the representable + /// one, which would assign a wrong identity and a wrong mass. + #[test] + fn ambiguity_resolves_to_the_closest_not_the_representable() { + // a2 is representable and further; y2-CO2-NH3 is closer and is not. + let (r, e) = resolve_annotation("a2/-0.0040,y2-CO2-NH3/-0.0001"); + assert!( + matches!(r, Resolved::UnknownLabel), + "the closest alternative wins even when unrepresentable" + ); + assert_eq!(e, Some(MassError::Da(-0.0001))); + + // When the closest one IS representable, it is used. + let (r, _) = resolve_annotation("a2/-0.0001,y2-CO2-NH3/-0.0040"); + assert!(matches!(r, Resolved::Annotated(_))); + } + + /// An exact tie pins no identity, so no theoretical m/z exists and the peak + /// cannot be stored without mixing observed and theoretical masses. + #[test] + fn tied_ambiguity_is_skipped() { + let (r, _) = resolve_annotation("a2/-0.0004,y2-CO2-NH3/-0.0004"); + assert!(matches!(r, Resolved::SkipAmbiguous)); + } + + #[test] + fn mass_error_recovers_theoretical() { + // Real SpectraST peak: y1 for C-terminal R, observed 175.1184. + let e = MassError::Da(-0.0005); + assert!((e.theoretical_from_observed(175.1184) - 175.1189).abs() < 1e-9); + } +} + +#[cfg(test)] +mod registry_tests { + use super::tests_support::fixture; + use crate::serde::{ + LibraryArena, + read_library_file, + }; + + /// The public entry point must dispatch mzSpecLib itself. It is sniffed + /// before the registry, so a regression here would silently fall through + /// to the always-true JSON reader and fail with a generic parse error. + #[test] + fn public_read_library_file_dispatches_mzspeclib() { + for name in ["diann.mzSpecLib.txt", "spectronaut.mzSpecLib.txt"] { + let arena = read_library_file(fixture(name)) + .unwrap_or_else(|e| panic!("{name} must load through the registry: {e:?}")); + let LibraryArena::Mzpaf { geom, frag_intens } = arena else { + panic!("{name} must land in the mzpaf arena"); + }; + assert!(geom.n_rows() > 0, "{name} produced no precursors"); + assert!( + frag_intens.is_some(), + "{name} must populate the reference-intensity sidecar" + ); + } + } +} + +#[cfg(test)] +mod tests_support { + pub fn fixture(name: &str) -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/mzspeclib_io_files") + .join(name) + } +} diff --git a/rust/timsquery/tests/mzspeclib_io_files/README.md b/rust/timsquery/tests/mzspeclib_io_files/README.md new file mode 100644 index 00000000..6756feb7 --- /dev/null +++ b/rust/timsquery/tests/mzspeclib_io_files/README.md @@ -0,0 +1,15 @@ +# mzSpecLib test fixtures + +Verbatim from [HUPO-PSI/mzSpecLib](https://github.com/HUPO-PSI/mzSpecLib) +`examples/`, Apache-2.0 (same licence as this project). + +| file | why | +|---|---| +| `diann.mzSpecLib.txt` | the shape `speclib_build` will emit; all peaks annotated, every mass error exactly `0.0` | +| `spectronaut.mzSpecLib.txt` | carries neutral losses (`-H2O`, `-NH3`), which exercise the unrepresentable-annotation path | + +Deliberately NOT vendored: the NIST and SpectraST examples. They are +dominated by internal fragments, immonium ions and unannotated (`?`) peaks, +and by consensus spectra whose observed m/z carries real calibration error. +Useful later for the resolution-policy tests, but they would make these two +fixtures harder to read for no gain. diff --git a/rust/timsquery/tests/mzspeclib_io_files/diann.mzSpecLib.txt b/rust/timsquery/tests/mzspeclib_io_files/diann.mzSpecLib.txt new file mode 100644 index 00000000..b835867a --- /dev/null +++ b/rust/timsquery/tests/mzspeclib_io_files/diann.mzSpecLib.txt @@ -0,0 +1,358 @@ + +MS:1003186|library format version=1.0 +MS:1003188|library name=phl004_canonical_sall_pv_plasma.head.diann +MS:1003207|library creation software=MS:1003253|DIA-NN + + + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAAASAGGK2 +MS:1000744|selected ion m/z=778.41296 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=20 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAAASAGGK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAAASAGGK/2 +MS:1001117|theoretical mass=1554.8114178831797 +[1]MS:1000885|protein accession=P0CG40 +[1]MS:1000886|protein name=SP9_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P0CG40 + +427.22995 1.0 b6/0.0 +498.26706 0.88039231 b7/0.0 +569.3042 0.66275221 b8/0.0 +356.19284 0.58039355 b5/0.0 +640.34131 0.44118175 b9/0.0 +703.37335 0.40000239 y9/0.0 +916.48468 0.40000239 y12/0.0 +1129.5961 0.36078224 y15/0.0 +774.41046 0.34117815 y10/0.0 +845.44757 0.30195799 y11/0.0 +490.26199 0.301357 y6/0.0 +419.22488 0.29999879 y5/0.0 +1058.5588 0.25881943 y14/0.0 +632.33624 0.2572208 y8/0.0 +853.45264 0.18174933 b12/0.0 +987.52179 0.17647269 y13/0.0 +782.41553 0.13921174 b11/0.0 +924.48975 0.13725254 b13/0.0 +711.37842 0.11764847 b10/0.0 +1200.6332 0.10526822 y16/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAAASAGGK3 +MS:1000744|selected ion m/z=519.27777 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=3 +MS:1003059|number of peaks=14 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAAASAGGK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAAASAGGK/3 +MS:1001117|theoretical mass=1554.8114178831797 +[1]MS:1000885|protein accession=P0CG40 +[1]MS:1000886|protein name=SP9_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P0CG40 + +703.37335 1.0 y9/0.0 +419.22488 0.75598669 y5/0.0 +498.26706 0.67160469 b7/0.0 +640.34131 0.61734796 b9/0.0 +569.3042 0.56162828 b8/0.0 +356.19284 0.51522595 b5/0.0 +427.22995 0.51522595 b6/0.0 +774.41046 0.48234525 y10/0.0 +561.29907 0.41774848 y7/0.0 +490.26199 0.37849048 y6/0.0 +648.33875 0.37133196 b18^2/0.0 +632.33624 0.32955998 y8/0.0 +711.37842 0.27849901 b10/0.0 +782.41553 0.23208249 b11/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAAGAGAGAK2 +MS:1000744|selected ion m/z=798.92627 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=24 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAAGAGAGAK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAAGAGAGAK/2 +MS:1001117|theoretical mass=1595.8379669841897 +[1]MS:1000885|protein accession=P55011 +[1]MS:1000886|protein name=S12A2_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P55011 + +427.22995 1.0 b6/0.0 +356.19284 0.91602999 b5/0.0 +498.26706 0.83227003 b7/0.0 +569.3042 0.79017997 b8/0.0 +640.34131 0.59228998 b9/0.0 +815.43701 0.54980999 y11/0.0 +886.47412 0.51920998 y12/0.0 +531.28851 0.50638998 y7/0.0 +957.51123 0.49983999 y13/0.0 +673.36279 0.46117997 y9/0.0 +1028.5483 0.44073999 y14/0.0 +711.37842 0.42937002 b10/0.0 +744.3999 0.41159999 y10/0.0 +602.32562 0.40491998 y8/0.0 +782.41553 0.38191 b11/0.0 +403.22995 0.36974999 y5/0.0 +1099.5854 0.34740001 y15/0.0 +1170.6226 0.31896001 y16/0.0 +853.45264 0.27395001 b12/0.0 +474.26706 0.26207 y6/0.0 +995.52686 0.20731001 b14/0.0 +1241.6597 0.2052 y17/0.0 +924.48975 0.18191999 b13/0.0 +1312.6968 0.114 y18/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAAGAGAGAK3 +MS:1000744|selected ion m/z=532.95325 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=3 +MS:1003059|number of peaks=20 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAAGAGAGAK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAAGAGAGAK/3 +MS:1001117|theoretical mass=1595.8379669841897 +[1]MS:1000885|protein accession=P55011 +[1]MS:1000886|protein name=S12A2_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P55011 + +531.28851 1.0 y7/0.0 +498.26706 0.82642001 b7/0.0 +427.22995 0.75252002 b6/0.0 +356.19284 0.71425998 b5/0.0 +602.32562 0.66922998 y8/0.0 +569.3042 0.56159002 b8/0.0 +673.36279 0.52272004 y9/0.0 +640.34131 0.47526002 b9/0.0 +711.37842 0.40158999 b10/0.0 +744.3999 0.39754999 y10/0.0 +403.22995 0.31963 y5/0.0 +462.7485 0.27653 b13^2/0.0 +782.41553 0.27631998 b11/0.0 +815.43701 0.24988998 y11/0.0 +474.26706 0.24581002 y6/0.0 +391.7114 0.18824001 b11^2/0.0 +853.45264 0.17081 b12/0.0 +886.47412 0.16329999 y12/0.0 +924.48975 0.10528 b13/0.0 +957.51123 0.091109999 y13/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAASGFAYPGTSER3 +MS:1000744|selected ion m/z=746.36969 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=3 +MS:1003059|number of peaks=13 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAASGFAYPGTSER +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAASGFAYPGTSER/3 +MS:1001117|theoretical mass=2236.087258363109 +[1]MS:1000885|protein accession=P35453 +[1]MS:1000886|protein name=HXD13_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P35453 + +646.31549 1.0 y6/0.0 +880.41589 0.72346348 y8/0.0 +498.26706 0.66272509 b7/0.0 +569.3042 0.60243261 b8/0.0 +356.19284 0.58793229 b5/0.0 +809.37878 0.48742947 y7/0.0 +711.37842 0.48370829 b10/0.0 +427.22995 0.48304707 b6/0.0 +1027.4844 0.38692665 y9/0.0 +853.45264 0.32992482 b12/0.0 +1171.5378 0.31720817 y11/0.0 +782.41553 0.21106207 b11/0.0 +640.34131 0.21104671 b9/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAASGFAYPGTSER4 +MS:1000744|selected ion m/z=560.02911 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=4 +MS:1003059|number of peaks=11 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAASGFAYPGTSER +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAASGFAYPGTSER/4 +MS:1001117|theoretical mass=2236.087258363109 +[1]MS:1000885|protein accession=P35453 +[1]MS:1000886|protein name=HXD13_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P35453 + +646.31549 1.0 y6/0.0 +356.19284 0.66913706 b5/0.0 +492.24124 0.59304374 y4/0.0 +498.26706 0.4210082 b7/0.0 +809.37878 0.4210082 y7/0.0 +569.3042 0.36934987 b8/0.0 +711.37842 0.344071 b10/0.0 +514.24579 0.33137658 y9^2/0.0 +427.22995 0.33042264 b6/0.0 +640.34131 0.31578368 b9/0.0 +586.27252 0.27729672 y11^2/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAK2 +MS:1000744|selected ion m/z=429.24561 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=10 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAK/2 +MS:1001117|theoretical mass=856.4766655447999 +[1]MS:1000885|protein accession=P50914,P50458,A6NHT5,P15502,DECOY_Q9Y651,DECOY_O60341 +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P50914,P50458,A6NHT5,P15502,DECOY_Q9Y651,DECOY_O60341 + +573.33551 1.0 y7/0.0 +502.29837 0.84049195 y6/0.0 +644.37262 0.62802154 y8/0.0 +715.40973 0.53955686 y9/0.0 +356.19284 0.52399796 b5/0.0 +431.26126 0.46901798 y5/0.0 +427.22995 0.32801768 b6/0.0 +360.22415 0.30400032 y4/0.0 +498.26706 0.22399409 b7/0.0 +786.44684 0.19198385 y10/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAR2 +MS:1000744|selected ion m/z=443.24869 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=9 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAR +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAR/2 +MS:1001117|theoretical mass=884.4828135543999 +[1]MS:1000885|protein accession=P47928,Q9Y651,DECOY_Q76L83,DECOY_Q8WXD9,DECOY_Q5VZB9,DECOY_P35453,DECOY_O14654,DECOY_P55011,DECOY_P0CG40 +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P47928,Q9Y651,DECOY_Q76L83,DECOY_Q8WXD9,DECOY_Q5VZB9,DECOY_P35453,DECOY_O14654,DECOY_P55011,DECOY_P0CG40 + +601.34161 1.0 y7/0.0 +530.3045 0.90602791 y6/0.0 +672.37872 0.83756346 y8/0.0 +356.19284 0.69194162 b5/0.0 +388.23029 0.52842641 y4/0.0 +459.2674 0.47519037 y5/0.0 +743.41583 0.39619291 y9/0.0 +427.22995 0.36180204 b6/0.0 +569.3042 0.20031726 b8/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAASGAAIPPLIPPR2 +MS:1000744|selected ion m/z=950.04419 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=25 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAASGAAIPPLIPPR +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAASGAAIPPLIPPR/2 +MS:1001117|theoretical mass=1898.0737805754497 +[1]MS:1000885|protein accession=O14654 +[1]MS:1000886|protein name=IRS4_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=O14654 + +789.49811 1.0 y7/0.0 +369.22449 0.58921003 y3/0.0 +427.22995 0.31046999 b6/0.0 +498.26706 0.30821002 b7/0.0 +569.3042 0.26800001 b8/0.0 +356.19284 0.25783998 b5/0.0 +640.34131 0.20063001 b9/0.0 +1101.6779 0.15594999 y11/0.0 +1188.71 0.15020999 y12/0.0 +902.58221 0.13153 y8/0.0 +1259.7471 0.1284 y13/0.0 +973.61932 0.11756 y9/0.0 +997.50616 0.11383001 b14/0.0 +711.37842 0.10726 b10/0.0 +1330.7842 0.10523 y14/0.0 +1110.5902 0.10514 b15/0.0 +926.46899 0.10461 b13/0.0 +1401.8213 0.080120005 y15/0.0 +855.43188 0.074140005 b12/0.0 +1044.6564 0.072379999 y10/0.0 +482.30853 0.067829996 y4/0.0 +798.41046 0.05923 b11/0.0 +1472.8584 0.055300001 y16/0.0 +692.44537 0.044360001 y6/0.0 +1543.8955 0.036620002 y17/0.0 + diff --git a/rust/timsquery/tests/mzspeclib_io_files/spectronaut.mzSpecLib.txt b/rust/timsquery/tests/mzspeclib_io_files/spectronaut.mzSpecLib.txt new file mode 100644 index 00000000..6236f10a --- /dev/null +++ b/rust/timsquery/tests/mzspeclib_io_files/spectronaut.mzSpecLib.txt @@ -0,0 +1,520 @@ + +MS:1003186|library format version=1.0 +MS:1003188|library name=human_serum.head.spectronaut +MS:1003207|library creation software=MS:1001327|Spectronaut + + + + +MS:1003061|library spectrum name=AQIPILR/2 +MS:1003208|experimental precursor monoisotopic m/z=405.7634379 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_AQIPILR_ +MS:1002476|ion mobility drift time=0.7629655 +MS:1001581|FAIMS compensation voltage=-60.0 +[2]MS:1000896|normalized retention time=28.658491 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=False +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P04114 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=6 + +MS:1000888|stripped peptide sequence=AQIPILR +MS:1003270|proforma peptidoform ion notation=AQIPILR/2 +MS:1001117|theoretical mass=809.5123227775299 +[1]MS:1000885|protein accession=P04114 +[1]MS:1000886|protein name=APOB_HUMAN +[1]MS:1001088|protein description=Apolipoprotein B-100 +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P04114 + +313.1870317 3.564491 b3/0.0 +401.2870801 3.3441753 y3/0.0 +498.339844 100.0 y4/0.0 +249.6735602 11.528888 y4^2/0.0 +611.423908 36.853043 y5/0.0 +481.3132956 5.708535 y4-NH3/0.0 + + +MS:1003061|library spectrum name=AGVLFGMSDR/2 +MS:1003208|experimental precursor monoisotopic m/z=526.763309 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_AGVLFGMSDR_ +MS:1002476|ion mobility drift time=0.8644136 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=46.12812 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=False +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P09172 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=13 + +MS:1000888|stripped peptide sequence=AGVLFGMSDR +MS:1003270|proforma peptidoform ion notation=AGVLFGMSDR/2 +MS:1001117|theoretical mass=1051.5120650773501 +[1]MS:1000885|protein accession=P09172 +[1]MS:1000886|protein name=DOPO_HUMAN +[1]MS:1001088|protein description=Dopamine beta-hydroxylase +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P09172 + +228.1342679 48.582985 b3/0.0 +377.1779236 4.8015785 y3/0.0 +341.2183319 4.1744895 b4/0.0 +508.2184085 5.056169 y4/0.0 +565.2398722 69.96272 y5/0.0 +712.3082861 100.0 y6/0.0 +825.3923501 51.118713 y7/0.0 +924.460764 1.5956572 y8/0.0 +360.1513752 3.1122544 y3-NH3/0.0 +491.1918601 1.0637656 y4-NH3/0.0 +694.2977213 1.3498487 y6-H2O/0.0 +695.2817378 2.9444132 y6-NH3/0.0 +359.1673588 2.1920948 y3-H2O/0.0 + + +MS:1003061|library spectrum name=DEDNNLLTEK/2 +MS:1003208|experimental precursor monoisotopic m/z=595.7804071 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_DEDNNLLTEK_ +MS:1002476|ion mobility drift time=0.90230334 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=13.850179 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P09486 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=19 + +MS:1000888|stripped peptide sequence=DEDNNLLTEK +MS:1003270|proforma peptidoform ion notation=DEDNNLLTEK/2 +MS:1001117|theoretical mass=1189.54626122625 +[1]MS:1000885|protein accession=P09486 +[1]MS:1000886|protein name=SPRC_HUMAN +[1]MS:1001088|protein description=SPARC +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P09486 + +360.1037556 14.370086 b3/0.0 +377.2030757 69.91263 y3/0.0 +490.2871397 47.727154 y4/0.0 +588.1896105 3.50105 b5/0.0 +603.3712037 20.282309 y5/0.0 +701.2736745 4.035067 b6/0.0 +717.4141311 24.80533 y6/0.0 +831.4570586 21.473965 y7/0.0 +946.4840016 100.0 y8/0.0 +342.0931908 5.760026 b3-H2O/0.0 +359.1925109 5.090061 y3-H2O/0.0 +472.2765749 4.874231 y4-H2O/0.0 +571.1630621 4.1113725 b5-NH3/0.0 +684.2471261 4.05031 b6-NH3/0.0 +700.3875827 3.518 y6-NH3/0.0 +814.4305102 7.3916 y7-NH3/0.0 +928.4734368 5.1134515 y8-H2O/0.0 +929.4574532 10.1338825 y8-NH3/0.0 +1075.526595 14.727533 y9/0.0 + + +MS:1003061|library spectrum name=GWVTDGFSSLK/2 +MS:1003208|experimental precursor monoisotopic m/z=598.8009456 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_GWVTDGFSSLK_ +MS:1002476|ion mobility drift time=0.92365193 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=62.688942 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P02656 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=14 + +MS:1000888|stripped peptide sequence=GWVTDGFSSLK +MS:1003270|proforma peptidoform ion notation=GWVTDGFSSLK/2 +MS:1001117|theoretical mass=1195.5873381925899 +[1]MS:1000885|protein accession=P02656 +[1]MS:1000886|protein name=APOC3_HUMAN +[1]MS:1001088|protein description=Apolipoprotein C-III +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P02656 + +343.1764671 3.8687427 b3/0.0 +347.2288965 6.1732 y3/0.0 +444.2241455 1.421 b4/0.0 +434.260925 9.774844 y4/0.0 +581.3293389 2.66143 y5/0.0 +638.3508026 19.109653 y6/0.0 +753.3777456 20.841564 y7/0.0 +854.4254241 60.931602 y8/0.0 +953.493838 36.67316 y9/0.0 +416.2503601 2.702636 y4-H2O/0.0 +735.3671808 2.1958628 y7-H2O/0.0 +836.4148593 4.374068 y8-H2O/0.0 +935.4832732 4.7364564 y9-H2O/0.0 +468.2452748 1.662614 y9-H2O^2/0.0 + + +MS:1003061|library spectrum name=VTSIQDWVQK/2 +MS:1003208|experimental precursor monoisotopic m/z=602.3220451 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_VTSIQDWVQK_ +MS:1002476|ion mobility drift time=0.9102522 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=39.73143 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P00738 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=22 + +MS:1000888|stripped peptide sequence=VTSIQDWVQK +MS:1003270|proforma peptidoform ion notation=VTSIQDWVQK/2 +MS:1001117|theoretical mass=1202.62953735774 +[1]MS:1000885|protein accession=P00738 +[1]MS:1000886|protein name=HPT_HUMAN +[1]MS:1001088|protein description=Haptoglobin +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P00738 + +288.1553973 13.402388 b3/0.0 +374.2397956 35.40592 y3/0.0 +560.3191085 46.133404 y4/0.0 +675.3460516 49.92076 y5/0.0 +803.4046291 88.30317 y6/0.0 +916.488693 35.01724 y7/0.0 +458.7479848 8.279677 y7^2/0.0 +1003.520721 100.0 y8/0.0 +1104.5684 49.551434 y9/0.0 +270.1448324 95.16555 b3-H2O/0.0 +785.3940643 24.796286 y6-H2O/0.0 +393.2006704 13.1079035 y6-H2O^2/0.0 +786.3780807 46.45572 y6-NH3/0.0 +985.5101566 7.3205233 y8-H2O/0.0 +493.2587166 4.255348 y8-H2O^2/0.0 +1086.557835 7.42398 y9-H2O/0.0 +201.1233688 88.65463 b4^2/0.0 +401.2394612 4.14027 b4/0.0 +552.7878382 4.422511 y9^2/0.0 +383.2288964 12.326735 b4-H2O/0.0 +986.4941731 24.063145 y8-NH3/0.0 +357.2132472 60.34427 y3-NH3/0.0 + + +MS:1003061|library spectrum name=QELSEAEQATR/2 +MS:1003208|experimental precursor monoisotopic m/z=631.3045807 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_QELSEAEQATR_ +MS:1002476|ion mobility drift time=0.92576075 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=-14.638051 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P01024 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=15 + +MS:1000888|stripped peptide sequence=QELSEAEQATR +MS:1003270|proforma peptidoform ion notation=QELSEAEQATR/2 +MS:1001117|theoretical mass=1260.594608401 +[1]MS:1000885|protein accession=P01024 +[1]MS:1000886|protein name=CO3_HUMAN +[1]MS:1001088|protein description=Complement C3 +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P01024 + +371.192511 3.1688333 b3/0.0 +347.2037444 10.465045 y3/0.0 +475.2623219 23.276728 y4/0.0 +604.304915 37.09764 y5/0.0 +675.3420288 64.2233 y6/0.0 +804.3846219 37.812637 y7/0.0 +891.4166503 96.74726 y8/0.0 +1004.500714 100.0 y9/0.0 +353.1819462 4.8028316 b3-H2O/0.0 +354.1659627 5.46505 b3-NH3/0.0 +458.2357735 1.8879279 y4-NH3/0.0 +658.3154804 1.120537 y6-NH3/0.0 +786.3740571 7.649042 y7-H2O/0.0 +873.4060855 4.228747 y8-H2O/0.0 +986.4901495 1.3095266 y9-H2O/0.0 + + +MS:1003061|library spectrum name=EEGTDLEVTANR/2 +MS:1003208|experimental precursor monoisotopic m/z=667.3151454 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_EEGTDLEVTANR_ +MS:1002476|ion mobility drift time=0.94296074 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=5.3526073 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P20742 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=9 + +MS:1000888|stripped peptide sequence=EEGTDLEVTANR +MS:1003270|proforma peptidoform ion notation=EEGTDLEVTANR/2 +MS:1001117|theoretical mass=1332.6157377683999 +[1]MS:1000885|protein accession=P20742 +[1]MS:1000886|protein name=PZP_HUMAN +[1]MS:1001088|protein description=Pregnancy zone protein +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P20742 + +360.1989934 20.52395 y3/0.0 +461.2466719 85.759026 y4/0.0 +560.3150858 82.32931 y5/0.0 +689.3576789 100.0 y6/0.0 +802.4417428 79.06382 y7/0.0 +917.4686859 72.09271 y8/0.0 +1018.516364 21.396385 y9/0.0 +1075.537828 83.90465 y10/0.0 +671.3471141 8.983159 y6-H2O/0.0 + + +MS:1003061|library spectrum name=C[Carbamidomethyl (C)]EEDEEFTC[Carbamidomethyl (C)]R/2 +MS:1003208|experimental precursor monoisotopic m/z=687.7504667 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_C[Carbamidomethyl (C)]EEDEEFTC[Carbamidomethyl (C)]R_ +MS:1002476|ion mobility drift time=0.9011946 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=-1.2823446 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P00747 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=17 + +MS:1000888|stripped peptide sequence=CEEDEEFTCR +MS:1003270|proforma peptidoform ion notation=C[Carbamidomethyl]EEDEEFTC[Carbamidomethyl]R/2 +MS:1001117|theoretical mass=1373.4863810338297 +[1]MS:1000885|protein accession=P00747 +[1]MS:1000886|protein name=PLMN_HUMAN +[1]MS:1001088|protein description=Plasminogen +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P00747 + +419.1231111 14.944622 b3/0.0 +436.1972791 30.738253 y3/0.0 +534.1500542 9.713619 b4/0.0 +583.2656931 51.723152 y4/0.0 +712.3082861 50.78903 y5/0.0 +841.3508792 59.771255 y6/0.0 +956.3778223 75.695114 y7/0.0 +1085.420415 100.0 y8/0.0 +1214.463008 15.559877 y9/0.0 +401.1125463 9.39616 b3-H2O/0.0 +516.1394894 5.2136855 b4-H2O/0.0 +566.2391447 6.1200476 y4-NH3/0.0 +694.2977213 6.602107 y5-H2O/0.0 +823.3403144 10.601803 y6-H2O/0.0 +938.3672575 6.748549 y7-H2O/0.0 +1067.409851 26.568663 y8-H2O/0.0 +1196.452444 10.719879 y9-H2O/0.0 + + +MS:1003061|library spectrum name=KQELSEAEQATR/2 +MS:1003208|experimental precursor monoisotopic m/z=695.3520622 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_KQELSEAEQATR_ +MS:1002476|ion mobility drift time=0.9663669 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=-26.474043 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P01024 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=29 + +MS:1000888|stripped peptide sequence=KQELSEAEQATR +MS:1003270|proforma peptidoform ion notation=KQELSEAEQATR/2 +MS:1001117|theoretical mass=1388.6895714149998 +[1]MS:1000885|protein accession=P01024 +[1]MS:1000886|protein name=CO3_HUMAN +[1]MS:1001088|protein description=Complement C3 +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P01024 + +386.2034101 7.7662497 b3/0.0 +347.2037444 3.2926106 y3/0.0 +499.2874741 6.734219 b4/0.0 +475.2623219 14.221682 y4/0.0 +586.3195025 1.5547751 b5/0.0 +604.304915 12.158777 y5/0.0 +715.3620955 3.230758 b6/0.0 +675.3420288 25.758886 y6/0.0 +786.3992093 1.9753934 b7/0.0 +804.3846219 22.087849 y7/0.0 +915.4418024 1.8401791 b8/0.0 +458.2245394 1.6086894 b8^2/0.0 +891.4166503 74.71736 y8/0.0 +1004.500714 54.28254 y9/0.0 +1114.537494 1.0495659 b10/0.0 +1133.543307 100.0 y10/0.0 +1261.601885 24.588951 y11/0.0 +631.3045807 2.1570513 y11^2/0.0 +368.1928453 3.4925315 b3-H2O/0.0 +369.1768617 2.9452903 b3-NH3/0.0 +481.2769092 1.0290943 b4-H2O/0.0 +458.2357735 1.8126523 y4-NH3/0.0 +873.4060855 2.0101676 y8-H2O/0.0 +1115.532743 6.641937 y10-H2O/0.0 +622.2992983 6.5424037 y11-H2O^2/0.0 +1243.59132 3.3419182 y11-H2O/0.0 +1244.575336 10.56645 y11-NH3/0.0 +482.2609257 1.1637812 b4-NH3/0.0 +330.177196 1.6216956 y3-NH3/0.0 + From 000ac1d17ec05b296bb20969157152069782fb66 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 17:04:30 -0700 Subject: [PATCH 07/27] docs: vendor the Carafe contract and pin it with tests Carafe drives timsquery_cli as a subprocess and parses the output with fastjson -- no field remapping, no schema negotiation. A renamed field or a dropped serde alias fails loudly on neither side: Carafe gets a null and NPEs somewhere unrelated. Keeping the assumptions in another repo made that a landmine. The tests use the literal payloads from the document, through the public entry points Carafe actually reaches (a file path into `read_library_file`, a JSON blob into `Tolerance`), so a refactor that keeps the internals working but moves the boundary still fails. Two things worth knowing that the tests now enforce: - `precursor`, `fragments` and `fragment_labels` are serde aliases of names used elsewhere in the codebase. "Unifying" them would break Carafe while passing every other test. - Carafe writes `percent`/`absolute` where the CLI's own templates write `pct`/`da`. Both spellings must keep working, so both are covered. The `ms` window is `[itol - itol_shift, itol + itol_shift]` -- a +-itol window recentred on a measured calibration offset, not a symmetric tolerance. The test pins both edges distinctly, since collapsing them would quietly recentre every extraction window. Not covered, because it needs the built binary and a real .d: aggregator names, the -o directory layout, and the results.json basename. --- docs/CARAFE_CONTRACT.md | 103 ++++++++++++++++++ rust/timsquery/tests/carafe_contract.rs | 132 ++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 docs/CARAFE_CONTRACT.md create mode 100644 rust/timsquery/tests/carafe_contract.rs diff --git a/docs/CARAFE_CONTRACT.md b/docs/CARAFE_CONTRACT.md new file mode 100644 index 00000000..10155ca2 --- /dev/null +++ b/docs/CARAFE_CONTRACT.md @@ -0,0 +1,103 @@ +# Carafe contract + +What Carafe assumes when calling `timsquery`. Break any of it → silent failure or NPE. +Refs: `util/CallTimsQuery.java`, `ai/AIGear.java` (~6660–6912), `dia/{PSMQuery,PSMQueryResult,XICQueryResult}.java`. + +> Vendored from the Carafe repo so the assumptions live next to the code that +> has to honour them. `rust/timsquery/tests/carafe_contract.rs` pins the parts +> that are mechanically checkable — field names, aliases, units — against the +> literal JSON in this document. If you edit a payload here, edit it there. + +## CLI + +``` + query-index -a -r -t -e -f ndjson -o +``` + +| Flag | Value | +|---|---| +| subcmd | `query-index` | +| `-a` | `spectrum-aggregator` \| `chromatogram-aggregator` (selects output schema) | +| `-r` | raw `.d` input | +| `-t` | tolerance JSON (§tolerances) | +| `-e` | query targets JSON (§targets) | +| `-f` | `ndjson` | +| `-o` | output **directory** (not file) | + +Binary path: `bin/timsquery/{windows,macos,linux}/timsquery_cli[.exe]`. + +**Fragile couplings** (most likely to break on a timsquery change): +- Aggregator names are literals — rename breaks Carafe, no fallback. +- `-o` is a dir; Carafe reads `/results.json` — basename must be exactly `results.json`. +- Output parsed line-by-line — must be ndjson, one object per line. +- Exit 0 = success; nonzero logged, no retry. + +## Tolerances (`-t`) + +```json +{ + "ms": { "ppm": [itol-itol_shift, itol+itol_shift] }, + "rt": { "minutes": [rt_win, rt_win] }, + "mobility": { "percent": [mobility, mobility] }, + "quad": { "absolute": [quad, quad] } +} +``` +`ms` unit key is dynamic (`itolu`, currently `ppm`). Each value is `[low, high]`. Defaults: itol 15, mobility 3.0, quad 0.1, rt_win 0.1 (spectra) / `CParameter.rt_win` (xic). + +### `ms` window derivation (`itol` / `itol_shift`) + +The `ms` window is `[itol - itol_shift, itol + itol_shift]`, built from two independent inputs: + +- **`itol`** = `CParameter.itol` — the configured fragment-ion tolerance (default 15). Static per run. +- **`itol_shift`** = per-run **median observed m/z error**, a data-driven calibration offset measured before the query (`AIGear.java` ~6835–6855): + - Carafe collects MS1 and MS2 mass errors from already-matched ions, takes the median of each (`ms1_error_shift`, `ms2_error_shift`). + - If precursor and fragment units match (`CParameter.tolu` == `CParameter.itolu`): `itol_shift = max(ms1_error_shift, ms2_error_shift)`. + - Else: `itol_shift = ms2_error_shift` (fragment only). + - No matched ions → `itol_shift = 0`, so the window collapses to `[itol, itol]`. + +timsquery interprets the pair as `[light_magnitude, heavy_magnitude]`: `[2, 7]` = the ion may be up to 2 ppm light **or** up to 7 ppm heavy (signed error range `[-2, +7]`). Under that convention Carafe's `[itol - itol_shift, itol + itol_shift]` is a `±itol` window recentered on the calibration offset `itol_shift`: + +- signed error window = `[itol_shift - itol, itol_shift + itol]` +- light magnitude = `itol - itol_shift`, heavy magnitude = `itol + itol_shift` + +Example: `itol` 15, `itol_shift` +2 → `[13, 17]` = "13 ppm light to 17 ppm heavy" = a ±15 ppm window centered on the +2 ppm systematic offset. This is the intended behavior — the offset shifts the window's center, `itol` sets its half-width. + +## Targets (`-e`, `psm_query.json`) + +JSON **array** of: +```json +{ "id": 0, "mobility": 0.95, "rt_seconds": 1234.5, "precursor": 650.32, + "precursor_charge": 2, "precursor_isotopes": [0,1,2], + "fragments": [175.1, 288.2], "fragment_labels": ["y1","y3^2"] } +``` +`id` = row index, echoed back in results. `rt_seconds` = RT_min × 60. `fragments`↔`fragment_labels` positional. Labels: charge 1 → `y3`, else `y3^2`; precursor → `p`/`p^z`. + +## Results (`-o`, `/results.json`) + +**ndjson** — one object per line, one line per `id`. Parsed `JSON.parseObject(line, …)`, keyed by `id`. +(`.json` name is misleading; it is not a JSON array.) + +### spectrum-aggregator → `PSMQueryResult` +```json +{ "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, "precursor_mz":650.32, + "precursor_charge":2, "precursor_intensities":[1200,800,300], "precursor_labels":[0,1,2], + "fragment_mzs":[175.1,288.2], "fragment_intensities":[500,0] } +``` +Scalar intensity per ion (no RT axis). `precursor_intensities`/`fragment_intensities` are **1-D**, positionally paired with their m/z arrays. + +### chromatogram-aggregator → `XICQueryResult` +```json +{ "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, + "precursor_mzs":[650.32,650.82], "precursor_intensities":[[…],[…]], + "fragment_mzs":[175.1,288.2], "fragment_labels":["y1","b2"], + "fragment_intensities":[[…],[…]], "retention_time_results_seconds":[1230,1231] } +``` +2-D matrices: `[ion][rt_point]`. Every row length == `retention_time_results_seconds.length`. + +## Invariants + +1. `id` echoed from input; unique + present. Dup → overwrite; missing → NPE downstream. +2. ndjson: one complete object per line, no array wrapper, no pretty-print. +3. Exact field names (fastjson, no remap). Note **singular vs plural** across modes: spectrum `precursor_mz`+`precursor_labels`(int[]); chromatogram `precursor_mzs`+`precursor_intensities`(2-D). Two distinct schemas. +4. m/z ↔ intensity arrays positionally paired (spectrum 1-D, chromatogram row-per-ion). +5. Output basename exactly `results.json` in `-o` dir. Exit 0 on success. diff --git a/rust/timsquery/tests/carafe_contract.rs b/rust/timsquery/tests/carafe_contract.rs new file mode 100644 index 00000000..5bf4577f --- /dev/null +++ b/rust/timsquery/tests/carafe_contract.rs @@ -0,0 +1,132 @@ +//! Executable form of `docs/CARAFE_CONTRACT.md`. +//! +//! Carafe drives `timsquery_cli` as a subprocess and parses its output with +//! fastjson, with no field remapping and no schema negotiation. A renamed +//! field or a dropped serde alias fails loudly on neither side — Carafe just +//! gets a null and NPEs somewhere unrelated. These tests pin the mechanically +//! checkable parts, using the literal payloads from the document so the two +//! cannot drift apart silently. +//! +//! Everything goes through the public entry points Carafe actually reaches +//! (a file path into `read_library_file`, a JSON blob into `Tolerance`) rather +//! than internal types, so a refactor that keeps the internals working but +//! changes the boundary still fails here. +//! +//! NOT covered, because it needs the built binary and a real `.d`: aggregator +//! names, the `-o` directory layout, and the `results.json` basename. + +use std::io::Write; +use timsquery::models::Tolerance; +use timsquery::serde::{ + LibraryArena, + read_library_file, +}; + +/// Verbatim from the contract's "Targets (`-e`, `psm_query.json`)" section. +const CARAFE_TARGETS: &str = r#"[ + { "id": 0, "mobility": 0.95, "rt_seconds": 1234.5, "precursor": 650.32, + "precursor_charge": 2, "precursor_isotopes": [0,1,2], + "fragments": [175.1, 288.2], "fragment_labels": ["y1","y3^2"] } +]"#; + +/// Verbatim from the contract's "Tolerances (`-t`)" section. Note `percent` +/// and `absolute`, not the `pct`/`da` spellings the CLI's own templates use — +/// both must deserialize. +const CARAFE_TOLERANCES: &str = r#"{ + "ms": { "ppm": [13.0, 17.0] }, + "rt": { "minutes": [0.1, 0.1] }, + "mobility": { "percent": [3.0, 3.0] }, + "quad": { "absolute": [0.1, 0.1] } +}"#; + +fn write_targets(json: &str) -> tempfile::NamedTempFile { + let mut f = tempfile::Builder::new() + .suffix(".json") + .tempfile() + .expect("tempfile"); + f.write_all(json.as_bytes()).expect("write targets"); + f.flush().expect("flush"); + f +} + +/// Every field name Carafe emits must land somewhere. `precursor`, +/// `fragments` and `fragment_labels` are serde aliases of names used elsewhere +/// in the codebase, so a cleanup that "unifies" them would break Carafe +/// without failing any other test. +#[test] +fn carafe_target_payload_loads_through_the_public_reader() { + let f = write_targets(CARAFE_TARGETS); + let arena = read_library_file(f.path()).expect("Carafe's target JSON must load"); + + // String fragment labels must resolve to ion annotations. If Carafe ever + // omitted `fragment_labels` the try-chain would silently fall through to + // the integer-labelled variant and synthesize positional labels — a wrong + // answer rather than an error. + let LibraryArena::Mzpaf { geom, .. } = arena else { + panic!("labelled targets must land in the ion-annotated arena"); + }; + assert_eq!(geom.n_rows(), 1); + assert_eq!(geom.precursor_mz[0], 650.32); + assert_eq!(geom.charge[0], 2); + assert_eq!(geom.rt_seconds[0], 1234.5); + assert_eq!(geom.mobility[0], 0.95); + + // Labels are positionally paired with their m/z values, and the `^N` + // charge suffix survives. + assert_eq!(geom.frag_mzs.len(), 2); + assert_eq!(geom.frag_mzs[0], 175.1); + assert_eq!(geom.frag_mzs[1], 288.2); + let labels: Vec = geom.frag_labels.iter().map(|l| l.to_string()).collect(); + assert_eq!(labels, vec!["y1".to_string(), "y3^2".to_string()]); +} + +/// `id` is echoed back and used as Carafe's map key. A missing or renamed +/// `id` NPEs downstream rather than erroring here. +#[test] +fn carafe_id_field_is_required() { + let without_id = CARAFE_TARGETS.replace("\"id\": 0,", ""); + let f = write_targets(&without_id); + assert!( + read_library_file(f.path()).is_err(), + "a target without `id` must fail rather than default it" + ); +} + +/// Carafe writes `percent` and `absolute`; the CLI's own templates write `pct` +/// and `da`. Dropping either spelling silently breaks one caller. +#[test] +fn carafe_tolerance_spellings_deserialize() { + let tol: Tolerance = + serde_json::from_str(CARAFE_TOLERANCES).expect("Carafe's tolerance JSON must deserialize"); + + // The `ms` window is `[itol - itol_shift, itol + itol_shift]`: a +-itol + // window recentred on a measured calibration offset, read as "13 ppm light + // to 17 ppm heavy". Both edges must stay distinct — collapsing them to a + // symmetric tolerance would quietly recentre every extraction window. + let rendered = format!("{tol:?}"); + assert!( + rendered.contains("13.0") && rendered.contains("17.0"), + "both m/z window edges must be preserved distinctly, got {rendered}" + ); + + let round = serde_json::to_string(&tol).expect("tolerance must re-serialize"); + let back: Tolerance = serde_json::from_str(&round).expect("and deserialize again"); + assert_eq!( + format!("{back:?}"), + rendered, + "tolerance must survive a round trip through its own output" + ); +} + +/// The CLI-template spellings must keep working too, so the two callers stay +/// interchangeable. +#[test] +fn cli_template_tolerance_spellings_still_deserialize() { + let cli_style = r#"{ + "ms": { "da": [0.04, 0.04] }, + "rt": "Unrestricted", + "mobility": { "pct": [20.0, 20.0] }, + "quad": { "da": [0.2, 0.2] } + }"#; + serde_json::from_str::(cli_style).expect("the CLI's own spellings must deserialize"); +} From cda7343c0cea77a86b89f5f9d4c092d72e099498 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 17:07:58 -0700 Subject: [PATCH 08/27] feat!: remove the msgpack speclib format Removes the MessagePack reader, the `SpeclibWriter` (whose only arm was msgpack+zstd), the `.msgpack{,.zst,.zstd}` extension detection, and the rmp-serde dependency. `.msgpack` paths now fall through to the timsquery bridge and fail there rather than being claimed by a native reader. BREAKING: `speclib_build_cli` does not compile. It was the sole user of `SpeclibWriter`, and its replacement writer emits mzSpecLib -- landing separately. Left broken deliberately rather than propped up with an interim ndjson writer that would be deleted a commit later. Everything else in the workspace builds and passes: timsquery, timsseek, micromzpaf and the remaining binaries are untouched. --- Cargo.lock | 20 --- README.md | 4 +- rust/micromzpaf/src/lib.rs | 7 +- rust/timsseek/Cargo.toml | 1 - rust/timsseek/examples/query_bench.rs | 2 +- rust/timsseek/src/data_sources/mod.rs | 1 - rust/timsseek/src/data_sources/speclib.rs | 146 ++----------------- rust/timsseek/src/models/query_item.rs | 2 +- rust/timsseek_cli/assets/default_config.toml | 2 +- 9 files changed, 23 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b69058b1..d3970058 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5815,25 +5815,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rmp" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" -dependencies = [ - "num-traits", -] - -[[package]] -name = "rmp-serde" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" -dependencies = [ - "rmp", - "serde", -] - [[package]] name = "robust" version = "1.2.0" @@ -6949,7 +6930,6 @@ dependencies = [ "rand 0.9.3", "rayon", "regex", - "rmp-serde", "rusqlite", "serde", "serde_json", diff --git a/README.md b/README.md index 20a03915..7cd7db17 100644 --- a/README.md +++ b/README.md @@ -85,11 +85,11 @@ Both CLIs accept `s3://` URIs anywhere a path is accepted (AWS / MinIO / R2). `. ```bash timsseek --raw-inputs s3://bkt/sample.d.tar \ - --speclib-uri s3://bkt/lib.msgpack.zst \ + --speclib-uri s3://bkt/lib.mzSpecLib.txt \ --output-uri s3://bkt/runs/out speclib_build_cli --fasta s3://bkt/proteome.fasta \ - --output s3://bkt/lib.msgpack.zst + --output s3://bkt/lib.mzSpecLib.txt ``` Auth via AWS default chain. MinIO/R2: set `AWS_ENDPOINT_URL`. See `docs/development.md` for `[staging]` config + env var list. diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index 0f46e6a9..8ddda676 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -527,9 +527,10 @@ pub struct ParsedAnnotation { /// Split the trailing `/[ppm]` off an annotation, if present. /// -/// Care is needed because `/` does not otherwise appear, but the numeric part -/// may be signed and the `ppm` suffix optional. -fn split_mass_error(s: &str) -> Result<(&str, Option), IonParsingError> { +/// Public because a library reader needs the error even when the ion itself is +/// unrepresentable: the error is what recovers theoretical m/z, so a peak that +/// ends up with an unknown label still gets a correct mass. +pub fn split_mass_error(s: &str) -> Result<(&str, Option), IonParsingError> { let Some((head, tail)) = s.rsplit_once('/') else { return Ok((s, None)); }; diff --git a/rust/timsseek/Cargo.toml b/rust/timsseek/Cargo.toml index 352e7c69..f774e856 100644 --- a/rust/timsseek/Cargo.toml +++ b/rust/timsseek/Cargo.toml @@ -6,7 +6,6 @@ license.workspace = true [dependencies] regex = "1.10.6" -rmp-serde = "1.1" zstd = "0.13" # Gradient boosted tree diff --git a/rust/timsseek/examples/query_bench.rs b/rust/timsseek/examples/query_bench.rs index 892e4272..015c901c 100644 --- a/rust/timsseek/examples/query_bench.rs +++ b/rust/timsseek/examples/query_bench.rs @@ -75,7 +75,7 @@ fn main() { ); let speclib_path = env( "BENCH_SPECLIB", - "/Users/sebastianpaez/fasta/asdad.msgpack.zstd", + "/Users/sebastianpaez/fasta/asdad.ndjson.zstd", ); let n: usize = env("QB_N", "2000").parse().unwrap(); let iters: usize = env("QB_ITERS", "1").parse().unwrap(); diff --git a/rust/timsseek/src/data_sources/mod.rs b/rust/timsseek/src/data_sources/mod.rs index 15a139c4..cd26f59a 100644 --- a/rust/timsseek/src/data_sources/mod.rs +++ b/rust/timsseek/src/data_sources/mod.rs @@ -11,5 +11,4 @@ pub use speclib::{ ReferenceEG, SerSpeclibElement, Speclib, - SpeclibWriter, }; diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 1acb1b03..8884acb2 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -297,26 +297,21 @@ pub type Speclib = ReferenceLibrary; pub enum SpeclibFormat { NdJson, NdJsonZstd, - MessagePack, - MessagePackZstd, } impl SpeclibFormat { /// Detect a native timsseek format by EXTENSION ONLY. Returns `None` for - /// anything else (including `.speclib`), which routes to the timsquery - /// bridge. + /// anything else (including `.speclib` and `.mzSpecLib.txt`), which routes + /// to the timsquery bridge. /// - /// Extension-only is deliberate: msgpack has no reliable magic byte, so a - /// content sniff would misclaim raw binaries like `.speclib` as msgpack. + /// Extension-only is a leftover from the msgpack era, where a content + /// sniff would have misclaimed raw binaries. ndjson could be sniffed, but + /// the bridge already handles anything this returns `None` for. pub fn detect_from_extension(path: &Path) -> Option { let path_str = path.to_string_lossy().to_lowercase(); // Accept both `.zst` and `.zstd` — DIA-NN/user pipelines use either. - if path_str.ends_with(".msgpack.zst") || path_str.ends_with(".msgpack.zstd") { - Some(SpeclibFormat::MessagePackZstd) - } else if path_str.ends_with(".msgpack") { - Some(SpeclibFormat::MessagePack) - } else if path_str.ends_with(".ndjson.zst") || path_str.ends_with(".ndjson.zstd") { + if path_str.ends_with(".ndjson.zst") || path_str.ends_with(".ndjson.zstd") { Some(SpeclibFormat::NdJsonZstd) } else if path_str.ends_with(".ndjson") { Some(SpeclibFormat::NdJson) @@ -355,19 +350,6 @@ impl<'a> SpeclibReader<'a> { })?; Box::new(NdJsonReader::new(BufReader::new(decoder))) } - SpeclibFormat::MessagePack => Box::new(MessagePackReader::new(reader)), - SpeclibFormat::MessagePackZstd => { - let decoder = zstd::Decoder::new(reader).map_err(|e| { - LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error creating ZSTD decoder", - } - })?; - Box::new(MessagePackReader::new(decoder)) - } }; Ok(SpeclibReader { inner }) @@ -425,47 +407,6 @@ impl Iterator for NdJsonReader { } } -struct MessagePackReader { - deserializer: rmp_serde::Deserializer>, -} - -impl MessagePackReader { - fn new(reader: R) -> Self { - Self { - deserializer: rmp_serde::Deserializer::new(reader), - } - } -} - -impl Iterator for MessagePackReader { - type Item = Result; - - fn next(&mut self) -> Option { - use serde::Deserialize; - - match SerSpeclibElement::deserialize(&mut self.deserializer) { - Ok(elem) => Some(Ok(elem)), - Err(rmp_serde::decode::Error::InvalidMarkerRead(ref io_err)) - if io_err.kind() == std::io::ErrorKind::UnexpectedEof => - { - None - } // EOF - Err(rmp_serde::decode::Error::InvalidDataRead(ref io_err)) - if io_err.kind() == std::io::ErrorKind::UnexpectedEof => - { - None - } // EOF - Err(e) => Some(Err(LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error reading MessagePack", - })), - } - } -} - impl Speclib { /// Whether every sequence in the library parsed (gates sequence-derived /// scoring features). Reads the sealed arena's `sequence_features` state. @@ -605,47 +546,6 @@ impl Speclib { ); } } - -pub struct SpeclibWriter { - inner: SpeclibWriterInner, -} - -enum SpeclibWriterInner { - MsgpackZstd(zstd::Encoder<'static, W>), -} - -impl SpeclibWriter { - pub fn new_msgpack_zstd(writer: W) -> Result { - let encoder = zstd::Encoder::new(writer, 3)?; - Ok(Self { - inner: SpeclibWriterInner::MsgpackZstd(encoder), - }) - } - - pub fn append(&mut self, elem: &SerSpeclibElement) -> Result<(), LibraryReadingError> { - match &mut self.inner { - SpeclibWriterInner::MsgpackZstd(encoder) => { - rmp_serde::encode::write(encoder, elem).map_err(|e| { - LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error writing MessagePack", - } - })?; - } - } - Ok(()) - } - - pub fn finish(self) -> Result { - match self.inner { - SpeclibWriterInner::MsgpackZstd(encoder) => encoder.finish(), - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -658,22 +558,20 @@ mod tests { fn test_detect_native_format_by_extension() { use std::path::Path; // Both .zst and .zstd must map to the native zstd readers. - for ext in ["lib.msgpack.zst", "lib.msgpack.zstd"] { - assert!(matches!( - SpeclibFormat::detect_from_extension(Path::new(ext)), - Some(SpeclibFormat::MessagePackZstd) - )); - } for ext in ["lib.ndjson.zst", "lib.ndjson.zstd"] { assert!(matches!( SpeclibFormat::detect_from_extension(Path::new(ext)), Some(SpeclibFormat::NdJsonZstd) )); } - assert!(matches!( - SpeclibFormat::detect_from_extension(Path::new("lib.msgpack")), - Some(SpeclibFormat::MessagePack) - )); + // msgpack is gone: these must fall through to the timsquery bridge + // rather than matching a native reader. + for ext in ["lib.msgpack", "lib.msgpack.zst", "lib.msgpack.zstd"] { + assert!( + SpeclibFormat::detect_from_extension(Path::new(ext)).is_none(), + "{ext} must no longer be claimed as a native format" + ); + } assert!(matches!( SpeclibFormat::detect_from_extension(Path::new("lib.ndjson")), Some(SpeclibFormat::NdJson) @@ -1071,22 +969,6 @@ mod tests { } } - #[test] - fn test_speclib_writer_roundtrip() { - let elem = SerSpeclibElement::sample(); - let mut buf = Vec::new(); - { - let mut writer = SpeclibWriter::new_msgpack_zstd(&mut buf).unwrap(); - writer.append(&elem).unwrap(); - writer.append(&elem).unwrap(); - writer.finish().unwrap(); - } - let reader = - SpeclibReader::new(std::io::Cursor::new(&buf), SpeclibFormat::MessagePackZstd).unwrap(); - let items: Vec<_> = reader.collect::, _>>().unwrap(); - assert_eq!(items.len(), 2); - } - /// End-to-end `Speclib::from_file` over the real DIA-NN HeLa `.speclib` /// fixture (the actual workload path). Proves: the arena narrows to a lazy /// library with targets, variant-0 is a target, and the intensity sidecar diff --git a/rust/timsseek/src/models/query_item.rs b/rust/timsseek/src/models/query_item.rs index 4f6824d9..60291d6a 100644 --- a/rust/timsseek/src/models/query_item.rs +++ b/rust/timsseek/src/models/query_item.rs @@ -76,7 +76,7 @@ impl Default for ExpectedIntensities { impl ExpectedIntensities { /// Construct from fragment and precursor pair iterators, erroring on any /// repeated key in either input. Preferred entry point for all library - /// load paths (speclib ndjson/msgpack, DIA-NN/Spectronaut/Skyline TSV). + /// load paths (speclib ndjson, mzSpecLib, DIA-NN/Spectronaut/Skyline TSV). pub fn try_from_pairs(frags: FI, precs: PI) -> Result where FI: IntoIterator, diff --git a/rust/timsseek_cli/assets/default_config.toml b/rust/timsseek_cli/assets/default_config.toml index 3a09acdf..f712e2c2 100644 --- a/rust/timsseek_cli/assets/default_config.toml +++ b/rust/timsseek_cli/assets/default_config.toml @@ -13,7 +13,7 @@ ## Input spectral library (optional — `--speclib-uri` overrides this). # [input] # type = "speclib" -# uri = "path/to/library.msgpack.zst" +# uri = "path/to/library.mzSpecLib.txt" [analysis] From 358d69b29da74710ce640f6f414c505e8686ce7d Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 17:14:09 -0700 Subject: [PATCH 09/27] refactor: tighten the mzSpecLib reader and IonAnnot internals Three behaviour fixes found while re-reading the diff: - `resolve_annotation` dropped alternatives whose mass-error suffix failed to parse, which could silently turn an ambiguous peak into an unambiguous one and resolve it against a partial set. A malformed suffix on any alternative now makes the whole peak unresolvable. It also reported that case as `SkipUnannotated`, miscounting an annotated peak as unannotated. - Tie detection compared with `f64::EPSILON`, which is neither exact nor scale-aware. These errors come from parsing decimal literals, so equal ones compare exactly; the fuzzy compare bought nothing and hid the intent. - A malformed ion-mobility value fell back to 0.0. Absent still means 0.0 (DIA-NN writes that), but present-and-unparseable now drops the spectrum rather than inventing a mobility. Also collapses the double scan for the minimum (count-then-find, with an expect between them) into one pass, moves the loss discriminant match next to the enum it must stay in sync with, and drops a `debug_assert` in the `loss()` getter guarding an invariant no constructor can violate plus one asserting a bound the type system already guarantees. `REGISTRY_BIT` went with them -- an unread constant enforces nothing, and the bit reservation is documented in the layout. Trims comments that restated the module docs verbatim; inline comment density is now in line with the neighbouring readers. --- rust/micromzpaf/src/lib.rs | 36 ++------ rust/micromzpaf/src/loss.rs | 23 +++++ rust/timsquery/src/serde/mzspeclib_io.rs | 106 +++++++++++------------ 3 files changed, 79 insertions(+), 86 deletions(-) diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index 8ddda676..8a05ba92 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -32,9 +32,9 @@ //! range-checks — see [`IonAnnot::try_new`]. //! //! Bit 30 is reserved: when set, `loss` would be an index into a growable -//! registry instead of a [`NeutralLoss`] discriminant. Nothing sets it today -//! and constructors assert it stays clear; it exists so an esoteric loss can be -//! added later without another layout change. +//! registry instead of a [`NeutralLoss`] discriminant. Nothing reads or writes +//! it today; it exists so an esoteric loss can be added later without another +//! layout change. //! //! # mzPAF compliance //! @@ -89,8 +89,6 @@ const LOSS_SHIFT: u32 = 12; const LOSS_BITS: u32 = 6; const PAYLOAD_SHIFT: u32 = 18; const PAYLOAD_BITS: u32 = 12; -/// Reserved: set would mean `loss` is a registry index. Always clear today. -const REGISTRY_BIT: u32 = 1 << 30; /// Widest charge the 4-bit zigzag field holds. Observed maximum is 3. pub const CHARGE_MIN: i8 = -7; @@ -362,10 +360,6 @@ impl IonAnnot { return Err(IonParsingError::IsotopeOutOfRange { isotope }); } debug_assert!(payload <= mask(PAYLOAD_BITS), "payload overflows its field"); - debug_assert!( - (loss as u32) <= mask(LOSS_BITS), - "loss discriminant overflows its field" - ); Ok(IonAnnot( ((kind as u32) << KIND_SHIFT) | ((zigzag(charge) & mask(CHARGE_BITS)) << CHARGE_SHIFT) @@ -395,29 +389,11 @@ impl IonAnnot { unzigzag((self.0 >> ISOTOPE_SHIFT) & mask(ISOTOPE_BITS)) } - /// The neutral loss this ion carries, [`NeutralLoss::None`] if any. + /// The neutral loss this ion carries, [`NeutralLoss::None`] if it carries + /// none. #[inline] pub fn loss(&self) -> NeutralLoss { - debug_assert!( - self.0 & REGISTRY_BIT == 0, - "registry-backed losses are reserved but not implemented" - ); - match (self.0 >> LOSS_SHIFT) & mask(LOSS_BITS) { - 0 => NeutralLoss::None, - 1 => NeutralLoss::Water, - 2 => NeutralLoss::Ammonia, - 3 => NeutralLoss::CarbonMonoxide, - 4 => NeutralLoss::CarbonDioxide, - 5 => NeutralLoss::WaterX2, - 6 => NeutralLoss::AmmoniaX2, - 7 => NeutralLoss::WaterAmmonia, - 8 => NeutralLoss::Methanesulfenic, - 9 => NeutralLoss::Carbamidomethylthiol, - 10 => NeutralLoss::PhosphoricAcid, - 11 => NeutralLoss::Metaphosphoric, - 12 => NeutralLoss::PhosphoricAcidWater, - _ => NeutralLoss::None, - } + NeutralLoss::from_discriminant(((self.0 >> LOSS_SHIFT) & mask(LOSS_BITS)) as u8) } pub fn terminality(&self) -> IonSeriesTerminality { diff --git a/rust/micromzpaf/src/loss.rs b/rust/micromzpaf/src/loss.rs index 68eb8d66..865cf4b6 100644 --- a/rust/micromzpaf/src/loss.rs +++ b/rust/micromzpaf/src/loss.rs @@ -258,6 +258,29 @@ const TABLE: &[(Composition, NeutralLoss, &str)] = &[ ]; impl NeutralLoss { + /// Inverse of the `#[repr(u8)]` discriminant, for unpacking out of a bit + /// field. Lives next to the enum so the two cannot drift apart. + /// + /// An unrecognized value maps to [`Self::None`]: the only way to produce + /// one is a reserved discriminant, which no constructor emits. + pub(crate) fn from_discriminant(d: u8) -> Self { + match d { + 1 => Self::Water, + 2 => Self::Ammonia, + 3 => Self::CarbonMonoxide, + 4 => Self::CarbonDioxide, + 5 => Self::WaterX2, + 6 => Self::AmmoniaX2, + 7 => Self::WaterAmmonia, + 8 => Self::Methanesulfenic, + 9 => Self::Carbamidomethylthiol, + 10 => Self::PhosphoricAcid, + 11 => Self::Metaphosphoric, + 12 => Self::PhosphoricAcidWater, + _ => Self::None, + } + } + /// Resolve a loss expression (without the leading `-`) to a discriminant. /// /// `None` means "parsed as a valid composition, but not one we represent" — diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs index 07be1fbb..fe0ac0dd 100644 --- a/rust/timsquery/src/serde/mzspeclib_io.rs +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -17,27 +17,26 @@ //! //! # Peak resolution //! -//! An mzSpecLib peak list carries *observed* m/z; the annotation's mass-error -//! suffix is what recovers the theoretical value the arena wants -//! (`theoretical = observed - error`). Two properties follow, and they drive -//! the whole policy: +//! Peak lists carry *observed* m/z; the annotation's mass-error suffix recovers +//! the theoretical value the arena wants (`theoretical = observed - error`). So +//! a theoretical mass only exists once a single identity is pinned, which sorts +//! peaks three ways: //! -//! 1. A theoretical m/z only exists once a single ion identity is pinned down. -//! A peak with no annotation (`?`) or an unresolvable ambiguity has no -//! error to subtract, so it is **skipped** rather than stored at its -//! observed m/z. Mixing observed and theoretical masses in one arena would -//! be invisible downstream. -//! 2. "Identity known" and "identity representable" are different. `y1-HCOOH` -//! has a known identity and therefore a computable theoretical mass, even -//! though [`IonAnnot`] cannot spell the loss. Those peaks are **kept** with -//! an unknown label: the mass stays exact, only the label is erased. +//! | | kept | m/z | +//! |---|---|---| +//! | resolved, representable | real label | theoretical | +//! | resolved, not representable (`y1-HCOOH`) | unknown label | theoretical | +//! | unannotated (`?`) or tied ambiguity | no | — | +//! +//! Row three is skipped rather than stored at observed m/z: an arena mixing +//! observed and theoretical masses would be invisible downstream. Row two is +//! kept because a known-but-unspellable identity still has an exact mass — only +//! the label is lost. //! -//! Ambiguous (comma-separated) annotations resolve to the alternative with the -//! smallest absolute mass error. If that alternative is not representable the -//! peak takes an unknown label — it is deliberately NOT downgraded to a -//! worse-matching but representable alternative, which would assign a wrong -//! chemical identity *and* a wrong theoretical mass. An exact tie pins no -//! identity at all, so it is skipped. +//! Ambiguous (comma-separated) annotations take the alternative with the +//! smallest absolute mass error. If that one is unrepresentable the peak gets an +//! unknown label rather than falling back to a worse-matching representable +//! alternative, which would assign both a wrong identity and a wrong mass. use crate::ion::IonAnnot; use crate::models::{ @@ -65,9 +64,7 @@ use tracing::{ /// First non-empty line of an mzSpecLib text file. const MAGIC: &str = ""; -// ── CV term ladders ────────────────────────────────────────────────────────── -// -// Ordered most- to least-specific. The first term present wins. +// CV term ladders, ordered most- to least-specific; the first present wins. /// Precursor m/z. Experimental monoisotopic is preferred over `selected ion /// m/z`, which on a quadrupole instrument is the isolation-window centre and @@ -253,51 +250,47 @@ fn resolve_annotation(annotation: &str) -> (Resolved, Option) { return (Resolved::SkipUnannotated, None); } - let alternatives: Vec<&str> = annotation.split(',').map(str::trim).collect(); - - // Split the mass error off every alternative first. This works even when - // the ion itself will not parse, which is exactly the case that still - // needs an exact mass. - let split: Vec<(&str, Option)> = alternatives - .iter() - .filter_map(|a| split_mass_error(a).ok()) - .collect(); - if split.is_empty() { - return (Resolved::SkipUnannotated, None); + // Splitting the error off comes first: it works even when the ion will not + // parse, which is exactly the case that still needs an exact mass. A + // malformed suffix on ANY alternative makes the whole peak unresolvable — + // dropping just that one would silently turn an ambiguous peak into an + // unambiguous one. + let mut alternatives = Vec::new(); + for alt in annotation.split(',') { + let Ok(parsed) = split_mass_error(alt.trim()) else { + return (Resolved::SkipAmbiguous, None); + }; + alternatives.push(parsed); } - let chosen = if split.len() == 1 { - split[0] + let (ion_str, mass_error) = if let [single] = alternatives[..] { + single } else { - // Ambiguous: the alternative whose observed mass sits closest to its - // own theoretical. A tie pins no identity, so nothing can be stored. + // Closest by absolute mass error. Comparing a Da magnitude against a + // ppm one would be meaningless, but a library uses one unit + // throughout. Errors are parsed decimal literals, so equal ones + // compare exactly and a tie pins no identity. let magnitude = |m: &Option| match m { - Some(MassError::Da(d)) => d.abs(), - Some(MassError::Ppm(p)) => p.abs(), + Some(MassError::Da(v) | MassError::Ppm(v)) => v.abs(), None => f64::INFINITY, }; - let best = split + let best = alternatives .iter() .map(|(_, e)| magnitude(e)) .fold(f64::INFINITY, f64::min); - let tied = split - .iter() - .filter(|(_, e)| (magnitude(e) - best).abs() <= f64::EPSILON) - .count(); - if tied != 1 || !best.is_finite() { + if !best.is_finite() { return (Resolved::SkipAmbiguous, None); } - *split - .iter() - .find(|(_, e)| (magnitude(e) - best).abs() <= f64::EPSILON) - .expect("a unique minimum was just counted") + let mut winners = alternatives.iter().filter(|(_, e)| magnitude(e) == best); + let winner = *winners.next().expect("the minimum came from this iterator"); + if winners.next().is_some() { + return (Resolved::SkipAmbiguous, None); + } + winner }; - - let (ion_str, mass_error) = chosen; match IonAnnot::try_from(ion_str) { Ok(ion) => (Resolved::Annotated(ion), mass_error), - // Known identity, unrepresentable spelling: keep the peak and its exact - // mass, lose only the label. + // Keep the peak and its exact mass, lose only the label. Err(_) => (Resolved::UnknownLabel, mass_error), } } @@ -339,12 +332,14 @@ fn convert_spectrum( }; let mobility = match raw.attrs.find(MOBILITY_INVERSE_REDUCED) { - Some(a) => a.value.parse().unwrap_or(0.0), + Some(a) => a.value.parse().ok()?, None => match raw.attrs.find(MOBILITY_DRIFT_TIME) { Some(a) => { stats.spectra_with_drift_time_mobility += 1; - a.value.parse().unwrap_or(0.0) + a.value.parse().ok()? } + // Absent is fine — an unset mobility is 0.0, same as DIA-NN writes. + // A *present but malformed* one drops the spectrum instead. None => 0.0, }, }; @@ -411,7 +406,6 @@ fn convert_spectrum( } }; - // Peak lists carry observed m/z; the arena wants theoretical. let mz = match mass_error { Some(e) => e.theoretical_from_observed(*observed_mz), None => *observed_mz, From 9ac6731b33f4b02919bb20dd5b730d3259e7938c Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 17:44:24 -0700 Subject: [PATCH 10/27] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20unb?= =?UTF-8?q?lock=20the=20default=20build,=20drop=20dead=20API,=20dedupe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blockers: - `cargo build` was broken for everyone: speclib_build_cli is in default-members and does not compile without the msgpack writer. Dropped from default-members until the mzSpecLib writer lands, so `cargo check` is clean and only an explicit `--workspace` hits it. - Serializing a default-constructed IonAnnot panicked. `Default` is not optional here -- `tinyvec::Array` requires `Item: Default`, `TimsElutionGroup` stores labels in a `TinyVec`, and `KeyLike` propagates the bound -- and `Serialize` renders through `format!`, so the panicking `Display` arm was reachable from any serde path. Renders inertly now. (Pre-existing on main, not introduced here.) Duplication: - `next_unknown_ordinal` was triplicated verbatim; hoisted to `serde/unknown_ordinal.rs` with each caller mapping into its own error. - `IonSeriesOrdinal` had grown copies of `IonAnnot`'s own `terminality` and `try_get_ordinal` that could drift. Parsing also round-tripped `"y12" -> IonSeriesOrdinal -> as_char_and_ordinal -> Kind` for no reason; it now reads the series char directly, which lets the whole duplicate surface go. `IonSeriesTerminality` had no users at all. - The `pro_forma` warning-dropping shim was repeated 3x; now `sequence::parse_proforma`. - Two fixture helpers and three test modules in mzspeclib_io collapsed to one. - mzSpecLib and `.speclib` bypassed the reader registry with hardcoded `if sniff_x` branches. `LibraryReader` gained a `read_arena` default, so every format now goes through `registry()` and adding a direct-arena format no longer means another branch. Dead code: `try_fill_labels_annot` (no callers), `SerSpeclibElement:: sample`/`sample_json` (pub, so no dead-code warning fired; only consumer was the deleted writer test), `NeutralLoss::composition` (now test-only), and the `Composition`/`INTERNAL_POS_MAX` public surface narrowed to the crate. `IonParsingError::Custom`'s one use claimed "exceeds i8 range" for a +-7 field; the range check below it already reports the real bound. Also: msgpack leftovers in speclib_build_cli's `--help`, config default and example TOML (users following those would have produced an unloadable file); the unreachable-by-intent RT unit arm now counts unknown units instead of silently assuming minutes; the 8-field anonymous tuple is a named `ArenaRow`; `from_discriminant` is covered for every variant; and the Carafe tests assert on `Tolerance` fields rather than `Debug` strings, with the missing-`id` case built as its own payload rather than by string surgery on the const. --- Cargo.lock | 1 + Cargo.toml | 4 +- bench/wandb_bench.py | 2 +- example_speclib_config.toml | 4 +- rust/micromzpaf/Cargo.toml | 3 + rust/micromzpaf/src/lib.rs | 238 ++---- rust/micromzpaf/src/loss.rs | 35 +- rust/speclib_build_cli/src/cli.rs | 2 +- rust/speclib_build_cli/src/config.rs | 2 +- rust/speclib_build_cli/src/entry.rs | 3 +- rust/speclib_build_cli/src/pipeline.rs | 2 +- rust/timsquery/src/lib.rs | 1 - rust/timsquery/src/serde/diann_io.rs | 30 +- .../src/serde/elution_group_inputs.rs | 52 +- rust/timsquery/src/serde/library_file.rs | 76 +- rust/timsquery/src/serde/mod.rs | 1 + rust/timsquery/src/serde/mzspeclib_io.rs | 97 +-- rust/timsquery/src/serde/mzspeclib_io.rs.bak | 751 ++++++++++++++++++ rust/timsquery/src/serde/skyline_io.rs | 25 +- rust/timsquery/src/serde/spectronaut_io.rs | 21 +- rust/timsquery/src/serde/unknown_ordinal.rs | 17 + rust/timsquery/tests/carafe_contract.rs | 35 +- .../tests/mzspeclib_io_files/README.md | 9 +- rust/timsseek/src/data_sources/speclib.rs | 53 +- .../fragment_mass/elution_group_converter.rs | 13 +- rust/timsseek/src/lib.rs | 1 - rust/timsseek/src/models/sequence.rs | 19 +- 27 files changed, 1034 insertions(+), 463 deletions(-) create mode 100644 rust/timsquery/src/serde/mzspeclib_io.rs.bak create mode 100644 rust/timsquery/src/serde/unknown_ordinal.rs diff --git a/Cargo.lock b/Cargo.lock index d3970058..d3e2d9ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4085,6 +4085,7 @@ name = "micromzpaf" version = "0.33.0" dependencies = [ "serde", + "serde_json", "thiserror 2.0.18", ] diff --git a/Cargo.toml b/Cargo.toml index fa8cc32e..49ed3e07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,9 @@ default-members = [ "rust/timsquery_cli", "rust/timsquery_viewer", "rust/apex_sim", - "rust/speclib_build_cli", + # speclib_build_cli is temporarily NOT here: its only writer was the msgpack + # `SpeclibWriter`, removed with that format. It does not compile until the + # mzSpecLib writer lands. Put it back then. "rust/tims_stage", "rust/timsseek_macros" ] diff --git a/bench/wandb_bench.py b/bench/wandb_bench.py index d9b17ef0..1bfe2f3a 100644 --- a/bench/wandb_bench.py +++ b/bench/wandb_bench.py @@ -225,7 +225,7 @@ def wandb_context(config_dict: dict[str, Any], wandb_kwargs=None): def main(wandb_kwargs: dict | None = None, koina_url: str | None = None): fasta_file = Path.home() / "fasta/hela_gt20peps.fasta" - speclib_path = Path.home() / "fasta/asdad.msgpack.zstd" + speclib_path = Path.home() / "fasta/asdad.mzSpecLib.txt" prefix = Path.home() / "data/decompressed_timstof/" dotd_files = [ diff --git a/example_speclib_config.toml b/example_speclib_config.toml index 717afdc6..cb752085 100644 --- a/example_speclib_config.toml +++ b/example_speclib_config.toml @@ -5,8 +5,8 @@ # All fields are optional — omit a section or key to use the compiled-in default. # ── Output ──────────────────────────────────────────────────────────────────── -# Path for the output spectral library (msgpack + zstd). -output = "library.msgpack.zst" +# Path for the output spectral library (mzSpecLib text). +output = "library.mzSpecLib.txt" # ── Digestion ───────────────────────────────────────────────────────────────── [digestion] diff --git a/rust/micromzpaf/Cargo.toml b/rust/micromzpaf/Cargo.toml index 34ecc552..d23bec52 100644 --- a/rust/micromzpaf/Cargo.toml +++ b/rust/micromzpaf/Cargo.toml @@ -4,6 +4,9 @@ version.workspace = true edition.workspace = true license.workspace = true +[dev-dependencies] +serde_json = { workspace = true } + [dependencies] serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index 8a05ba92..70a8cc68 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -9,11 +9,11 @@ //! # Bit layout //! //! ```text -//! bit: 31 30 29 18 17 12 11 8 7 4 3 0 -//! ┌──┬──┬────────────────┬──────────┬────────┬────────┬───────┐ -//! │▒▒│R │ payload │ loss │isotope │ charge │ kind │ -//! │1b│1b│ 12b │ 6b │ 4b zz │ 4b zz │ 4b │ -//! └──┴──┴────────────────┴──────────┴────────┴────────┴───────┘ +//! bit: 31 30 29 18 17 12 11 8 7 4 3 0 +//! ┌───────┬────────────────┬──────────┬────────┬────────┬───────┐ +//! │ spare │ payload │ loss │isotope │ charge │ kind │ +//! │ 2b │ 12b │ 6b │ 4b zz │ 4b zz │ 4b │ +//! └───────┴────────────────┴──────────┴────────┴────────┴───────┘ //! ``` //! //! `payload` is reinterpreted per `kind` — a tagged union inside the word: @@ -31,11 +31,6 @@ //! the field truncates rather than wrapping loudly, every constructor //! range-checks — see [`IonAnnot::try_new`]. //! -//! Bit 30 is reserved: when set, `loss` would be an index into a growable -//! registry instead of a [`NeutralLoss`] discriminant. Nothing reads or writes -//! it today; it exists so an esoteric loss can be added later without another -//! layout change. -//! //! # mzPAF compliance //! //! Supported: the a/b/c/d/v/w/x/y/z series, precursor (`p`), unknown (`?`), @@ -64,17 +59,13 @@ pub mod loss; -pub use loss::{ - Composition, - NeutralLoss, -}; +pub use loss::NeutralLoss; use serde::{ Deserialize, Serialize, }; use std::fmt::Display; use std::hash::Hash; -use std::str::FromStr; use thiserror::Error; // ── Bit layout ─────────────────────────────────────────────────────────────── @@ -97,7 +88,7 @@ pub const CHARGE_MAX: i8 = 7; pub const ISOTOPE_MIN: i8 = -7; pub const ISOTOPE_MAX: i8 = 7; /// Widest residue index an internal fragment endpoint holds (6 bits). -pub const INTERNAL_POS_MAX: u8 = 63; +pub(crate) const INTERNAL_POS_MAX: u8 = 63; #[inline] const fn mask(bits: u32) -> u32 { @@ -212,8 +203,7 @@ impl Kind { /// Compact representation of fragment annotations. /// /// A packed `u32`; see the crate docs for the bit layout. Ordering is by the -/// packed word rather than field-by-field — nothing depends on the previous -/// ordering (only tests sorted these), but it is not the same order. +/// packed word, not field-by-field. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] pub struct IonAnnot(u32); @@ -396,30 +386,15 @@ impl IonAnnot { NeutralLoss::from_discriminant(((self.0 >> LOSS_SHIFT) & mask(LOSS_BITS)) as u8) } - pub fn terminality(&self) -> IonSeriesTerminality { - match self.kind() { - Kind::A | Kind::B | Kind::C | Kind::D => IonSeriesTerminality::NTerm, - Kind::V | Kind::W | Kind::X | Kind::Y | Kind::Z => IonSeriesTerminality::CTerm, - _ => IonSeriesTerminality::None, - } - } - /// Shift the isotope by `offset_neutrons`. /// /// Errors when the result leaves [`ISOTOPE_MIN`]..=[`ISOTOPE_MAX`]. That - /// bound is narrower than the `i8` this used to hold, but an order of - /// magnitude past any observed isotope offset. + /// bound is an order of magnitude past any observed isotope offset. pub fn try_with_offset_neutrons(&self, offset_neutrons: i8) -> Result { - let new_isotope = self - .get_isotope() - .checked_add(offset_neutrons) - .ok_or_else(|| IonParsingError::Custom { - error: format!( - "Isotope offset overflow: {} + {} exceeds i8 range", - self.get_isotope(), - offset_neutrons - ), - })?; + // Saturating rather than checked: the field is far narrower than `i8`, + // so the range check below is the one that matters and it reports the + // bound that actually applies. + let new_isotope = self.get_isotope().saturating_add(offset_neutrons); if !(ISOTOPE_MIN..=ISOTOPE_MAX).contains(&new_isotope) { return Err(IonParsingError::IsotopeOutOfRange { isotope: new_isotope, @@ -624,9 +599,25 @@ impl IonAnnot { }; } - let series_ord = IonSeriesOrdinal::from_str(core)?; - let (ion_type, ordinal) = series_ord.as_char_and_ordinal(); - Self::try_new_with_loss(ion_type, ordinal, charge, isotope, loss) + // Backbone / precursor / unknown: a series char then an ordinal. + let mut chars = core.chars(); + let series = chars.next().ok_or(IonParsingError::ParsingError { + error: value.to_string(), + context: Some("Empty string"), + })?; + let rest = chars.as_str(); + let ordinal = if rest.is_empty() { + None + } else { + Some( + rest.parse::() + .map_err(|e| IonParsingError::ParsingError { + error: format!("{rest} -> {e:?}"), + context: Some("Unable to parse the ordinal number"), + })?, + ) + }; + Self::try_new_with_loss(series, ordinal, charge, isotope, loss) } } @@ -686,15 +677,6 @@ pub enum IonParsingError { Custom { error: String }, } -/// Refers to what terminus of the original peptide retains the -/// charge after a fragmentation. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum IonSeriesTerminality { - NTerm, - CTerm, - None, -} - /// The logical series-and-payload view of an [`IonAnnot`]. /// /// This is a *view*: `IonAnnot` stores a packed word and reconstructs this on @@ -748,96 +730,6 @@ pub enum IonSeriesOrdinal { None, } -impl IonSeriesOrdinal { - pub fn try_new(series: char, ordinal: Option) -> Result { - let tmp = match (series, ordinal) { - ('a', Some(ordinal)) => Self::a { ordinal }, - ('b', Some(ordinal)) => Self::b { ordinal }, - ('c', Some(ordinal)) => Self::c { ordinal }, - ('d', Some(ordinal)) => Self::d { ordinal }, - ('v', Some(ordinal)) => Self::v { ordinal }, - ('w', Some(ordinal)) => Self::w { ordinal }, - ('x', Some(ordinal)) => Self::x { ordinal }, - ('y', Some(ordinal)) => Self::y { ordinal }, - ('z', Some(ordinal)) => Self::z { ordinal }, - ('?', Some(ordinal)) => Self::unknown { ordinal }, - ('p', None) => Self::precursor, - ('p', Some(ordinal)) => { - return Err(IonParsingError::OrdinalOutOfRange { - ordinal: ordinal as i32, - series: Some(series), - }); - } - _ => { - return Err(IonParsingError::UnsupportedFragmentType { - fragment_type: series, - }); - } - }; - - Ok(tmp) - } - - /// Series character and ordinal, for handing back to [`IonAnnot::try_new`]. - fn as_char_and_ordinal(&self) -> (char, Option) { - match self { - Self::a { ordinal } => ('a', Some(*ordinal)), - Self::b { ordinal } => ('b', Some(*ordinal)), - Self::c { ordinal } => ('c', Some(*ordinal)), - Self::d { ordinal } => ('d', Some(*ordinal)), - Self::v { ordinal } => ('v', Some(*ordinal)), - Self::w { ordinal } => ('w', Some(*ordinal)), - Self::x { ordinal } => ('x', Some(*ordinal)), - Self::y { ordinal } => ('y', Some(*ordinal)), - Self::z { ordinal } => ('z', Some(*ordinal)), - Self::unknown { ordinal } => ('?', Some(*ordinal)), - Self::precursor => ('p', None), - Self::internal { start, .. } => ('m', Some(*start)), - Self::immonium { residue } => (*residue, None), - Self::None => ('\0', None), - } - } - - pub fn terminality(&self) -> IonSeriesTerminality { - match self { - IonSeriesOrdinal::a { .. } - | IonSeriesOrdinal::b { .. } - | IonSeriesOrdinal::c { .. } - | IonSeriesOrdinal::d { .. } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::v { .. } - | IonSeriesOrdinal::w { .. } - | IonSeriesOrdinal::x { .. } - | IonSeriesOrdinal::y { .. } - | IonSeriesOrdinal::z { .. } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::unknown { .. } - | IonSeriesOrdinal::precursor - | IonSeriesOrdinal::internal { .. } - | IonSeriesOrdinal::immonium { .. } => IonSeriesTerminality::None, - IonSeriesOrdinal::None => panic!("IonSeriesOrdinal::None should not be used directly"), - } - } - - pub fn try_get_ordinal(&self) -> Option { - match self { - IonSeriesOrdinal::a { ordinal } - | IonSeriesOrdinal::b { ordinal } - | IonSeriesOrdinal::c { ordinal } - | IonSeriesOrdinal::d { ordinal } - | IonSeriesOrdinal::v { ordinal } - | IonSeriesOrdinal::w { ordinal } - | IonSeriesOrdinal::x { ordinal } - | IonSeriesOrdinal::y { ordinal } - | IonSeriesOrdinal::z { ordinal } => Some(*ordinal), - // ?1 does not mean its an ordinal, just a placeholder - IonSeriesOrdinal::unknown { .. } - | IonSeriesOrdinal::precursor - | IonSeriesOrdinal::internal { .. } - | IonSeriesOrdinal::immonium { .. } - | IonSeriesOrdinal::None => None, - } - } -} - impl Display for IonSeriesOrdinal { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -854,37 +746,13 @@ impl Display for IonSeriesOrdinal { IonSeriesOrdinal::precursor => write!(f, "p"), IonSeriesOrdinal::internal { start, end } => write!(f, "m{}:{}", start, end), IonSeriesOrdinal::immonium { residue } => write!(f, "I{}", residue), - IonSeriesOrdinal::None => panic!("IonSeriesOrdinal::None should not be used directly"), - } - } -} - -impl FromStr for IonSeriesOrdinal { - type Err = IonParsingError; - - fn from_str(s: &str) -> Result { - // "b12" split into "b" and "12" - if s.is_empty() { - return Err(IonParsingError::ParsingError { - error: s.to_string(), - context: Some("Empty string"), - }); - } - let (series_chunk, ordinal_chunk) = s.split_at(1); - let series_id = series_chunk.chars().next(); - let series_ordinal = ordinal_chunk.parse::(); - - match (series_id, series_ordinal) { - (None, _) => Err(IonParsingError::ParsingError { - error: s.to_string(), - context: Some("Empty string"), - }), - (Some(x), Ok(y)) => IonSeriesOrdinal::try_new(x, Some(y)), - (Some('p'), Err(_)) => Ok(IonSeriesOrdinal::precursor), - (Some(_), Err(err)) => Err(IonParsingError::ParsingError { - error: format!("{ordinal_chunk} -> {err:?}"), - context: Some("Unable to parse the ordinal number"), - }), + // Reached only via `IonAnnot::default()`, which packs to zero. + // That `Default` is not optional: `tinyvec::Array` requires + // `Item: Default`, `TimsElutionGroup` stores labels in a + // `TinyVec<[T; 13]>`, and timsquery's `KeyLike` propagates the + // bound. Since `Serialize` renders through `format!`, panicking + // here is reachable from any serde path — so render inertly. + IonSeriesOrdinal::None => write!(f, "?0"), } } } @@ -907,9 +775,30 @@ mod tests { } #[test] - fn test_ion_series_ord_from_str() { - let ion: IonSeriesOrdinal = IonSeriesOrdinal::from_str("b12").unwrap(); - assert_eq!(ion, IonSeriesOrdinal::b { ordinal: 12 }); + fn series_ordinal_is_a_view_of_the_packed_word() { + assert_eq!( + ion("b12").series_ordinal(), + IonSeriesOrdinal::b { ordinal: 12 } + ); + } + + /// `IonAnnot: Default` packs to zero, which decodes to `Kind::None` and a + /// charge of 0. `Serialize` renders through `format!`, so a panicking + /// `Display` arm is reachable from any serde path — `TinyVec` alone can + /// hand out a default. + /// + /// The rendered value is degenerate and deliberately does NOT round-trip: + /// charge 0 is rejected by every constructor. Not panicking is the + /// property being pinned. + #[test] + fn default_annotation_renders_instead_of_panicking() { + let d = IonAnnot::default(); + assert_eq!(d.to_string(), "?0^0"); + assert_eq!(serde_json::to_string(&d).unwrap(), "\"?0^0\""); + assert!( + IonAnnot::try_from("?0^0").is_err(), + "the default is not a valid annotation, only a printable one" + ); } #[test] @@ -960,7 +849,6 @@ mod tests { assert_eq!(a.get_isotope(), isotope); assert_eq!(a.try_get_ordinal(), Some(ordinal)); assert_eq!(a.loss(), loss); - assert_eq!(a.terminality(), IonSeriesTerminality::CTerm); } } } diff --git a/rust/micromzpaf/src/loss.rs b/rust/micromzpaf/src/loss.rs index 865cf4b6..f0130b7f 100644 --- a/rust/micromzpaf/src/loss.rs +++ b/rust/micromzpaf/src/loss.rs @@ -29,7 +29,7 @@ use crate::IonParsingError; /// from C/H/N/O/S/P, and keeping it to six `u8`s makes equality a single /// 6-byte compare during the parse-time table lookup. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct Composition { +pub(crate) struct Composition { pub c: u8, pub h: u8, pub n: u8, @@ -39,7 +39,7 @@ pub struct Composition { } impl Composition { - pub const fn new(c: u8, h: u8, n: u8, o: u8, s: u8, p: u8) -> Self { + pub(crate) const fn new(c: u8, h: u8, n: u8, o: u8, s: u8, p: u8) -> Self { Self { c, h, n, o, s, p } } @@ -122,7 +122,7 @@ impl Composition { /// /// Because terms are summed, ordering and multiplier spelling collapse for /// free: `H2O-NH3` == `NH3-H2O`, and `2H2O` == `H2O-H2O`. - pub fn parse_expression(s: &str) -> Result { + pub(crate) fn parse_expression(s: &str) -> Result { let mut total = Composition::default(); for term in s.split('-') { let term = term.trim(); @@ -287,7 +287,7 @@ impl NeutralLoss { /// distinct from `Err`, which means the text was not a loss expression at /// all. Callers route the former to an unknown label and the latter to a /// parse failure. - pub fn from_expression(s: &str) -> Result, IonParsingError> { + pub(crate) fn from_expression(s: &str) -> Result, IonParsingError> { let comp = Composition::parse_expression(s)?; Ok(TABLE .iter() @@ -296,7 +296,8 @@ impl NeutralLoss { } /// The composition this loss removes. - pub fn composition(self) -> Composition { + #[cfg(test)] + pub(crate) fn composition(self) -> Composition { if self == NeutralLoss::None { return Composition::default(); } @@ -308,7 +309,7 @@ impl NeutralLoss { } /// Canonical spelling, without the leading `-`. Empty for [`Self::None`]. - pub fn canonical(self) -> &'static str { + pub(crate) fn canonical(self) -> &'static str { if self == NeutralLoss::None { return ""; } @@ -417,7 +418,10 @@ mod tests { assert!(NeutralLoss::from_expression("H2O-").is_err()); } - /// Every table entry must survive canonical -> composition -> discriminant. + /// Every table entry must survive canonical -> composition -> discriminant, + /// and back out through the bit field. `from_discriminant` hand-mirrors the + /// `#[repr(u8)]` values, so adding a loss without updating it would + /// silently decode as [`NeutralLoss::None`] — this is what catches that. #[test] fn table_round_trips_through_canonical_spelling() { for (comp, loss, canon) in TABLE { @@ -426,9 +430,24 @@ mod tests { Some(*loss), "canonical spelling {canon} must resolve to its own loss" ); - assert_eq!(loss.composition(), *comp); assert_eq!(loss.canonical(), *canon); + assert_eq!(loss.composition(), *comp); + assert_eq!( + NeutralLoss::from_discriminant(*loss as u8), + *loss, + "{canon} does not survive the discriminant round trip" + ); } + assert_eq!(NeutralLoss::from_discriminant(0), NeutralLoss::None); + // Every non-None variant must be in TABLE, or it has no spelling and no + // composition and could never be produced by parsing. + assert_eq!( + TABLE.len(), + (1..=u8::MAX) + .filter(|d| NeutralLoss::from_discriminant(*d) != NeutralLoss::None) + .count(), + "a variant is decodable but missing from TABLE" + ); } /// Compositions must be unique: two entries sharing one would make the diff --git a/rust/speclib_build_cli/src/cli.rs b/rust/speclib_build_cli/src/cli.rs index ce27fd62..e60e733c 100644 --- a/rust/speclib_build_cli/src/cli.rs +++ b/rust/speclib_build_cli/src/cli.rs @@ -21,7 +21,7 @@ pub struct Cli { // ── Output ───────────────────────────────────────────────────────────── /// Output URI for the spectral library (local path or s3://...; default: - /// library.msgpack.zst). + /// library.mzSpecLib.txt). #[arg(long, short = 'o')] pub output: Option, diff --git a/rust/speclib_build_cli/src/config.rs b/rust/speclib_build_cli/src/config.rs index 0c24bff1..13f9ce51 100644 --- a/rust/speclib_build_cli/src/config.rs +++ b/rust/speclib_build_cli/src/config.rs @@ -62,7 +62,7 @@ fn default_min_ions() -> usize { 3 } fn default_output() -> String { - "library.msgpack.zst".to_string() + "library.mzSpecLib.txt".to_string() } // ── Sub-structs ────────────────────────────────────────────────────────────── diff --git a/rust/speclib_build_cli/src/entry.rs b/rust/speclib_build_cli/src/entry.rs index de7f3605..d10f1ce9 100644 --- a/rust/speclib_build_cli/src/entry.rs +++ b/rust/speclib_build_cli/src/entry.rs @@ -54,8 +54,7 @@ use timsseek::models::sequence::normalize_to_proforma; fn compute_precursor_mz(modified_seq: &str, charge: u8) -> Option { use mzcore::prelude::*; let proforma = normalize_to_proforma(modified_seq); - // `pro_forma` also returns non-fatal warnings; only the peptidoform matters. - let (peptide, _warnings) = Peptidoform::pro_forma(&proforma, timsseek::ontologies()).ok()?; + let peptide = timsseek::models::sequence::parse_proforma(&proforma)?; let linear = peptide.as_linear()?; let formulas = linear.formulas(); if formulas.is_empty() { diff --git a/rust/speclib_build_cli/src/pipeline.rs b/rust/speclib_build_cli/src/pipeline.rs index 01537c50..c07943b5 100644 --- a/rust/speclib_build_cli/src/pipeline.rs +++ b/rust/speclib_build_cli/src/pipeline.rs @@ -271,7 +271,7 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box for DiannPrecursorParsingError { } } -/// Advance the per-precursor `?`-ordinal counter, refusing to wrap. -/// -/// `IonAnnot` ordinals are `u8`, so a precursor can carry at most -/// [`u8::MAX`] distinguishable unknown ions. Past that there is no way to keep -/// labels unique, and a silently reused ordinal corrupts scoring -/// (`linear_get` is first-match). Fail here, where the row index is still in -/// hand, rather than downstream in `try_from_pairs`. -fn next_unknown_ordinal(current: u8) -> Result { - current.checked_add(1).ok_or_else(|| { - error!( - "More than {} unknown-ion fragments in a single precursor; cannot assign unique labels", - u8::MAX - ); - DiannPrecursorParsingError::IonOverCapacity - }) -} - impl From for DiannReadingError { fn from(_err: DiannPrecursorParsingError) -> Self { DiannReadingError::DiannPrecursorParsingError @@ -361,11 +345,7 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(rows.len()); - // `?` labels are distinguished only by their ordinal, used here as a - // per-precursor counter. That counter IS what upholds the per-precursor - // label-uniqueness invariant (see `ExpectedIntensities::try_from_pairs`): - // wrapping past `u8::MAX` would re-emit `?1` and fail the load much later - // with a duplicate-key error pointing at the symptom, not at this row. + // Per-precursor `?` counter — see `next_unknown_ordinal`. let mut num_unknown_losses: u8 = 0; for (i, row) in rows.iter().enumerate() { @@ -386,7 +366,8 @@ fn parse_precursor_group( row.fragment_loss_type, i ); - num_unknown_losses = next_unknown_ordinal(num_unknown_losses)?; + num_unknown_losses = next_unknown_ordinal(num_unknown_losses) + .ok_or(DiannPrecursorParsingError::IonOverCapacity)?; let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); @@ -703,7 +684,8 @@ fn parse_precursor_group_from_parquet( columns.fragment_loss_types[idx], i ); - num_unknown_losses = next_unknown_ordinal(num_unknown_losses)?; + num_unknown_losses = next_unknown_ordinal(num_unknown_losses) + .ok_or(DiannPrecursorParsingError::IonOverCapacity)?; let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); diff --git a/rust/timsquery/src/serde/elution_group_inputs.rs b/rust/timsquery/src/serde/elution_group_inputs.rs index f109c7f4..0338f908 100644 --- a/rust/timsquery/src/serde/elution_group_inputs.rs +++ b/rust/timsquery/src/serde/elution_group_inputs.rs @@ -1,4 +1,3 @@ -use crate::ion::IonAnnot; use crate::tinyvec::{ TinyVec, tiny_vec, @@ -81,34 +80,6 @@ impl ElutionGroupInput { fragment_labels: Some(fragment_labels), }) } - - pub fn try_fill_labels_annot( - self, - ) -> Result, ElutionGroupInputError> { - let tmp = self.try_fill_labels_u8()?; - let new_frags = tmp - .fragment_labels - .unwrap() - .into_iter() - // isotope 0 — these are monoisotopic placeholders. Every other `?` - // construction in the readers uses 0; the previous `1` here would - // have labelled every synthesized fragment as the M+1 peak. - .map(|lbl| IonAnnot::try_new('?', Some(lbl), 1, 0)) - .collect::, _>>() - .map_err(|e| ElutionGroupInputError::IonConversionError { - inner: format!("{e:?}"), - })?; - Ok(ElutionGroupInput { - id: tmp.id, - mobility: tmp.mobility, - rt_seconds: tmp.rt_seconds, - precursor: tmp.precursor, - precursor_charge: tmp.precursor_charge, - precursor_isotopes: tmp.precursor_isotopes, - fragments: tmp.fragments, - fragment_labels: Some(new_frags), - }) - } } impl + KeyLike> TryFrom> for TimsElutionGroup { @@ -186,8 +157,8 @@ mod tests { assert_eq!(unique.len(), 256, "synthesized labels must all be distinct"); } - /// One past capacity previously wrapped (`i as u8`), silently emitting a - /// second `0` label and corrupting scoring downstream. It must now fail. + /// One past capacity must fail rather than wrap: a second `0` label would + /// collide with the first and corrupt scoring downstream. #[test] fn fill_labels_u8_rejects_overflow_instead_of_wrapping() { let err = input_with_n_fragments(257) @@ -201,23 +172,4 @@ mod tests { "expected TooManyFragmentsToLabel, got {err:?}" ); } - - /// Synthesized `?` annotations are monoisotopic placeholders; a nonzero - /// isotope would mislabel every fragment as the M+1 peak. - #[test] - fn fill_labels_annot_uses_monoisotopic_placeholders() { - let filled = input_with_n_fragments(3) - .try_fill_labels_annot() - .expect("3 fragments label fine"); - let labels = filled.fragment_labels.unwrap(); - let expected: Vec = (0u8..3) - .map(|i| IonAnnot::try_new('?', Some(i), 1, 0).unwrap()) - .collect(); - assert_eq!( - labels, expected, - "synthesized placeholders must be `?` at isotope 0, not M+1" - ); - let unique: std::collections::HashSet<_> = labels.iter().copied().collect(); - assert_eq!(unique.len(), 3, "synthesized labels must all be distinct"); - } } diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index c91ac13b..0c68b0c1 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -404,15 +404,58 @@ pub trait LibraryReader: Send + Sync { /// Cheap probe: header bytes / extension / first data row. Must not read the /// whole file. fn sniff(&self, path: &Path) -> bool; - fn read(&self, path: &Path) -> Result; + /// Read into the arena. + /// + /// Most formats produce an [`ElutionGroupCollection`] and let + /// [`LibraryArena::from_elution_groups`] adapt it; implement + /// [`Self::read`] for those. The binary `.speclib` and mzSpecLib readers + /// build the arena directly (with the reference-intensity sidecar) and + /// override this instead. + fn read_arena(&self, path: &Path) -> Result { + LibraryArena::from_elution_groups(self.read(path)?) + } + /// The legacy path. Direct-arena readers leave this unimplemented. + fn read(&self, _path: &Path) -> Result { + unreachable!("a reader must implement either `read` or `read_arena`") + } } +struct MzSpecLibReader; +struct DiannSpeclibReader; struct DiannParquetReader; struct DiannTsvReader; struct SpectronautReader; struct SkylineReader; struct JsonReader; +impl LibraryReader for MzSpecLibReader { + fn name(&self) -> &'static str { + "mzspeclib" + } + + fn sniff(&self, path: &Path) -> bool { + sniff_mzspeclib_library_file(path) + } + + fn read_arena(&self, path: &Path) -> Result { + read_mzspeclib_library_file(path) + } +} + +impl LibraryReader for DiannSpeclibReader { + fn name(&self) -> &'static str { + "diann-speclib" + } + + fn sniff(&self, path: &Path) -> bool { + sniff_diann_speclib_library_file(path) + } + + fn read_arena(&self, path: &Path) -> Result { + read_diann_speclib_library_file(path) + } +} + impl LibraryReader for DiannParquetReader { fn name(&self) -> &'static str { "diann-parquet" @@ -517,8 +560,17 @@ impl LibraryReader for JsonReader { } } +/// Readers in dispatch order: most specific first, ending with the +/// always-sniffs-true JSON fallback. +/// +/// mzSpecLib and `.speclib` lead because their probes are exact (a magic first +/// line and a version-gated header), so they cannot steal another format's +/// file — and `.speclib`'s read is the only one that surfaces an +/// `UnsupportedSpeclibVersion` diagnostic, which a later reader would mask. fn registry() -> &'static [&'static dyn LibraryReader] { &[ + &MzSpecLibReader, + &DiannSpeclibReader, &DiannParquetReader, &DiannTsvReader, &SpectronautReader, @@ -529,30 +581,12 @@ fn registry() -> &'static [&'static dyn LibraryReader] { pub fn read_library_file>(path: T) -> Result { let path = path.as_ref(); - // mzSpecLib is sniffed alongside `.speclib` rather than through the - // registry: like the DIA-NN binary reader it builds the arena directly - // (with the reference-intensity sidecar) instead of going through the - // legacy `ElutionGroupCollection`. Its magic first line makes the probe - // exact, so an early check cannot steal another format's file. - if sniff_mzspeclib_library_file(path) { - info!("Dispatching library read to mzspeclib (direct arena build)"); - return read_mzspeclib_library_file(path); - } - // The DIA-NN `.speclib` reader builds the columnar arena directly (with the - // reference-intensity sidecar); every other format still produces the legacy - // `ElutionGroupCollection`, adapted into the arena here. `.speclib` is - // sniffed first because its `read` path is the only one that can surface an - // `UnsupportedSpeclibVersion` diagnostic (the sniff has no version gate). - if sniff_diann_speclib_library_file(path) { - info!("Dispatching library read to diann-speclib (direct arena build)"); - return read_diann_speclib_library_file(path); - } let mut last_err = None; for reader in registry() { if reader.sniff(path) { info!("Dispatching library read to {}", reader.name()); - match reader.read(path) { - Ok(egs) => return LibraryArena::from_elution_groups(egs), + match reader.read_arena(path) { + Ok(arena) => return Ok(arena), // A sniff can fire on a file the reader then fails to parse // (overlapping sniffs). Fall through to the next candidate // instead of committing to the first sniff. Keep the FIRST diff --git a/rust/timsquery/src/serde/mod.rs b/rust/timsquery/src/serde/mod.rs index dc18d5a4..f65e70fe 100644 --- a/rust/timsquery/src/serde/mod.rs +++ b/rust/timsquery/src/serde/mod.rs @@ -7,6 +7,7 @@ mod library_file; pub mod mzspeclib_io; mod skyline_io; mod spectronaut_io; +mod unknown_ordinal; pub use chromatogram_output::*; pub use index_serde::*; diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs index fe0ac0dd..660af579 100644 --- a/rust/timsquery/src/serde/mzspeclib_io.rs +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -120,10 +120,24 @@ pub struct MzSpecLibStats { pub spectra_without_rt: usize, /// Spectra whose mobility came from a drift time rather than 1/K0. pub spectra_with_drift_time_mobility: usize, + /// Spectra whose retention time carried a unit this reader does not know. + pub spectra_with_unknown_rt_unit: usize, /// Precursors dropped for having no usable peak left. pub dropped_empty_precursors: usize, } +/// One spectrum converted into the shape [`QueryCollection::push_row`] takes. +struct ArenaRow { + precursor_mz: f64, + charge: u8, + rt_seconds: f32, + mobility: f32, + frags: Vec<(IonAnnot, f64)>, + intensities: Vec, + stripped: String, + modified: String, +} + impl MzSpecLibStats { fn anything_to_report(&self) -> bool { self.kept_unknown_label > 0 @@ -132,6 +146,7 @@ impl MzSpecLibStats { || self.dropped_duplicate_label > 0 || self.spectra_without_rt > 0 || self.spectra_with_drift_time_mobility > 0 + || self.spectra_with_unknown_rt_unit > 0 || self.dropped_empty_precursors > 0 } @@ -148,7 +163,7 @@ impl MzSpecLibStats { "mzSpecLib {}: kept {} annotated + {} with unknown labels; \ skipped {} unannotated, {} ambiguous, {} duplicate-label; \ {} spectra without RT, {} using drift time as mobility, \ - {} precursors dropped as empty", + {} with an unknown RT unit, {} precursors dropped as empty", path.display(), self.kept_annotated, self.kept_unknown_label, @@ -157,6 +172,7 @@ impl MzSpecLibStats { self.dropped_duplicate_label, self.spectra_without_rt, self.spectra_with_drift_time_mobility, + self.spectra_with_unknown_rt_unit, self.dropped_empty_precursors, ); } @@ -299,19 +315,7 @@ fn resolve_annotation(annotation: &str) -> (Resolved, Option) { /// /// Returns `None` when the spectrum lacks something structural (precursor m/z, /// charge, sequence) or ends up with no usable peak. -fn convert_spectrum( - raw: &RawSpectrum, - stats: &mut MzSpecLibStats, -) -> Option<( - f64, - u8, - f32, - f32, - Vec<(IonAnnot, f64)>, - Vec, - String, - String, -)> { +fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { let precursor_mz = raw.attrs.f64_of(PRECURSOR_MZ_TERMS)?; let charge: u8 = raw.attrs.find(CHARGE_TERM)?.value.parse().ok()?; @@ -322,7 +326,13 @@ fn convert_spectrum( match raw.attrs.unit_for(attr) { Some(UNIT_SECOND) => v, Some(UNIT_MINUTE) | None => v * 60.0, - Some(_) => v * 60.0, + // An unrecognized unit is counted, not silently treated as + // minutes: guessing wrong here is a 60x error in the RT the + // whole extraction window is built around. + Some(_) => { + stats.spectra_with_unknown_rt_unit += 1; + v * 60.0 + } } } None => { @@ -426,16 +436,16 @@ fn convert_spectrum( return None; } - Some(( + Some(ArenaRow { precursor_mz, charge, - rt_seconds as f32, + rt_seconds: rt_seconds as f32, mobility, frags, - intens, + intensities: intens, stripped, modified, - )) + }) } /// Cheap probe: the format's magic first line. @@ -483,20 +493,18 @@ pub fn read_mzspeclib_library_file>( frag_intens: &mut Vec, stats: &mut MzSpecLibStats| { let Some(raw) = cur else { return }; - let Some((mz, charge, rt, mobility, frags, intens, stripped, modified)) = - convert_spectrum(&raw, stats) - else { + let Some(row) = convert_spectrum(&raw, stats) else { return; }; - frag_intens.extend_from_slice(&intens); + frag_intens.extend_from_slice(&row.intensities); geom.push_row( - mz, - charge, - rt, - mobility, - &frags, - &stripped, - &modified, + row.precursor_mz, + row.charge, + row.rt_seconds, + row.mobility, + &row.frags, + &row.stripped, + &row.modified, &[], false, ); @@ -621,9 +629,9 @@ mod tests { ); } - /// Spectronaut's export carries `-H2O`/`-NH3` losses, which the packed - /// `IonAnnot` now represents, so they keep their real labels rather than - /// falling back to unknown. + /// Spectronaut's export carries `-H2O`/`-NH3` losses. `IonAnnot` + /// represents those, so they must keep real labels rather than degrading + /// to unknown. #[test] fn spectronaut_losses_keep_real_labels() { let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); @@ -719,21 +727,13 @@ mod tests { let e = MassError::Da(-0.0005); assert!((e.theoretical_from_observed(175.1184) - 175.1189).abs() < 1e-9); } -} - -#[cfg(test)] -mod registry_tests { - use super::tests_support::fixture; - use crate::serde::{ - LibraryArena, - read_library_file, - }; /// The public entry point must dispatch mzSpecLib itself. It is sniffed - /// before the registry, so a regression here would silently fall through - /// to the always-true JSON reader and fail with a generic parse error. + /// before the registry, so a regression here falls through to the + /// always-true JSON reader and fails with a generic parse error. #[test] fn public_read_library_file_dispatches_mzspeclib() { + use crate::serde::read_library_file; for name in ["diann.mzSpecLib.txt", "spectronaut.mzSpecLib.txt"] { let arena = read_library_file(fixture(name)) .unwrap_or_else(|e| panic!("{name} must load through the registry: {e:?}")); @@ -748,12 +748,3 @@ mod registry_tests { } } } - -#[cfg(test)] -mod tests_support { - pub fn fixture(name: &str) -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/mzspeclib_io_files") - .join(name) - } -} diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs.bak b/rust/timsquery/src/serde/mzspeclib_io.rs.bak new file mode 100644 index 00000000..7c8e984c --- /dev/null +++ b/rust/timsquery/src/serde/mzspeclib_io.rs.bak @@ -0,0 +1,751 @@ + || self.spectra_with_drift_time_mobility > 0 + || self.spectra_with_unknown_rt_unit > 0 + || self.dropped_empty_precursors > 0//! Reader for the mzSpecLib text format (HUPO-PSI). +//! +//! # Why every field needs a fallback ladder +//! +//! mzSpecLib says *how* to write a controlled-vocabulary term, not *which* +//! term a writer must use. The two reference exports disagree on nearly +//! everything this reader needs: +//! +//! | field | DIA-NN writes | Spectronaut writes | +//! |---|---|---| +//! | precursor m/z | `MS:1000744` selected ion m/z | `MS:1003208` experimental precursor monoisotopic m/z | +//! | retention time | *nothing at all* | `MS:1000896` normalized retention time, in minutes | +//! | ion mobility | `MS:1002476` (written as `0.0`) | `MS:1002476` | +//! +//! So each field is resolved by trying terms in priority order, and RT carries +//! a unit that must be honoured rather than assumed. +//! +//! # Peak resolution +//! +//! Peak lists carry *observed* m/z; the annotation's mass-error suffix recovers +//! the theoretical value the arena wants (`theoretical = observed - error`). So +//! a theoretical mass only exists once a single identity is pinned, which sorts +//! peaks three ways: +//! +//! | | kept | m/z | +//! |---|---|---| +//! | resolved, representable | real label | theoretical | +//! | resolved, not representable (`y1-HCOOH`) | unknown label | theoretical | +//! | unannotated (`?`) or tied ambiguity | no | — | +//! +//! Row three is skipped rather than stored at observed m/z: an arena mixing +//! observed and theoretical masses would be invisible downstream. Row two is +//! kept because a known-but-unspellable identity still has an exact mass — only +//! the label is lost. +//! +//! Ambiguous (comma-separated) annotations take the alternative with the +//! smallest absolute mass error. If that one is unrepresentable the peak gets an +//! unknown label rather than falling back to a worse-matching representable +//! alternative, which would assign both a wrong identity and a wrong mass. + +use crate::ion::IonAnnot; +use crate::models::{ + LibCapabilities, + QueryCollection, +}; +use crate::serde::library_file::{ + LibraryArena, + LibraryReadingError, +}; +use micromzpaf::{ + MassError, + split_mass_error, +}; +use std::io::{ + BufRead, + BufReader, +}; +use std::path::Path; +use tracing::{ + info, + warn, +}; + +/// First non-empty line of an mzSpecLib text file. +const MAGIC: &str = ""; + +// CV term ladders, ordered most- to least-specific; the first present wins. + +/// Precursor m/z. Experimental monoisotopic is preferred over `selected ion +/// m/z`, which on a quadrupole instrument is the isolation-window centre and +/// need not be the monoisotopic peak. +const PRECURSOR_MZ_TERMS: &[&str] = &[ + "MS:1003208", // experimental precursor monoisotopic m/z + "MS:1003053", // theoretical monoisotopic m/z + "MS:1000744", // selected ion m/z +]; +/// Retention time. `normalized retention time` is an iRT-style index rather +/// than a clock reading, but it is what Spectronaut exports and the only RT +/// signal available in those files. +const RT_TERMS: &[&str] = &[ + "MS:1000894", // retention time + "MS:1000896", // normalized retention time +]; +/// Ion mobility. Note these are different quantities, not spellings of one: +/// `MS:1002815` is inverse reduced ion mobility (1/K0), which is what the +/// arena wants, while `MS:1002476` is a drift time. They are tried in that +/// order and a drift-time-only library is counted, since treating a drift time +/// as 1/K0 is only valid for instruments that report it that way. +const MOBILITY_INVERSE_REDUCED: &str = "MS:1002815"; +const MOBILITY_DRIFT_TIME: &str = "MS:1002476"; + +const CHARGE_TERM: &str = "MS:1000041"; +const STRIPPED_SEQ_TERM: &str = "MS:1000888"; +const PROFORMA_TERM: &str = "MS:1003270"; +const UNIT_TERM: &str = "UO:0000000"; + +const UNIT_MINUTE: &str = "UO:0000031"; +const UNIT_SECOND: &str = "UO:0000010"; + +/// Per-library tally of everything that did not land verbatim in the arena. +/// +/// Reported once at the end of a load rather than per row: a consensus library +/// can carry thousands of unannotated peaks, and a line each would bury the +/// signal. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct MzSpecLibStats { + /// Peaks stored with their parsed annotation. + pub kept_annotated: usize, + /// Identity known but not representable (a loss outside the table, a + /// modified immonium). Stored with an unknown label and an exact mass. + pub kept_unknown_label: usize, + /// `?` — no annotation, so no mass error, so no theoretical m/z. + pub skipped_unannotated: usize, + /// Comma-separated alternatives that tied on absolute mass error. + pub skipped_ambiguous: usize, + /// Peaks dropped because their label collided with one already in the + /// precursor after the unknown-label rewrite. + pub dropped_duplicate_label: usize, + /// Spectra with no retention-time term at all. + pub spectra_without_rt: usize, + /// Spectra whose mobility came from a drift time rather than 1/K0. + pub spectra_with_drift_time_mobility: usize, + /// Spectra whose retention time carried a unit this reader does not know. + pub spectra_with_unknown_rt_unit: usize, + /// Precursors dropped for having no usable peak left. + pub dropped_empty_precursors: usize, +} + +/// One spectrum converted into the shape [`QueryCollection::push_row`] takes. +struct ArenaRow { + precursor_mz: f64, + charge: u8, + rt_seconds: f32, + mobility: f32, + frags: Vec<(IonAnnot, f64)>, + intensities: Vec, + stripped: String, + modified: String, +} + +impl MzSpecLibStats { + fn anything_to_report(&self) -> bool { + self.kept_unknown_label > 0 + || self.skipped_unannotated > 0 + || self.skipped_ambiguous > 0 + || self.dropped_duplicate_label > 0 + || self.spectra_without_rt > 0 + || self.spectra_with_drift_time_mobility > 0 + || self.dropped_empty_precursors > 0 + } + + fn report(&self, path: &Path) { + if !self.anything_to_report() { + info!( + "mzSpecLib {}: {} peaks, all annotated and representable", + path.display(), + self.kept_annotated + ); + return; + } + warn!( + "mzSpecLib {}: kept {} annotated + {} with unknown labels; \ + skipped {} unannotated, {} ambiguous, {} duplicate-label; \ + {} spectra without RT, {} using drift time as mobility, \ + {} with an unknown RT unit, {} precursors dropped as empty", + path.display(), + self.kept_annotated, + self.kept_unknown_label, + self.skipped_unannotated, + self.skipped_ambiguous, + self.dropped_duplicate_label, + self.spectra_without_rt, + self.spectra_with_drift_time_mobility, + self.spectra_with_unknown_rt_unit, + self.dropped_empty_precursors, + ); + } +} + +/// One `ACC|name=value` attribute, with its optional `[n]` group tag. +#[derive(Debug, Clone)] +struct Attr { + group: Option, + accession: String, + value: String, +} + +impl Attr { + fn parse(line: &str) -> Option { + let (group, rest) = match line.strip_prefix('[') { + Some(r) => { + let (g, r) = r.split_once(']')?; + (Some(g.parse().ok()?), r) + } + None => (None, line), + }; + let (key, value) = rest.split_once('=')?; + let accession = key.split('|').next()?.to_string(); + Some(Attr { + group, + accession, + value: value.to_string(), + }) + } + + /// The accession out of a `ACC|name` *value* (as opposed to a key), for + /// terms whose value is itself a CV term — e.g. `unit=UO:0000031|minute`. + fn value_accession(&self) -> &str { + self.value.split('|').next().unwrap_or(&self.value) + } +} + +/// Attributes collected for one spectrum (its own plus its analyte's). +#[derive(Debug, Default)] +struct AttrBag(Vec); + +impl AttrBag { + fn find(&self, accession: &str) -> Option<&Attr> { + self.0.iter().find(|a| a.accession == accession) + } + + fn first_of(&self, accessions: &[&str]) -> Option<&Attr> { + accessions.iter().find_map(|a| self.find(a)) + } + + fn f64_of(&self, accessions: &[&str]) -> Option { + self.first_of(accessions)?.value.parse().ok() + } + + /// The unit term attached to `attr` via its `[n]` group, if any. + fn unit_for(&self, attr: &Attr) -> Option<&str> { + let group = attr.group?; + self.0 + .iter() + .find(|a| a.group == Some(group) && a.accession == UNIT_TERM) + .map(|a| a.value_accession()) + } +} + +/// A spectrum accumulated from the text stream, before conversion. +#[derive(Debug, Default)] +struct RawSpectrum { + attrs: AttrBag, + /// `(observed mz, intensity, annotation)` + peaks: Vec<(f64, f32, String)>, +} + +/// What resolving one peak's annotation produced. +enum Resolved { + /// Parsed cleanly; store with this label. + Annotated(IonAnnot), + /// Identity known, not representable. Store with an unknown label, exact + /// mass. + UnknownLabel, + /// No single identity, so no theoretical mass. Skip. + SkipUnannotated, + SkipAmbiguous, +} + +/// Resolve one annotation string into a storage decision plus the mass error +/// needed to recover theoretical m/z. +fn resolve_annotation(annotation: &str) -> (Resolved, Option) { + let annotation = annotation.trim(); + if annotation.is_empty() || annotation == "?" { + return (Resolved::SkipUnannotated, None); + } + + // Splitting the error off comes first: it works even when the ion will not + // parse, which is exactly the case that still needs an exact mass. A + // malformed suffix on ANY alternative makes the whole peak unresolvable — + // dropping just that one would silently turn an ambiguous peak into an + // unambiguous one. + let mut alternatives = Vec::new(); + for alt in annotation.split(',') { + let Ok(parsed) = split_mass_error(alt.trim()) else { + return (Resolved::SkipAmbiguous, None); + }; + alternatives.push(parsed); + } + + let (ion_str, mass_error) = if let [single] = alternatives[..] { + single + } else { + // Closest by absolute mass error. Comparing a Da magnitude against a + // ppm one would be meaningless, but a library uses one unit + // throughout. Errors are parsed decimal literals, so equal ones + // compare exactly and a tie pins no identity. + let magnitude = |m: &Option| match m { + Some(MassError::Da(v) | MassError::Ppm(v)) => v.abs(), + None => f64::INFINITY, + }; + let best = alternatives + .iter() + .map(|(_, e)| magnitude(e)) + .fold(f64::INFINITY, f64::min); + if !best.is_finite() { + return (Resolved::SkipAmbiguous, None); + } + let mut winners = alternatives.iter().filter(|(_, e)| magnitude(e) == best); + let winner = *winners.next().expect("the minimum came from this iterator"); + if winners.next().is_some() { + return (Resolved::SkipAmbiguous, None); + } + winner + }; + match IonAnnot::try_from(ion_str) { + Ok(ion) => (Resolved::Annotated(ion), mass_error), + // Keep the peak and its exact mass, lose only the label. + Err(_) => (Resolved::UnknownLabel, mass_error), + } +} + +/// Convert one accumulated spectrum into arena rows. +/// +/// Returns `None` when the spectrum lacks something structural (precursor m/z, +/// charge, sequence) or ends up with no usable peak. +fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { + let precursor_mz = raw.attrs.f64_of(PRECURSOR_MZ_TERMS)?; + let charge: u8 = raw.attrs.find(CHARGE_TERM)?.value.parse().ok()?; + + let rt_seconds = match raw.attrs.first_of(RT_TERMS) { + Some(attr) => { + let v: f64 = attr.value.parse().ok()?; + // Honour the unit rather than assuming: Spectronaut writes minutes. + match raw.attrs.unit_for(attr) { + Some(UNIT_SECOND) => v, + Some(UNIT_MINUTE) | None => v * 60.0, + // An unrecognized unit is counted, not silently treated as + // minutes: guessing wrong here is a 60x error in the RT the + // whole extraction window is built around. + Some(_) => { + stats.spectra_with_unknown_rt_unit += 1; + v * 60.0 + } + } + } + None => { + stats.spectra_without_rt += 1; + 0.0 + } + }; + + let mobility = match raw.attrs.find(MOBILITY_INVERSE_REDUCED) { + Some(a) => a.value.parse().ok()?, + None => match raw.attrs.find(MOBILITY_DRIFT_TIME) { + Some(a) => { + stats.spectra_with_drift_time_mobility += 1; + a.value.parse().ok()? + } + // Absent is fine — an unset mobility is 0.0, same as DIA-NN writes. + // A *present but malformed* one drops the spectrum instead. + None => 0.0, + }, + }; + + let stripped = raw + .attrs + .find(STRIPPED_SEQ_TERM) + .map(|a| a.value.clone()) + .unwrap_or_default(); + // The proforma term carries a trailing `/charge` that is not part of the + // peptidoform. + let modified = raw + .attrs + .find(PROFORMA_TERM) + .map(|a| { + a.value + .rsplit_once('/') + .map(|(p, _)| p.to_string()) + .unwrap_or_else(|| a.value.clone()) + }) + .unwrap_or_else(|| stripped.clone()); + if stripped.is_empty() && modified.is_empty() { + return None; + } + + let mut frags: Vec<(IonAnnot, f64)> = Vec::with_capacity(raw.peaks.len()); + let mut intens: Vec = Vec::with_capacity(raw.peaks.len()); + let mut unknown_counter: u8 = 0; + + for (observed_mz, intensity, annotation) in &raw.peaks { + let (resolved, mass_error) = resolve_annotation(annotation); + let label = match resolved { + Resolved::Annotated(ion) => { + stats.kept_annotated += 1; + ion + } + Resolved::UnknownLabel => { + // The ordinal is a per-precursor uniqueness counter. Past 255 + // there is no way to keep labels distinct, so drop rather than + // reuse one. + let Some(next) = unknown_counter.checked_add(1) else { + stats.dropped_duplicate_label += 1; + continue; + }; + unknown_counter = next; + match IonAnnot::try_new('?', Some(unknown_counter), 1, 0) { + Ok(i) => { + stats.kept_unknown_label += 1; + i + } + Err(_) => { + stats.dropped_duplicate_label += 1; + continue; + } + } + } + Resolved::SkipUnannotated => { + stats.skipped_unannotated += 1; + continue; + } + Resolved::SkipAmbiguous => { + stats.skipped_ambiguous += 1; + continue; + } + }; + + let mz = match mass_error { + Some(e) => e.theoretical_from_observed(*observed_mz), + None => *observed_mz, + }; + + // Labels must stay unique within the precursor (`linear_get` is + // first-match), so a collision drops the later peak. + if frags.iter().any(|(l, _)| *l == label) { + stats.dropped_duplicate_label += 1; + continue; + } + frags.push((label, mz)); + intens.push(*intensity); + } + + if frags.is_empty() { + stats.dropped_empty_precursors += 1; + return None; + } + + Some(ArenaRow { + precursor_mz, + charge, + rt_seconds: rt_seconds as f32, + mobility, + frags, + intensities: intens, + stripped, + modified, + }) +} + +/// Cheap probe: the format's magic first line. +pub fn sniff_mzspeclib_library_file>(path: T) -> bool { + let Ok(file) = std::fs::File::open(path.as_ref()) else { + return false; + }; + let mut reader = BufReader::new(file); + let mut line = String::new(); + // Only the first non-empty line is inspected, so this stays O(1) on a + // multi-gigabyte library. + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => return false, + Ok(_) => { + let t = line.trim(); + if t.is_empty() { + continue; + } + return t == MAGIC; + } + Err(_) => return false, + } + } +} + +/// Read an mzSpecLib text file into the columnar arena. +pub fn read_mzspeclib_library_file>( + path: T, +) -> Result { + let path = path.as_ref(); + let file = std::fs::File::open(path).map_err(LibraryReadingError::IoError)?; + let reader = BufReader::new(file); + + let mut geom = QueryCollection::with_capabilities(LibCapabilities::default_diann_no_decoys()); + let mut frag_intens: Vec = Vec::new(); + let mut stats = MzSpecLibStats::default(); + + let mut current: Option = None; + let mut in_peaks = false; + + let flush = |cur: Option, + geom: &mut QueryCollection, + frag_intens: &mut Vec, + stats: &mut MzSpecLibStats| { + let Some(raw) = cur else { return }; + let Some(row) = convert_spectrum(&raw, stats) else { + return; + }; + frag_intens.extend_from_slice(&row.intensities); + geom.push_row( + row.precursor_mz, + row.charge, + row.rt_seconds, + row.mobility, + &row.frags, + &row.stripped, + &row.modified, + &[], + false, + ); + }; + + for line in reader.lines() { + let line = line.map_err(LibraryReadingError::IoError)?; + let trimmed = line.trim_end(); + + if trimmed.starts_with("" { + in_peaks = true; + continue; + } + // `` and `` attributes are folded into the + // spectrum's bag: this reader wants the union, not the hierarchy. + if trimmed.starts_with('<') { + in_peaks = false; + continue; + } + if trimmed.is_empty() { + in_peaks = false; + continue; + } + + let Some(spec) = current.as_mut() else { + continue; // library-level header + }; + + if in_peaks { + let mut cols = trimmed.split('\t'); + let (Some(mz), Some(intensity)) = (cols.next(), cols.next()) else { + continue; + }; + let (Ok(mz), Ok(intensity)) = + (mz.trim().parse::(), intensity.trim().parse::()) + else { + continue; + }; + let annotation = cols.next().unwrap_or("?").to_string(); + spec.peaks.push((mz, intensity, annotation)); + } else if let Some(attr) = Attr::parse(trimmed) { + spec.attrs.0.push(attr); + } + } + flush(current.take(), &mut geom, &mut frag_intens, &mut stats); + + stats.report(path); + + if geom.n_rows() == 0 { + return Err(LibraryReadingError::SpeclibParse(format!( + "mzSpecLib {} yielded no usable precursors", + path.display() + ))); + } + if frag_intens.len() != geom.frag_labels.len() { + return Err(LibraryReadingError::SpeclibParse(format!( + "reference-intensity sidecar ({}) must stay parallel to the fragment-label arena ({})", + frag_intens.len(), + geom.frag_labels.len(), + ))); + } + + geom.seal(); + Ok(LibraryArena::Mzpaf { + geom, + frag_intens: Some(frag_intens), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture(name: &str) -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/mzspeclib_io_files") + .join(name) + } + + #[test] + fn sniffs_only_mzspeclib() { + assert!(sniff_mzspeclib_library_file(fixture("diann.mzSpecLib.txt"))); + assert!(sniff_mzspeclib_library_file(fixture( + "spectronaut.mzSpecLib.txt" + ))); + // A DIA-NN TSV must not be claimed. + let tsv = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/diann_io_files/sample_lib.tsv"); + assert!(!sniff_mzspeclib_library_file(tsv)); + } + + #[test] + fn reads_diann_export() { + let arena = read_mzspeclib_library_file(fixture("diann.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, frag_intens } = arena else { + panic!("mzSpecLib must build an mzpaf arena"); + }; + assert!(geom.n_rows() > 0); + let intens = frag_intens.expect("reference intensities are populated"); + assert_eq!(intens.len(), geom.frag_labels.len()); + } + + /// DIA-NN's export writes every mass error as exactly `0.0`, so observed + /// and theoretical coincide and the m/z must pass through untouched. + #[test] + fn zero_mass_error_leaves_mz_unchanged() { + let arena = read_mzspeclib_library_file(fixture("diann.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + // First spectrum's first peak in the fixture: 427.22995, annotated b6/0.0 + let first = geom.frag_mzs[0]; + assert!( + (first - 427.22995).abs() < 1e-6, + "expected the observed m/z verbatim, got {first}" + ); + } + + /// Spectronaut's export carries `-H2O`/`-NH3` losses. `IonAnnot` + /// represents those, so they must keep real labels rather than degrading + /// to unknown. + #[test] + fn spectronaut_losses_keep_real_labels() { + let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + let has_loss = geom + .frag_labels + .iter() + .any(|l| l.loss() != micromzpaf::NeutralLoss::None); + assert!(has_loss, "expected at least one loss-bearing label"); + let unknowns = geom + .frag_labels + .iter() + .filter(|l| l.try_get_ordinal().is_none() && l.loss() == micromzpaf::NeutralLoss::None) + .count(); + assert_eq!(unknowns, 0, "no peak should need an unknown label here"); + } + + /// RT is unit-tagged; Spectronaut writes minutes and the arena wants + /// seconds. Getting this wrong is a silent 60x error. + #[test] + fn retention_time_honours_its_unit() { + let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + // Fixture's first spectrum: normalized retention time = 28.658491 min. + let rt = geom.rt_seconds[0]; + assert!( + (rt - 28.658491 * 60.0).abs() < 0.01, + "expected minutes converted to seconds, got {rt}" + ); + } + + #[test] + fn resolves_unambiguous_representable() { + let (r, e) = resolve_annotation("y5/-0.0005"); + assert!(matches!(r, Resolved::Annotated(_))); + assert_eq!(e, Some(MassError::Da(-0.0005))); + } + + /// Known identity, unrepresentable spelling: keep the peak and the exact + /// mass, erase only the label. + #[test] + fn unrepresentable_loss_keeps_peak_with_unknown_label() { + let (r, e) = resolve_annotation("y1-HCOOH/0.0003"); + assert!(matches!(r, Resolved::UnknownLabel)); + assert_eq!( + e, + Some(MassError::Da(0.0003)), + "the mass error must survive so theoretical m/z stays exact" + ); + } + + #[test] + fn unannotated_peak_is_skipped() { + assert!(matches!( + resolve_annotation("?").0, + Resolved::SkipUnannotated + )); + } + + /// Closest-by-error wins; if that alternative is unrepresentable the peak + /// takes an unknown label rather than falling back to the representable + /// one, which would assign a wrong identity and a wrong mass. + #[test] + fn ambiguity_resolves_to_the_closest_not_the_representable() { + // a2 is representable and further; y2-CO2-NH3 is closer and is not. + let (r, e) = resolve_annotation("a2/-0.0040,y2-CO2-NH3/-0.0001"); + assert!( + matches!(r, Resolved::UnknownLabel), + "the closest alternative wins even when unrepresentable" + ); + assert_eq!(e, Some(MassError::Da(-0.0001))); + + // When the closest one IS representable, it is used. + let (r, _) = resolve_annotation("a2/-0.0001,y2-CO2-NH3/-0.0040"); + assert!(matches!(r, Resolved::Annotated(_))); + } + + /// An exact tie pins no identity, so no theoretical m/z exists and the peak + /// cannot be stored without mixing observed and theoretical masses. + #[test] + fn tied_ambiguity_is_skipped() { + let (r, _) = resolve_annotation("a2/-0.0004,y2-CO2-NH3/-0.0004"); + assert!(matches!(r, Resolved::SkipAmbiguous)); + } + + #[test] + fn mass_error_recovers_theoretical() { + // Real SpectraST peak: y1 for C-terminal R, observed 175.1184. + let e = MassError::Da(-0.0005); + assert!((e.theoretical_from_observed(175.1184) - 175.1189).abs() < 1e-9); + } + + /// The public entry point must dispatch mzSpecLib itself. It is sniffed + /// before the registry, so a regression here falls through to the + /// always-true JSON reader and fails with a generic parse error. + #[test] + fn public_read_library_file_dispatches_mzspeclib() { + use crate::serde::read_library_file; + for name in ["diann.mzSpecLib.txt", "spectronaut.mzSpecLib.txt"] { + let arena = read_library_file(fixture(name)) + .unwrap_or_else(|e| panic!("{name} must load through the registry: {e:?}")); + let LibraryArena::Mzpaf { geom, frag_intens } = arena else { + panic!("{name} must land in the mzpaf arena"); + }; + assert!(geom.n_rows() > 0, "{name} produced no precursors"); + assert!( + frag_intens.is_some(), + "{name} must populate the reference-intensity sidecar" + ); + } + } +} diff --git a/rust/timsquery/src/serde/skyline_io.rs b/rust/timsquery/src/serde/skyline_io.rs index b5c13a29..a93b43f4 100644 --- a/rust/timsquery/src/serde/skyline_io.rs +++ b/rust/timsquery/src/serde/skyline_io.rs @@ -3,6 +3,7 @@ use crate::ion::{ IonAnnot, IonParsingError, }; +use crate::serde::unknown_ordinal::next_unknown_ordinal; use serde::{ Deserialize, Deserializer, @@ -59,27 +60,6 @@ impl From for SkylinePrecursorParsingError { } } -/// Advance the per-precursor `?`-ordinal counter, refusing to wrap. -/// -/// `IonAnnot` ordinals are `u8`, so a precursor can carry at most -/// [`u8::MAX`] distinguishable unknown ions. Past that there is no way to keep -/// labels unique, and a silently reused ordinal corrupts scoring -/// (`linear_get` is first-match). Fail here, where the row index is still in -/// hand, rather than downstream in `try_from_pairs`. -/// -/// This previously used `saturating_add`, which does not wrap but still pins -/// every ordinal past the limit to [`u8::MAX`] — producing the same duplicate -/// labels, just more quietly. -fn next_unknown_ordinal(current: u8) -> Result { - current.checked_add(1).ok_or_else(|| { - error!( - "More than {} unknown-ion fragments in a single precursor; cannot assign unique labels", - u8::MAX - ); - SkylinePrecursorParsingError::IonOverCapacity - }) -} - impl From for SkylineReadingError { fn from(_err: SkylinePrecursorParsingError) -> Self { SkylineReadingError::SkylinePrecursorParsingError @@ -372,7 +352,8 @@ fn parse_precursor_group( falling back to unknown ion", frag_type, i ); - num_unknown_losses = next_unknown_ordinal(num_unknown_losses)?; + num_unknown_losses = next_unknown_ordinal(num_unknown_losses) + .ok_or(SkylinePrecursorParsingError::IonOverCapacity)?; IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)? } }; diff --git a/rust/timsquery/src/serde/spectronaut_io.rs b/rust/timsquery/src/serde/spectronaut_io.rs index d1d389d5..ad4fb92f 100644 --- a/rust/timsquery/src/serde/spectronaut_io.rs +++ b/rust/timsquery/src/serde/spectronaut_io.rs @@ -3,6 +3,7 @@ use crate::ion::{ IonAnnot, IonParsingError, }; +use crate::serde::unknown_ordinal::next_unknown_ordinal; use serde::Deserialize; use std::path::Path; use tinyvec::tiny_vec; @@ -59,23 +60,6 @@ impl From for SpectronautPrecursorParsingError { } } -/// Advance the per-precursor `?`-ordinal counter, refusing to wrap. -/// -/// `IonAnnot` ordinals are `u8`, so a precursor can carry at most -/// [`u8::MAX`] distinguishable unknown ions. Past that there is no way to keep -/// labels unique, and a silently reused ordinal corrupts scoring -/// (`linear_get` is first-match). Fail here, where the row index is still in -/// hand, rather than downstream in `try_from_pairs`. -fn next_unknown_ordinal(current: u8) -> Result { - current.checked_add(1).ok_or_else(|| { - error!( - "More than {} unknown-ion fragments in a single precursor; cannot assign unique labels", - u8::MAX - ); - SpectronautPrecursorParsingError::IonOverCapacity - }) -} - impl From for SpectronautReadingError { fn from(_err: SpectronautPrecursorParsingError) -> Self { SpectronautReadingError::SpectronautPrecursorParsingError @@ -324,7 +308,8 @@ fn parse_precursor_group( row.fragment_loss_type, i ); - num_unknown_losses = next_unknown_ordinal(num_unknown_losses)?; + num_unknown_losses = next_unknown_ordinal(num_unknown_losses) + .ok_or(SpectronautPrecursorParsingError::IonOverCapacity)?; let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); diff --git a/rust/timsquery/src/serde/unknown_ordinal.rs b/rust/timsquery/src/serde/unknown_ordinal.rs new file mode 100644 index 00000000..c1843a60 --- /dev/null +++ b/rust/timsquery/src/serde/unknown_ordinal.rs @@ -0,0 +1,17 @@ +//! Shared counter for `?`-labelled unknown ions. + +/// Advance a per-precursor `?`-ordinal counter, refusing to wrap. +/// +/// `IonAnnot` ordinals are `u8`, so a precursor can carry at most +/// [`u8::MAX`] distinguishable unknown ions. Past that there is no way to +/// keep labels unique, and duplicates are the one thing the arena cannot +/// tolerate — see [`ExpectedIntensities::try_from_pairs`] for why. +/// +/// `None` means "cannot allocate another": callers turn it into their own +/// over-capacity error while the row index is still in hand, rather than +/// letting the duplicate surface downstream. +/// +/// [`ExpectedIntensities::try_from_pairs`]: https://docs.rs/timsseek +pub(crate) fn next_unknown_ordinal(current: u8) -> Option { + current.checked_add(1) +} diff --git a/rust/timsquery/tests/carafe_contract.rs b/rust/timsquery/tests/carafe_contract.rs index 5bf4577f..7bc3144e 100644 --- a/rust/timsquery/tests/carafe_contract.rs +++ b/rust/timsquery/tests/carafe_contract.rs @@ -16,7 +16,13 @@ //! names, the `-o` directory layout, and the `results.json` basename. use std::io::Write; -use timsquery::models::Tolerance; +use timsquery::models::tolerance::{ + MobilityTolerance, + MzTolerance, + QuadTolerance, + RtTolerance, + Tolerance, +}; use timsquery::serde::{ LibraryArena, read_library_file, @@ -80,12 +86,19 @@ fn carafe_target_payload_loads_through_the_public_reader() { assert_eq!(labels, vec!["y1".to_string(), "y3^2".to_string()]); } -/// `id` is echoed back and used as Carafe's map key. A missing or renamed -/// `id` NPEs downstream rather than erroring here. +/// `id` is echoed back and used as Carafe's map key. A missing `id` NPEs +/// downstream rather than erroring here, so it must not silently default. #[test] fn carafe_id_field_is_required() { - let without_id = CARAFE_TARGETS.replace("\"id\": 0,", ""); - let f = write_targets(&without_id); + // Build the payload without `id` rather than string-surgering the const, + // so the test cannot pass because the edit produced malformed JSON. + let without_id = r#"[ + { "mobility": 0.95, "rt_seconds": 1234.5, "precursor": 650.32, + "precursor_charge": 2, "precursor_isotopes": [0,1,2], + "fragments": [175.1, 288.2], "fragment_labels": ["y1","y3^2"] } + ]"#; + serde_json::from_str::(without_id).expect("still valid JSON"); + let f = write_targets(without_id); assert!( read_library_file(f.path()).is_err(), "a target without `id` must fail rather than default it" @@ -103,17 +116,15 @@ fn carafe_tolerance_spellings_deserialize() { // window recentred on a measured calibration offset, read as "13 ppm light // to 17 ppm heavy". Both edges must stay distinct — collapsing them to a // symmetric tolerance would quietly recentre every extraction window. - let rendered = format!("{tol:?}"); - assert!( - rendered.contains("13.0") && rendered.contains("17.0"), - "both m/z window edges must be preserved distinctly, got {rendered}" - ); + assert_eq!(tol.ms, MzTolerance::Ppm((13.0, 17.0))); + assert_eq!(tol.rt, RtTolerance::Minutes((0.1, 0.1))); + assert_eq!(tol.mobility, MobilityTolerance::Pct((3.0, 3.0))); + assert_eq!(tol.quad, QuadTolerance::Absolute((0.1, 0.1))); let round = serde_json::to_string(&tol).expect("tolerance must re-serialize"); let back: Tolerance = serde_json::from_str(&round).expect("and deserialize again"); assert_eq!( - format!("{back:?}"), - rendered, + back, tol, "tolerance must survive a round trip through its own output" ); } diff --git a/rust/timsquery/tests/mzspeclib_io_files/README.md b/rust/timsquery/tests/mzspeclib_io_files/README.md index 6756feb7..5f1aa80f 100644 --- a/rust/timsquery/tests/mzspeclib_io_files/README.md +++ b/rust/timsquery/tests/mzspeclib_io_files/README.md @@ -6,10 +6,13 @@ Verbatim from [HUPO-PSI/mzSpecLib](https://github.com/HUPO-PSI/mzSpecLib) | file | why | |---|---| | `diann.mzSpecLib.txt` | the shape `speclib_build` will emit; all peaks annotated, every mass error exactly `0.0` | -| `spectronaut.mzSpecLib.txt` | carries neutral losses (`-H2O`, `-NH3`), which exercise the unrepresentable-annotation path | +| `spectronaut.mzSpecLib.txt` | carries `-H2O`/`-NH3` losses and a unit-tagged retention time in minutes | + +Both are fully representable, so neither exercises the unknown-label or +skip paths — those are covered by unit tests over `resolve_annotation`. Deliberately NOT vendored: the NIST and SpectraST examples. They are dominated by internal fragments, immonium ions and unannotated (`?`) peaks, and by consensus spectra whose observed m/z carries real calibration error. -Useful later for the resolution-policy tests, but they would make these two -fixtures harder to read for no gain. +Worth adding when the resolution policy needs end-to-end coverage; today +they would only make these two harder to read. diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 8884acb2..9de067cd 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -43,54 +43,6 @@ impl SerSpeclibElement { elution_group, } } - - pub fn sample() -> Self { - SerSpeclibElement { - precursor: PrecursorEntry { - sequence: "PEPTIDESEK".into(), - charge: 2, - decoy: false, - decoy_group: 32, - }, - elution_group: ReferenceEG { - id: 32, - precursor_mz: 512.2, - precursor_labels: vec![0, 2], - fragment_mzs: vec![312.2, 675.7], - fragment_labels: vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - precursor_intensities: vec![1.0, 0.5], - fragment_intensities: vec![0.8, 0.3], - mobility_ook0: 0.75, - rt_seconds: 120.0, - }, - } - } - - pub fn sample_json() -> &'static str { - r#"{ - "precursor": { - "sequence": "PEPTIDEPINK", - "charge": 2, - "decoy": false, - "decoy_group": 0 - }, - "elution_group": { - "id": 0, - "precursor_mz": 876.5432, - "precursor_labels": [ 0, 1 ], - "fragment_mzs": [ 123.0, 123.0, 123.0 ], - "fragment_labels": ["a1", "b1", "c1^2"], - "precursor_intensities": [1.0, 1.0], - "fragment_intensities": [1.0, 1.0, 1.0], - "precursor_charge": 2, - "mobility_ook0": 0.8, - "rt_seconds": 0.0 - } - }"# - } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -304,9 +256,8 @@ impl SpeclibFormat { /// anything else (including `.speclib` and `.mzSpecLib.txt`), which routes /// to the timsquery bridge. /// - /// Extension-only is a leftover from the msgpack era, where a content - /// sniff would have misclaimed raw binaries. ndjson could be sniffed, but - /// the bridge already handles anything this returns `None` for. + /// Matched by extension rather than content; anything unmatched routes to + /// the timsquery bridge, which sniffs properly. pub fn detect_from_extension(path: &Path) -> Option { let path_str = path.to_string_lossy().to_lowercase(); diff --git a/rust/timsseek/src/fragment_mass/elution_group_converter.rs b/rust/timsseek/src/fragment_mass/elution_group_converter.rs index 87354b58..3e173985 100644 --- a/rust/timsseek/src/fragment_mass/elution_group_converter.rs +++ b/rust/timsseek/src/fragment_mass/elution_group_converter.rs @@ -3,7 +3,6 @@ use mzcore::prelude::{ AmbiguousMolecule, Element, MolecularFormula, - Peptidoform, }; /// Super simple 1/k0 prediction. @@ -119,16 +118,8 @@ pub fn count_carbon_sulphur_in_sequence(sequence: &str) -> Result<(u16, u16), St } fn count_carbon_sulphur_in_sequence_mzcore(sequence: &str) -> Result<(u16, u16), String> { - let peptide = match Peptidoform::pro_forma(sequence, crate::models::sequence::ontologies()) { - // `pro_forma` also yields non-fatal warnings; the formula is all we need. - Ok((pep, _warnings)) => pep, - Err(e) => { - return Err(format!( - "Error parsing peptide sequence {}: {:?}", - sequence, e - )); - } - }; + let peptide = crate::models::sequence::parse_proforma(sequence) + .ok_or_else(|| format!("Error parsing peptide sequence {sequence}"))?; let peptide = match peptide.as_linear() { Some(pep) => pep, None => return Err("Peptide is not linear.".to_string()), diff --git a/rust/timsseek/src/lib.rs b/rust/timsseek/src/lib.rs index 21221853..31269619 100644 --- a/rust/timsseek/src/lib.rs +++ b/rust/timsseek/src/lib.rs @@ -34,6 +34,5 @@ pub use scoring::{ pub use timsquery::ion::{ IonAnnot, IonParsingError, - IonSeriesTerminality, }; pub use traits::ScorerQueriable; diff --git a/rust/timsseek/src/models/sequence.rs b/rust/timsseek/src/models/sequence.rs index 1b235acc..62617bf0 100644 --- a/rust/timsseek/src/models/sequence.rs +++ b/rust/timsseek/src/models/sequence.rs @@ -12,8 +12,7 @@ use std::sync::{ /// Process-wide modification ontologies, built on first use. /// -/// mzcore requires an explicit `Ontologies` value at every `pro_forma` call -/// (rustyms, its predecessor, used global lazy statics and took `None`). +/// mzcore requires an explicit `Ontologies` value at every `pro_forma` call. /// Building it costs ~210 ms and ~200 MB, so it is deliberately behind a /// `OnceLock` reached ONLY from [`parse_sequence_mzcore`] — the fallback past /// the byte-walk fast path. A library whose sequences all match the fast @@ -27,6 +26,19 @@ pub fn ontologies() -> &'static mzcore::ontology::Ontologies { }) } +/// Parse a ProForma string against the shared [`ontologies`]. +/// +/// mzcore returns non-fatal parse warnings alongside the peptidoform; none of +/// the callers can act on them, so they are dropped in one place instead of +/// each site carrying its own `(pep, _warnings)` destructure. +pub fn parse_proforma( + sequence: &str, +) -> Option> { + let (peptidoform, _warnings) = + mzcore::sequence::Peptidoform::pro_forma(sequence, ontologies()).ok()?; + Some(peptidoform) +} + /// Amino acid stored as alphabet offset `c - b'A'` (0..=25). `u8::MAX` /// means "unrecognized / non-alpha". Unreachable slots in count buffers /// (B=1, J=9, O=14, U=20, X=23, Z=25) stay zero. @@ -278,9 +290,8 @@ fn parse_sequence_fast(s: &str) -> Option { /// this function's contract is a binary parsed/not-parsed verdict. fn parse_sequence_mzcore(normalized: &str) -> Option { use mzcore::prelude::IsAminoAcid; - use mzcore::sequence::Peptidoform; - let (pf, _warnings) = Peptidoform::pro_forma(normalized, ontologies()).ok()?; + let pf = parse_proforma(normalized)?; let linear = pf.into_linear()?; let mut residues: SmallVec<[AminoAcid; 32]> = SmallVec::new(); From 7912f0a0f5ff488f021b6a9d2b53aab02f98f34a Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 23:53:15 -0700 Subject: [PATCH 11/27] fix: restore an ndjson writer so the workspace and release build `speclib_build_cli` was left uncompilable when the msgpack `SpeclibWriter` was removed, and then taken out of `default-members` to hide it. That did not contain the damage: release.yml builds `speclib_build` on all four targets, `--workspace` was broken for every cargo command, and the crate's mzcore port had never been type-checked because rustc aborted at name resolution. `SpeclibWriter::new_ndjson_zstd` is the inverse of the `SpeclibReader` that already existed, so what it emits is what the loader reads back. Config, CLI help, README and run.bash go back to ndjson; they advertised mzSpecLib, which nothing in the tree can write. Replace when the real writer lands. Also here, same file: `SpeclibReader` dispatched between `NdJsonReader` and itself once msgpack left, so it boxes a `dyn BufRead` rather than a `dyn Iterator` and absorbs the reader it was forwarding to. Its blank-line skip recursed per line and could overflow the stack; it loops now. The remote-output tempfile suffix is fixed rather than derived from the destination URI -- `Path::extension` on `lib.ndjson.zst` yields `zst`, dropping the part that names the format. Deletes a committed `mzspeclib_io.rs.bak` (an in-place-sed artifact) and ignores `*.bak`. --- .gitignore | 3 + Cargo.toml | 4 +- README.md | 9 +- bench/wandb_bench.py | 2 +- example_speclib_config.toml | 4 +- run.bash | 8 +- rust/speclib_build_cli/src/cli.rs | 2 +- rust/speclib_build_cli/src/config.rs | 2 +- rust/speclib_build_cli/src/pipeline.rs | 12 +- rust/timsquery/src/serde/mzspeclib_io.rs.bak | 751 ------------------- rust/timsseek/examples/query_bench.rs | 2 +- rust/timsseek/src/data_sources/speclib.rs | 204 +++-- rust/timsseek_cli/assets/default_config.toml | 2 +- 13 files changed, 163 insertions(+), 842 deletions(-) delete mode 100644 rust/timsquery/src/serde/mzspeclib_io.rs.bak diff --git a/.gitignore b/.gitignore index 9443e54f..001bd603 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,6 @@ scratchpad_ab/ ab_bench.out # Local bench outputs, logs, feature dumps (repo-local, never tracked) bench_out/ + +# In-place-edit artifacts +*.bak diff --git a/Cargo.toml b/Cargo.toml index 49ed3e07..fa8cc32e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,9 +33,7 @@ default-members = [ "rust/timsquery_cli", "rust/timsquery_viewer", "rust/apex_sim", - # speclib_build_cli is temporarily NOT here: its only writer was the msgpack - # `SpeclibWriter`, removed with that format. It does not compile until the - # mzSpecLib writer lands. Put it back then. + "rust/speclib_build_cli", "rust/tims_stage", "rust/timsseek_macros" ] diff --git a/README.md b/README.md index 7cd7db17..4d5bd63b 100644 --- a/README.md +++ b/README.md @@ -85,11 +85,11 @@ Both CLIs accept `s3://` URIs anywhere a path is accepted (AWS / MinIO / R2). `. ```bash timsseek --raw-inputs s3://bkt/sample.d.tar \ - --speclib-uri s3://bkt/lib.mzSpecLib.txt \ + --speclib-uri s3://bkt/lib.ndjson.zst \ --output-uri s3://bkt/runs/out speclib_build_cli --fasta s3://bkt/proteome.fasta \ - --output s3://bkt/lib.mzSpecLib.txt + --output s3://bkt/lib.ndjson.zst ``` Auth via AWS default chain. MinIO/R2: set `AWS_ENDPOINT_URL`. See `docs/development.md` for `[staging]` config + env var list. @@ -98,6 +98,11 @@ Auth via AWS default chain. MinIO/R2: set `AWS_ENDPOINT_URL`. See `docs/developm See [docs/development.md](docs/development.md) for dev utilities, compile flags, env vars, Taskfile targets, and scripts. +[docs/CARAFE_CONTRACT.md](docs/CARAFE_CONTRACT.md) pins the CLI surface +[Carafe](https://github.com/Noble-Lab/Carafe) drives as a subprocess — read it +before renaming a JSON field or an output path. It is enforced by +`rust/timsquery/tests/carafe_contract.rs`. + ## License This project is licensed under the Apache License, Version 2.0. diff --git a/bench/wandb_bench.py b/bench/wandb_bench.py index 1bfe2f3a..1c6b6ddf 100644 --- a/bench/wandb_bench.py +++ b/bench/wandb_bench.py @@ -225,7 +225,7 @@ def wandb_context(config_dict: dict[str, Any], wandb_kwargs=None): def main(wandb_kwargs: dict | None = None, koina_url: str | None = None): fasta_file = Path.home() / "fasta/hela_gt20peps.fasta" - speclib_path = Path.home() / "fasta/asdad.mzSpecLib.txt" + speclib_path = Path.home() / "fasta/asdad.ndjson.zst" prefix = Path.home() / "data/decompressed_timstof/" dotd_files = [ diff --git a/example_speclib_config.toml b/example_speclib_config.toml index cb752085..ef3faf2e 100644 --- a/example_speclib_config.toml +++ b/example_speclib_config.toml @@ -5,8 +5,8 @@ # All fields are optional — omit a section or key to use the compiled-in default. # ── Output ──────────────────────────────────────────────────────────────────── -# Path for the output spectral library (mzSpecLib text). -output = "library.mzSpecLib.txt" +# Path for the output spectral library (zstd-compressed NDJSON). +output = "library.ndjson.zst" # ── Digestion ───────────────────────────────────────────────────────────────── [digestion] diff --git a/run.bash b/run.bash index 868d2cf3..7b25a56b 100644 --- a/run.bash +++ b/run.bash @@ -7,7 +7,7 @@ if [ -n "${FULL_RUN}" ]; then echo "Full run" sleep 2 FASTA_FILE="$HOME/fasta/20231030_LINEARIZED_UP000005640_9606.fasta" - SPECLIB_NAME="data_ignore/20231030_LINEARIZED_UP000005640_9606.msgpack.zstd" + SPECLIB_NAME="data_ignore/20231030_LINEARIZED_UP000005640_9606.ndjson.zst" DOTD_FILE="/Users/sebastianpaez/git/ionmesh/benchmark/240402_PRTC_01_S1-A1_1_11342.d" RESULTS_DIR="data_ignore/hela_search_results" SUMMARY_DIR="data_ignore/hela_search_summary" @@ -16,7 +16,7 @@ elif [ -n "${FULL_MCCOSS}" ]; then sleep 2 DOTD_FILE="$HOME/data/bo_maccoss/N20211212chenc_WOSP00101_DIA_60min_K562_rep1_1_Slot2-37_1_9898.d" FASTA_FILE="$HOME/fasta/20231030_LINEARIZED_UP000005640_9606.fasta" - SPECLIB_NAME="data_ignore/20231030_LINEARIZED_UP000005640_9606.msgpack.zstd" + SPECLIB_NAME="data_ignore/20231030_LINEARIZED_UP000005640_9606.ndjson.zst" RESULTS_DIR="data_ignore/mccoss_search_results" SUMMARY_DIR="data_ignore/mccoss_search_summary" elif [ -n "${VIMENTIN_ONLY}" ]; then @@ -24,7 +24,7 @@ elif [ -n "${VIMENTIN_ONLY}" ]; then sleep 2 DOTD_FILE="$HOME/git/ionmesh/benchmark/240402_PRTC_01_S1-A1_1_11342.d" FASTA_FILE="$HOME/fasta/VIMENTIN.fasta" - SPECLIB_NAME="data_ignore/vimentin.msgpack.zstd" + SPECLIB_NAME="data_ignore/vimentin.ndjson.zst" RESULTS_DIR="data_ignore/vimentin_search_results" SUMMARY_DIR="data_ignore/vimentin_search_summary" else @@ -32,7 +32,7 @@ else echo "Quick run" sleep 2 FASTA_FILE="$HOME/fasta/hela_gt20peps.fasta" - SPECLIB_NAME="data_ignore/asdad.msgpack.zstd" + SPECLIB_NAME="data_ignore/asdad.ndjson.zst" DOTD_FILE="$HOME/git.bkp/ionmesh/benchmark/240402_PRTC_01_S1-A1_1_11342.d" RESULTS_DIR="data_ignore/top_proteins_hela" SUMMARY_DIR="data_ignore/top_proteins_hela_summary" diff --git a/rust/speclib_build_cli/src/cli.rs b/rust/speclib_build_cli/src/cli.rs index e60e733c..016e13a4 100644 --- a/rust/speclib_build_cli/src/cli.rs +++ b/rust/speclib_build_cli/src/cli.rs @@ -21,7 +21,7 @@ pub struct Cli { // ── Output ───────────────────────────────────────────────────────────── /// Output URI for the spectral library (local path or s3://...; default: - /// library.mzSpecLib.txt). + /// library.ndjson.zst). #[arg(long, short = 'o')] pub output: Option, diff --git a/rust/speclib_build_cli/src/config.rs b/rust/speclib_build_cli/src/config.rs index 13f9ce51..2ab496b3 100644 --- a/rust/speclib_build_cli/src/config.rs +++ b/rust/speclib_build_cli/src/config.rs @@ -62,7 +62,7 @@ fn default_min_ions() -> usize { 3 } fn default_output() -> String { - "library.mzSpecLib.txt".to_string() + "library.ndjson.zst".to_string() } // ── Sub-structs ────────────────────────────────────────────────────────────── diff --git a/rust/speclib_build_cli/src/pipeline.rs b/rust/speclib_build_cli/src/pipeline.rs index c07943b5..0c9a2639 100644 --- a/rust/speclib_build_cli/src/pipeline.rs +++ b/rust/speclib_build_cli/src/pipeline.rs @@ -268,13 +268,13 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box = if remote_output { - let ext = std::path::Path::new(output_uri.trim_end_matches('/')) - .extension() - .and_then(|s| s.to_str()) - .unwrap_or("mzSpecLib.txt"); + // Fixed rather than derived from the destination URI: the writer only + // emits zstd-wrapped NDJSON, and the upload below carries the caller's + // own name anyway. (`Path::extension` would have yielded `zst` for + // `lib.ndjson.zst`, losing the part that identifies the format.) let tf = tempfile::Builder::new() .prefix("speclib-out-") - .suffix(&format!(".{ext}")) + .suffix(".ndjson.zst") .tempfile()?; Some(tf) } else { @@ -293,7 +293,7 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box 0 - || self.spectra_with_unknown_rt_unit > 0 - || self.dropped_empty_precursors > 0//! Reader for the mzSpecLib text format (HUPO-PSI). -//! -//! # Why every field needs a fallback ladder -//! -//! mzSpecLib says *how* to write a controlled-vocabulary term, not *which* -//! term a writer must use. The two reference exports disagree on nearly -//! everything this reader needs: -//! -//! | field | DIA-NN writes | Spectronaut writes | -//! |---|---|---| -//! | precursor m/z | `MS:1000744` selected ion m/z | `MS:1003208` experimental precursor monoisotopic m/z | -//! | retention time | *nothing at all* | `MS:1000896` normalized retention time, in minutes | -//! | ion mobility | `MS:1002476` (written as `0.0`) | `MS:1002476` | -//! -//! So each field is resolved by trying terms in priority order, and RT carries -//! a unit that must be honoured rather than assumed. -//! -//! # Peak resolution -//! -//! Peak lists carry *observed* m/z; the annotation's mass-error suffix recovers -//! the theoretical value the arena wants (`theoretical = observed - error`). So -//! a theoretical mass only exists once a single identity is pinned, which sorts -//! peaks three ways: -//! -//! | | kept | m/z | -//! |---|---|---| -//! | resolved, representable | real label | theoretical | -//! | resolved, not representable (`y1-HCOOH`) | unknown label | theoretical | -//! | unannotated (`?`) or tied ambiguity | no | — | -//! -//! Row three is skipped rather than stored at observed m/z: an arena mixing -//! observed and theoretical masses would be invisible downstream. Row two is -//! kept because a known-but-unspellable identity still has an exact mass — only -//! the label is lost. -//! -//! Ambiguous (comma-separated) annotations take the alternative with the -//! smallest absolute mass error. If that one is unrepresentable the peak gets an -//! unknown label rather than falling back to a worse-matching representable -//! alternative, which would assign both a wrong identity and a wrong mass. - -use crate::ion::IonAnnot; -use crate::models::{ - LibCapabilities, - QueryCollection, -}; -use crate::serde::library_file::{ - LibraryArena, - LibraryReadingError, -}; -use micromzpaf::{ - MassError, - split_mass_error, -}; -use std::io::{ - BufRead, - BufReader, -}; -use std::path::Path; -use tracing::{ - info, - warn, -}; - -/// First non-empty line of an mzSpecLib text file. -const MAGIC: &str = ""; - -// CV term ladders, ordered most- to least-specific; the first present wins. - -/// Precursor m/z. Experimental monoisotopic is preferred over `selected ion -/// m/z`, which on a quadrupole instrument is the isolation-window centre and -/// need not be the monoisotopic peak. -const PRECURSOR_MZ_TERMS: &[&str] = &[ - "MS:1003208", // experimental precursor monoisotopic m/z - "MS:1003053", // theoretical monoisotopic m/z - "MS:1000744", // selected ion m/z -]; -/// Retention time. `normalized retention time` is an iRT-style index rather -/// than a clock reading, but it is what Spectronaut exports and the only RT -/// signal available in those files. -const RT_TERMS: &[&str] = &[ - "MS:1000894", // retention time - "MS:1000896", // normalized retention time -]; -/// Ion mobility. Note these are different quantities, not spellings of one: -/// `MS:1002815` is inverse reduced ion mobility (1/K0), which is what the -/// arena wants, while `MS:1002476` is a drift time. They are tried in that -/// order and a drift-time-only library is counted, since treating a drift time -/// as 1/K0 is only valid for instruments that report it that way. -const MOBILITY_INVERSE_REDUCED: &str = "MS:1002815"; -const MOBILITY_DRIFT_TIME: &str = "MS:1002476"; - -const CHARGE_TERM: &str = "MS:1000041"; -const STRIPPED_SEQ_TERM: &str = "MS:1000888"; -const PROFORMA_TERM: &str = "MS:1003270"; -const UNIT_TERM: &str = "UO:0000000"; - -const UNIT_MINUTE: &str = "UO:0000031"; -const UNIT_SECOND: &str = "UO:0000010"; - -/// Per-library tally of everything that did not land verbatim in the arena. -/// -/// Reported once at the end of a load rather than per row: a consensus library -/// can carry thousands of unannotated peaks, and a line each would bury the -/// signal. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct MzSpecLibStats { - /// Peaks stored with their parsed annotation. - pub kept_annotated: usize, - /// Identity known but not representable (a loss outside the table, a - /// modified immonium). Stored with an unknown label and an exact mass. - pub kept_unknown_label: usize, - /// `?` — no annotation, so no mass error, so no theoretical m/z. - pub skipped_unannotated: usize, - /// Comma-separated alternatives that tied on absolute mass error. - pub skipped_ambiguous: usize, - /// Peaks dropped because their label collided with one already in the - /// precursor after the unknown-label rewrite. - pub dropped_duplicate_label: usize, - /// Spectra with no retention-time term at all. - pub spectra_without_rt: usize, - /// Spectra whose mobility came from a drift time rather than 1/K0. - pub spectra_with_drift_time_mobility: usize, - /// Spectra whose retention time carried a unit this reader does not know. - pub spectra_with_unknown_rt_unit: usize, - /// Precursors dropped for having no usable peak left. - pub dropped_empty_precursors: usize, -} - -/// One spectrum converted into the shape [`QueryCollection::push_row`] takes. -struct ArenaRow { - precursor_mz: f64, - charge: u8, - rt_seconds: f32, - mobility: f32, - frags: Vec<(IonAnnot, f64)>, - intensities: Vec, - stripped: String, - modified: String, -} - -impl MzSpecLibStats { - fn anything_to_report(&self) -> bool { - self.kept_unknown_label > 0 - || self.skipped_unannotated > 0 - || self.skipped_ambiguous > 0 - || self.dropped_duplicate_label > 0 - || self.spectra_without_rt > 0 - || self.spectra_with_drift_time_mobility > 0 - || self.dropped_empty_precursors > 0 - } - - fn report(&self, path: &Path) { - if !self.anything_to_report() { - info!( - "mzSpecLib {}: {} peaks, all annotated and representable", - path.display(), - self.kept_annotated - ); - return; - } - warn!( - "mzSpecLib {}: kept {} annotated + {} with unknown labels; \ - skipped {} unannotated, {} ambiguous, {} duplicate-label; \ - {} spectra without RT, {} using drift time as mobility, \ - {} with an unknown RT unit, {} precursors dropped as empty", - path.display(), - self.kept_annotated, - self.kept_unknown_label, - self.skipped_unannotated, - self.skipped_ambiguous, - self.dropped_duplicate_label, - self.spectra_without_rt, - self.spectra_with_drift_time_mobility, - self.spectra_with_unknown_rt_unit, - self.dropped_empty_precursors, - ); - } -} - -/// One `ACC|name=value` attribute, with its optional `[n]` group tag. -#[derive(Debug, Clone)] -struct Attr { - group: Option, - accession: String, - value: String, -} - -impl Attr { - fn parse(line: &str) -> Option { - let (group, rest) = match line.strip_prefix('[') { - Some(r) => { - let (g, r) = r.split_once(']')?; - (Some(g.parse().ok()?), r) - } - None => (None, line), - }; - let (key, value) = rest.split_once('=')?; - let accession = key.split('|').next()?.to_string(); - Some(Attr { - group, - accession, - value: value.to_string(), - }) - } - - /// The accession out of a `ACC|name` *value* (as opposed to a key), for - /// terms whose value is itself a CV term — e.g. `unit=UO:0000031|minute`. - fn value_accession(&self) -> &str { - self.value.split('|').next().unwrap_or(&self.value) - } -} - -/// Attributes collected for one spectrum (its own plus its analyte's). -#[derive(Debug, Default)] -struct AttrBag(Vec); - -impl AttrBag { - fn find(&self, accession: &str) -> Option<&Attr> { - self.0.iter().find(|a| a.accession == accession) - } - - fn first_of(&self, accessions: &[&str]) -> Option<&Attr> { - accessions.iter().find_map(|a| self.find(a)) - } - - fn f64_of(&self, accessions: &[&str]) -> Option { - self.first_of(accessions)?.value.parse().ok() - } - - /// The unit term attached to `attr` via its `[n]` group, if any. - fn unit_for(&self, attr: &Attr) -> Option<&str> { - let group = attr.group?; - self.0 - .iter() - .find(|a| a.group == Some(group) && a.accession == UNIT_TERM) - .map(|a| a.value_accession()) - } -} - -/// A spectrum accumulated from the text stream, before conversion. -#[derive(Debug, Default)] -struct RawSpectrum { - attrs: AttrBag, - /// `(observed mz, intensity, annotation)` - peaks: Vec<(f64, f32, String)>, -} - -/// What resolving one peak's annotation produced. -enum Resolved { - /// Parsed cleanly; store with this label. - Annotated(IonAnnot), - /// Identity known, not representable. Store with an unknown label, exact - /// mass. - UnknownLabel, - /// No single identity, so no theoretical mass. Skip. - SkipUnannotated, - SkipAmbiguous, -} - -/// Resolve one annotation string into a storage decision plus the mass error -/// needed to recover theoretical m/z. -fn resolve_annotation(annotation: &str) -> (Resolved, Option) { - let annotation = annotation.trim(); - if annotation.is_empty() || annotation == "?" { - return (Resolved::SkipUnannotated, None); - } - - // Splitting the error off comes first: it works even when the ion will not - // parse, which is exactly the case that still needs an exact mass. A - // malformed suffix on ANY alternative makes the whole peak unresolvable — - // dropping just that one would silently turn an ambiguous peak into an - // unambiguous one. - let mut alternatives = Vec::new(); - for alt in annotation.split(',') { - let Ok(parsed) = split_mass_error(alt.trim()) else { - return (Resolved::SkipAmbiguous, None); - }; - alternatives.push(parsed); - } - - let (ion_str, mass_error) = if let [single] = alternatives[..] { - single - } else { - // Closest by absolute mass error. Comparing a Da magnitude against a - // ppm one would be meaningless, but a library uses one unit - // throughout. Errors are parsed decimal literals, so equal ones - // compare exactly and a tie pins no identity. - let magnitude = |m: &Option| match m { - Some(MassError::Da(v) | MassError::Ppm(v)) => v.abs(), - None => f64::INFINITY, - }; - let best = alternatives - .iter() - .map(|(_, e)| magnitude(e)) - .fold(f64::INFINITY, f64::min); - if !best.is_finite() { - return (Resolved::SkipAmbiguous, None); - } - let mut winners = alternatives.iter().filter(|(_, e)| magnitude(e) == best); - let winner = *winners.next().expect("the minimum came from this iterator"); - if winners.next().is_some() { - return (Resolved::SkipAmbiguous, None); - } - winner - }; - match IonAnnot::try_from(ion_str) { - Ok(ion) => (Resolved::Annotated(ion), mass_error), - // Keep the peak and its exact mass, lose only the label. - Err(_) => (Resolved::UnknownLabel, mass_error), - } -} - -/// Convert one accumulated spectrum into arena rows. -/// -/// Returns `None` when the spectrum lacks something structural (precursor m/z, -/// charge, sequence) or ends up with no usable peak. -fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { - let precursor_mz = raw.attrs.f64_of(PRECURSOR_MZ_TERMS)?; - let charge: u8 = raw.attrs.find(CHARGE_TERM)?.value.parse().ok()?; - - let rt_seconds = match raw.attrs.first_of(RT_TERMS) { - Some(attr) => { - let v: f64 = attr.value.parse().ok()?; - // Honour the unit rather than assuming: Spectronaut writes minutes. - match raw.attrs.unit_for(attr) { - Some(UNIT_SECOND) => v, - Some(UNIT_MINUTE) | None => v * 60.0, - // An unrecognized unit is counted, not silently treated as - // minutes: guessing wrong here is a 60x error in the RT the - // whole extraction window is built around. - Some(_) => { - stats.spectra_with_unknown_rt_unit += 1; - v * 60.0 - } - } - } - None => { - stats.spectra_without_rt += 1; - 0.0 - } - }; - - let mobility = match raw.attrs.find(MOBILITY_INVERSE_REDUCED) { - Some(a) => a.value.parse().ok()?, - None => match raw.attrs.find(MOBILITY_DRIFT_TIME) { - Some(a) => { - stats.spectra_with_drift_time_mobility += 1; - a.value.parse().ok()? - } - // Absent is fine — an unset mobility is 0.0, same as DIA-NN writes. - // A *present but malformed* one drops the spectrum instead. - None => 0.0, - }, - }; - - let stripped = raw - .attrs - .find(STRIPPED_SEQ_TERM) - .map(|a| a.value.clone()) - .unwrap_or_default(); - // The proforma term carries a trailing `/charge` that is not part of the - // peptidoform. - let modified = raw - .attrs - .find(PROFORMA_TERM) - .map(|a| { - a.value - .rsplit_once('/') - .map(|(p, _)| p.to_string()) - .unwrap_or_else(|| a.value.clone()) - }) - .unwrap_or_else(|| stripped.clone()); - if stripped.is_empty() && modified.is_empty() { - return None; - } - - let mut frags: Vec<(IonAnnot, f64)> = Vec::with_capacity(raw.peaks.len()); - let mut intens: Vec = Vec::with_capacity(raw.peaks.len()); - let mut unknown_counter: u8 = 0; - - for (observed_mz, intensity, annotation) in &raw.peaks { - let (resolved, mass_error) = resolve_annotation(annotation); - let label = match resolved { - Resolved::Annotated(ion) => { - stats.kept_annotated += 1; - ion - } - Resolved::UnknownLabel => { - // The ordinal is a per-precursor uniqueness counter. Past 255 - // there is no way to keep labels distinct, so drop rather than - // reuse one. - let Some(next) = unknown_counter.checked_add(1) else { - stats.dropped_duplicate_label += 1; - continue; - }; - unknown_counter = next; - match IonAnnot::try_new('?', Some(unknown_counter), 1, 0) { - Ok(i) => { - stats.kept_unknown_label += 1; - i - } - Err(_) => { - stats.dropped_duplicate_label += 1; - continue; - } - } - } - Resolved::SkipUnannotated => { - stats.skipped_unannotated += 1; - continue; - } - Resolved::SkipAmbiguous => { - stats.skipped_ambiguous += 1; - continue; - } - }; - - let mz = match mass_error { - Some(e) => e.theoretical_from_observed(*observed_mz), - None => *observed_mz, - }; - - // Labels must stay unique within the precursor (`linear_get` is - // first-match), so a collision drops the later peak. - if frags.iter().any(|(l, _)| *l == label) { - stats.dropped_duplicate_label += 1; - continue; - } - frags.push((label, mz)); - intens.push(*intensity); - } - - if frags.is_empty() { - stats.dropped_empty_precursors += 1; - return None; - } - - Some(ArenaRow { - precursor_mz, - charge, - rt_seconds: rt_seconds as f32, - mobility, - frags, - intensities: intens, - stripped, - modified, - }) -} - -/// Cheap probe: the format's magic first line. -pub fn sniff_mzspeclib_library_file>(path: T) -> bool { - let Ok(file) = std::fs::File::open(path.as_ref()) else { - return false; - }; - let mut reader = BufReader::new(file); - let mut line = String::new(); - // Only the first non-empty line is inspected, so this stays O(1) on a - // multi-gigabyte library. - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => return false, - Ok(_) => { - let t = line.trim(); - if t.is_empty() { - continue; - } - return t == MAGIC; - } - Err(_) => return false, - } - } -} - -/// Read an mzSpecLib text file into the columnar arena. -pub fn read_mzspeclib_library_file>( - path: T, -) -> Result { - let path = path.as_ref(); - let file = std::fs::File::open(path).map_err(LibraryReadingError::IoError)?; - let reader = BufReader::new(file); - - let mut geom = QueryCollection::with_capabilities(LibCapabilities::default_diann_no_decoys()); - let mut frag_intens: Vec = Vec::new(); - let mut stats = MzSpecLibStats::default(); - - let mut current: Option = None; - let mut in_peaks = false; - - let flush = |cur: Option, - geom: &mut QueryCollection, - frag_intens: &mut Vec, - stats: &mut MzSpecLibStats| { - let Some(raw) = cur else { return }; - let Some(row) = convert_spectrum(&raw, stats) else { - return; - }; - frag_intens.extend_from_slice(&row.intensities); - geom.push_row( - row.precursor_mz, - row.charge, - row.rt_seconds, - row.mobility, - &row.frags, - &row.stripped, - &row.modified, - &[], - false, - ); - }; - - for line in reader.lines() { - let line = line.map_err(LibraryReadingError::IoError)?; - let trimmed = line.trim_end(); - - if trimmed.starts_with("" { - in_peaks = true; - continue; - } - // `` and `` attributes are folded into the - // spectrum's bag: this reader wants the union, not the hierarchy. - if trimmed.starts_with('<') { - in_peaks = false; - continue; - } - if trimmed.is_empty() { - in_peaks = false; - continue; - } - - let Some(spec) = current.as_mut() else { - continue; // library-level header - }; - - if in_peaks { - let mut cols = trimmed.split('\t'); - let (Some(mz), Some(intensity)) = (cols.next(), cols.next()) else { - continue; - }; - let (Ok(mz), Ok(intensity)) = - (mz.trim().parse::(), intensity.trim().parse::()) - else { - continue; - }; - let annotation = cols.next().unwrap_or("?").to_string(); - spec.peaks.push((mz, intensity, annotation)); - } else if let Some(attr) = Attr::parse(trimmed) { - spec.attrs.0.push(attr); - } - } - flush(current.take(), &mut geom, &mut frag_intens, &mut stats); - - stats.report(path); - - if geom.n_rows() == 0 { - return Err(LibraryReadingError::SpeclibParse(format!( - "mzSpecLib {} yielded no usable precursors", - path.display() - ))); - } - if frag_intens.len() != geom.frag_labels.len() { - return Err(LibraryReadingError::SpeclibParse(format!( - "reference-intensity sidecar ({}) must stay parallel to the fragment-label arena ({})", - frag_intens.len(), - geom.frag_labels.len(), - ))); - } - - geom.seal(); - Ok(LibraryArena::Mzpaf { - geom, - frag_intens: Some(frag_intens), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn fixture(name: &str) -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/mzspeclib_io_files") - .join(name) - } - - #[test] - fn sniffs_only_mzspeclib() { - assert!(sniff_mzspeclib_library_file(fixture("diann.mzSpecLib.txt"))); - assert!(sniff_mzspeclib_library_file(fixture( - "spectronaut.mzSpecLib.txt" - ))); - // A DIA-NN TSV must not be claimed. - let tsv = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/diann_io_files/sample_lib.tsv"); - assert!(!sniff_mzspeclib_library_file(tsv)); - } - - #[test] - fn reads_diann_export() { - let arena = read_mzspeclib_library_file(fixture("diann.mzSpecLib.txt")).unwrap(); - let LibraryArena::Mzpaf { geom, frag_intens } = arena else { - panic!("mzSpecLib must build an mzpaf arena"); - }; - assert!(geom.n_rows() > 0); - let intens = frag_intens.expect("reference intensities are populated"); - assert_eq!(intens.len(), geom.frag_labels.len()); - } - - /// DIA-NN's export writes every mass error as exactly `0.0`, so observed - /// and theoretical coincide and the m/z must pass through untouched. - #[test] - fn zero_mass_error_leaves_mz_unchanged() { - let arena = read_mzspeclib_library_file(fixture("diann.mzSpecLib.txt")).unwrap(); - let LibraryArena::Mzpaf { geom, .. } = arena else { - unreachable!() - }; - // First spectrum's first peak in the fixture: 427.22995, annotated b6/0.0 - let first = geom.frag_mzs[0]; - assert!( - (first - 427.22995).abs() < 1e-6, - "expected the observed m/z verbatim, got {first}" - ); - } - - /// Spectronaut's export carries `-H2O`/`-NH3` losses. `IonAnnot` - /// represents those, so they must keep real labels rather than degrading - /// to unknown. - #[test] - fn spectronaut_losses_keep_real_labels() { - let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); - let LibraryArena::Mzpaf { geom, .. } = arena else { - unreachable!() - }; - let has_loss = geom - .frag_labels - .iter() - .any(|l| l.loss() != micromzpaf::NeutralLoss::None); - assert!(has_loss, "expected at least one loss-bearing label"); - let unknowns = geom - .frag_labels - .iter() - .filter(|l| l.try_get_ordinal().is_none() && l.loss() == micromzpaf::NeutralLoss::None) - .count(); - assert_eq!(unknowns, 0, "no peak should need an unknown label here"); - } - - /// RT is unit-tagged; Spectronaut writes minutes and the arena wants - /// seconds. Getting this wrong is a silent 60x error. - #[test] - fn retention_time_honours_its_unit() { - let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); - let LibraryArena::Mzpaf { geom, .. } = arena else { - unreachable!() - }; - // Fixture's first spectrum: normalized retention time = 28.658491 min. - let rt = geom.rt_seconds[0]; - assert!( - (rt - 28.658491 * 60.0).abs() < 0.01, - "expected minutes converted to seconds, got {rt}" - ); - } - - #[test] - fn resolves_unambiguous_representable() { - let (r, e) = resolve_annotation("y5/-0.0005"); - assert!(matches!(r, Resolved::Annotated(_))); - assert_eq!(e, Some(MassError::Da(-0.0005))); - } - - /// Known identity, unrepresentable spelling: keep the peak and the exact - /// mass, erase only the label. - #[test] - fn unrepresentable_loss_keeps_peak_with_unknown_label() { - let (r, e) = resolve_annotation("y1-HCOOH/0.0003"); - assert!(matches!(r, Resolved::UnknownLabel)); - assert_eq!( - e, - Some(MassError::Da(0.0003)), - "the mass error must survive so theoretical m/z stays exact" - ); - } - - #[test] - fn unannotated_peak_is_skipped() { - assert!(matches!( - resolve_annotation("?").0, - Resolved::SkipUnannotated - )); - } - - /// Closest-by-error wins; if that alternative is unrepresentable the peak - /// takes an unknown label rather than falling back to the representable - /// one, which would assign a wrong identity and a wrong mass. - #[test] - fn ambiguity_resolves_to_the_closest_not_the_representable() { - // a2 is representable and further; y2-CO2-NH3 is closer and is not. - let (r, e) = resolve_annotation("a2/-0.0040,y2-CO2-NH3/-0.0001"); - assert!( - matches!(r, Resolved::UnknownLabel), - "the closest alternative wins even when unrepresentable" - ); - assert_eq!(e, Some(MassError::Da(-0.0001))); - - // When the closest one IS representable, it is used. - let (r, _) = resolve_annotation("a2/-0.0001,y2-CO2-NH3/-0.0040"); - assert!(matches!(r, Resolved::Annotated(_))); - } - - /// An exact tie pins no identity, so no theoretical m/z exists and the peak - /// cannot be stored without mixing observed and theoretical masses. - #[test] - fn tied_ambiguity_is_skipped() { - let (r, _) = resolve_annotation("a2/-0.0004,y2-CO2-NH3/-0.0004"); - assert!(matches!(r, Resolved::SkipAmbiguous)); - } - - #[test] - fn mass_error_recovers_theoretical() { - // Real SpectraST peak: y1 for C-terminal R, observed 175.1184. - let e = MassError::Da(-0.0005); - assert!((e.theoretical_from_observed(175.1184) - 175.1189).abs() < 1e-9); - } - - /// The public entry point must dispatch mzSpecLib itself. It is sniffed - /// before the registry, so a regression here falls through to the - /// always-true JSON reader and fails with a generic parse error. - #[test] - fn public_read_library_file_dispatches_mzspeclib() { - use crate::serde::read_library_file; - for name in ["diann.mzSpecLib.txt", "spectronaut.mzSpecLib.txt"] { - let arena = read_library_file(fixture(name)) - .unwrap_or_else(|e| panic!("{name} must load through the registry: {e:?}")); - let LibraryArena::Mzpaf { geom, frag_intens } = arena else { - panic!("{name} must land in the mzpaf arena"); - }; - assert!(geom.n_rows() > 0, "{name} produced no precursors"); - assert!( - frag_intens.is_some(), - "{name} must populate the reference-intensity sidecar" - ); - } - } -} diff --git a/rust/timsseek/examples/query_bench.rs b/rust/timsseek/examples/query_bench.rs index 015c901c..3fb17641 100644 --- a/rust/timsseek/examples/query_bench.rs +++ b/rust/timsseek/examples/query_bench.rs @@ -75,7 +75,7 @@ fn main() { ); let speclib_path = env( "BENCH_SPECLIB", - "/Users/sebastianpaez/fasta/asdad.ndjson.zstd", + "/Users/sebastianpaez/fasta/asdad.ndjson.zst", ); let n: usize = env("QB_N", "2000").parse().unwrap(); let iters: usize = env("QB_ITERS", "1").parse().unwrap(); diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 9de067cd..0511e2a0 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -17,6 +17,7 @@ use std::io::{ BufRead, BufReader, Read, + Write, }; use std::path::{ Path, @@ -254,10 +255,7 @@ pub enum SpeclibFormat { impl SpeclibFormat { /// Detect a native timsseek format by EXTENSION ONLY. Returns `None` for /// anything else (including `.speclib` and `.mzSpecLib.txt`), which routes - /// to the timsquery bridge. - /// - /// Matched by extension rather than content; anything unmatched routes to - /// the timsquery bridge, which sniffs properly. + /// to the timsquery bridge — that is where content sniffing happens. pub fn detect_from_extension(path: &Path) -> Option { let path_str = path.to_string_lossy().to_lowercase(); @@ -277,8 +275,11 @@ impl SpeclibFormat { /// The native path builds the columnar arena directly from these elements (see /// `Speclib::from_file_with_format`), so the reader stays at the serializable /// element and does not eagerly build per-row scoring items. +/// +/// Both formats are NDJSON; zstd only adds a decoder underneath, so the +/// boxing is over the byte source rather than over the line parser. pub struct SpeclibReader<'a> { - inner: Box> + Send + 'a>, + reader: Box, } impl<'a> SpeclibReader<'a> { @@ -286,24 +287,23 @@ impl<'a> SpeclibReader<'a> { reader: R, format: SpeclibFormat, ) -> Result { - let inner: Box> + Send> = - match format { - SpeclibFormat::NdJson => Box::new(NdJsonReader::new(BufReader::new(reader))), - SpeclibFormat::NdJsonZstd => { - let decoder = zstd::Decoder::new(reader).map_err(|e| { - LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error creating ZSTD decoder", - } - })?; - Box::new(NdJsonReader::new(BufReader::new(decoder))) - } - }; + let reader: Box = match format { + SpeclibFormat::NdJson => Box::new(BufReader::new(reader)), + SpeclibFormat::NdJsonZstd => { + let decoder = zstd::Decoder::new(reader).map_err(|e| { + LibraryReadingError::SpeclibParsingError { + source: serde_json::Error::io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + e, + )), + context: "Error creating ZSTD decoder", + } + })?; + Box::new(BufReader::new(decoder)) + } + }; - Ok(SpeclibReader { inner }) + Ok(SpeclibReader { reader }) } } @@ -311,50 +311,70 @@ impl Iterator for SpeclibReader<'_> { type Item = Result; fn next(&mut self) -> Option { - self.inner.next() + // Looping rather than recursing: a file with a long run of blank lines + // would otherwise recurse once per line and blow the stack. + loop { + let mut line = String::new(); + match self.reader.read_line(&mut line) { + Ok(0) => return None, // EOF + Ok(_) => { + if line.trim().is_empty() { + continue; + } + return Some(serde_json::from_str(&line).map_err(|e| { + LibraryReadingError::SpeclibParsingError { + source: e, + context: "Error parsing NDJSON line", + } + })); + } + Err(e) => { + return Some(Err(LibraryReadingError::FileReadingError { + source: e, + context: "Error reading line", + path: PathBuf::new(), + })); + } + } + } } } -struct NdJsonReader { - reader: R, +/// Writes a native timsseek library: one JSON object per line, zstd-wrapped. +/// +/// The exact inverse of [`SpeclibReader`] on [`SpeclibFormat::NdJsonZstd`], so +/// what `speclib_build_cli` emits is what `Speclib::from_file` reads back. +pub struct SpeclibWriter { + encoder: zstd::Encoder<'static, W>, } -impl NdJsonReader { - fn new(reader: R) -> Self { - Self { reader } +impl SpeclibWriter { + pub fn new_ndjson_zstd(writer: W) -> Result { + Ok(Self { + encoder: zstd::Encoder::new(writer, 3)?, + }) } -} - -impl Iterator for NdJsonReader { - type Item = Result; - - fn next(&mut self) -> Option { - let mut line = String::new(); - match self.reader.read_line(&mut line) { - Ok(0) => None, // EOF - Ok(_) => { - if line.trim().is_empty() { - return self.next(); // Skip empty lines - } - - let elem: SerSpeclibElement = match serde_json::from_str(&line) { - Ok(x) => x, - Err(e) => { - return Some(Err(LibraryReadingError::SpeclibParsingError { - source: e, - context: "Error parsing NDJSON line", - })); - } - }; - Some(Ok(elem)) - } - Err(e) => Some(Err(LibraryReadingError::FileReadingError { + pub fn append(&mut self, elem: &SerSpeclibElement) -> Result<(), LibraryReadingError> { + let io_err = |e: std::io::Error| LibraryReadingError::FileReadingError { + source: e, + context: "Error writing NDJSON", + path: PathBuf::new(), + }; + serde_json::to_writer(&mut self.encoder, elem).map_err(|e| { + LibraryReadingError::SpeclibParsingError { source: e, - context: "Error reading line", - path: PathBuf::new(), - })), - } + context: "Error serializing NDJSON line", + } + })?; + // The newline is the record separator; without it the whole library is + // one unreadable line. + self.encoder.write_all(b"\n").map_err(io_err) + } + + /// Flushes the zstd frame. Skipping this truncates the library. + pub fn finish(self) -> Result { + self.encoder.finish() } } @@ -505,6 +525,55 @@ mod tests { RefQuery, }; + /// `speclib_build_cli` writes with [`SpeclibWriter`] and timsseek reads + /// with [`SpeclibReader`]; nothing else checks that the two agree, and a + /// mismatch only shows up as an unreadable library at the end of a long + /// Koina run. + #[test] + fn writer_output_reads_back_through_the_reader() { + let element = SerSpeclibElement::new( + PrecursorEntry::new("PEPTIDEK".to_string(), 2, false, 0), + ReferenceEG::new( + 7, + 450.5, + vec![0, 1], + vec![175.1, 288.2], + vec![ + IonAnnot::try_from("y1").unwrap(), + IonAnnot::try_from("b3^2").unwrap(), + ], + vec![1.0, 0.5], + vec![0.9, 0.4], + 0.95, + 1234.5, + ), + ); + + let mut writer = SpeclibWriter::new_ndjson_zstd(Vec::new()).expect("encoder"); + writer.append(&element).expect("append"); + // Twice, so the newline separator is exercised rather than the file + // happening to hold one record. + writer.append(&element).expect("append"); + let bytes = writer.finish().expect("finish"); + + let read: Vec = + SpeclibReader::new(bytes.as_slice(), SpeclibFormat::NdJsonZstd) + .expect("reader") + .collect::>() + .expect("every record must parse"); + + assert_eq!(read.len(), 2); + assert_eq!(read[0].precursor.sequence, "PEPTIDEK"); + assert_eq!(read[0].elution_group.fragment_mzs, vec![175.1, 288.2]); + let labels: Vec = read[0] + .elution_group + .fragment_labels + .iter() + .map(|l| l.to_string()) + .collect(); + assert_eq!(labels, vec!["y1".to_string(), "b3^2".to_string()]); + } + #[test] fn test_detect_native_format_by_extension() { use std::path::Path; @@ -515,21 +584,18 @@ mod tests { Some(SpeclibFormat::NdJsonZstd) )); } - // msgpack is gone: these must fall through to the timsquery bridge - // rather than matching a native reader. - for ext in ["lib.msgpack", "lib.msgpack.zst", "lib.msgpack.zstd"] { - assert!( - SpeclibFormat::detect_from_extension(Path::new(ext)).is_none(), - "{ext} must no longer be claimed as a native format" - ); - } assert!(matches!( SpeclibFormat::detect_from_extension(Path::new("lib.ndjson")), Some(SpeclibFormat::NdJson) )); - // A .speclib must NOT be claimed as native -> routes to the bridge. - assert!(SpeclibFormat::detect_from_extension(Path::new("lib.speclib")).is_none()); - assert!(SpeclibFormat::detect_from_extension(Path::new("lib.tsv")).is_none()); + // Everything else routes to the timsquery bridge, which sniffs by + // content. Claiming one of these here would bypass that. + for ext in ["lib.speclib", "lib.mzSpecLib.txt", "lib.tsv"] { + assert!( + SpeclibFormat::detect_from_extension(Path::new(ext)).is_none(), + "{ext} must not be claimed as a native format" + ); + } } /// `Speclib` is now a type alias for `ReferenceLibrary` (Task 9 collapsed diff --git a/rust/timsseek_cli/assets/default_config.toml b/rust/timsseek_cli/assets/default_config.toml index f712e2c2..2fd3b94e 100644 --- a/rust/timsseek_cli/assets/default_config.toml +++ b/rust/timsseek_cli/assets/default_config.toml @@ -13,7 +13,7 @@ ## Input spectral library (optional — `--speclib-uri` overrides this). # [input] # type = "speclib" -# uri = "path/to/library.mzSpecLib.txt" +# uri = "path/to/library.ndjson.zst" [analysis] From 9934aeba108d5b6908434f1b66c0c76e9d368d83 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 23:53:35 -0700 Subject: [PATCH 12/27] refactor: fold `Kind` back into `IonSeriesOrdinal`, tally what gets dropped The bit-packing work introduced `Kind` as a second discriminant enum beside `IonSeriesOrdinal`, leaving six hand-maintained tables over the same 14 cases with nothing enforcing they agree -- and tests covering only 6 of the series, so transposing two arms would mislabel a whole ion series silently. `IonSeriesOrdinal` now carries `to_parts`/`from_parts`/`from_series_char` and `Kind` is gone. Two new tests walk every variant, through the packed word and through its mzPAF spelling. `try_get_ordinal` goes back to an exhaustive match, so a new series cannot silently get `None` from a storage-shaped predicate. `UnknownIonCounter` replaces `next_unknown_ordinal`, which was `u8::checked_add` with a paragraph attached while the actual ~15-line block stayed triplicated. Five call sites become one line each and overflow is unrepresentable rather than something each reader remembers to check. The mzSpecLib reader had hand-rolled its own copy next to the helper. `LibraryReader` had two mutually-recursive defaulted methods with an `unreachable!()` standing in for a type-level requirement -- on an exported trait. One required method returning `LibraryArena`; the five legacy readers wrap their own result. `MzSpecLibStats` reported "all clean" on a file that had silently dropped spectra: `convert_spectrum` bailed on three structural failures and incremented nothing. Counters now come from one macro-declared list that `anything_to_report` and the log line both derive from, malformed spectra get their own counter, and overflow is no longer filed under "duplicate label" (whose `try_new` arm was dead anyway). `resolve_annotation` returns the mass error inside the variants that keep the peak instead of a parallel `Option` that was meaningless for the two skip cases. `OrdinalOutOfRange { ordinal: -1 }` becomes `MissingOrdinal`. `ParsedAnnotation`/`parse_mzpaf` and `IonParsingError::Custom` had no callers; `flush` was a closure capturing nothing; the line loop's `in_peaks` bool is a `Section`. --- rust/micromzpaf/src/lib.rs | 434 ++++++++-------- rust/micromzpaf/src/loss.rs | 59 +-- rust/timsquery/src/lib.rs | 1 + rust/timsquery/src/serde/diann_io.rs | 16 +- rust/timsquery/src/serde/diann_speclib_io.rs | 14 +- .../src/serde/elution_group_inputs.rs | 13 +- rust/timsquery/src/serde/library_file.rs | 70 ++- rust/timsquery/src/serde/mod.rs | 3 +- rust/timsquery/src/serde/mzspeclib_io.rs | 468 ++++++++++-------- rust/timsquery/src/serde/skyline_io.rs | 8 +- rust/timsquery/src/serde/spectronaut_io.rs | 9 +- rust/timsquery/src/serde/unknown_ordinal.rs | 17 - .../tests/mzspeclib_io_files/README.md | 2 +- 13 files changed, 594 insertions(+), 520 deletions(-) delete mode 100644 rust/timsquery/src/serde/unknown_ordinal.rs diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index 70a8cc68..f9533101 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -105,101 +105,6 @@ const fn unzigzag(u: u32) -> i8 { (((u >> 1) as i32) ^ -((u & 1) as i32)) as i8 } -/// Discriminants for the `kind` field. Not public: the public view is -/// [`IonSeriesOrdinal`], which fuses kind with its payload. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(u8)] -enum Kind { - None = 0, - A = 1, - B = 2, - C = 3, - D = 4, - V = 5, - W = 6, - X = 7, - Y = 8, - Z = 9, - Precursor = 10, - Unknown = 11, - Internal = 12, - Immonium = 13, -} - -impl Kind { - const fn from_raw(v: u32) -> Kind { - match v { - 1 => Kind::A, - 2 => Kind::B, - 3 => Kind::C, - 4 => Kind::D, - 5 => Kind::V, - 6 => Kind::W, - 7 => Kind::X, - 8 => Kind::Y, - 9 => Kind::Z, - 10 => Kind::Precursor, - 11 => Kind::Unknown, - 12 => Kind::Internal, - 13 => Kind::Immonium, - _ => Kind::None, - } - } - - const fn series_char(self) -> char { - match self { - Kind::A => 'a', - Kind::B => 'b', - Kind::C => 'c', - Kind::D => 'd', - Kind::V => 'v', - Kind::W => 'w', - Kind::X => 'x', - Kind::Y => 'y', - Kind::Z => 'z', - Kind::Precursor => 'p', - Kind::Unknown => '?', - Kind::Internal => 'm', - Kind::Immonium => 'I', - Kind::None => '\0', - } - } - - const fn from_series_char(c: char) -> Option { - match c { - 'a' => Some(Kind::A), - 'b' => Some(Kind::B), - 'c' => Some(Kind::C), - 'd' => Some(Kind::D), - 'v' => Some(Kind::V), - 'w' => Some(Kind::W), - 'x' => Some(Kind::X), - 'y' => Some(Kind::Y), - 'z' => Some(Kind::Z), - 'p' => Some(Kind::Precursor), - '?' => Some(Kind::Unknown), - _ => None, - } - } - - /// Does this kind carry an 8-bit ordinal in its payload? - const fn has_ordinal(self) -> bool { - matches!( - self, - Kind::A - | Kind::B - | Kind::C - | Kind::D - | Kind::V - | Kind::W - | Kind::X - | Kind::Y - | Kind::Z - | Kind::Unknown - ) - } -} - /// Compact representation of fragment annotations. /// /// A packed `u32`; see the crate docs for the bit layout. Ordering is by the @@ -259,33 +164,12 @@ impl IonAnnot { isotope: i8, loss: NeutralLoss, ) -> Result { - let kind = - Kind::from_series_char(ion_type).ok_or(IonParsingError::UnsupportedFragmentType { - fragment_type: ion_type, - })?; - let payload = match (kind, ordinal) { - (k, Some(o)) if k.has_ordinal() => o as u32, - (Kind::Precursor, None) => 0, - (Kind::Precursor, Some(o)) => { - return Err(IonParsingError::OrdinalOutOfRange { - ordinal: o as i32, - series: Some(ion_type), - }); - } - (_, None) => { - return Err(IonParsingError::OrdinalOutOfRange { - ordinal: -1, - series: Some(ion_type), - }); - } - (_, Some(o)) => { - return Err(IonParsingError::OrdinalOutOfRange { - ordinal: o as i32, - series: Some(ion_type), - }); - } - }; - Self::pack(kind, payload, loss, charge, isotope) + Self::pack( + IonSeriesOrdinal::from_series_char(ion_type, ordinal)?, + loss, + charge, + isotope, + ) } /// Build an internal fragment spanning residues `start..=end`. @@ -302,12 +186,16 @@ impl IonAnnot { ) -> Result { if start > INTERNAL_POS_MAX || end > INTERNAL_POS_MAX { return Err(IonParsingError::OrdinalOutOfRange { - ordinal: start.max(end) as i32, - series: Some('m'), + ordinal: start.max(end), + series: 'm', }); } - let payload = (start as u32) | ((end as u32) << 6); - Self::pack(Kind::Internal, payload, loss, charge, isotope) + Self::pack( + IonSeriesOrdinal::internal { start, end }, + loss, + charge, + isotope, + ) } /// Build a bare immonium ion for an uppercase residue code. @@ -326,22 +214,22 @@ impl IonAnnot { fragment_type: residue, }); } - let payload = (residue as u8 - b'A') as u32; - Self::pack(Kind::Immonium, payload, loss, charge, isotope) + Self::pack( + IonSeriesOrdinal::immonium { residue }, + loss, + charge, + isotope, + ) } fn pack( - kind: Kind, - payload: u32, + series: IonSeriesOrdinal, loss: NeutralLoss, charge: i8, isotope: i8, ) -> Result { if charge == 0 { - return Err(IonParsingError::ParsingError { - error: format!("{}", kind.series_char()), - context: Some("Charge cannot be 0"), - }); + return Err(IonParsingError::ChargeCannotBeZero); } if !(CHARGE_MIN..=CHARGE_MAX).contains(&charge) { return Err(IonParsingError::ChargeOutOfRange { charge }); @@ -349,9 +237,10 @@ impl IonAnnot { if !(ISOTOPE_MIN..=ISOTOPE_MAX).contains(&isotope) { return Err(IonParsingError::IsotopeOutOfRange { isotope }); } + let (kind, payload) = series.to_parts(); debug_assert!(payload <= mask(PAYLOAD_BITS), "payload overflows its field"); Ok(IonAnnot( - ((kind as u32) << KIND_SHIFT) + (kind << KIND_SHIFT) | ((zigzag(charge) & mask(CHARGE_BITS)) << CHARGE_SHIFT) | ((zigzag(isotope) & mask(ISOTOPE_BITS)) << ISOTOPE_SHIFT) | ((loss as u32 & mask(LOSS_BITS)) << LOSS_SHIFT) @@ -359,11 +248,6 @@ impl IonAnnot { )) } - #[inline] - fn kind(self) -> Kind { - Kind::from_raw((self.0 >> KIND_SHIFT) & mask(KIND_BITS)) - } - #[inline] fn payload(self) -> u32 { (self.0 >> PAYLOAD_SHIFT) & mask(PAYLOAD_BITS) @@ -411,39 +295,28 @@ impl IonAnnot { /// `None` for precursor, unknown, internal and immonium: `?1` is a /// uniqueness counter, not a position in a ladder. pub fn try_get_ordinal(&self) -> Option { - let k = self.kind(); - if k.has_ordinal() && k != Kind::Unknown { - Some(self.payload() as u8) - } else { - None + use IonSeriesOrdinal as S; + match self.series_ordinal() { + S::a { ordinal } + | S::b { ordinal } + | S::c { ordinal } + | S::d { ordinal } + | S::v { ordinal } + | S::w { ordinal } + | S::x { ordinal } + | S::y { ordinal } + | S::z { ordinal } => Some(ordinal), + S::unknown { .. } + | S::precursor + | S::internal { .. } + | S::immonium { .. } + | S::None => None, } } /// The logical series-and-payload view of this annotation. pub fn series_ordinal(&self) -> IonSeriesOrdinal { - let k = self.kind(); - let p = self.payload(); - match k { - Kind::A => IonSeriesOrdinal::a { ordinal: p as u8 }, - Kind::B => IonSeriesOrdinal::b { ordinal: p as u8 }, - Kind::C => IonSeriesOrdinal::c { ordinal: p as u8 }, - Kind::D => IonSeriesOrdinal::d { ordinal: p as u8 }, - Kind::V => IonSeriesOrdinal::v { ordinal: p as u8 }, - Kind::W => IonSeriesOrdinal::w { ordinal: p as u8 }, - Kind::X => IonSeriesOrdinal::x { ordinal: p as u8 }, - Kind::Y => IonSeriesOrdinal::y { ordinal: p as u8 }, - Kind::Z => IonSeriesOrdinal::z { ordinal: p as u8 }, - Kind::Unknown => IonSeriesOrdinal::unknown { ordinal: p as u8 }, - Kind::Precursor => IonSeriesOrdinal::precursor, - Kind::Internal => IonSeriesOrdinal::internal { - start: (p & mask(6)) as u8, - end: ((p >> 6) & mask(6)) as u8, - }, - Kind::Immonium => IonSeriesOrdinal::immonium { - residue: (b'A' + (p & mask(5)) as u8) as char, - }, - Kind::None => IonSeriesOrdinal::None, - } + IonSeriesOrdinal::from_parts((self.0 >> KIND_SHIFT) & mask(KIND_BITS), self.payload()) } } @@ -469,13 +342,6 @@ impl MassError { } } -/// One parsed mzPAF annotation plus its optional mass-error suffix. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct ParsedAnnotation { - pub ion: IonAnnot, - pub mass_error: Option, -} - /// Split the trailing `/[ppm]` off an annotation, if present. /// /// Public because a library reader needs the error even when the ion itself is @@ -504,19 +370,11 @@ pub fn split_mass_error(s: &str) -> Result<(&str, Option), IonParsing } impl IonAnnot { - /// Parse a full mzPAF annotation, including any mass-error suffix. + /// Parse the ion itself, with no mass-error suffix. /// /// A comma-separated list of alternatives is NOT handled here — that is /// ambiguity, and resolving it needs the caller's policy. Split on `,` and /// parse each alternative. - pub fn parse_mzpaf(value: &str) -> Result { - let (rest, mass_error) = split_mass_error(value)?; - Ok(ParsedAnnotation { - ion: Self::parse_ion(rest)?, - mass_error, - }) - } - fn parse_ion(value: &str) -> Result { // charge: trailing ^N let (rest, charge) = match value.split_once('^') { @@ -625,9 +483,9 @@ impl TryFrom<&str> for IonAnnot { type Error = IonParsingError; /// Parses an annotation, discarding any mass-error suffix. Use - /// [`IonAnnot::parse_mzpaf`] to keep it. + /// [`split_mass_error`] first to keep it. fn try_from(value: &str) -> Result { - Ok(Self::parse_mzpaf(value)?.ion) + Self::parse_ion(split_mass_error(value)?.0) } } @@ -656,10 +514,14 @@ impl Display for IonAnnot { #[derive(Debug, Error)] pub enum IonParsingError { - #[error("Ordinal {ordinal} out of range for series '{series:?}'")] - OrdinalOutOfRange { ordinal: i32, series: Option }, + #[error("Ordinal {ordinal} out of range for series '{series}'")] + OrdinalOutOfRange { ordinal: u8, series: char }, + #[error("Series '{series}' requires an ordinal")] + MissingOrdinal { series: char }, #[error("Unsupported fragment type: '{fragment_type}'")] UnsupportedFragmentType { fragment_type: char }, + #[error("Charge cannot be 0")] + ChargeCannotBeZero, #[error("Charge {charge} outside the representable range")] ChargeOutOfRange { charge: i8 }, #[error("Isotope offset {isotope} outside the representable range")] @@ -668,13 +530,41 @@ pub enum IonParsingError { UnsupportedNeutralLoss { loss: String }, #[error("Modified immonium ions are not representable: '{annotation}'")] UnsupportedModifiedImmonium { annotation: String }, + #[error("Ran out of distinct unknown-ion labels: the 8-bit ordinal is exhausted")] + UnknownIonsExhausted, #[error("Parsing error: {error}{}", .context.map(|c| format!(" ({})", c)).unwrap_or_default())] ParsingError { error: String, context: Option<&'static str>, }, - #[error("{error}")] - Custom { error: String }, +} + +/// Hands out `?1`, `?2`, ... for peaks whose annotation this crate cannot +/// represent. +/// +/// Fragment labels must be unique within a precursor — lookup is by first +/// match, so a repeated label makes every later peak carrying it unreachable. +/// A monotonic counter makes that uniqueness structural, and returning an +/// error once the 8-bit ordinal is spent keeps the overflow from being +/// something each reader has to remember to check. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnknownIonCounter(u8); + +impl UnknownIonCounter { + pub fn new() -> Self { + Self::default() + } + + /// The next unused unknown label at `charge`. + pub fn next(&mut self, charge: i8) -> Result { + let ordinal = self + .0 + .checked_add(1) + .ok_or(IonParsingError::UnknownIonsExhausted)?; + let annot = IonAnnot::try_new('?', Some(ordinal), charge, 0)?; + self.0 = ordinal; + Ok(annot) + } } /// The logical series-and-payload view of an [`IonAnnot`]. @@ -730,6 +620,91 @@ pub enum IonSeriesOrdinal { None, } +impl IonSeriesOrdinal { + /// Split into the `kind` discriminant and its `payload`, the two fields + /// [`IonAnnot`] packs. + /// + /// This and [`Self::from_parts`] are the only place the numbering lives. + /// Both are exhaustive over this enum, so adding a variant is a compile + /// error here rather than a silently mislabelled ion series. + const fn to_parts(self) -> (u32, u32) { + match self { + Self::a { ordinal } => (1, ordinal as u32), + Self::b { ordinal } => (2, ordinal as u32), + Self::c { ordinal } => (3, ordinal as u32), + Self::d { ordinal } => (4, ordinal as u32), + Self::v { ordinal } => (5, ordinal as u32), + Self::w { ordinal } => (6, ordinal as u32), + Self::x { ordinal } => (7, ordinal as u32), + Self::y { ordinal } => (8, ordinal as u32), + Self::z { ordinal } => (9, ordinal as u32), + Self::precursor => (10, 0), + Self::unknown { ordinal } => (11, ordinal as u32), + Self::internal { start, end } => (12, (start as u32) | ((end as u32) << 6)), + Self::immonium { residue } => (13, (residue as u8 - b'A') as u32), + Self::None => (0, 0), + } + } + + /// Inverse of [`Self::to_parts`]. An unrecognised discriminant decodes to + /// [`Self::None`] rather than panicking: it can only come from a + /// corrupted word, and the render path must stay total. + const fn from_parts(kind: u32, payload: u32) -> Self { + let ordinal = payload as u8; + match kind { + 1 => Self::a { ordinal }, + 2 => Self::b { ordinal }, + 3 => Self::c { ordinal }, + 4 => Self::d { ordinal }, + 5 => Self::v { ordinal }, + 6 => Self::w { ordinal }, + 7 => Self::x { ordinal }, + 8 => Self::y { ordinal }, + 9 => Self::z { ordinal }, + 10 => Self::precursor, + 11 => Self::unknown { ordinal }, + 12 => Self::internal { + start: (payload & mask(6)) as u8, + end: ((payload >> 6) & mask(6)) as u8, + }, + 13 => Self::immonium { + residue: (b'A' + (payload & mask(5)) as u8) as char, + }, + _ => Self::None, + } + } + + /// Build the series view from an mzPAF series letter and its ordinal. + /// + /// `p` is the only letter that takes no ordinal; every other one requires + /// one. Internal fragments and immonium ions are spelled differently and + /// have their own constructors. + fn from_series_char(c: char, ordinal: Option) -> Result { + if c == 'p' { + return match ordinal { + None => Ok(Self::precursor), + Some(ordinal) => Err(IonParsingError::OrdinalOutOfRange { ordinal, series: c }), + }; + } + let ordinal = ordinal.ok_or(IonParsingError::MissingOrdinal { series: c })?; + Ok(match c { + 'a' => Self::a { ordinal }, + 'b' => Self::b { ordinal }, + 'c' => Self::c { ordinal }, + 'd' => Self::d { ordinal }, + 'v' => Self::v { ordinal }, + 'w' => Self::w { ordinal }, + 'x' => Self::x { ordinal }, + 'y' => Self::y { ordinal }, + 'z' => Self::z { ordinal }, + '?' => Self::unknown { ordinal }, + _ => { + return Err(IonParsingError::UnsupportedFragmentType { fragment_type: c }); + } + }) + } +} + impl Display for IonSeriesOrdinal { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -934,25 +909,88 @@ mod tests { #[test] fn parses_and_applies_the_mass_error_suffix() { - let p = IonAnnot::parse_mzpaf("y1/-0.0005").unwrap(); - assert_eq!(p.ion, ion("y1")); - assert_eq!(p.mass_error, Some(MassError::Da(-0.0005))); + let (rest, err) = split_mass_error("y1/-0.0005").unwrap(); + assert_eq!(ion(rest), ion("y1")); + assert_eq!(err, Some(MassError::Da(-0.0005))); // theoretical = observed - error; verified against a real SpectraST // peak: y1 for C-terminal R, observed 175.1184, theoretical 175.11895. - let theo = p.mass_error.unwrap().theoretical_from_observed(175.1184); + let theo = err.unwrap().theoretical_from_observed(175.1184); assert!((theo - 175.1189).abs() < 1e-9, "got {theo}"); - let q = IonAnnot::parse_mzpaf("y6/1.2ppm").unwrap(); - assert_eq!(q.mass_error, Some(MassError::Ppm(1.2))); - let theo = q.mass_error.unwrap().theoretical_from_observed(700.0); + let (_, err) = split_mass_error("y6/1.2ppm").unwrap(); + assert_eq!(err, Some(MassError::Ppm(1.2))); + let theo = err.unwrap().theoretical_from_observed(700.0); assert!((theo - 699.99916).abs() < 1e-4, "got {theo}"); // Absent suffix is not an error. - assert_eq!(IonAnnot::parse_mzpaf("y6").unwrap().mass_error, None); + assert_eq!(split_mass_error("y6").unwrap(), ("y6", None)); // TryFrom discards it rather than failing. assert_eq!(IonAnnot::try_from("y1/-0.0005").unwrap(), ion("y1")); } + /// One representative of every `IonSeriesOrdinal` variant, each with a + /// distinct payload so a transposition in `to_parts`/`from_parts` cannot + /// cancel out. + const ALL_SERIES: &[IonSeriesOrdinal] = &[ + IonSeriesOrdinal::a { ordinal: 1 }, + IonSeriesOrdinal::b { ordinal: 2 }, + IonSeriesOrdinal::c { ordinal: 3 }, + IonSeriesOrdinal::d { ordinal: 4 }, + IonSeriesOrdinal::v { ordinal: 5 }, + IonSeriesOrdinal::w { ordinal: 6 }, + IonSeriesOrdinal::x { ordinal: 7 }, + IonSeriesOrdinal::y { ordinal: 8 }, + IonSeriesOrdinal::z { ordinal: 9 }, + IonSeriesOrdinal::precursor, + IonSeriesOrdinal::unknown { ordinal: 10 }, + IonSeriesOrdinal::internal { start: 2, end: 11 }, + IonSeriesOrdinal::immonium { residue: 'W' }, + IonSeriesOrdinal::None, + ]; + + /// `to_parts` and `from_parts` are hand-written inverses. Without this, + /// swapping two arms (`v` encoding as `w`) mislabels a whole ion series + /// and every other test still passes. + #[test] + fn every_series_variant_round_trips_through_the_packed_word() { + for &series in ALL_SERIES { + let (kind, payload) = series.to_parts(); + assert_eq!(IonSeriesOrdinal::from_parts(kind, payload), series); + assert!(kind <= mask(KIND_BITS), "{series:?} kind overflows"); + assert!( + payload <= mask(PAYLOAD_BITS), + "{series:?} payload overflows" + ); + } + + let mut kinds: Vec = ALL_SERIES.iter().map(|s| s.to_parts().0).collect(); + kinds.sort_unstable(); + kinds.dedup(); + assert_eq!( + kinds.len(), + ALL_SERIES.len(), + "two series share a discriminant" + ); + } + + /// The other three tables — `Display`, `from_series_char` and the parser — + /// must agree with the packing for every variant, not just the handful the + /// other tests happen to spell out. + #[test] + fn every_series_variant_round_trips_through_its_mzpaf_spelling() { + for &series in ALL_SERIES { + // `None` is the `Default` filler; it renders inertly but is not a + // real annotation, so it has no spelling to parse back. + if series == IonSeriesOrdinal::None { + continue; + } + let annot = IonAnnot::pack(series, NeutralLoss::None, 1, 0).expect("valid"); + assert_eq!(annot.series_ordinal(), series); + let text = annot.to_string(); + assert_eq!(ion(&text).series_ordinal(), series, "{text}"); + } + } + /// An unrepresentable loss must fail loudly. Parsing `y1-HCOOH` as plain /// `y1` would put a loss peak's m/z on the `y1` label and collide with the /// real `y1`. diff --git a/rust/micromzpaf/src/loss.rs b/rust/micromzpaf/src/loss.rs index f0130b7f..eb864b85 100644 --- a/rust/micromzpaf/src/loss.rs +++ b/rust/micromzpaf/src/loss.rs @@ -23,47 +23,36 @@ use std::fmt::Display; use crate::IonParsingError; +/// Slots in [`Composition`], in the order [`Composition::new`] takes them. +const C: usize = 0; +const H: usize = 1; +const N: usize = 2; +const O: usize = 3; +const S: usize = 4; +const P: usize = 5; + /// Atom counts for the elements that appear in peptide neutral losses. /// /// Deliberately not a general chemical formula: these losses only ever draw /// from C/H/N/O/S/P, and keeping it to six `u8`s makes equality a single /// 6-byte compare during the parse-time table lookup. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub(crate) struct Composition { - pub c: u8, - pub h: u8, - pub n: u8, - pub o: u8, - pub s: u8, - pub p: u8, -} +pub(crate) struct Composition([u8; 6]); impl Composition { pub(crate) const fn new(c: u8, h: u8, n: u8, o: u8, s: u8, p: u8) -> Self { - Self { c, h, n, o, s, p } + Self([c, h, n, o, s, p]) } /// Multiply every count, saturating. Used for the `2H2O` multiplier form. fn scaled(self, k: u8) -> Self { - Self { - c: self.c.saturating_mul(k), - h: self.h.saturating_mul(k), - n: self.n.saturating_mul(k), - o: self.o.saturating_mul(k), - s: self.s.saturating_mul(k), - p: self.p.saturating_mul(k), - } + Self(self.0.map(|n| n.saturating_mul(k))) } - fn plus(self, o: Self) -> Self { - Self { - c: self.c.saturating_add(o.c), - h: self.h.saturating_add(o.h), - n: self.n.saturating_add(o.n), - o: self.o.saturating_add(o.o), - s: self.s.saturating_add(o.s), - p: self.p.saturating_add(o.p), - } + fn plus(self, other: Self) -> Self { + Self(std::array::from_fn(|i| { + self.0[i].saturating_add(other.0[i]) + })) } /// Parse a bare formula like `H2O`, `CH4OS`, `C2H5NOS`. @@ -99,12 +88,12 @@ impl Composition { })? }; let slot = match elem { - b'C' => &mut out.c, - b'H' => &mut out.h, - b'N' => &mut out.n, - b'O' => &mut out.o, - b'S' => &mut out.s, - b'P' => &mut out.p, + b'C' => C, + b'H' => H, + b'N' => N, + b'O' => O, + b'S' => S, + b'P' => P, _ => { return Err(IonParsingError::ParsingError { error: s.to_string(), @@ -112,7 +101,7 @@ impl Composition { }); } }; - *slot = slot.saturating_add(count); + out.0[slot] = out.0[slot].saturating_add(count); } Ok(out) } @@ -133,7 +122,9 @@ impl Composition { }); } // Leading digits are a repeat count for the whole term. - let digits = term.len() - term.trim_start_matches(|c: char| c.is_ascii_digit()).len(); + let digits = term + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(term.len()); let (mult, formula) = if digits > 0 { let m: u8 = term[..digits] .parse() diff --git a/rust/timsquery/src/lib.rs b/rust/timsquery/src/lib.rs index 17999732..f5fb8b35 100644 --- a/rust/timsquery/src/lib.rs +++ b/rust/timsquery/src/lib.rs @@ -58,6 +58,7 @@ pub mod ion { IonAnnot, IonParsingError, IonSeriesOrdinal, + UnknownIonCounter, }; } diff --git a/rust/timsquery/src/serde/diann_io.rs b/rust/timsquery/src/serde/diann_io.rs index 25b4b03e..a654a4ce 100644 --- a/rust/timsquery/src/serde/diann_io.rs +++ b/rust/timsquery/src/serde/diann_io.rs @@ -2,8 +2,8 @@ use crate::TimsElutionGroup; use crate::ion::{ IonAnnot, IonParsingError, + UnknownIonCounter, }; -use crate::serde::unknown_ordinal::next_unknown_ordinal; use arrow::array::{ Float32Array, Float64Array, @@ -345,8 +345,7 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(rows.len()); - // Per-precursor `?` counter — see `next_unknown_ordinal`. - let mut num_unknown_losses: u8 = 0; + let mut unknown_ions = UnknownIonCounter::new(); for (i, row) in rows.iter().enumerate() { let fragment_mz = row.fragment_mz; @@ -366,9 +365,7 @@ fn parse_precursor_group( row.fragment_loss_type, i ); - num_unknown_losses = next_unknown_ordinal(num_unknown_losses) - .ok_or(DiannPrecursorParsingError::IonOverCapacity)?; - let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; + let ion_annot = unknown_ions.next(frag_charge as i8)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); relative_intensities.push((ion_annot, rel_intensity)); @@ -661,8 +658,7 @@ fn parse_precursor_group_from_parquet( let mut fragment_mzs = Vec::with_capacity(indices.len()); buffers.fragment_labels.clear(); let mut rel_intensities = Vec::with_capacity(indices.len()); - // Per-precursor `?` counter — see `next_unknown_ordinal`. - let mut num_unknown_losses: u8 = 0; + let mut unknown_ions = UnknownIonCounter::new(); for (i, &idx) in indices.iter().enumerate() { let fragment_mz = columns.product_mzs[idx] as f64; @@ -684,9 +680,7 @@ fn parse_precursor_group_from_parquet( columns.fragment_loss_types[idx], i ); - num_unknown_losses = next_unknown_ordinal(num_unknown_losses) - .ok_or(DiannPrecursorParsingError::IonOverCapacity)?; - let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; + let ion_annot = unknown_ions.next(frag_charge as i8)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); rel_intensities.push((ion_annot, rel_intensity)); diff --git a/rust/timsquery/src/serde/diann_speclib_io.rs b/rust/timsquery/src/serde/diann_speclib_io.rs index 785f2b53..300b71a9 100644 --- a/rust/timsquery/src/serde/diann_speclib_io.rs +++ b/rust/timsquery/src/serde/diann_speclib_io.rs @@ -28,6 +28,7 @@ use super::library_file::{ LibraryArena, LibraryReadingError, + finish_mzpaf_arena, }; use crate::ion::IonAnnot; use crate::models::{ @@ -891,7 +892,7 @@ pub fn read_diann_speclib_library_file>( // mmap the file (not an owned read) — pages fault in on demand, no // file-sized resident buffer. - let (mut geom, frag_intens, stats, at_eof) = SpecLib::open_mmap(path)?.parse_parallel()?; + let (geom, frag_intens, stats, at_eof) = SpecLib::open_mmap(path)?.parse_parallel()?; if !at_eof { // A parse that doesn't land on EOF means the entries were misaligned @@ -925,16 +926,7 @@ pub fn read_diann_speclib_library_file>( path.display() ); - assert_eq!( - frag_intens.len(), - geom.frag_labels.len(), - "reference-intensity sidecar must stay parallel to the fragment-label arena" - ); - geom.seal(); - Ok(LibraryArena::Mzpaf { - geom, - frag_intens: Some(frag_intens), - }) + finish_mzpaf_arena(geom, frag_intens) } #[cfg(test)] diff --git a/rust/timsquery/src/serde/elution_group_inputs.rs b/rust/timsquery/src/serde/elution_group_inputs.rs index 0338f908..c1d622e3 100644 --- a/rust/timsquery/src/serde/elution_group_inputs.rs +++ b/rust/timsquery/src/serde/elution_group_inputs.rs @@ -59,15 +59,12 @@ impl ElutionGroupInput { if self.fragment_labels.is_some() { return Err(ElutionGroupInputError::AlreadyHasFragmentLabels); } - if num_fragments > u8::MAX as usize + 1 { - return Err(ElutionGroupInputError::TooManyFragmentsToLabel { + let fragment_labels: Vec = (0..num_fragments) + .map(u8::try_from) + .collect::>() + .map_err(|_| ElutionGroupInputError::TooManyFragmentsToLabel { count: num_fragments, - }); - } - // Iterate in `usize` and narrow per element: `0..(num_fragments as u8)` - // would be an EMPTY range at exactly 256, since `256 as u8` is 0. The - // guard above is what makes the narrowing lossless. - let fragment_labels: Vec = (0..num_fragments).map(|i| i as u8).collect(); + })?; Ok(ElutionGroupInput { id: self.id, diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index 0c68b0c1..bd28067f 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -396,6 +396,31 @@ impl LibraryArena { } } +/// Seal a directly-built mzpaf arena together with its reference-intensity +/// sidecar. +/// +/// The sidecar is indexed by the same offsets as `frag_labels`, so a length +/// mismatch means some path pushed a label without an intensity (or the +/// reverse) and every downstream fragment lookup is off by that much. Returned +/// as an error rather than asserted: it is a property of the file being read. +pub(super) fn finish_mzpaf_arena( + mut geom: QueryCollection, + frag_intens: Vec, +) -> Result { + if frag_intens.len() != geom.frag_labels.len() { + return Err(LibraryReadingError::SpeclibParse(format!( + "reference-intensity sidecar ({}) must stay parallel to the fragment-label arena ({})", + frag_intens.len(), + geom.frag_labels.len(), + ))); + } + geom.seal(); + Ok(LibraryArena::Mzpaf { + geom, + frag_intens: Some(frag_intens), + }) +} + /// A single spectral-library format reader. Adding a format = one struct + one /// line in [`registry`], instead of editing an enum, a method, and an ordered /// try-chain. @@ -404,20 +429,13 @@ pub trait LibraryReader: Send + Sync { /// Cheap probe: header bytes / extension / first data row. Must not read the /// whole file. fn sniff(&self, path: &Path) -> bool; - /// Read into the arena. + /// Read the whole library into the arena. /// - /// Most formats produce an [`ElutionGroupCollection`] and let - /// [`LibraryArena::from_elution_groups`] adapt it; implement - /// [`Self::read`] for those. The binary `.speclib` and mzSpecLib readers - /// build the arena directly (with the reference-intensity sidecar) and - /// override this instead. - fn read_arena(&self, path: &Path) -> Result { - LibraryArena::from_elution_groups(self.read(path)?) - } - /// The legacy path. Direct-arena readers leave this unimplemented. - fn read(&self, _path: &Path) -> Result { - unreachable!("a reader must implement either `read` or `read_arena`") - } + /// Formats that go through [`ElutionGroupCollection`] adapt it with + /// [`LibraryArena::from_elution_groups`]; the binary `.speclib` and + /// mzSpecLib readers build the arena directly, because only that path + /// carries the reference-intensity sidecar. + fn read(&self, path: &Path) -> Result; } struct MzSpecLibReader; @@ -437,7 +455,7 @@ impl LibraryReader for MzSpecLibReader { sniff_mzspeclib_library_file(path) } - fn read_arena(&self, path: &Path) -> Result { + fn read(&self, path: &Path) -> Result { read_mzspeclib_library_file(path) } } @@ -451,7 +469,7 @@ impl LibraryReader for DiannSpeclibReader { sniff_diann_speclib_library_file(path) } - fn read_arena(&self, path: &Path) -> Result { + fn read(&self, path: &Path) -> Result { read_diann_speclib_library_file(path) } } @@ -465,13 +483,13 @@ impl LibraryReader for DiannParquetReader { sniff_diann_parquet_library_file(path) } - fn read(&self, path: &Path) -> Result { + fn read(&self, path: &Path) -> Result { let egs = read_diann_parquet(path).map_err(|e| { warn!("Failed to read DIA-NN parquet library file: {:?}", e); LibraryReadingError::UnableToParseElutionGroups })?; let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - Ok(ElutionGroupCollection::MzpafLabels( + LibraryArena::from_elution_groups(ElutionGroupCollection::MzpafLabels( egs, Some(FileReadingExtras::Diann(extras)), )) @@ -487,13 +505,13 @@ impl LibraryReader for DiannTsvReader { sniff_diann_library_file(path) } - fn read(&self, path: &Path) -> Result { + fn read(&self, path: &Path) -> Result { let egs = read_diann_tsv(path).map_err(|e| { warn!("Failed to read DIA-NN TSV library file: {:?}", e); LibraryReadingError::UnableToParseElutionGroups })?; let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - Ok(ElutionGroupCollection::MzpafLabels( + LibraryArena::from_elution_groups(ElutionGroupCollection::MzpafLabels( egs, Some(FileReadingExtras::Diann(extras)), )) @@ -509,13 +527,13 @@ impl LibraryReader for SpectronautReader { sniff_spectronaut_library_file(path).is_ok() } - fn read(&self, path: &Path) -> Result { + fn read(&self, path: &Path) -> Result { let egs = read_spectronaut_tsv(path).map_err(|e| { warn!("Failed to read Spectronaut TSV library file: {:?}", e); LibraryReadingError::UnableToParseElutionGroups })?; let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - Ok(ElutionGroupCollection::MzpafLabels( + LibraryArena::from_elution_groups(ElutionGroupCollection::MzpafLabels( egs, Some(FileReadingExtras::Spectronaut(extras)), )) @@ -531,13 +549,13 @@ impl LibraryReader for SkylineReader { sniff_skyline_library_file(path).is_ok() } - fn read(&self, path: &Path) -> Result { + fn read(&self, path: &Path) -> Result { let egs = read_skyline_csv(path).map_err(|e| { warn!("Failed to read Skyline transition list: {:?}", e); LibraryReadingError::UnableToParseElutionGroups })?; let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - Ok(ElutionGroupCollection::MzpafLabels( + LibraryArena::from_elution_groups(ElutionGroupCollection::MzpafLabels( egs, Some(FileReadingExtras::Skyline(extras)), )) @@ -555,8 +573,8 @@ impl LibraryReader for JsonReader { true } - fn read(&self, path: &Path) -> Result { - ElutionGroupCollection::try_read_json(path) + fn read(&self, path: &Path) -> Result { + LibraryArena::from_elution_groups(ElutionGroupCollection::try_read_json(path)?) } } @@ -585,7 +603,7 @@ pub fn read_library_file>(path: T) -> Result return Ok(arena), // A sniff can fire on a file the reader then fails to parse // (overlapping sniffs). Fall through to the next candidate diff --git a/rust/timsquery/src/serde/mod.rs b/rust/timsquery/src/serde/mod.rs index f65e70fe..903eb6d5 100644 --- a/rust/timsquery/src/serde/mod.rs +++ b/rust/timsquery/src/serde/mod.rs @@ -4,10 +4,9 @@ pub mod diann_speclib_io; mod elution_group_inputs; pub mod index_serde; mod library_file; -pub mod mzspeclib_io; +mod mzspeclib_io; mod skyline_io; mod spectronaut_io; -mod unknown_ordinal; pub use chromatogram_output::*; pub use index_serde::*; diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs index 660af579..561d26eb 100644 --- a/rust/timsquery/src/serde/mzspeclib_io.rs +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -38,7 +38,10 @@ //! unknown label rather than falling back to a worse-matching representable //! alternative, which would assign both a wrong identity and a wrong mass. -use crate::ion::IonAnnot; +use crate::ion::{ + IonAnnot, + UnknownIonCounter, +}; use crate::models::{ LibCapabilities, QueryCollection, @@ -46,6 +49,7 @@ use crate::models::{ use crate::serde::library_file::{ LibraryArena, LibraryReadingError, + finish_mzpaf_arena, }; use micromzpaf::{ MassError, @@ -97,33 +101,62 @@ const UNIT_TERM: &str = "UO:0000000"; const UNIT_MINUTE: &str = "UO:0000031"; const UNIT_SECOND: &str = "UO:0000010"; -/// Per-library tally of everything that did not land verbatim in the arena. +/// Declare the per-library tally so that the counters, the "is anything +/// wrong?" test and the log line all come from one list. /// -/// Reported once at the end of a load rather than per row: a consensus library -/// can carry thousands of unannotated peaks, and a line each would bury the -/// signal. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct MzSpecLibStats { - /// Peaks stored with their parsed annotation. - pub kept_annotated: usize, +/// Spelled as a macro because the alternative — a struct plus a hand-written +/// `||` chain plus a hand-written `warn!` — is three places to update per +/// counter and the compiler checks none of them. That already went wrong once. +macro_rules! anomaly_counters { + ($( $(#[$doc:meta])* $field:ident => $label:literal, )+) => { + /// Per-library tally of everything that did not land verbatim in the + /// arena. + /// + /// Reported once at the end of a load rather than per row: a consensus + /// library can carry thousands of unannotated peaks, and a line each + /// would bury the signal. + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] + pub(crate) struct MzSpecLibStats { + /// Peaks stored with their parsed annotation. The only counter + /// here that is not an anomaly. + pub kept_annotated: usize, + $( $(#[$doc])* pub $field: usize, )+ + } + + impl MzSpecLibStats { + /// Every anomaly counter paired with how to say it. + fn anomalies(&self) -> impl Iterator { + [ $( ($label, self.$field), )+ ].into_iter() + } + } + }; +} + +anomaly_counters! { /// Identity known but not representable (a loss outside the table, a /// modified immonium). Stored with an unknown label and an exact mass. - pub kept_unknown_label: usize, + kept_unknown_label => "kept with an unknown label", /// `?` — no annotation, so no mass error, so no theoretical m/z. - pub skipped_unannotated: usize, + skipped_unannotated => "skipped as unannotated", /// Comma-separated alternatives that tied on absolute mass error. - pub skipped_ambiguous: usize, + skipped_ambiguous => "skipped as ambiguous", /// Peaks dropped because their label collided with one already in the - /// precursor after the unknown-label rewrite. - pub dropped_duplicate_label: usize, + /// precursor. + dropped_duplicate_label => "dropped for a duplicate label", + /// Peaks dropped because the precursor had already spent all 255 unknown + /// labels, so no distinct one was left. + dropped_unknown_over_capacity => "dropped with the unknown labels exhausted", /// Spectra with no retention-time term at all. - pub spectra_without_rt: usize, + spectra_without_rt => "spectra without an RT", /// Spectra whose mobility came from a drift time rather than 1/K0. - pub spectra_with_drift_time_mobility: usize, + spectra_with_drift_time_mobility => "spectra using a drift time as mobility", /// Spectra whose retention time carried a unit this reader does not know. - pub spectra_with_unknown_rt_unit: usize, + spectra_with_unknown_rt_unit => "spectra with an unknown RT unit", + /// Spectra dropped for missing or unparseable precursor m/z, charge or + /// sequence. + dropped_malformed_spectrum => "spectra dropped as malformed", /// Precursors dropped for having no usable peak left. - pub dropped_empty_precursors: usize, + dropped_empty_precursors => "precursors dropped as empty", } /// One spectrum converted into the shape [`QueryCollection::push_row`] takes. @@ -139,42 +172,26 @@ struct ArenaRow { } impl MzSpecLibStats { - fn anything_to_report(&self) -> bool { - self.kept_unknown_label > 0 - || self.skipped_unannotated > 0 - || self.skipped_ambiguous > 0 - || self.dropped_duplicate_label > 0 - || self.spectra_without_rt > 0 - || self.spectra_with_drift_time_mobility > 0 - || self.spectra_with_unknown_rt_unit > 0 - || self.dropped_empty_precursors > 0 - } - fn report(&self, path: &Path) { - if !self.anything_to_report() { + let flagged: Vec = self + .anomalies() + .filter(|(_, n)| *n > 0) + .map(|(label, n)| format!("{n} {label}")) + .collect(); + if flagged.is_empty() { info!( "mzSpecLib {}: {} peaks, all annotated and representable", path.display(), self.kept_annotated ); - return; + } else { + warn!( + "mzSpecLib {}: kept {} annotated peaks; {}", + path.display(), + self.kept_annotated, + flagged.join(", "), + ); } - warn!( - "mzSpecLib {}: kept {} annotated + {} with unknown labels; \ - skipped {} unannotated, {} ambiguous, {} duplicate-label; \ - {} spectra without RT, {} using drift time as mobility, \ - {} with an unknown RT unit, {} precursors dropped as empty", - path.display(), - self.kept_annotated, - self.kept_unknown_label, - self.skipped_unannotated, - self.skipped_ambiguous, - self.dropped_duplicate_label, - self.spectra_without_rt, - self.spectra_with_drift_time_mobility, - self.spectra_with_unknown_rt_unit, - self.dropped_empty_precursors, - ); } } @@ -246,24 +263,35 @@ struct RawSpectrum { peaks: Vec<(f64, f32, String)>, } +/// Why a peak cannot be stored at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SkipReason { + /// No annotation, so no mass error, so no theoretical m/z. + Unannotated, + /// Alternatives that pin no single identity, so likewise no theoretical + /// m/z. Storing the observed one would mix the two. + Ambiguous, +} + /// What resolving one peak's annotation produced. +/// +/// The mass error rides along on the two variants that keep the peak, because +/// it is what recovers theoretical m/z from the observed value in the file. It +/// is absent from `Skip` because a skipped peak has no mass to recover. enum Resolved { /// Parsed cleanly; store with this label. - Annotated(IonAnnot), - /// Identity known, not representable. Store with an unknown label, exact - /// mass. - UnknownLabel, - /// No single identity, so no theoretical mass. Skip. - SkipUnannotated, - SkipAmbiguous, + Annotated(IonAnnot, Option), + /// Identity known, not representable. Store with an unknown label and the + /// exact mass. + UnknownLabel(Option), + Skip(SkipReason), } -/// Resolve one annotation string into a storage decision plus the mass error -/// needed to recover theoretical m/z. -fn resolve_annotation(annotation: &str) -> (Resolved, Option) { +/// Resolve one annotation string into a storage decision. +fn resolve_annotation(annotation: &str) -> Resolved { let annotation = annotation.trim(); if annotation.is_empty() || annotation == "?" { - return (Resolved::SkipUnannotated, None); + return Resolved::Skip(SkipReason::Unannotated); } // Splitting the error off comes first: it works even when the ion will not @@ -274,7 +302,7 @@ fn resolve_annotation(annotation: &str) -> (Resolved, Option) { let mut alternatives = Vec::new(); for alt in annotation.split(',') { let Ok(parsed) = split_mass_error(alt.trim()) else { - return (Resolved::SkipAmbiguous, None); + return Resolved::Skip(SkipReason::Ambiguous); }; alternatives.push(parsed); } @@ -295,27 +323,44 @@ fn resolve_annotation(annotation: &str) -> (Resolved, Option) { .map(|(_, e)| magnitude(e)) .fold(f64::INFINITY, f64::min); if !best.is_finite() { - return (Resolved::SkipAmbiguous, None); + return Resolved::Skip(SkipReason::Ambiguous); } let mut winners = alternatives.iter().filter(|(_, e)| magnitude(e) == best); let winner = *winners.next().expect("the minimum came from this iterator"); if winners.next().is_some() { - return (Resolved::SkipAmbiguous, None); + return Resolved::Skip(SkipReason::Ambiguous); } winner }; match IonAnnot::try_from(ion_str) { - Ok(ion) => (Resolved::Annotated(ion), mass_error), + Ok(ion) => Resolved::Annotated(ion, mass_error), // Keep the peak and its exact mass, lose only the label. - Err(_) => (Resolved::UnknownLabel, mass_error), + Err(_) => Resolved::UnknownLabel(mass_error), } } -/// Convert one accumulated spectrum into arena rows. +/// Convert one accumulated spectrum into an arena row, counting whichever way +/// it failed. /// -/// Returns `None` when the spectrum lacks something structural (precursor m/z, -/// charge, sequence) or ends up with no usable peak. +/// The counting lives here rather than inside [`spectrum_row`] so that every +/// `?` in there lands on a tally. Without it a library that dropped half its +/// spectra for a missing charge still reports "all annotated and +/// representable", which is the one thing this module exists to prevent. fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { + let Some(row) = spectrum_row(raw, stats) else { + stats.dropped_malformed_spectrum += 1; + return None; + }; + if row.frags.is_empty() { + stats.dropped_empty_precursors += 1; + return None; + } + Some(row) +} + +/// The conversion proper: `None` means the spectrum lacked something +/// structural (precursor m/z, charge, a parseable RT or mobility, a sequence). +fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { let precursor_mz = raw.attrs.f64_of(PRECURSOR_MZ_TERMS)?; let charge: u8 = raw.attrs.find(CHARGE_TERM)?.value.parse().ok()?; @@ -377,40 +422,29 @@ fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option = Vec::with_capacity(raw.peaks.len()); let mut intens: Vec = Vec::with_capacity(raw.peaks.len()); - let mut unknown_counter: u8 = 0; + let mut unknown_ions = UnknownIonCounter::new(); for (observed_mz, intensity, annotation) in &raw.peaks { - let (resolved, mass_error) = resolve_annotation(annotation); - let label = match resolved { - Resolved::Annotated(ion) => { + let (label, mass_error) = match resolve_annotation(annotation) { + Resolved::Annotated(ion, error) => { stats.kept_annotated += 1; - ion + (ion, error) } - Resolved::UnknownLabel => { - // The ordinal is a per-precursor uniqueness counter. Past 255 - // there is no way to keep labels distinct, so drop rather than - // reuse one. - let Some(next) = unknown_counter.checked_add(1) else { - stats.dropped_duplicate_label += 1; + Resolved::UnknownLabel(error) => match unknown_ions.next(1) { + Ok(ion) => { + stats.kept_unknown_label += 1; + (ion, error) + } + Err(_) => { + stats.dropped_unknown_over_capacity += 1; continue; - }; - unknown_counter = next; - match IonAnnot::try_new('?', Some(unknown_counter), 1, 0) { - Ok(i) => { - stats.kept_unknown_label += 1; - i - } - Err(_) => { - stats.dropped_duplicate_label += 1; - continue; - } } - } - Resolved::SkipUnannotated => { + }, + Resolved::Skip(SkipReason::Unannotated) => { stats.skipped_unannotated += 1; continue; } - Resolved::SkipAmbiguous => { + Resolved::Skip(SkipReason::Ambiguous) => { stats.skipped_ambiguous += 1; continue; } @@ -422,7 +456,9 @@ fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option Option>(path: T) -> bool { let Ok(file) = std::fs::File::open(path.as_ref()) else { return false; }; - let mut reader = BufReader::new(file); - let mut line = String::new(); - // Only the first non-empty line is inspected, so this stays O(1) on a + // Lazy, so only the first non-empty line is read: this stays O(1) on a // multi-gigabyte library. - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => return false, - Ok(_) => { - let t = line.trim(); - if t.is_empty() { - continue; - } - return t == MAGIC; - } - Err(_) => return false, - } - } + BufReader::new(file) + .lines() + .map_while(Result::ok) + .find(|l| !l.trim().is_empty()) + .is_some_and(|l| l.trim() == MAGIC) +} + +/// Which part of a spectrum block the line loop is inside. +enum Section { + Attributes, + Peaks, +} + +/// Convert an accumulated spectrum and append it to the arena. +fn flush( + current: Option, + geom: &mut QueryCollection, + frag_intens: &mut Vec, + stats: &mut MzSpecLibStats, +) { + let Some(raw) = current else { return }; + let Some(row) = convert_spectrum(&raw, stats) else { + return; + }; + frag_intens.extend_from_slice(&row.intensities); + geom.push_row( + row.precursor_mz, + row.charge, + row.rt_seconds, + row.mobility, + &row.frags, + &row.stripped, + &row.modified, + &[], + false, + ); } /// Read an mzSpecLib text file into the columnar arena. @@ -486,52 +537,29 @@ pub fn read_mzspeclib_library_file>( let mut stats = MzSpecLibStats::default(); let mut current: Option = None; - let mut in_peaks = false; - - let flush = |cur: Option, - geom: &mut QueryCollection, - frag_intens: &mut Vec, - stats: &mut MzSpecLibStats| { - let Some(raw) = cur else { return }; - let Some(row) = convert_spectrum(&raw, stats) else { - return; - }; - frag_intens.extend_from_slice(&row.intensities); - geom.push_row( - row.precursor_mz, - row.charge, - row.rt_seconds, - row.mobility, - &row.frags, - &row.stripped, - &row.modified, - &[], - false, - ); - }; + let mut section = Section::Attributes; for line in reader.lines() { let line = line.map_err(LibraryReadingError::IoError)?; let trimmed = line.trim_end(); - if trimmed.starts_with("" { - in_peaks = true; - continue; - } - // `` and `` attributes are folded into the - // spectrum's bag: this reader wants the union, not the hierarchy. + // Any `<...>` header ends the peak list; only `` opens one. + // `` and `` attributes are folded into + // the spectrum's bag: this reader wants the union, not the hierarchy. if trimmed.starts_with('<') { - in_peaks = false; + if trimmed.starts_with("" { + Section::Peaks + } else { + Section::Attributes + }; continue; } if trimmed.is_empty() { - in_peaks = false; + section = Section::Attributes; continue; } @@ -539,20 +567,25 @@ pub fn read_mzspeclib_library_file>( continue; // library-level header }; - if in_peaks { - let mut cols = trimmed.split('\t'); - let (Some(mz), Some(intensity)) = (cols.next(), cols.next()) else { - continue; - }; - let (Ok(mz), Ok(intensity)) = - (mz.trim().parse::(), intensity.trim().parse::()) - else { - continue; - }; - let annotation = cols.next().unwrap_or("?").to_string(); - spec.peaks.push((mz, intensity, annotation)); - } else if let Some(attr) = Attr::parse(trimmed) { - spec.attrs.0.push(attr); + match section { + Section::Peaks => { + let mut cols = trimmed.split('\t'); + let (Some(mz), Some(intensity)) = (cols.next(), cols.next()) else { + continue; + }; + let (Ok(mz), Ok(intensity)) = + (mz.trim().parse::(), intensity.trim().parse::()) + else { + continue; + }; + let annotation = cols.next().unwrap_or("?").to_string(); + spec.peaks.push((mz, intensity, annotation)); + } + Section::Attributes => { + if let Some(attr) = Attr::parse(trimmed) { + spec.attrs.0.push(attr); + } + } } } flush(current.take(), &mut geom, &mut frag_intens, &mut stats); @@ -565,19 +598,7 @@ pub fn read_mzspeclib_library_file>( path.display() ))); } - if frag_intens.len() != geom.frag_labels.len() { - return Err(LibraryReadingError::SpeclibParse(format!( - "reference-intensity sidecar ({}) must stay parallel to the fragment-label arena ({})", - frag_intens.len(), - geom.frag_labels.len(), - ))); - } - - geom.seal(); - Ok(LibraryArena::Mzpaf { - geom, - frag_intens: Some(frag_intens), - }) + finish_mzpaf_arena(geom, frag_intens) } #[cfg(test)] @@ -667,31 +688,70 @@ mod tests { ); } + /// A spectrum missing a structural field is dropped by `?` deep inside + /// `spectrum_row`. Nothing there increments a counter, so this is what + /// stops the load from reporting "all annotated and representable" while + /// having silently lost half the library. + #[test] + fn structurally_broken_spectra_are_counted_not_swallowed() { + let attr = |accession: &str, value: &str| Attr { + group: None, + accession: accession.to_string(), + value: value.to_string(), + }; + let good = |charge: &str| RawSpectrum { + attrs: AttrBag(vec![ + attr(PRECURSOR_MZ_TERMS[0], "500.25"), + attr(CHARGE_TERM, charge), + attr(STRIPPED_SEQ_TERM, "PEPTIDEK"), + attr(RT_TERMS[0], "10.0"), + ]), + peaks: vec![(175.1, 1.0, "y1/0.0".to_string())], + }; + + let mut stats = MzSpecLibStats::default(); + assert!(convert_spectrum(&good("2"), &mut stats).is_some()); + // Unparseable charge — the `?` that used to vanish. + assert!(convert_spectrum(&good("not-a-number"), &mut stats).is_none()); + // No peak survives resolution, which is a different failure. + let mut empty = good("2"); + empty.peaks = vec![(175.1, 1.0, "?".to_string())]; + assert!(convert_spectrum(&empty, &mut stats).is_none()); + + assert_eq!(stats.kept_annotated, 1); + assert_eq!(stats.dropped_malformed_spectrum, 1); + assert_eq!(stats.dropped_empty_precursors, 1); + assert_eq!(stats.skipped_unannotated, 1); + assert!( + stats.anomalies().any(|(_, n)| n > 0), + "the report must not claim a clean load" + ); + } + #[test] fn resolves_unambiguous_representable() { - let (r, e) = resolve_annotation("y5/-0.0005"); - assert!(matches!(r, Resolved::Annotated(_))); - assert_eq!(e, Some(MassError::Da(-0.0005))); + assert!(matches!( + resolve_annotation("y5/-0.0005"), + Resolved::Annotated(_, Some(MassError::Da(-0.0005))) + )); } /// Known identity, unrepresentable spelling: keep the peak and the exact - /// mass, erase only the label. + /// mass, erase only the label. The mass error must survive so theoretical + /// m/z stays exact. #[test] fn unrepresentable_loss_keeps_peak_with_unknown_label() { - let (r, e) = resolve_annotation("y1-HCOOH/0.0003"); - assert!(matches!(r, Resolved::UnknownLabel)); - assert_eq!( - e, - Some(MassError::Da(0.0003)), - "the mass error must survive so theoretical m/z stays exact" - ); + assert!(matches!( + resolve_annotation("y1-HCOOH/0.0003"), + Resolved::UnknownLabel(Some(MassError::Da(0.0003))) + )); } #[test] fn unannotated_peak_is_skipped() { assert!(matches!( - resolve_annotation("?").0, - Resolved::SkipUnannotated + resolve_annotation("?"), + Resolved::Skip(SkipReason::Unannotated) )); } @@ -701,24 +761,29 @@ mod tests { #[test] fn ambiguity_resolves_to_the_closest_not_the_representable() { // a2 is representable and further; y2-CO2-NH3 is closer and is not. - let (r, e) = resolve_annotation("a2/-0.0040,y2-CO2-NH3/-0.0001"); assert!( - matches!(r, Resolved::UnknownLabel), + matches!( + resolve_annotation("a2/-0.0040,y2-CO2-NH3/-0.0001"), + Resolved::UnknownLabel(Some(MassError::Da(-0.0001))) + ), "the closest alternative wins even when unrepresentable" ); - assert_eq!(e, Some(MassError::Da(-0.0001))); // When the closest one IS representable, it is used. - let (r, _) = resolve_annotation("a2/-0.0001,y2-CO2-NH3/-0.0040"); - assert!(matches!(r, Resolved::Annotated(_))); + assert!(matches!( + resolve_annotation("a2/-0.0001,y2-CO2-NH3/-0.0040"), + Resolved::Annotated(..) + )); } /// An exact tie pins no identity, so no theoretical m/z exists and the peak /// cannot be stored without mixing observed and theoretical masses. #[test] fn tied_ambiguity_is_skipped() { - let (r, _) = resolve_annotation("a2/-0.0004,y2-CO2-NH3/-0.0004"); - assert!(matches!(r, Resolved::SkipAmbiguous)); + assert!(matches!( + resolve_annotation("a2/-0.0004,y2-CO2-NH3/-0.0004"), + Resolved::Skip(SkipReason::Ambiguous) + )); } #[test] @@ -728,9 +793,10 @@ mod tests { assert!((e.theoretical_from_observed(175.1184) - 175.1189).abs() < 1e-9); } - /// The public entry point must dispatch mzSpecLib itself. It is sniffed - /// before the registry, so a regression here falls through to the - /// always-true JSON reader and fails with a generic parse error. + /// The public entry point must dispatch mzSpecLib itself. It is the + /// registry's first entry precisely because the JSON reader at the end + /// accepts anything, so a regression in the sniff falls through to it and + /// surfaces as a generic parse error rather than as "wrong reader". #[test] fn public_read_library_file_dispatches_mzspeclib() { use crate::serde::read_library_file; diff --git a/rust/timsquery/src/serde/skyline_io.rs b/rust/timsquery/src/serde/skyline_io.rs index a93b43f4..696ea2cd 100644 --- a/rust/timsquery/src/serde/skyline_io.rs +++ b/rust/timsquery/src/serde/skyline_io.rs @@ -2,8 +2,8 @@ use crate::TimsElutionGroup; use crate::ion::{ IonAnnot, IonParsingError, + UnknownIonCounter, }; -use crate::serde::unknown_ordinal::next_unknown_ordinal; use serde::{ Deserialize, Deserializer, @@ -313,7 +313,7 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(fragment_rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(fragment_rows.len()); - let mut num_unknown_losses = 0u8; + let mut unknown_ions = UnknownIonCounter::new(); for (i, row) in fragment_rows.iter().enumerate() { let fragment_mz = row.product_mz; @@ -352,9 +352,7 @@ fn parse_precursor_group( falling back to unknown ion", frag_type, i ); - num_unknown_losses = next_unknown_ordinal(num_unknown_losses) - .ok_or(SkylinePrecursorParsingError::IonOverCapacity)?; - IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)? + unknown_ions.next(frag_charge as i8)? } }; diff --git a/rust/timsquery/src/serde/spectronaut_io.rs b/rust/timsquery/src/serde/spectronaut_io.rs index ad4fb92f..2c44ec4b 100644 --- a/rust/timsquery/src/serde/spectronaut_io.rs +++ b/rust/timsquery/src/serde/spectronaut_io.rs @@ -2,8 +2,8 @@ use crate::TimsElutionGroup; use crate::ion::{ IonAnnot, IonParsingError, + UnknownIonCounter, }; -use crate::serde::unknown_ordinal::next_unknown_ordinal; use serde::Deserialize; use std::path::Path; use tinyvec::tiny_vec; @@ -287,8 +287,7 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(included_rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(included_rows.len()); - // Per-precursor `?` counter — see `next_unknown_ordinal`. - let mut num_unknown_losses: u8 = 0; + let mut unknown_ions = UnknownIonCounter::new(); for (i, row) in included_rows.iter().enumerate() { let fragment_mz = row.fragment_mz; @@ -308,9 +307,7 @@ fn parse_precursor_group( row.fragment_loss_type, i ); - num_unknown_losses = next_unknown_ordinal(num_unknown_losses) - .ok_or(SpectronautPrecursorParsingError::IonOverCapacity)?; - let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; + let ion_annot = unknown_ions.next(frag_charge as i8)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); relative_intensities.push((ion_annot, rel_intensity)); diff --git a/rust/timsquery/src/serde/unknown_ordinal.rs b/rust/timsquery/src/serde/unknown_ordinal.rs deleted file mode 100644 index c1843a60..00000000 --- a/rust/timsquery/src/serde/unknown_ordinal.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Shared counter for `?`-labelled unknown ions. - -/// Advance a per-precursor `?`-ordinal counter, refusing to wrap. -/// -/// `IonAnnot` ordinals are `u8`, so a precursor can carry at most -/// [`u8::MAX`] distinguishable unknown ions. Past that there is no way to -/// keep labels unique, and duplicates are the one thing the arena cannot -/// tolerate — see [`ExpectedIntensities::try_from_pairs`] for why. -/// -/// `None` means "cannot allocate another": callers turn it into their own -/// over-capacity error while the row index is still in hand, rather than -/// letting the duplicate surface downstream. -/// -/// [`ExpectedIntensities::try_from_pairs`]: https://docs.rs/timsseek -pub(crate) fn next_unknown_ordinal(current: u8) -> Option { - current.checked_add(1) -} diff --git a/rust/timsquery/tests/mzspeclib_io_files/README.md b/rust/timsquery/tests/mzspeclib_io_files/README.md index 5f1aa80f..30994f3e 100644 --- a/rust/timsquery/tests/mzspeclib_io_files/README.md +++ b/rust/timsquery/tests/mzspeclib_io_files/README.md @@ -5,7 +5,7 @@ Verbatim from [HUPO-PSI/mzSpecLib](https://github.com/HUPO-PSI/mzSpecLib) | file | why | |---|---| -| `diann.mzSpecLib.txt` | the shape `speclib_build` will emit; all peaks annotated, every mass error exactly `0.0` | +| `diann.mzSpecLib.txt` | a DIA-NN export: all peaks annotated, every mass error exactly `0.0` | | `spectronaut.mzSpecLib.txt` | carries `-H2O`/`-NH3` losses and a unit-tagged retention time in minutes | Both are fully representable, so neither exercises the unknown-label or From 925cc02e90115a06f218022d76cfbcd23f75c855 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Wed, 26 Aug 2026 23:53:48 -0700 Subject: [PATCH 13/27] refactor: use mzcore's own static ontologies, keep the parse error `ontologies()` rebuilt what mzcore already ships as `STATIC_ONTOLOGIES`, a `LazyLock` over the same `init_static`, adding only a `tracing::debug!`. Its doc also claimed it was reached only from `parse_sequence_mzcore`; two other call sites reach it through `parse_proforma`. `parse_proforma` returned `Option`, so `count_carbon_sulphur_in_sequence` lost mzcore's diagnostic and reported only the sequence. It returns `Result` now; the two callers that cannot use the message discard it explicitly. Not changed, deliberately: most of the ~200 MB is GNOme glycan data that `modification_to_mod` discards, and mzcore can build a Unimod-only index -- but dropping an ontology also drops the sequences that reference it, turning a mod this code already ignores into a peptide it cannot parse. Noted where it would be done. --- rust/speclib_build_cli/src/entry.rs | 2 +- .../fragment_mass/elution_group_converter.rs | 2 +- rust/timsseek/src/lib.rs | 1 - rust/timsseek/src/models/sequence.rs | 61 +++++++++---------- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/rust/speclib_build_cli/src/entry.rs b/rust/speclib_build_cli/src/entry.rs index d10f1ce9..68ef68a8 100644 --- a/rust/speclib_build_cli/src/entry.rs +++ b/rust/speclib_build_cli/src/entry.rs @@ -54,7 +54,7 @@ use timsseek::models::sequence::normalize_to_proforma; fn compute_precursor_mz(modified_seq: &str, charge: u8) -> Option { use mzcore::prelude::*; let proforma = normalize_to_proforma(modified_seq); - let peptide = timsseek::models::sequence::parse_proforma(&proforma)?; + let peptide = timsseek::models::sequence::parse_proforma(&proforma).ok()?; let linear = peptide.as_linear()?; let formulas = linear.formulas(); if formulas.is_empty() { diff --git a/rust/timsseek/src/fragment_mass/elution_group_converter.rs b/rust/timsseek/src/fragment_mass/elution_group_converter.rs index 3e173985..2a26d2e3 100644 --- a/rust/timsseek/src/fragment_mass/elution_group_converter.rs +++ b/rust/timsseek/src/fragment_mass/elution_group_converter.rs @@ -119,7 +119,7 @@ pub fn count_carbon_sulphur_in_sequence(sequence: &str) -> Result<(u16, u16), St fn count_carbon_sulphur_in_sequence_mzcore(sequence: &str) -> Result<(u16, u16), String> { let peptide = crate::models::sequence::parse_proforma(sequence) - .ok_or_else(|| format!("Error parsing peptide sequence {sequence}"))?; + .map_err(|e| format!("Error parsing peptide sequence {sequence}: {e}"))?; let peptide = match peptide.as_linear() { Some(pep) => pep, None => return Err("Peptide is not linear.".to_string()), diff --git a/rust/timsseek/src/lib.rs b/rust/timsseek/src/lib.rs index 31269619..88908d39 100644 --- a/rust/timsseek/src/lib.rs +++ b/rust/timsseek/src/lib.rs @@ -11,7 +11,6 @@ pub mod scoring; pub mod traits; pub mod utils; pub use micromzpaf; -pub use models::sequence::ontologies; pub use data_sources::{ ExpectedIntensity, diff --git a/rust/timsseek/src/models/sequence.rs b/rust/timsseek/src/models/sequence.rs index 62617bf0..ac0bf8ce 100644 --- a/rust/timsseek/src/models/sequence.rs +++ b/rust/timsseek/src/models/sequence.rs @@ -5,38 +5,39 @@ use crate::models::decoy::DecoyMarking; use serde::Serialize; use smallvec::SmallVec; -use std::sync::{ - Arc, - OnceLock, -}; +use std::sync::Arc; -/// Process-wide modification ontologies, built on first use. +/// Parse a ProForma string against mzcore's shared ontologies. /// -/// mzcore requires an explicit `Ontologies` value at every `pro_forma` call. -/// Building it costs ~210 ms and ~200 MB, so it is deliberately behind a -/// `OnceLock` reached ONLY from [`parse_sequence_mzcore`] — the fallback past -/// the byte-walk fast path. A library whose sequences all match the fast -/// grammar never pays it; a real DIA-NN `.speclib` load peaks at ~10 MB and -/// never initializes this. -pub fn ontologies() -> &'static mzcore::ontology::Ontologies { - static ONTOLOGIES: OnceLock = OnceLock::new(); - ONTOLOGIES.get_or_init(|| { - tracing::debug!("initializing mzcore ontologies (first non-fast-path sequence)"); - mzcore::ontology::Ontologies::init_static() - }) -} - -/// Parse a ProForma string against the shared [`ontologies`]. +/// mzcore requires an explicit `Ontologies` value at every `pro_forma` call and +/// ships `STATIC_ONTOLOGIES` for exactly this; it is a `LazyLock`, so the +/// ~210 ms / ~200 MB build happens on first use and only on a path that gets +/// here. That matters: this is the fallback *past* the byte-walk fast path in +/// [`parse_sequence`], so a library whose sequences all match the fast grammar +/// never pays it (a real DIA-NN `.speclib` load peaks at ~10 MB and never +/// touches this). The two other callers — `count_carbon_sulphur_in_sequence`'s +/// mzcore fallback and `speclib_build_cli` — do pay it. +/// +/// Most of that footprint is GNOme glycan data that [`modification_to_mod`] +/// discards, and mzcore can build a Unimod-only index. Not done here: dropping +/// an ontology also drops the sequences that reference it, turning a mod this +/// code already ignores into a peptide it cannot parse at all. /// -/// mzcore returns non-fatal parse warnings alongside the peptidoform; none of -/// the callers can act on them, so they are dropped in one place instead of -/// each site carrying its own `(pep, _warnings)` destructure. +/// mzcore also returns non-fatal parse warnings alongside the peptidoform; none +/// of the callers can act on them, so they are dropped in one place rather than +/// at each site. pub fn parse_proforma( sequence: &str, -) -> Option> { - let (peptidoform, _warnings) = - mzcore::sequence::Peptidoform::pro_forma(sequence, ontologies()).ok()?; - Some(peptidoform) +) -> Result, String> { + mzcore::sequence::Peptidoform::pro_forma(sequence, &mzcore::ontology::STATIC_ONTOLOGIES) + .map(|(peptidoform, _warnings)| peptidoform) + .map_err(|errors| { + errors + .iter() + .map(|e| e.to_string()) + .collect::>() + .join("; ") + }) } /// Amino acid stored as alphabet offset `c - b'A'` (0..=25). `u8::MAX` @@ -291,7 +292,7 @@ fn parse_sequence_fast(s: &str) -> Option { fn parse_sequence_mzcore(normalized: &str) -> Option { use mzcore::prelude::IsAminoAcid; - let pf = parse_proforma(normalized)?; + let pf = parse_proforma(normalized).ok()?; let linear = pf.into_linear()?; let mut residues: SmallVec<[AminoAcid; 32]> = SmallVec::new(); @@ -339,8 +340,6 @@ fn modification_to_mod(m: &mzcore::sequence::Modification) -> Option { _ => return None, // Cross-link / ambiguous — out of v1 scope }; match simple.as_ref() { - // mzcore carries the mass tag and the source digit count alongside the - // mass itself; only the mass matters here. SimpleModificationInner::Mass(_tag, mass, _digits) => Some(Mod::Mass(mass.value as f32)), SimpleModificationInner::Database { id, .. } => { if id.ontology != Ontology::Unimod { @@ -699,7 +698,7 @@ mod tests { /// through the UNIMOD ontology to the same numeric id the `[UNIMOD:n]` /// spelling yields. This is the one behavior that has no fast-path /// equivalent, so nothing else covers it; it is also the only test that - /// forces `ontologies()` to actually initialize. + /// forces mzcore's `STATIC_ONTOLOGIES` to actually initialize. #[test] fn mzcore_fallback_resolves_named_mods_via_ontology() { for (named, expected) in [ From 6338de5c65d56ccca3a8d21052f9e9a2f5a9503c Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 07:57:49 -0700 Subject: [PATCH 14/27] perf(timsseek): drop GNOme from the ProForma ontologies GNOme is 191_529 entries / 26.4 MB of the 27.8 MB mzcore loads, and the build goes from ~2.6 s to ~48 ms without it (debug, measured). The earlier reasoning for keeping it was wrong. It assumed a glycan mod was merely ignored, so that dropping the ontology would turn a usable peptide into an unparseable one. It is not ignored: `modification_to_mod` returns `None` for anything non-Unimod and `parse_sequence_mzcore` propagates that `None` for the entire peptide. GNOme's only effect was to let a `[GNO:...]` sequence parse and then be discarded one step later. PSI-MOD, XL-MOD and RESID stay (~1.4 MB combined) so the formula path in `count_carbon_sulphur_in_sequence` still sees them. The one behavioural difference is that a GNO-accession glycopeptide now takes the averagine isotope fallback instead of a composition envelope -- the documented path, already tallied as `n_averagine_fallback`. The new test pins the reasoning rather than the footprint: Unimod (named, numeric, bare mass) still resolves, the three remaining ontologies still reach mzcore and are still rejected downstream, and `[Glycan:HexNAc]` is untouched because a composition needs no index. --- rust/timsseek/src/models/sequence.rs | 107 ++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 17 deletions(-) diff --git a/rust/timsseek/src/models/sequence.rs b/rust/timsseek/src/models/sequence.rs index ac0bf8ce..567c0b85 100644 --- a/rust/timsseek/src/models/sequence.rs +++ b/rust/timsseek/src/models/sequence.rs @@ -5,23 +5,49 @@ use crate::models::decoy::DecoyMarking; use serde::Serialize; use smallvec::SmallVec; -use std::sync::Arc; +use std::sync::{ + Arc, + OnceLock, +}; -/// Parse a ProForma string against mzcore's shared ontologies. +/// Modification ontologies for ProForma parsing: everything mzcore ships +/// except GNOme. /// -/// mzcore requires an explicit `Ontologies` value at every `pro_forma` call and -/// ships `STATIC_ONTOLOGIES` for exactly this; it is a `LazyLock`, so the -/// ~210 ms / ~200 MB build happens on first use and only on a path that gets -/// here. That matters: this is the fallback *past* the byte-walk fast path in -/// [`parse_sequence`], so a library whose sequences all match the fast grammar -/// never pays it (a real DIA-NN `.speclib` load peaks at ~10 MB and never -/// touches this). The two other callers — `count_carbon_sulphur_in_sequence`'s -/// mzcore fallback and `speclib_build_cli` — do pay it. +/// mzcore's own `STATIC_ONTOLOGIES` loads all six, and GNOme is 191_529 entries +/// / 26.4 MB of the 27.8 MB total. Skipping it takes the build from ~2.6 s to +/// ~48 ms. /// -/// Most of that footprint is GNOme glycan data that [`modification_to_mod`] -/// discards, and mzcore can build a Unimod-only index. Not done here: dropping -/// an ontology also drops the sequences that reference it, turning a mod this -/// code already ignores into a peptide it cannot parse at all. +/// Dropping an ontology normally costs you the sequences that reference it, but +/// not here. A GNO-accession glycopeptide is already unusable: every mod goes +/// through [`modification_to_mod`], which returns `None` for anything +/// non-Unimod, and [`parse_sequence_mzcore`] propagates that `None` for the +/// whole peptide. So GNOme's only effect was to make such a peptide parse and +/// then be discarded one step later. +/// +/// The single behavioural difference is +/// [`count_carbon_sulphur_in_sequence`](crate::fragment_mass::elution_group_converter::count_carbon_sulphur_in_sequence): +/// a `[GNO:...]` sequence no longer yields a composition, so its isotope +/// envelope comes from averagine instead — the documented fallback, already +/// tallied as `n_averagine_fallback`. PSI-MOD, XL-MOD and RESID stay loaded +/// (~1.4 MB combined) so that path is unchanged for them. +fn ontologies() -> &'static mzcore::ontology::Ontologies { + static ONTOLOGIES: OnceLock = OnceLock::new(); + ONTOLOGIES.get_or_init(|| { + let mut ontologies = mzcore::ontology::Ontologies::empty(); + *ontologies.unimod_mut() = mzcv::CVIndex::init_static(); + *ontologies.psimod_mut() = mzcv::CVIndex::init_static(); + *ontologies.xlmod_mut() = mzcv::CVIndex::init_static(); + *ontologies.resid_mut() = mzcv::CVIndex::init_static(); + ontologies + }) +} + +/// Parse a ProForma string against [`ontologies`]. +/// +/// Built on first use, and this is the fallback *past* the byte-walk fast path +/// in [`parse_sequence`] — so a library whose sequences all match the fast +/// grammar never pays for it at all. A real DIA-NN `.speclib` load peaks at +/// ~10 MB and never gets here. /// /// mzcore also returns non-fatal parse warnings alongside the peptidoform; none /// of the callers can act on them, so they are dropped in one place rather than @@ -29,7 +55,7 @@ use std::sync::Arc; pub fn parse_proforma( sequence: &str, ) -> Result, String> { - mzcore::sequence::Peptidoform::pro_forma(sequence, &mzcore::ontology::STATIC_ONTOLOGIES) + mzcore::sequence::Peptidoform::pro_forma(sequence, ontologies()) .map(|(peptidoform, _warnings)| peptidoform) .map_err(|errors| { errors @@ -694,11 +720,58 @@ mod tests { } } + /// [`ontologies`] omits GNOme, which is 95% of what mzcore's own + /// `STATIC_ONTOLOGIES` loads. This pins the reasoning: every non-Unimod + /// mod already yields no parsed sequence, because `modification_to_mod` + /// returns `None` and `parse_sequence_mzcore` propagates it for the whole + /// peptide. Dropping GNOme moves where a `[GNO:...]` sequence fails, not + /// whether it fails. + #[test] + fn dropping_gnome_costs_no_sequence_that_was_usable() { + // Unimod, by name and by id, plus a bare mass: all still resolve. + for usable in [ + "PEPTIDEK", + "PEPTC[UNIMOD:4]IDEK", + "PEPTC[Carbamidomethyl]IDEK", + "PEPT[+79.966]IDEK", + ] { + assert!( + parse_sequence(usable).is_some(), + "{usable:?} must still parse" + ); + } + + // Ontologies still loaded: these reach mzcore, and are then rejected + // by `modification_to_mod` for not being Unimod. Kept loaded so the + // formula path (`count_carbon_sulphur_in_sequence`) still sees them. + for non_unimod in [ + "PEPTK[MOD:00046]IDEK", + "PEPTK[XLMOD:02001]IDEK", + "PEPTK[RESID:AA0038]IDEK", + ] { + assert!( + parse_proforma(non_unimod).is_ok(), + "{non_unimod:?} must still reach mzcore" + ); + assert!( + parse_sequence(non_unimod).is_none(), + "{non_unimod:?} yields no usable sequence either way" + ); + } + + // The one casualty. It failed before this change too, just later. + assert!( + parse_sequence("PEPTN[GNO:G59626AS]IDEK").is_none(), + "a GNO glycopeptide was never usable" + ); + // A glycan *composition* needs no index, so it is unaffected. + assert!(parse_proforma("PEPTN[Glycan:HexNAc]IDEK").is_ok()); + } + /// The fallback must not just accept a named mod — it must resolve it /// through the UNIMOD ontology to the same numeric id the `[UNIMOD:n]` /// spelling yields. This is the one behavior that has no fast-path - /// equivalent, so nothing else covers it; it is also the only test that - /// forces mzcore's `STATIC_ONTOLOGIES` to actually initialize. + /// equivalent, so nothing else covers it. #[test] fn mzcore_fallback_resolves_named_mods_via_ontology() { for (named, expected) in [ From a3ac755b5bc50224015a33e000382c1ea2ca2ba9 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 08:10:58 -0700 Subject: [PATCH 15/27] fix: correct the README speclib name, dedupe the last arena tail The README's build example named the output `vimentin.ndjson`, but the writer always emits zstd, so the resulting file could not be read back. `mzpaf_with_intensities` still inlined the length check and seal that `finish_mzpaf_arena` now owns. --- README.md | 2 +- rust/micromzpaf/src/lib.rs | 2 +- rust/timsquery/src/serde/library_file.rs | 14 +------------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4d5bd63b..6beb14b7 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ integrate other sources of predictions for it.) ```bash DOTD_FILE="$HOME/data/my_data.d" FASTA_FILE="$HOME/fasta/VIMENTIN.fasta" -SPECLIB_NAME="vimentin.ndjson" +SPECLIB_NAME="vimentin.ndjson.zst" RESULTS_DIR="vimentin_search_results" # Build the spectral lib using Koina (Prosit) for fragment/RT prediction. diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index f9533101..488865c3 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -757,7 +757,7 @@ mod tests { ); } - /// `IonAnnot: Default` packs to zero, which decodes to `Kind::None` and a + /// `IonAnnot: Default` packs to zero, which decodes to `IonSeriesOrdinal::None` and a /// charge of 0. `Serialize` renders through `format!`, so a panicking /// `Display` arm is reachable from any serde path — `TinyVec` alone can /// hand out a default. diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index bd28067f..6333a712 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -307,19 +307,7 @@ impl LibraryArena { ); } - if frag_intens.len() != geom.frag_labels.len() { - return Err(LibraryReadingError::SpeclibParse(format!( - "reference-intensity sidecar ({}) must stay parallel to the fragment-label arena ({})", - frag_intens.len(), - geom.frag_labels.len(), - ))); - } - - geom.seal(); - Ok(LibraryArena::Mzpaf { - geom, - frag_intens: Some(frag_intens), - }) + finish_mzpaf_arena(geom, frag_intens) } /// Adapt the legacy [`ElutionGroupCollection`] (produced by the non-speclib From 491a606399dd9904f451cb64caf1bd4a935f1cf5 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 08:43:09 -0700 Subject: [PATCH 16/27] fix: stop panicking on non-ASCII mod bodies, sniff zstd, name the msgpack removal `classify_mod` sliced a library-supplied bracket body at a fixed byte index, which aborts the search when byte 7 lands inside a multi-byte char. `SpeclibReader` now takes the compression from the zstd magic number instead of the file name, which removes `SpeclibFormat` and lets a mislabelled file still load. A leftover `.msgpack.zst` reported "stream did not contain valid UTF-8"; it now names the removed format and how to rebuild. `FileReadingError::path` is an `Option`, so the two sites that had no path say so instead of printing an empty one, and zstd failures get their own variant rather than being laundered through `serde_json::Error::io`. --- rust/timsseek/src/data_sources/speclib.rs | 212 ++++++++++++++-------- rust/timsseek/src/errors.rs | 9 +- rust/timsseek/src/models/sequence.rs | 29 ++- 3 files changed, 168 insertions(+), 82 deletions(-) diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 0511e2a0..d022e183 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -30,7 +30,7 @@ use timsquery::utils::constants::PROTON_MASS; /// The serializable, on-disk form of a native speclib element. Kept backwards /// compatible; the load path builds the columnar `ReferenceLibrary` arena -/// directly from these elements (see `Speclib::from_file_with_format`). +/// directly from these elements (see `Speclib::from_native_file`). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SerSpeclibElement { precursor: PrecursorEntry, @@ -246,61 +246,60 @@ fn finalize_reference_library( /// iterates `RefQuery` flyweights via [`ReferenceLibrary::item_at`]. pub type Speclib = ReferenceLibrary; -#[derive(Debug, Clone, Copy)] -pub enum SpeclibFormat { - NdJson, - NdJsonZstd, -} - -impl SpeclibFormat { - /// Detect a native timsseek format by EXTENSION ONLY. Returns `None` for - /// anything else (including `.speclib` and `.mzSpecLib.txt`), which routes - /// to the timsquery bridge — that is where content sniffing happens. - pub fn detect_from_extension(path: &Path) -> Option { - let path_str = path.to_string_lossy().to_lowercase(); - - // Accept both `.zst` and `.zstd` — DIA-NN/user pipelines use either. - if path_str.ends_with(".ndjson.zst") || path_str.ends_with(".ndjson.zstd") { - Some(SpeclibFormat::NdJsonZstd) - } else if path_str.ends_with(".ndjson") { - Some(SpeclibFormat::NdJson) - } else { - None - } - } +/// Whether `path` names a native timsseek library, by EXTENSION ONLY. +/// +/// This answers *which reader family*, not *which encoding*. Content sniffing +/// cannot answer it: a DIA-NN `.speclib` and a native library are both opaque +/// byte streams, and the point of the extension rule is that a native extension +/// commits to the native reader and surfaces its error rather than falling +/// through the timsquery registry to report some other reader's complaint. +/// Whether the bytes are zstd-wrapped is a separate question, and +/// [`SpeclibReader`] answers that one from the magic number. +fn is_native_extension(path: &Path) -> bool { + let path_str = path.to_string_lossy().to_lowercase(); + // `.zst` and `.zstd` are both in the wild. + let stem = path_str + .strip_suffix(".zst") + .or_else(|| path_str.strip_suffix(".zstd")) + .unwrap_or(path_str.as_str()); + stem.ends_with(".ndjson") } /// Streams raw `SerSpeclibElement`s out of a native timsseek library file. /// /// The native path builds the columnar arena directly from these elements (see -/// `Speclib::from_file_with_format`), so the reader stays at the serializable +/// `Speclib::from_native_file`), so the reader stays at the serializable /// element and does not eagerly build per-row scoring items. /// -/// Both formats are NDJSON; zstd only adds a decoder underneath, so the +/// The payload is always NDJSON; zstd only adds a decoder underneath, so the /// boxing is over the byte source rather than over the line parser. pub struct SpeclibReader<'a> { reader: Box, } +/// Leading bytes of a zstd frame. +const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD]; + impl<'a> SpeclibReader<'a> { - pub fn new( - reader: R, - format: SpeclibFormat, - ) -> Result { - let reader: Box = match format { - SpeclibFormat::NdJson => Box::new(BufReader::new(reader)), - SpeclibFormat::NdJsonZstd => { - let decoder = zstd::Decoder::new(reader).map_err(|e| { - LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error creating ZSTD decoder", - } - })?; - Box::new(BufReader::new(decoder)) - } + /// Compression is detected from the first four bytes, not the file name, so + /// a mislabelled `.ndjson` that is really zstd (or the reverse) still reads. + pub fn new(reader: R) -> Result { + let mut buffered = BufReader::new(reader); + let compressed = buffered + .fill_buf() + .map_err(|source| LibraryReadingError::FileReadingError { + source, + context: "Error reading the start of the speclib", + path: None, + })? + .starts_with(&ZSTD_MAGIC); + + let reader: Box = if compressed { + let decoder = zstd::Decoder::with_buffer(buffered) + .map_err(|source| LibraryReadingError::Decompression { source })?; + Box::new(BufReader::new(decoder)) + } else { + Box::new(buffered) }; Ok(SpeclibReader { reader }) @@ -332,7 +331,7 @@ impl Iterator for SpeclibReader<'_> { return Some(Err(LibraryReadingError::FileReadingError { source: e, context: "Error reading line", - path: PathBuf::new(), + path: None, })); } } @@ -342,7 +341,7 @@ impl Iterator for SpeclibReader<'_> { /// Writes a native timsseek library: one JSON object per line, zstd-wrapped. /// -/// The exact inverse of [`SpeclibReader`] on [`SpeclibFormat::NdJsonZstd`], so +/// The exact inverse of [`SpeclibReader`], so /// what `speclib_build_cli` emits is what `Speclib::from_file` reads back. pub struct SpeclibWriter { encoder: zstd::Encoder<'static, W>, @@ -359,7 +358,7 @@ impl SpeclibWriter { let io_err = |e: std::io::Error| LibraryReadingError::FileReadingError { source: e, context: "Error writing NDJSON", - path: PathBuf::new(), + path: None, }; serde_json::to_writer(&mut self.encoder, elem).map_err(|e| { LibraryReadingError::SpeclibParsingError { @@ -401,17 +400,26 @@ impl Speclib { path: &Path, decoy_policy: crate::models::DecoyPolicy, ) -> Result { - // Native timsseek formats are matched by EXTENSION ONLY: a native - // extension commits to the native reader and surfaces its error. A - // `.speclib` matches no native extension and falls through to the - // bridge -> timsquery registry -> binary reader. - if let Some(format) = SpeclibFormat::detect_from_extension(path) { - tracing::info!( - "Loading native speclib format ({:?}) from {}", - format, - path.display() - ); - return Self::from_file_with_format(path, format, decoy_policy); + // msgpack was removed in the mzcore migration. Without this arm the + // file reaches the JSON reader and is reported as invalid UTF-8, which + // names neither the real problem nor the fix. + let path_str = path.to_string_lossy().to_lowercase(); + if path_str.contains(".msgpack") { + return Err(LibraryReadingError::UnsupportedFormat { + message: format!( + "{}: msgpack speclibs are no longer supported. Rebuild with \ + speclib_build_cli, which now emits .ndjson.zst", + path.display() + ), + }); + } + + // See `is_native_extension`: a native extension commits to the native + // reader and surfaces its error. A `.speclib` matches no native + // extension and falls through to the bridge -> timsquery registry. + if is_native_extension(path) { + tracing::info!("Loading native speclib from {}", path.display()); + return Self::from_native_file(path, decoy_policy); } // Terminal source: bridge to the timsquery reader registry (DIA-NN @@ -435,19 +443,20 @@ impl Speclib { Ok(lib) } - pub fn from_file_with_format( + /// Load a native timsseek library (NDJSON, optionally zstd-wrapped — + /// [`SpeclibReader`] sniffs which). + fn from_native_file( path: &Path, - format: SpeclibFormat, decoy_policy: crate::models::DecoyPolicy, ) -> Result { let file = std::fs::File::open(path).map_err(|e| LibraryReadingError::FileReadingError { source: e, context: "Error opening speclib file", - path: PathBuf::from(path), + path: Some(PathBuf::from(path)), })?; - let reader = SpeclibReader::new(file, format)?; + let reader = SpeclibReader::new(file)?; // Build the columnar arena directly from the streamed elements (same // lazy shape as the `.speclib` path), instead of collecting per-row @@ -556,11 +565,10 @@ mod tests { writer.append(&element).expect("append"); let bytes = writer.finish().expect("finish"); - let read: Vec = - SpeclibReader::new(bytes.as_slice(), SpeclibFormat::NdJsonZstd) - .expect("reader") - .collect::>() - .expect("every record must parse"); + let read: Vec = SpeclibReader::new(bytes.as_slice()) + .expect("reader") + .collect::>() + .expect("every record must parse"); assert_eq!(read.len(), 2); assert_eq!(read[0].precursor.sequence, "PEPTIDEK"); @@ -575,29 +583,73 @@ mod tests { } #[test] - fn test_detect_native_format_by_extension() { + fn native_extensions_route_to_the_native_reader() { use std::path::Path; - // Both .zst and .zstd must map to the native zstd readers. - for ext in ["lib.ndjson.zst", "lib.ndjson.zstd"] { - assert!(matches!( - SpeclibFormat::detect_from_extension(Path::new(ext)), - Some(SpeclibFormat::NdJsonZstd) - )); + for ext in [ + "lib.ndjson", + "lib.ndjson.zst", + "lib.ndjson.zstd", + "LIB.NDJSON", + ] { + assert!(is_native_extension(Path::new(ext)), "{ext} is native"); } - assert!(matches!( - SpeclibFormat::detect_from_extension(Path::new("lib.ndjson")), - Some(SpeclibFormat::NdJson) - )); // Everything else routes to the timsquery bridge, which sniffs by // content. Claiming one of these here would bypass that. - for ext in ["lib.speclib", "lib.mzSpecLib.txt", "lib.tsv"] { + for ext in ["lib.speclib", "lib.mzSpecLib.txt", "lib.tsv", "lib.zst"] { assert!( - SpeclibFormat::detect_from_extension(Path::new(ext)).is_none(), + !is_native_extension(Path::new(ext)), "{ext} must not be claimed as a native format" ); } } + /// Compression is decided by the magic number, so the two encodings are + /// interchangeable regardless of what the file is called. + #[test] + fn the_reader_sniffs_zstd_rather_than_trusting_the_name() { + let plain = b"{\"precursor\":{\"sequence\":\"PEPTIDEK\",\"charge\":2,\"decoy\":false,\ + \"decoy_group\":0},\"elution_group\":{\"id\":0,\"precursor_mz\":500.0,\ + \"precursor_labels\":[],\"fragment_mzs\":[175.1],\ + \"fragment_labels\":[\"y1\"],\"precursor_intensities\":[],\ + \"fragment_intensities\":[1.0],\"mobility_ook0\":0.9,\ + \"rt_seconds\":10.0}}\n"; + + let from_plain: Vec = SpeclibReader::new(&plain[..]) + .expect("uncompressed reader") + .collect::>() + .expect("plain NDJSON parses"); + + let compressed = zstd::encode_all(&plain[..], 3).expect("encode"); + assert!(compressed.starts_with(&ZSTD_MAGIC)); + let from_zstd: Vec = SpeclibReader::new(compressed.as_slice()) + .expect("compressed reader") + .collect::>() + .expect("zstd NDJSON parses"); + + assert_eq!(from_plain.len(), 1); + assert_eq!( + from_plain[0].precursor.sequence, + from_zstd[0].precursor.sequence + ); + } + + /// A leftover `.msgpack.zst` must say what happened, not "invalid UTF-8". + #[test] + fn msgpack_libraries_report_the_format_removal() { + let err = Speclib::from_file( + Path::new("/nonexistent/lib.msgpack.zst"), + crate::models::DecoyPolicy::default(), + ) + .expect_err("msgpack is no longer supported"); + match err { + LibraryReadingError::UnsupportedFormat { message } => { + assert!(message.contains("msgpack"), "{message}"); + assert!(message.contains("speclib_build_cli"), "{message}"); + } + other => panic!("expected UnsupportedFormat, got {other:?}"), + } + } + /// `Speclib` is now a type alias for `ReferenceLibrary` (Task 9 collapsed /// the enum), so a loaded library is already the lazy arena. This identity /// helper is kept so the fixture assertions below read as @@ -1024,7 +1076,7 @@ mod tests { /// native path produces a lazy `ReferenceLibrary` with the right length, target/ /// decoy flags, and per-fragment reference intensities. #[test] - fn from_file_with_format_native_ndjson_builds_lazy_arena() { + fn native_ndjson_load_builds_lazy_arena() { use crate::data_sources::reference_library::ScoredIdentity; let target = SerSpeclibElement::new( diff --git a/rust/timsseek/src/errors.rs b/rust/timsseek/src/errors.rs index d37d2311..f638e72d 100644 --- a/rust/timsseek/src/errors.rs +++ b/rust/timsseek/src/errors.rs @@ -39,7 +39,14 @@ pub enum LibraryReadingError { FileReadingError { source: std::io::Error, context: &'static str, - path: PathBuf, + /// `None` where the failure happens on an already-open stream, which + /// no longer knows where it came from. + path: Option, + }, + /// The zstd frame could not be opened or decoded. Distinct from + /// [`Self::SpeclibParsingError`]: the bytes never became text. + Decompression { + source: std::io::Error, }, TimsQueryLibraryError { source: timsquery::serde::LibraryReadingError, diff --git a/rust/timsseek/src/models/sequence.rs b/rust/timsseek/src/models/sequence.rs index 567c0b85..d9ec1f1e 100644 --- a/rust/timsseek/src/models/sequence.rs +++ b/rust/timsseek/src/models/sequence.rs @@ -235,7 +235,14 @@ pub fn parse_sequence(normalized: &str) -> Option { /// which forces the mzcore fallback in [`parse_sequence`]. fn classify_mod(body: &str) -> Option { let body = body.trim(); - if body.len() >= 7 && body[..7].eq_ignore_ascii_case("UNIMOD:") { + // Match on bytes: `body` is arbitrary text from the library, so `body[..7]` + // would panic on a multi-byte char straddling byte 7. Once the prefix + // matches it is ASCII, so byte 7 is a char boundary and `body[7..]` is safe. + if body + .as_bytes() + .get(..7) + .is_some_and(|p| p.eq_ignore_ascii_case(b"UNIMOD:")) + { return body[7..].trim().parse::().ok().map(Mod::Unimod); } match body.as_bytes().first() { @@ -804,6 +811,26 @@ mod tests { assert_eq!(named, numeric); } + #[test] + fn non_ascii_bracket_bodies_are_rejected_not_panicked_on() { + // Bracket bodies come straight from the library file, so they can hold + // any UTF-8. A verdict of `None` is fine; aborting the search is not. + for s in [ + "PEPT[abcdef√]IDEK", + "PEPT[√]IDEK", + "[abcdef√]-PEPTIDEK", + "PEPTIDEK-[abcdef√]", + "PEPT[unimod:√]IDEK", + ] { + assert!(parse_sequence_fast(s).is_none(), "{s:?} must defer"); + } + // The prefix match stays case-insensitive over the ASCII it accepts. + assert_eq!( + parse_sequence("PEPTC[unimod:4]IDEK"), + parse_sequence("PEPTC[UNIMOD:4]IDEK") + ); + } + #[test] fn fast_matches_mzcore_on_recognized_grammar() { // Differential test: wherever the fast path claims an input, it must From df251595dac8bd75506df995417c80c17730e0a8 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 08:51:04 -0700 Subject: [PATCH 17/27] refactor(timsquery): drop the integer-label deserialization paths `u8` was the first arm of both JSON ladders, so numeric or absent `fragment_labels` always produced `TinyIntLabels`, which `from_elution_groups` rejects. The whole integer half could only ever return `Err`, including the label-synthesis helper and its two tests. Unlabelled input now fails as `MissingFragmentLabels` instead: a label names a series and an ordinal, and a positional index is not one. `try_read_json` propagates that instead of flattening it to `UnableToParseElutionGroups`. --- .../src/serde/elution_group_inputs.rs | 89 ++++++------------- rust/timsquery/src/serde/library_file.rs | 63 ++----------- rust/timsquery/tests/carafe_contract.rs | 6 +- 3 files changed, 36 insertions(+), 122 deletions(-) diff --git a/rust/timsquery/src/serde/elution_group_inputs.rs b/rust/timsquery/src/serde/elution_group_inputs.rs index c1d622e3..e172b975 100644 --- a/rust/timsquery/src/serde/elution_group_inputs.rs +++ b/rust/timsquery/src/serde/elution_group_inputs.rs @@ -13,15 +13,13 @@ pub enum ElutionGroupInputError { expected: usize, found: usize, }, - AlreadyHasFragmentLabels, IonConversionError { inner: String, }, + /// The input shipped no `fragment_labels`. There is no way to synthesize + /// them: a label names an ion series and ordinal, and a positional index + /// carries no chemistry. MissingFragmentLabels, - /// More fragments than a `u8` label space can index uniquely. - TooManyFragmentsToLabel { - count: usize, - }, } /// User-friendly format for specifying elution groups in an input file @@ -42,43 +40,6 @@ pub struct ElutionGroupInput { pub fragment_labels: Option>, } -impl ElutionGroupInput { - pub fn needs_fragment_labels(&self) -> bool { - self.fragment_labels.is_none() - } - - /// Synthesize positional `u8` labels for an input that shipped none. - /// - /// Labels must be unique within the elution group (see - /// `ExpectedIntensities::try_from_pairs`), and `u8` can only index 256 of - /// them. `i as u8` would wrap silently past that — emitting `0,1,..,255,0,1,..` - /// and failing far downstream with a duplicate-key error — so the overflow - /// is rejected here instead. - pub fn try_fill_labels_u8(self) -> Result, ElutionGroupInputError> { - let num_fragments = self.fragments.len(); - if self.fragment_labels.is_some() { - return Err(ElutionGroupInputError::AlreadyHasFragmentLabels); - } - let fragment_labels: Vec = (0..num_fragments) - .map(u8::try_from) - .collect::>() - .map_err(|_| ElutionGroupInputError::TooManyFragmentsToLabel { - count: num_fragments, - })?; - - Ok(ElutionGroupInput { - id: self.id, - mobility: self.mobility, - rt_seconds: self.rt_seconds, - precursor: self.precursor, - precursor_charge: self.precursor_charge, - precursor_isotopes: self.precursor_isotopes, - fragments: self.fragments, - fragment_labels: Some(fragment_labels), - }) - } -} - impl + KeyLike> TryFrom> for TimsElutionGroup { type Error = ElutionGroupInputError; @@ -126,8 +87,9 @@ impl + KeyLike> TryFrom> for Tims #[cfg(test)] mod tests { use super::*; + use crate::ion::IonAnnot; - fn input_with_n_fragments(n: usize) -> ElutionGroupInput { + fn input(fragment_labels: Option>) -> ElutionGroupInput { ElutionGroupInput { id: 0, mobility: 0.8, @@ -135,38 +97,37 @@ mod tests { precursor: 500.0, precursor_charge: 2, precursor_isotopes: None, - fragments: vec![100.0; n], - fragment_labels: None, + fragments: vec![100.0, 200.0], + fragment_labels, } } - /// Synthesized labels must be unique — that is the invariant - /// `ExpectedIntensities::try_from_pairs` relies on. 256 fragments is the - /// exact capacity of the `u8` label space. + /// Unlabelled input is rejected by name rather than by whatever fails + /// first downstream. Nothing can stand in for a missing label. #[test] - fn fill_labels_u8_is_unique_at_capacity() { - let filled = input_with_n_fragments(256) - .try_fill_labels_u8() - .expect("256 fragments fit the u8 label space"); - let labels = filled.fragment_labels.unwrap(); - assert_eq!(labels.len(), 256); - let unique: std::collections::HashSet<_> = labels.iter().copied().collect(); - assert_eq!(unique.len(), 256, "synthesized labels must all be distinct"); + fn missing_fragment_labels_is_its_own_error() { + let err = TimsElutionGroup::::try_from(input(None)) + .expect_err("no labels means no elution group"); + assert!( + matches!(err, ElutionGroupInputError::MissingFragmentLabels), + "expected MissingFragmentLabels, got {err:?}" + ); } - /// One past capacity must fail rather than wrap: a second `0` label would - /// collide with the first and corrupt scoring downstream. #[test] - fn fill_labels_u8_rejects_overflow_instead_of_wrapping() { - let err = input_with_n_fragments(257) - .try_fill_labels_u8() - .expect_err("257 fragments cannot be labelled uniquely with a u8"); + fn label_count_must_match_fragment_count() { + let one_label = vec![IonAnnot::try_from("y1").unwrap()]; + let err = TimsElutionGroup::::try_from(input(Some(one_label))) + .expect_err("1 label for 2 fragments"); assert!( matches!( err, - ElutionGroupInputError::TooManyFragmentsToLabel { count: 257 } + ElutionGroupInputError::MismatchedFragmentLabelsLength { + expected: 2, + found: 1 + } ), - "expected TooManyFragmentsToLabel, got {err:?}" + "expected MismatchedFragmentLabelsLength, got {err:?}" ); } } diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index 6333a712..3f329d46 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -77,8 +77,6 @@ pub enum FileReadingExtras { pub enum ElutionGroupCollection { StringLabels(Vec>, Option), MzpafLabels(Vec>, Option), - TinyIntLabels(Vec>, Option), - IntLabels(Vec>, Option), } impl ElutionGroupCollection { @@ -86,8 +84,6 @@ impl ElutionGroupCollection { match self { ElutionGroupCollection::StringLabels(egs, _) => egs.len(), ElutionGroupCollection::MzpafLabels(egs, _) => egs.len(), - ElutionGroupCollection::TinyIntLabels(egs, _) => egs.len(), - ElutionGroupCollection::IntLabels(egs, _) => egs.len(), } } @@ -99,45 +95,18 @@ impl ElutionGroupCollection { info!("Successfully deserialized elution groups directly"); return Ok(egs); } - // Next try deserialization via inputed format - match Self::try_deser_inputed(&file_content) { - Ok(egs) => { - info!("Successfully deserialized elution groups via inputed format"); - Ok(egs) - } - Err(_) => Err(LibraryReadingError::UnableToParseElutionGroups), - } + // Next try deserialization via inputed format. Its error is returned + // as-is: it names the field that is wrong, which + // `UnableToParseElutionGroups` does not. + let egs = Self::try_deser_inputed(&file_content)?; + info!("Successfully deserialized elution groups via inputed format"); + Ok(egs) } fn try_deser_inputed(content: &str) -> Result { - // We can try from smallest to largest overhead - // Here we try to do the deser into ElutionGroupInput variants first + // mzpaf before string: `"y1"` deserializes as either, and only the + // mzpaf form carries ion chemistry. debug!("Attempting deserialization of elution group inputs"); - debug!("Attempting to deserialize elution group inputs with tiny int labels"); - if let Ok(eg_inputs) = serde_json::from_str::>>(content) { - // Here we can handle filling the inputs if they are needed... - let eg_inputs = if eg_inputs.first().is_some_and(|x| x.needs_fragment_labels()) { - debug!("Filling missing fragment labels with tiny int labels"); - eg_inputs - .into_iter() - .map(|x| x.try_fill_labels_u8()) - .collect::>() - } else { - Ok(eg_inputs) - }; - - let out: Result>, ElutionGroupInputError> = eg_inputs? - .into_iter() - .map( as TryInto>>::try_into) - .collect(); - return Ok(ElutionGroupCollection::TinyIntLabels(out?, None)); - } - debug!("Attempting to deserialize elution group inputs with int labels"); - if let Ok(eg_inputs) = serde_json::from_str::>>(content) { - let out: Result>, ElutionGroupInputError> = - eg_inputs.into_iter().map(|x| x.try_into()).collect(); - return Ok(ElutionGroupCollection::IntLabels(out?, None)); - } debug!("Attempting to deserialize elution group inputs with mzpaf labels"); if let Ok(eg_inputs) = serde_json::from_str::>>(content) { let out: Result>, ElutionGroupInputError> = @@ -154,18 +123,8 @@ impl ElutionGroupCollection { } fn try_deser_direct(content: &str) -> Result { - // We can try from smallest to largest overhead - // Here we try to do the direct deser into ElutionGroupCollection variants - // u8 -> u32 -> IonAnnot -> String + // mzpaf before string, for the same reason as `try_deser_inputed`. debug!("Attempting direct deserialization of elution groups"); - debug!("Attempting to deserialize elution groups with tiny int labels"); - if let Ok(egs) = serde_json::from_str::>>(content) { - return Ok(ElutionGroupCollection::TinyIntLabels(egs, None)); - } - debug!("Attempting to deserialize elution groups with int labels"); - if let Ok(egs) = serde_json::from_str::>>(content) { - return Ok(ElutionGroupCollection::IntLabels(egs, None)); - } debug!("Attempting to deserialize elution groups with mzpaf labels"); if let Ok(egs) = serde_json::from_str::>>(content) { return Ok(ElutionGroupCollection::MzpafLabels(egs, None)); @@ -376,10 +335,6 @@ impl LibraryArena { geom.seal(); Ok(LibraryArena::Str { geom }) } - ElutionGroupCollection::TinyIntLabels(..) | ElutionGroupCollection::IntLabels(..) => { - warn!("integer-labelled libraries have no LibraryArena variant; rejecting"); - Err(LibraryReadingError::UnableToParseElutionGroups) - } } } } diff --git a/rust/timsquery/tests/carafe_contract.rs b/rust/timsquery/tests/carafe_contract.rs index 7bc3144e..2c63fa97 100644 --- a/rust/timsquery/tests/carafe_contract.rs +++ b/rust/timsquery/tests/carafe_contract.rs @@ -64,10 +64,8 @@ fn carafe_target_payload_loads_through_the_public_reader() { let f = write_targets(CARAFE_TARGETS); let arena = read_library_file(f.path()).expect("Carafe's target JSON must load"); - // String fragment labels must resolve to ion annotations. If Carafe ever - // omitted `fragment_labels` the try-chain would silently fall through to - // the integer-labelled variant and synthesize positional labels — a wrong - // answer rather than an error. + // String fragment labels must resolve to ion annotations, not to the + // string-labelled arena, which carries no ion chemistry. let LibraryArena::Mzpaf { geom, .. } = arena else { panic!("labelled targets must land in the ion-annotated arena"); }; From b1a3242e5a3858370a8e34089a9aedf2a6ccbf93 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 08:58:28 -0700 Subject: [PATCH 18/27] refactor(micromzpaf)!: split `Series` out, delete `IonSeriesOrdinal::None` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nine backbone series differed only by a letter but were spelled out in four hand-mirrored tables (`to_parts`, `from_parts`, `from_series_char`, `Display`) — ~36 of ~52 arms carrying no information, kept in agreement only by a round-trip test. They are now one `Series` enum with one letter table, and each of those four sites drops to five arms. That gives `FragmentLabel` the `series()` accessor its own TODO asked for. `IonSeriesOrdinal::None` existed to satisfy a `Default` bound that `#[derive(Default)]` on the `u32` newtype already provides; it had been a field of `IonAnnot` before the packing change, and was not one after. Instead `unknown` takes discriminant 0 and charge is stored biased by one, so the all-zero word — which `tinyvec` can hand out at any time — is `?0` at charge 1. `IonAnnot::default()` is now a real annotation that parses back, rather than a value only `Display` accepted. The packed layout is not a wire format: `IonAnnot` serializes as its mzPAF string, so the charge bias changes no file on disk. Also names `INTERNAL_POS_BITS`/`IMMONIUM_BITS` — the two field widths had four unnamed copies between them — and covers both at their ceilings, which `internal { start: 2, end: 11 }` never reached. --- rust/micromzpaf/src/lib.rs | 376 +++++++++++--------- rust/timsquery/src/lib.rs | 1 + rust/timsquery/src/traits/fragment_label.rs | 12 +- 3 files changed, 214 insertions(+), 175 deletions(-) diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index 488865c3..eb2deb2f 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -25,6 +25,11 @@ //! | immonium | residue index 5b | //! | precursor | unused | //! +//! `unknown` is discriminant 0 and `charge` is stored biased by one, so the +//! all-zero word is the valid annotation `?0` at charge 1. That matters +//! because `IonAnnot: Default` is forced by `tinyvec::Array` and a default can +//! reach any serde path. +//! //! `charge` and `isotope` are zigzag-encoded so they stay signed in 4 bits. //! Their ranges (±7) are far wider than anything observed: the HUPO-PSI corpus //! tops out at charge 3 and isotope 3, with no negative charges at all. Because @@ -87,8 +92,12 @@ pub const CHARGE_MAX: i8 = 7; /// Widest isotope offset the 4-bit zigzag field holds. Observed maximum is 3. pub const ISOTOPE_MIN: i8 = -7; pub const ISOTOPE_MAX: i8 = 7; -/// Widest residue index an internal fragment endpoint holds (6 bits). -pub(crate) const INTERNAL_POS_MAX: u8 = 63; +/// Width of each internal-fragment endpoint inside `payload`. +const INTERNAL_POS_BITS: u32 = 6; +/// Width of the immonium residue index inside `payload`. +const IMMONIUM_BITS: u32 = 5; +/// Widest residue index an internal fragment endpoint holds. +pub(crate) const INTERNAL_POS_MAX: u8 = mask(INTERNAL_POS_BITS) as u8; #[inline] const fn mask(bits: u32) -> u32 { @@ -105,6 +114,21 @@ const fn unzigzag(u: u32) -> i8 { (((u >> 1) as i32) ^ -((u & 1) as i32)) as i8 } +/// Charge is stored biased by one, so the zero field decodes to charge 1. +/// +/// Charge 0 is rejected by every constructor, so it is not a value the field +/// needs to represent — and spending the zero word on it would make +/// `IonAnnot::default()` render an annotation that cannot be parsed back. +/// `CHARGE_MIN..=CHARGE_MAX` minus one still zigzags inside 4 bits. +#[inline] +const fn zigzag_charge(charge: i8) -> u32 { + zigzag(charge - 1) +} +#[inline] +const fn unzigzag_charge(u: u32) -> i8 { + unzigzag(u) + 1 +} + /// Compact representation of fragment annotations. /// /// A packed `u32`; see the crate docs for the bit layout. Ordering is by the @@ -241,7 +265,7 @@ impl IonAnnot { debug_assert!(payload <= mask(PAYLOAD_BITS), "payload overflows its field"); Ok(IonAnnot( (kind << KIND_SHIFT) - | ((zigzag(charge) & mask(CHARGE_BITS)) << CHARGE_SHIFT) + | ((zigzag_charge(charge) & mask(CHARGE_BITS)) << CHARGE_SHIFT) | ((zigzag(isotope) & mask(ISOTOPE_BITS)) << ISOTOPE_SHIFT) | ((loss as u32 & mask(LOSS_BITS)) << LOSS_SHIFT) | ((payload & mask(PAYLOAD_BITS)) << PAYLOAD_SHIFT), @@ -255,7 +279,7 @@ impl IonAnnot { #[inline] pub fn get_charge(&self) -> i8 { - unzigzag((self.0 >> CHARGE_SHIFT) & mask(CHARGE_BITS)) + unzigzag_charge((self.0 >> CHARGE_SHIFT) & mask(CHARGE_BITS)) } #[inline] @@ -297,20 +321,19 @@ impl IonAnnot { pub fn try_get_ordinal(&self) -> Option { use IonSeriesOrdinal as S; match self.series_ordinal() { - S::a { ordinal } - | S::b { ordinal } - | S::c { ordinal } - | S::d { ordinal } - | S::v { ordinal } - | S::w { ordinal } - | S::x { ordinal } - | S::y { ordinal } - | S::z { ordinal } => Some(ordinal), - S::unknown { .. } - | S::precursor - | S::internal { .. } - | S::immonium { .. } - | S::None => None, + S::backbone { ordinal, .. } => Some(ordinal), + S::unknown { .. } | S::precursor | S::internal { .. } | S::immonium { .. } => None, + } + } + + /// The backbone series this annotation belongs to, if any. + /// + /// `None` for precursor, unknown, internal and immonium ions, none of + /// which sit on a backbone ladder. + pub fn try_get_series(&self) -> Option { + match self.series_ordinal() { + IonSeriesOrdinal::backbone { series, .. } => Some(series), + _ => None, } } @@ -567,40 +590,74 @@ impl UnknownIonCounter { } } +/// A backbone fragment ion series. +/// +/// The nine mzPAF backbone series differ only by their letter, so they are one +/// enum with one letter table rather than nine variants repeated across every +/// match in this module. +#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] +#[allow(non_camel_case_types)] +#[repr(u8)] +pub enum Series { + a = 1, + b, + c, + d, + v, + w, + x, + y, + z, +} + +impl Series { + /// Every series, in discriminant order. Parallel to [`Self::CHARS`]. + pub const ALL: [Self; 9] = [ + Self::a, + Self::b, + Self::c, + Self::d, + Self::v, + Self::w, + Self::x, + Self::y, + Self::z, + ]; + /// The mzPAF letters, in discriminant order. The single place the + /// letter↔discriminant pairing lives. + const CHARS: &'static [u8; 9] = b"abcdvwxyz"; + + /// The mzPAF letter for this series. + pub const fn as_char(self) -> char { + Self::CHARS[self as usize - 1] as char + } + + /// The series for an mzPAF letter, or `None` if it names no backbone series. + fn from_char(c: char) -> Option { + let idx = Self::CHARS.iter().position(|&b| b as char == c)?; + Some(Self::ALL[idx]) + } +} + +impl Display for Series { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_char()) + } +} + /// The logical series-and-payload view of an [`IonAnnot`]. /// /// This is a *view*: `IonAnnot` stores a packed word and reconstructs this on /// demand. Constructing one directly does not allocate an annotation. -#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Default)] +#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy)] #[allow(non_camel_case_types)] pub enum IonSeriesOrdinal { - a { - ordinal: u8, - }, - b { - ordinal: u8, - }, - c { - ordinal: u8, - }, - d { - ordinal: u8, - }, - v { - ordinal: u8, - }, - w { - ordinal: u8, - }, - x { - ordinal: u8, - }, - y { - ordinal: u8, - }, - z { + /// One of the nine backbone series, at position `ordinal` in the ladder. + backbone { + series: Series, ordinal: u8, }, + /// An unannotated peak. `ordinal` is a uniqueness counter, not a position. unknown { ordinal: u8, }, @@ -614,10 +671,6 @@ pub enum IonSeriesOrdinal { immonium { residue: char, }, - - /// This variant should not be used directly ... its mainly added to satisfy trait constraints by TinyVec - #[default] - None, } impl IonSeriesOrdinal { @@ -625,52 +678,43 @@ impl IonSeriesOrdinal { /// [`IonAnnot`] packs. /// /// This and [`Self::from_parts`] are the only place the numbering lives. - /// Both are exhaustive over this enum, so adding a variant is a compile - /// error here rather than a silently mislabelled ion series. + /// The nine backbone series share one arm, so their discriminants come + /// from [`Series`] itself and cannot drift out of step with their letters. const fn to_parts(self) -> (u32, u32) { match self { - Self::a { ordinal } => (1, ordinal as u32), - Self::b { ordinal } => (2, ordinal as u32), - Self::c { ordinal } => (3, ordinal as u32), - Self::d { ordinal } => (4, ordinal as u32), - Self::v { ordinal } => (5, ordinal as u32), - Self::w { ordinal } => (6, ordinal as u32), - Self::x { ordinal } => (7, ordinal as u32), - Self::y { ordinal } => (8, ordinal as u32), - Self::z { ordinal } => (9, ordinal as u32), + Self::backbone { series, ordinal } => (series as u32, ordinal as u32), + // Discriminant 0, so the all-zero word — `IonAnnot::default()` — + // is the unknown ion `?0` rather than an undecodable value. + Self::unknown { ordinal } => (0, ordinal as u32), Self::precursor => (10, 0), - Self::unknown { ordinal } => (11, ordinal as u32), - Self::internal { start, end } => (12, (start as u32) | ((end as u32) << 6)), + Self::internal { start, end } => { + (12, (start as u32) | ((end as u32) << INTERNAL_POS_BITS)) + } Self::immonium { residue } => (13, (residue as u8 - b'A') as u32), - Self::None => (0, 0), } } - /// Inverse of [`Self::to_parts`]. An unrecognised discriminant decodes to - /// [`Self::None`] rather than panicking: it can only come from a - /// corrupted word, and the render path must stay total. + /// Inverse of [`Self::to_parts`]. Total by construction: `unknown` is the + /// catch-all discriminant, so a value this build does not recognise — only + /// reachable from a corrupted word — decodes as an unknown ion instead of + /// panicking on a path `Display` (and therefore `Serialize`) reaches. const fn from_parts(kind: u32, payload: u32) -> Self { let ordinal = payload as u8; match kind { - 1 => Self::a { ordinal }, - 2 => Self::b { ordinal }, - 3 => Self::c { ordinal }, - 4 => Self::d { ordinal }, - 5 => Self::v { ordinal }, - 6 => Self::w { ordinal }, - 7 => Self::x { ordinal }, - 8 => Self::y { ordinal }, - 9 => Self::z { ordinal }, + 1..=9 => Self::backbone { + // `kind` is in range, so this is the inverse of `series as u32`. + series: Series::ALL[kind as usize - 1], + ordinal, + }, 10 => Self::precursor, - 11 => Self::unknown { ordinal }, 12 => Self::internal { - start: (payload & mask(6)) as u8, - end: ((payload >> 6) & mask(6)) as u8, + start: (payload & mask(INTERNAL_POS_BITS)) as u8, + end: ((payload >> INTERNAL_POS_BITS) & mask(INTERNAL_POS_BITS)) as u8, }, 13 => Self::immonium { - residue: (b'A' + (payload & mask(5)) as u8) as char, + residue: (b'A' + (payload & mask(IMMONIUM_BITS)) as u8) as char, }, - _ => Self::None, + _ => Self::unknown { ordinal }, } } @@ -687,51 +731,26 @@ impl IonSeriesOrdinal { }; } let ordinal = ordinal.ok_or(IonParsingError::MissingOrdinal { series: c })?; - Ok(match c { - 'a' => Self::a { ordinal }, - 'b' => Self::b { ordinal }, - 'c' => Self::c { ordinal }, - 'd' => Self::d { ordinal }, - 'v' => Self::v { ordinal }, - 'w' => Self::w { ordinal }, - 'x' => Self::x { ordinal }, - 'y' => Self::y { ordinal }, - 'z' => Self::z { ordinal }, - '?' => Self::unknown { ordinal }, - _ => { - return Err(IonParsingError::UnsupportedFragmentType { fragment_type: c }); - } - }) + if c == '?' { + return Ok(Self::unknown { ordinal }); + } + Series::from_char(c) + .map(|series| Self::backbone { series, ordinal }) + .ok_or(IonParsingError::UnsupportedFragmentType { fragment_type: c }) } } impl Display for IonSeriesOrdinal { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - IonSeriesOrdinal::a { ordinal } => write!(f, "a{}", ordinal), - IonSeriesOrdinal::b { ordinal } => write!(f, "b{}", ordinal), - IonSeriesOrdinal::c { ordinal } => write!(f, "c{}", ordinal), - IonSeriesOrdinal::d { ordinal } => write!(f, "d{}", ordinal), - IonSeriesOrdinal::v { ordinal } => write!(f, "v{}", ordinal), - IonSeriesOrdinal::w { ordinal } => write!(f, "w{}", ordinal), - IonSeriesOrdinal::x { ordinal } => write!(f, "x{}", ordinal), - IonSeriesOrdinal::y { ordinal } => write!(f, "y{}", ordinal), - IonSeriesOrdinal::z { ordinal } => write!(f, "z{}", ordinal), - IonSeriesOrdinal::unknown { ordinal } => write!(f, "?{}", ordinal), - IonSeriesOrdinal::precursor => write!(f, "p"), - IonSeriesOrdinal::internal { start, end } => write!(f, "m{}:{}", start, end), - IonSeriesOrdinal::immonium { residue } => write!(f, "I{}", residue), - // Reached only via `IonAnnot::default()`, which packs to zero. - // That `Default` is not optional: `tinyvec::Array` requires - // `Item: Default`, `TimsElutionGroup` stores labels in a - // `TinyVec<[T; 13]>`, and timsquery's `KeyLike` propagates the - // bound. Since `Serialize` renders through `format!`, panicking - // here is reachable from any serde path — so render inertly. - IonSeriesOrdinal::None => write!(f, "?0"), + Self::backbone { series, ordinal } => write!(f, "{}{}", series.as_char(), ordinal), + Self::unknown { ordinal } => write!(f, "?{}", ordinal), + Self::precursor => write!(f, "p"), + Self::internal { start, end } => write!(f, "m{}:{}", start, end), + Self::immonium { residue } => write!(f, "I{}", residue), } } } - #[cfg(test)] mod tests { use super::*; @@ -749,31 +768,22 @@ mod tests { assert_eq!(size_of::<(IonAnnot, f32)>(), 8); } - #[test] - fn series_ordinal_is_a_view_of_the_packed_word() { - assert_eq!( - ion("b12").series_ordinal(), - IonSeriesOrdinal::b { ordinal: 12 } - ); - } - - /// `IonAnnot: Default` packs to zero, which decodes to `IonSeriesOrdinal::None` and a - /// charge of 0. `Serialize` renders through `format!`, so a panicking - /// `Display` arm is reachable from any serde path — `TinyVec` alone can - /// hand out a default. + /// `IonAnnot: Default` is not optional — `tinyvec::Array` requires + /// `Item: Default`, `TimsElutionGroup` stores labels in a `TinyVec`, and + /// timsquery's `KeyLike` propagates the bound. So a default can reach any + /// serde path, and the zero word has to mean something. /// - /// The rendered value is degenerate and deliberately does NOT round-trip: - /// charge 0 is rejected by every constructor. Not panicking is the - /// property being pinned. + /// It means `?0`: charge is stored as `zigzag(charge - 1)`, so the zero + /// field is charge 1 rather than the impossible charge 0. The default is + /// therefore a real annotation and `Serialize`/`Deserialize` are inverses + /// on it, instead of rendering a value no constructor accepts. #[test] - fn default_annotation_renders_instead_of_panicking() { + fn the_default_annotation_is_a_real_annotation() { let d = IonAnnot::default(); - assert_eq!(d.to_string(), "?0^0"); - assert_eq!(serde_json::to_string(&d).unwrap(), "\"?0^0\""); - assert!( - IonAnnot::try_from("?0^0").is_err(), - "the default is not a valid annotation, only a printable one" - ); + assert_eq!(d.to_string(), "?0"); + assert_eq!(d.get_charge(), 1); + assert_eq!(serde_json::to_string(&d).unwrap(), "\"?0\""); + assert_eq!(IonAnnot::try_from("?0").expect("the default parses"), d); } #[test] @@ -928,32 +938,42 @@ mod tests { assert_eq!(IonAnnot::try_from("y1/-0.0005").unwrap(), ion("y1")); } - /// One representative of every `IonSeriesOrdinal` variant, each with a - /// distinct payload so a transposition in `to_parts`/`from_parts` cannot - /// cancel out. - const ALL_SERIES: &[IonSeriesOrdinal] = &[ - IonSeriesOrdinal::a { ordinal: 1 }, - IonSeriesOrdinal::b { ordinal: 2 }, - IonSeriesOrdinal::c { ordinal: 3 }, - IonSeriesOrdinal::d { ordinal: 4 }, - IonSeriesOrdinal::v { ordinal: 5 }, - IonSeriesOrdinal::w { ordinal: 6 }, - IonSeriesOrdinal::x { ordinal: 7 }, - IonSeriesOrdinal::y { ordinal: 8 }, - IonSeriesOrdinal::z { ordinal: 9 }, - IonSeriesOrdinal::precursor, - IonSeriesOrdinal::unknown { ordinal: 10 }, - IonSeriesOrdinal::internal { start: 2, end: 11 }, - IonSeriesOrdinal::immonium { residue: 'W' }, - IonSeriesOrdinal::None, - ]; + /// One representative of every `IonSeriesOrdinal` case: all nine backbone + /// series (each with a distinct ordinal, so a transposition in + /// `to_parts`/`from_parts` cannot cancel out) plus the four others, at + /// their field boundaries. + fn all_series() -> Vec { + let mut out: Vec = Series::ALL + .iter() + .enumerate() + .map(|(i, &series)| IonSeriesOrdinal::backbone { + series, + ordinal: i as u8 + 1, + }) + .collect(); + out.extend([ + IonSeriesOrdinal::precursor, + IonSeriesOrdinal::unknown { ordinal: 10 }, + IonSeriesOrdinal::internal { start: 2, end: 11 }, + // Both endpoints at their 6-bit ceiling: the widest payload the + // 12 bits hold, and the case a narrowed field would silently clip. + IonSeriesOrdinal::internal { + start: INTERNAL_POS_MAX, + end: INTERNAL_POS_MAX, + }, + IonSeriesOrdinal::immonium { residue: 'A' }, + // Highest residue index the 5-bit immonium field must hold. + IonSeriesOrdinal::immonium { residue: 'Z' }, + ]); + out + } /// `to_parts` and `from_parts` are hand-written inverses. Without this, /// swapping two arms (`v` encoding as `w`) mislabels a whole ion series /// and every other test still passes. #[test] fn every_series_variant_round_trips_through_the_packed_word() { - for &series in ALL_SERIES { + for series in all_series() { let (kind, payload) = series.to_parts(); assert_eq!(IonSeriesOrdinal::from_parts(kind, payload), series); assert!(kind <= mask(KIND_BITS), "{series:?} kind overflows"); @@ -963,27 +983,26 @@ mod tests { ); } - let mut kinds: Vec = ALL_SERIES.iter().map(|s| s.to_parts().0).collect(); + // Every backbone series must land on its own discriminant; the other + // four are singletons and are covered by the round trip above. + let mut kinds: Vec = Series::ALL + .iter() + .map(|&series| { + IonSeriesOrdinal::backbone { series, ordinal: 1 } + .to_parts() + .0 + }) + .collect(); kinds.sort_unstable(); kinds.dedup(); - assert_eq!( - kinds.len(), - ALL_SERIES.len(), - "two series share a discriminant" - ); + assert_eq!(kinds.len(), Series::ALL.len(), "two series share a kind"); } - /// The other three tables — `Display`, `from_series_char` and the parser — - /// must agree with the packing for every variant, not just the handful the - /// other tests happen to spell out. + /// `Display`, `from_series_char` and the parser must agree with the + /// packing for every case, not just the handful the other tests spell out. #[test] fn every_series_variant_round_trips_through_its_mzpaf_spelling() { - for &series in ALL_SERIES { - // `None` is the `Default` filler; it renders inertly but is not a - // real annotation, so it has no spelling to parse back. - if series == IonSeriesOrdinal::None { - continue; - } + for series in all_series() { let annot = IonAnnot::pack(series, NeutralLoss::None, 1, 0).expect("valid"); assert_eq!(annot.series_ordinal(), series); let text = annot.to_string(); @@ -991,6 +1010,19 @@ mod tests { } } + /// The letters are the mzPAF spelling of the discriminants, and + /// `from_char`/`as_char` are hand-written inverses of each other. + #[test] + fn series_letters_and_discriminants_agree() { + assert_eq!(Series::ALL.len(), Series::CHARS.len()); + for &series in &Series::ALL { + assert_eq!(Series::from_char(series.as_char()), Some(series)); + } + for c in ['p', '?', 'm', 'I', 'q', 'A'] { + assert_eq!(Series::from_char(c), None, "{c} is not a backbone series"); + } + } + /// An unrepresentable loss must fail loudly. Parsing `y1-HCOOH` as plain /// `y1` would put a loss peak's m/z on the `y1` label and collide with the /// real `y1`. diff --git a/rust/timsquery/src/lib.rs b/rust/timsquery/src/lib.rs index f5fb8b35..f3223ee0 100644 --- a/rust/timsquery/src/lib.rs +++ b/rust/timsquery/src/lib.rs @@ -58,6 +58,7 @@ pub mod ion { IonAnnot, IonParsingError, IonSeriesOrdinal, + Series, UnknownIonCounter, }; } diff --git a/rust/timsquery/src/traits/fragment_label.rs b/rust/timsquery/src/traits/fragment_label.rs index 9abf2f0a..dca01e1d 100644 --- a/rust/timsquery/src/traits/fragment_label.rs +++ b/rust/timsquery/src/traits/fragment_label.rs @@ -1,5 +1,6 @@ use crate::IonAnnot; use crate::traits::KeyLike; +use micromzpaf::Series; use std::sync::Arc; /// Capability to apply a decoy m/z shift to a fragment label's m/z. @@ -20,11 +21,12 @@ pub trait DecoyShift { /// `FragmentLabel`. NOTE: decoy m/z shifting is gated on `DecoyShift`, not this /// trait — a decoy variant computes its shift through `DecoyShift::decoy_shift_mz`, /// which every label implements. -// TODO(fragment-features): add series() accessor once a standalone series -// enum exists (today series and ordinal are fused in `IonSeriesOrdinal`), and -// wire this bound into the score that consumes it. +// TODO(fragment-features): wire this bound into the score that consumes it. pub trait FragmentLabel: KeyLike + DecoyShift { fn try_get_ordinal(&self) -> Option; + /// The backbone series, or `None` for ions that sit on no ladder + /// (precursor, unknown, internal, immonium). + fn try_get_series(&self) -> Option; fn get_charge(&self) -> i8; } @@ -45,6 +47,10 @@ impl FragmentLabel for IonAnnot { IonAnnot::try_get_ordinal(self) } + fn try_get_series(&self) -> Option { + IonAnnot::try_get_series(self) + } + fn get_charge(&self) -> i8 { IonAnnot::get_charge(self) } From 63b2d6d5596b928ccb77726160763f57d8423328 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 09:04:55 -0700 Subject: [PATCH 19/27] fix(timsquery): scope mzSpecLib attribute groups to their block, fix the tally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[n]` group ids are block-scoped in mzSpecLib, but every block's attributes landed in one bag keyed on the id alone. `spectronaut.mzSpecLib.txt` already uses `[2]` for the RT+unit pair in its Spectrum block and for the NCBI TaxID in its Analyte block; it works only because the latter carries no unit term. Reading the wrong unit is a silent 60x error in the RT. `find` now also prefers the Spectrum block instead of trusting file order, and attributes in an unapplied `` block are counted rather than dropped silently. The tally had three defects. A malformed spectrum was reported as malformed AND as whatever the RT/mobility counters had already recorded on the way to the bail-out. A collided peak was counted as both kept and dropped. And a DIA-NN library warned "9 spectra using a drift time as mobility" on every load, because DIA-NN writes `0.0` there to mean "unset" — so the clean-load branch was unreachable for the format. Annotated peaks with no `/error` suffix were stored at observed m/z with no counter, which is the one case that produces the observed/theoretical mixture the module header forbids. Now counted. A malformed suffix reports as malformed rather than as ambiguous. Documents that `MS:1003072|spectrum origin type` — which declares whether the peak m/z are observed at all — and the `DECOY` attribute set are both unimplemented, rather than leaving the header's blanket claim standing. --- rust/timsquery/src/serde/mzspeclib_io.rs | 336 ++++++++++++++++++++--- 1 file changed, 298 insertions(+), 38 deletions(-) diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs index 561d26eb..2e7dd6df 100644 --- a/rust/timsquery/src/serde/mzspeclib_io.rs +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -17,26 +17,43 @@ //! //! # Peak resolution //! -//! Peak lists carry *observed* m/z; the annotation's mass-error suffix recovers -//! the theoretical value the arena wants (`theoretical = observed - error`). So -//! a theoretical mass only exists once a single identity is pinned, which sorts -//! peaks three ways: +//! The arena wants *theoretical* m/z. A peak list may carry either: whether it +//! does is declared by `MS:1003072|spectrum origin type`, which this reader +//! does not consult (see below). What it uses instead is the annotation's +//! mass-error suffix, `theoretical = observed - error` — correct for an +//! observed list, and a no-op on a theoretical one, where the error is `0.0`. //! -//! | | kept | m/z | +//! So a theoretical mass exists only once a single identity is pinned: +//! +//! | annotation | kept | m/z | //! |---|---|---| //! | resolved, representable | real label | theoretical | //! | resolved, not representable (`y1-HCOOH`) | unknown label | theoretical | -//! | unannotated (`?`) or tied ambiguity | no | — | +//! | resolved, no `/error` suffix | as above | observed, and counted | +//! | unannotated (`?`), tied ambiguity, malformed suffix | no | — | //! -//! Row three is skipped rather than stored at observed m/z: an arena mixing -//! observed and theoretical masses would be invisible downstream. Row two is -//! kept because a known-but-unspellable identity still has an exact mass — only -//! the label is lost. +//! A peak with no single identity is skipped rather than stored at observed +//! m/z: an arena mixing observed and theoretical masses would be invisible +//! downstream. A known-but-unspellable identity is kept because it still has an +//! exact mass — only the label is lost. The suffix is optional in mzPAF, so the +//! third row is possible and is the one case where the mixture does happen; +//! `kept_at_observed_mz` is how it shows up. //! //! Ambiguous (comma-separated) annotations take the alternative with the //! smallest absolute mass error. If that one is unrepresentable the peak gets an //! unknown label rather than falling back to a worse-matching representable //! alternative, which would assign both a wrong identity and a wrong mass. +//! +//! # Not implemented +//! +//! - **`MS:1003072|spectrum origin type`.** It distinguishes `MS:1003073` +//! (observed), `MS:1003074` (predicted) and `MS:1003424` (theoretical m/z, +//! observed intensity), i.e. exactly whether the subtraction above is needed. +//! Both vendored fixtures write `/0.0` throughout, so the subtraction is a +//! no-op on them either way. +//! - **Decoys.** `` + `MS:1003212` marks decoy +//! spectra in SpectraST exports. Every row here is pushed as a target; see +//! `ignored_attribute_set_entries`. use crate::ion::{ IonAnnot, @@ -128,6 +145,14 @@ macro_rules! anomaly_counters { fn anomalies(&self) -> impl Iterator { [ $( ($label, self.$field), )+ ].into_iter() } + + /// Fold `other` in. Used to hold a spectrum's counts aside until + /// it is known to be kept, so a dropped spectrum does not also + /// report the peaks it would have contributed. + fn merge(&mut self, other: &Self) { + self.kept_annotated += other.kept_annotated; + $( self.$field += other.$field; )+ + } } }; } @@ -140,6 +165,14 @@ anomaly_counters! { skipped_unannotated => "skipped as unannotated", /// Comma-separated alternatives that tied on absolute mass error. skipped_ambiguous => "skipped as ambiguous", + /// An `/error` suffix that would not parse. The peak has no recoverable + /// mass, so it cannot be stored at any label. + skipped_malformed_mass_error => "skipped for a malformed mass error", + /// Peaks kept at their OBSERVED m/z because the annotation carried no + /// `/error` suffix (which mzPAF makes optional). Everything else in the + /// arena is theoretical, so this is the one counter that means the arena + /// mixes the two. + kept_at_observed_mz => "kept at observed m/z (no mass-error suffix)", /// Peaks dropped because their label collided with one already in the /// precursor. dropped_duplicate_label => "dropped for a duplicate label", @@ -157,6 +190,11 @@ anomaly_counters! { dropped_malformed_spectrum => "spectra dropped as malformed", /// Precursors dropped for having no usable peak left. dropped_empty_precursors => "precursors dropped as empty", + /// Attributes inside an `` block, which this reader does + /// not apply. Both vendored fixtures have these blocks empty; a writer + /// that hoists an RT unit group or a DECOY marker into one would otherwise + /// lose it with no trace. + ignored_attribute_set_entries => "attribute-set entries ignored", } /// One spectrum converted into the shape [`QueryCollection::push_row`] takes. @@ -195,20 +233,51 @@ impl MzSpecLibStats { } } -/// One `ACC|name=value` attribute, with its optional `[n]` group tag. +/// Which `<...>` block an attribute was written in. +/// +/// `[n]` group ids are scoped to their block, so two blocks can both use `[2]` +/// for unrelated groups — `spectronaut.mzSpecLib.txt` already does. Carrying +/// the block makes a group id unique within a spectrum. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BlockId { + Spectrum, + Analyte(u32), + Interpretation(u32), +} + +impl BlockId { + /// Parse a ``-style header. `None` for a header that opens no + /// attribute block (``, ``). + fn parse(header: &str) -> Option { + let inner = header.strip_prefix('<')?.strip_suffix('>')?; + if inner == "Spectrum" || inner.starts_with("Spectrum=") { + return Some(Self::Spectrum); + } + let (kind, id) = inner.split_once('=')?; + let id = id.parse().ok()?; + match kind { + "Analyte" => Some(Self::Analyte(id)), + "Interpretation" => Some(Self::Interpretation(id)), + _ => None, + } + } +} + +/// One `ACC|name=value` attribute, with its `[n]` group tag scoped to the +/// block it was written in. #[derive(Debug, Clone)] struct Attr { - group: Option, + group: Option<(BlockId, u32)>, accession: String, value: String, } impl Attr { - fn parse(line: &str) -> Option { + fn parse(line: &str, block: BlockId) -> Option { let (group, rest) = match line.strip_prefix('[') { Some(r) => { let (g, r) = r.split_once(']')?; - (Some(g.parse().ok()?), r) + (Some((block, g.parse().ok()?)), r) } None => (None, line), }; @@ -233,8 +302,13 @@ impl Attr { struct AttrBag(Vec); impl AttrBag { + /// Spectrum-block attributes win over Analyte/Interpretation ones. Without + /// this the answer would depend on which block the writer emitted first. fn find(&self, accession: &str) -> Option<&Attr> { - self.0.iter().find(|a| a.accession == accession) + let matching = || self.0.iter().filter(|a| a.accession == accession); + matching() + .find(|a| a.group.is_none_or(|(b, _)| b == BlockId::Spectrum)) + .or_else(|| matching().next()) } fn first_of(&self, accessions: &[&str]) -> Option<&Attr> { @@ -246,6 +320,10 @@ impl AttrBag { } /// The unit term attached to `attr` via its `[n]` group, if any. + /// + /// The group is matched on `(block, id)`, not `id` alone: an unrelated + /// `[2]` in the Analyte block must not supply the unit for a `[2]` in the + /// Spectrum block. Getting that wrong is a silent 60x RT error. fn unit_for(&self, attr: &Attr) -> Option<&str> { let group = attr.group?; self.0 @@ -271,6 +349,9 @@ enum SkipReason { /// Alternatives that pin no single identity, so likewise no theoretical /// m/z. Storing the observed one would mix the two. Ambiguous, + /// The `/error` suffix was present but unparseable. Distinct from + /// `Ambiguous`: one malformed annotation is malformed, not ambiguous. + MalformedMassError, } /// What resolving one peak's annotation produced. @@ -302,7 +383,7 @@ fn resolve_annotation(annotation: &str) -> Resolved { let mut alternatives = Vec::new(); for alt in annotation.split(',') { let Ok(parsed) = split_mass_error(alt.trim()) else { - return Resolved::Skip(SkipReason::Ambiguous); + return Resolved::Skip(SkipReason::MalformedMassError); }; alternatives.push(parsed); } @@ -347,10 +428,20 @@ fn resolve_annotation(annotation: &str) -> Resolved { /// spectra for a missing charge still reports "all annotated and /// representable", which is the one thing this module exists to prevent. fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { - let Some(row) = spectrum_row(raw, stats) else { + // A malformed spectrum bails out of `spectrum_row` via `?`, possibly after + // incrementing RT/mobility counters on the way. Those are held aside and + // discarded on the bail-out, so it is reported once as malformed rather + // than also as "without an RT" and "using a drift time". + // + // An *empty* precursor is structurally fine, so its per-peak counts are + // kept: they are the explanation for why it came out empty. + let mut local = MzSpecLibStats::default(); + let Some(row) = spectrum_row(raw, &mut local) else { stats.dropped_malformed_spectrum += 1; return None; }; + stats.merge(&local); + if row.frags.is_empty() { stats.dropped_empty_precursors += 1; return None; @@ -390,8 +481,15 @@ fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option a.value.parse().ok()?, None => match raw.attrs.find(MOBILITY_DRIFT_TIME) { Some(a) => { - stats.spectra_with_drift_time_mobility += 1; - a.value.parse().ok()? + let drift: f32 = a.value.parse().ok()?; + // DIA-NN writes `MS:1002476|ion mobility drift time=0.0` on + // every spectrum, meaning "unset". Counting that as a real + // drift time warns on every DIA-NN load and buries the case + // this counter exists for. + if drift != 0.0 { + stats.spectra_with_drift_time_mobility += 1; + } + drift } // Absent is fine — an unset mobility is 0.0, same as DIA-NN writes. // A *present but malformed* one drops the spectrum instead. @@ -425,16 +523,10 @@ fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { - stats.kept_annotated += 1; - (ion, error) - } + let (label, mass_error, annotated) = match resolve_annotation(annotation) { + Resolved::Annotated(ion, error) => (ion, error, true), Resolved::UnknownLabel(error) => match unknown_ions.next(1) { - Ok(ion) => { - stats.kept_unknown_label += 1; - (ion, error) - } + Ok(ion) => (ion, error, false), Err(_) => { stats.dropped_unknown_over_capacity += 1; continue; @@ -448,21 +540,39 @@ fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option e.theoretical_from_observed(*observed_mz), - None => *observed_mz, + Resolved::Skip(SkipReason::MalformedMassError) => { + stats.skipped_malformed_mass_error += 1; + continue; + } }; // Labels must stay unique within the precursor (`linear_get` is - // first-match), so a collision drops the later peak. Unknown labels - // come off a monotonic counter and cannot collide; this only ever - // fires for annotated ones. + // first-match), so a collision drops the later peak. Checked before + // the kept counters: otherwise a collided peak is tallied as both + // kept and dropped. Unknown labels come off a monotonic counter and + // cannot collide; this only ever fires for annotated ones. if frags.iter().any(|(l, _)| *l == label) { stats.dropped_duplicate_label += 1; continue; } + + let mz = match mass_error { + Some(e) => e.theoretical_from_observed(*observed_mz), + // The `/error` suffix is optional in mzPAF. Without it the + // observed m/z is the best available value, but it is NOT the + // theoretical one the rest of the arena holds, so the mixture is + // counted rather than passed off as exact. + None => { + stats.kept_at_observed_mz += 1; + *observed_mz + } + }; + + if annotated { + stats.kept_annotated += 1; + } else { + stats.kept_unknown_label += 1; + } frags.push((label, mz)); intens.push(*intensity); } @@ -538,6 +648,10 @@ pub fn read_mzspeclib_library_file>( let mut current: Option = None; let mut section = Section::Attributes; + // Which block the attributes now being read belong to. `[n]` group ids are + // scoped to it, so it has to be threaded into every `Attr::parse`. + let mut block = BlockId::Spectrum; + let mut in_attribute_set = false; for line in reader.lines() { let line = line.map_err(LibraryReadingError::IoError)?; @@ -545,7 +659,8 @@ pub fn read_mzspeclib_library_file>( // Any `<...>` header ends the peak list; only `` opens one. // `` and `` attributes are folded into - // the spectrum's bag: this reader wants the union, not the hierarchy. + // the spectrum's bag: this reader wants the union, not the hierarchy, + // but each keeps its own group scope. if trimmed.starts_with('<') { if trimmed.starts_with(">( } else { Section::Attributes }; + // `` declares defaults applied to whole classes + // of spectra by name. Honouring them is not implemented, so a + // non-empty one is counted rather than silently dropped: it can + // carry the RT unit, or the DECOY marker. + in_attribute_set = trimmed.starts_with(">( } let Some(spec) = current.as_mut() else { + if in_attribute_set && Attr::parse(trimmed, BlockId::Spectrum).is_some() { + stats.ignored_attribute_set_entries += 1; + } continue; // library-level header }; @@ -582,7 +706,7 @@ pub fn read_mzspeclib_library_file>( spec.peaks.push((mz, intensity, annotation)); } Section::Attributes => { - if let Some(attr) = Attr::parse(trimmed) { + if let Some(attr) = Attr::parse(trimmed, block) { spec.attrs.0.push(attr); } } @@ -688,6 +812,142 @@ mod tests { ); } + /// `[n]` group ids are scoped to their block, so an Analyte `[2]` must not + /// supply the unit for a Spectrum `[2]`. + /// + /// `spectronaut.mzSpecLib.txt` already carries both: `[2]` is the RT + + /// unit pair in its Spectrum block and the NCBI TaxID in its Analyte + /// block. It survives today only because the Analyte's `[2]` happens to + /// carry no unit term. If it did, the RT would be read in the wrong unit — + /// a silent 60x error in the value the extraction window is built on. + #[test] + fn group_ids_do_not_leak_between_blocks() { + let grouped = |block: BlockId, id: u32, accession: &str, value: &str| Attr { + group: Some((block, id)), + accession: accession.to_string(), + value: value.to_string(), + }; + let rt = grouped(BlockId::Spectrum, 2, RT_TERMS[1], "10.0"); + let bag = AttrBag(vec![ + rt.clone(), + // Same group id, different block, and it does carry a unit. + grouped(BlockId::Analyte(1), 2, "MS:1001467", "9606"), + grouped(BlockId::Analyte(1), 2, UNIT_TERM, UNIT_SECOND), + ]); + + assert_eq!( + bag.unit_for(&rt), + None, + "the Analyte block's unit must not reach the Spectrum block's group" + ); + + // And when the unit is in the same block, it is found. + let mut same_block = bag; + same_block + .0 + .push(grouped(BlockId::Spectrum, 2, UNIT_TERM, UNIT_SECOND)); + assert_eq!(same_block.unit_for(&rt), Some(UNIT_SECOND)); + } + + /// The `/error` suffix is optional in mzPAF, so a bare `y5` is legal and + /// its m/z stays observed while the rest of the arena is theoretical. + /// Both vendored fixtures write `/0.0` everywhere, so nothing else covers + /// this. A malformed suffix is a different verdict from an ambiguous one. + #[test] + fn a_missing_mass_error_is_counted_and_a_malformed_one_is_skipped() { + assert!(matches!( + resolve_annotation("y5"), + Resolved::Annotated(_, None) + )); + assert!(matches!( + resolve_annotation("y5/not-a-number"), + Resolved::Skip(SkipReason::MalformedMassError) + )); + + let attr = |accession: &str, value: &str| Attr { + group: None, + accession: accession.to_string(), + value: value.to_string(), + }; + let raw = RawSpectrum { + attrs: AttrBag(vec![ + attr(PRECURSOR_MZ_TERMS[0], "500.25"), + attr(CHARGE_TERM, "2"), + attr(STRIPPED_SEQ_TERM, "PEPTIDEK"), + attr(RT_TERMS[0], "10.0"), + ]), + peaks: vec![ + (175.1, 1.0, "y1/0.0".to_string()), + (288.2, 1.0, "y2".to_string()), + ], + }; + + let mut stats = MzSpecLibStats::default(); + let row = convert_spectrum(&raw, &mut stats).expect("both peaks are usable"); + assert_eq!(row.frags.len(), 2); + assert_eq!(stats.kept_annotated, 2); + assert_eq!( + stats.kept_at_observed_mz, 1, + "only the suffix-less peak keeps its observed m/z" + ); + } + + /// A collided peak must be reported once, as dropped — not as kept AND + /// dropped, which is what counting before the uniqueness check produced. + #[test] + fn a_duplicate_label_is_counted_once_as_dropped() { + let attr = |accession: &str, value: &str| Attr { + group: None, + accession: accession.to_string(), + value: value.to_string(), + }; + let raw = RawSpectrum { + attrs: AttrBag(vec![ + attr(PRECURSOR_MZ_TERMS[0], "500.25"), + attr(CHARGE_TERM, "2"), + attr(STRIPPED_SEQ_TERM, "PEPTIDEK"), + attr(RT_TERMS[0], "10.0"), + ]), + peaks: vec![ + (175.1, 1.0, "y1/0.0".to_string()), + (175.2, 2.0, "y1/0.0".to_string()), + ], + }; + + let mut stats = MzSpecLibStats::default(); + let row = convert_spectrum(&raw, &mut stats).expect("the first peak is usable"); + assert_eq!(row.frags.len(), 1); + assert_eq!(stats.kept_annotated, 1); + assert_eq!(stats.dropped_duplicate_label, 1); + } + + /// DIA-NN writes `MS:1002476|ion mobility drift time=0.0` on every + /// spectrum, meaning "unset". Counting that warns on every DIA-NN load. + #[test] + fn a_zero_drift_time_is_absent_not_a_drift_time() { + let mut stats = MzSpecLibStats::default(); + let attr = |accession: &str, value: &str| Attr { + group: None, + accession: accession.to_string(), + value: value.to_string(), + }; + let with_drift = |drift: &str| RawSpectrum { + attrs: AttrBag(vec![ + attr(PRECURSOR_MZ_TERMS[0], "500.25"), + attr(CHARGE_TERM, "2"), + attr(STRIPPED_SEQ_TERM, "PEPTIDEK"), + attr(RT_TERMS[0], "10.0"), + attr(MOBILITY_DRIFT_TIME, drift), + ]), + peaks: vec![(175.1, 1.0, "y1/0.0".to_string())], + }; + + assert!(convert_spectrum(&with_drift("0.0"), &mut stats).is_some()); + assert_eq!(stats.spectra_with_drift_time_mobility, 0); + assert!(convert_spectrum(&with_drift("0.85"), &mut stats).is_some()); + assert_eq!(stats.spectra_with_drift_time_mobility, 1); + } + /// A spectrum missing a structural field is dropped by `?` deep inside /// `spectrum_row`. Nothing there increments a counter, so this is what /// stops the load from reporting "all annotated and representable" while From a1bd7eb4c9aa6256597d07f5db4b3e518f79e424 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 09:11:17 -0700 Subject: [PATCH 20/27] fix(timsquery): one owner for per-precursor fragment-label uniqueness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three readers had three answers to "labels must be unique within a precursor": the mzSpecLib reader dropped the later peak, the DIA-NN `.speclib` reader collapsed onto the more intense one, and `mzpaf_with_intensities` — which the DIA-NN/Spectronaut/Skyline TSV readers all feed — did nothing at all. The third is not "fails later". `ExpectedIntensities::try_from_pairs` rejects duplicate keys and timsseek's scoring pipeline `.expect()`s it, so two TSV rows with the same series + ordinal + charge for one precursor panic mid-search, per candidate. Nothing upstream of that dedupes, and the `HashMap` intensity lookup silently collapsed such rows on the way through. `FragmentSet` now owns the invariant and the parallel `(labels, mzs, intensities)` bookkeeping for all three, collapsing onto the more intense peak: intensity is what gets scored, and file order is not meaningful. Each caller keeps its own counter name for the collision. --- rust/timsquery/src/serde/diann_speclib_io.rs | 34 ++--- rust/timsquery/src/serde/library_file.rs | 149 ++++++++++++++++++- rust/timsquery/src/serde/mzspeclib_io.rs | 49 +++--- 3 files changed, 180 insertions(+), 52 deletions(-) diff --git a/rust/timsquery/src/serde/diann_speclib_io.rs b/rust/timsquery/src/serde/diann_speclib_io.rs index 300b71a9..6ca4babc 100644 --- a/rust/timsquery/src/serde/diann_speclib_io.rs +++ b/rust/timsquery/src/serde/diann_speclib_io.rs @@ -26,6 +26,8 @@ //! than exposed as a random-access view. use super::library_file::{ + FragmentSet, + Inserted, LibraryArena, LibraryReadingError, finish_mzpaf_arena, @@ -487,7 +489,7 @@ impl SpecLib { ), Vec::::new(), SpeclibDecodeStats::default(), - Vec::new(), + FragmentSet::default(), ) }, |(mut geom, mut frag_intens, mut stats, mut scratch), @@ -508,7 +510,7 @@ impl SpecLib { ), Vec::::new(), SpeclibDecodeStats::default(), - Vec::new(), + FragmentSet::default(), ) }, |(mut a_geom, mut a_int, sa, scratch), (b_geom, b_int, sb, _)| { @@ -688,7 +690,7 @@ fn map_entry( geom: &mut QueryCollection, frag_intens: &mut Vec, stats: &mut SpeclibDecodeStats, - scratch: &mut Vec<(IonAnnot, f64, f32)>, + scratch: &mut FragmentSet, ) -> Result<(), LibraryReadingError> { let pep = &entry.peptide; let name = entry.name; @@ -726,9 +728,9 @@ fn map_entry( residue_count(&stripped_peptide) }; - // (IonAnnot, fragment mz as f64, height). Dedup by IonAnnot keeping max - // height so a duplicate label can't fail the whole load via - // `ExpectedIntensities::try_from_pairs`. Reuses the caller's scratch buffer. + // `FragmentSet` owns the per-precursor label-uniqueness invariant that + // `ExpectedIntensities::try_from_pairs` depends on. Reuses the caller's + // scratch buffer so the allocation is per worker, not per entry. scratch.clear(); let kept = scratch; @@ -795,25 +797,15 @@ fn map_entry( } }; - if let Some(slot) = kept.iter_mut().find(|(k, _, _)| *k == ion) { + if kept.insert(ion, f.mz() as f64, f.height()) == Inserted::Collapsed { stats.dedup_dropped += 1; - if f.height() > slot.2 { - slot.1 = f.mz() as f64; - slot.2 = f.height(); - } - } else { - kept.push((ion, f.mz() as f64, f.height())); } } - // (label, mz) pairs for the arena, with the parallel reference-intensity - // sidecar filled in the same order — dedup already collapsed duplicate - // labels, so this order is what lands in `geom.frag_labels`. - let mut frags: Vec<(IonAnnot, f64)> = Vec::with_capacity(kept.len()); - for &(ion, mz, height) in kept.iter() { - frags.push((ion, mz)); - frag_intens.push(height); - } + // The reference-intensity sidecar is filled in the same order the labels + // land in `geom.frag_labels`. + kept.extend_sidecar(frag_intens); + let frags = kept.frags(); // Record charge is i32; `push_target` wants u8. Charge was range-checked to // 1..=255 above, so the cast is safe. diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index 3f329d46..925130ef 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -238,27 +238,36 @@ impl LibraryArena { let mut geom = QueryCollection::with_capabilities(LibCapabilities::default_diann_no_decoys()); let mut frag_intens: Vec = Vec::new(); + let mut n_collapsed = 0usize; for (eg, row) in egs.iter().zip(rows) { // Reference intensities keyed by fragment label (see fn docs). let lookup: std::collections::HashMap = row.relative_intensities.into_iter().collect(); - let frags: Vec<(IonAnnot, f64)> = eg.iter_fragments().map(|(l, mz)| (*l, mz)).collect(); - for (label, _) in &frags { + + // Through `FragmentSet` rather than straight into a Vec: two TSV + // rows with the same series + ordinal + charge for one precursor + // produce two identical labels, which panics in scoring. + let mut set = FragmentSet::with_capacity(eg.iter_fragments().count()); + for (label, mz) in eg.iter_fragments() { let intensity = lookup.get(label).ok_or_else(|| { LibraryReadingError::SpeclibParse(format!( "fragment {label:?} of precursor {:?} has no reference intensity", row.modified )) })?; - frag_intens.push(*intensity); + if set.insert(*label, mz, *intensity) == Inserted::Collapsed { + n_collapsed += 1; + } } + + set.extend_sidecar(&mut frag_intens); geom.push_row( eg.precursor_mz(), eg.precursor_charge(), eg.rt_seconds(), eg.mobility_ook0(), - &frags, + set.frags(), &row.stripped, &row.modified, &[], @@ -266,6 +275,10 @@ impl LibraryArena { ); } + if n_collapsed > 0 { + warn!("{n_collapsed} fragments collapsed onto a duplicate label"); + } + finish_mzpaf_arena(geom, frag_intens) } @@ -339,6 +352,83 @@ impl LibraryArena { } } +/// What [`FragmentSet::insert`] did with a fragment. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Inserted { + /// A label not yet in this precursor; stored. + Added, + /// The label was already present, so the two peaks were collapsed onto the + /// more intense one. The caller counts this under its own name. + Collapsed, +} + +/// The fragments of one precursor, with labels unique by construction. +/// +/// Fragment labels MUST be unique within a precursor. `linear_get` is +/// first-match, so a duplicate silently shadows one peak — and +/// `ExpectedIntensities::try_from_pairs` rejects duplicates outright, which +/// timsseek's scoring pipeline `.expect()`s. A duplicate reaching the arena is +/// therefore a panic mid-search, per candidate. +/// +/// Every reader that builds an mzpaf arena goes through this type, so the +/// invariant holds in one place instead of being re-derived (or, in the TSV +/// readers' case, forgotten) at each site. Collisions collapse onto the more +/// intense peak rather than keeping whichever came first: intensity is the +/// signal being scored, and file order is not meaningful. +#[derive(Debug, Default)] +pub(super) struct FragmentSet { + frags: Vec<(IonAnnot, f64)>, + intensities: Vec, +} + +impl FragmentSet { + pub(super) fn with_capacity(n: usize) -> Self { + Self { + frags: Vec::with_capacity(n), + intensities: Vec::with_capacity(n), + } + } + + pub(super) fn insert(&mut self, label: IonAnnot, mz: f64, intensity: f32) -> Inserted { + if let Some(idx) = self.frags.iter().position(|(l, _)| *l == label) { + if intensity > self.intensities[idx] { + self.frags[idx].1 = mz; + self.intensities[idx] = intensity; + } + return Inserted::Collapsed; + } + self.frags.push((label, mz)); + self.intensities.push(intensity); + Inserted::Added + } + + pub(super) fn is_empty(&self) -> bool { + self.frags.is_empty() + } + + /// Empty without releasing the allocation, so one set can be reused across + /// entries (the DIA-NN `.speclib` reader keeps one per rayon worker). + pub(super) fn clear(&mut self) { + self.frags.clear(); + self.intensities.clear(); + } + + #[cfg(test)] + pub(super) fn len(&self) -> usize { + self.frags.len() + } + + pub(super) fn frags(&self) -> &[(IonAnnot, f64)] { + &self.frags + } + + /// Append this precursor's intensities to a whole-library sidecar, in the + /// same order `frags()` will be pushed. + pub(super) fn extend_sidecar(&self, sidecar: &mut Vec) { + sidecar.extend_from_slice(&self.intensities); + } +} + /// Seal a directly-built mzpaf arena together with its reference-intensity /// sidecar. /// @@ -566,3 +656,54 @@ pub fn read_library_file>(path: T) -> Result IonAnnot { + IonAnnot::try_from(s).expect("valid annotation") + } + + /// Fragment labels must be unique within a precursor: `linear_get` is + /// first-match, and `ExpectedIntensities::try_from_pairs` rejects + /// duplicates outright — which timsseek's scoring pipeline `.expect()`s. + /// A duplicate reaching the arena is a panic mid-search, so this is the + /// invariant that stops it. + #[test] + fn duplicate_labels_collapse_onto_the_more_intense_peak() { + let mut set = FragmentSet::with_capacity(3); + assert_eq!(set.insert(ion("y1"), 175.1, 0.5), Inserted::Added); + assert_eq!(set.insert(ion("b3^2"), 200.0, 0.9), Inserted::Added); + + // Weaker duplicate: kept peak is unchanged. + assert_eq!(set.insert(ion("y1"), 999.9, 0.1), Inserted::Collapsed); + assert_eq!(set.frags()[0], (ion("y1"), 175.1)); + + // Stronger duplicate: takes over both m/z and intensity. + assert_eq!(set.insert(ion("y1"), 175.2, 0.8), Inserted::Collapsed); + assert_eq!(set.frags()[0], (ion("y1"), 175.2)); + + assert_eq!(set.len(), 2, "a collision must not grow the set"); + let mut sidecar = vec![0.0]; + set.extend_sidecar(&mut sidecar); + assert_eq!( + sidecar, + vec![0.0, 0.8, 0.9], + "the sidecar stays parallel to frags(), appended in order" + ); + + // Charge is part of the label, so these do not collide. + assert_eq!(set.insert(ion("y1^2"), 88.0, 0.3), Inserted::Added); + assert_eq!(set.len(), 3); + } + + #[test] + fn clear_keeps_the_set_reusable() { + let mut set = FragmentSet::with_capacity(2); + set.insert(ion("y1"), 175.1, 1.0); + set.clear(); + assert!(set.is_empty()); + assert_eq!(set.insert(ion("y1"), 175.1, 1.0), Inserted::Added); + } +} diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs index 2e7dd6df..6a251c35 100644 --- a/rust/timsquery/src/serde/mzspeclib_io.rs +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -64,6 +64,8 @@ use crate::models::{ QueryCollection, }; use crate::serde::library_file::{ + FragmentSet, + Inserted, LibraryArena, LibraryReadingError, finish_mzpaf_arena, @@ -203,8 +205,7 @@ struct ArenaRow { charge: u8, rt_seconds: f32, mobility: f32, - frags: Vec<(IonAnnot, f64)>, - intensities: Vec, + frags: FragmentSet, stripped: String, modified: String, } @@ -518,8 +519,7 @@ fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option = Vec::with_capacity(raw.peaks.len()); - let mut intens: Vec = Vec::with_capacity(raw.peaks.len()); + let mut frags = FragmentSet::with_capacity(raw.peaks.len()); let mut unknown_ions = UnknownIonCounter::new(); for (observed_mz, intensity, annotation) in &raw.peaks { @@ -546,35 +546,31 @@ fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option e.theoretical_from_observed(*observed_mz), - // The `/error` suffix is optional in mzPAF. Without it the - // observed m/z is the best available value, but it is NOT the - // theoretical one the rest of the arena holds, so the mixture is - // counted rather than passed off as exact. - None => { - stats.kept_at_observed_mz += 1; - *observed_mz - } + None => *observed_mz, }; + // `FragmentSet` owns the per-precursor uniqueness invariant; a + // collision collapses onto the more intense peak. Unknown labels come + // off a monotonic counter and cannot collide, so this only ever fires + // for annotated ones. Every kept counter is behind this check, so a + // collided peak is tallied once, as dropped. + if frags.insert(label, mz, *intensity) == Inserted::Collapsed { + stats.dropped_duplicate_label += 1; + continue; + } + if mass_error.is_none() { + stats.kept_at_observed_mz += 1; + } if annotated { stats.kept_annotated += 1; } else { stats.kept_unknown_label += 1; } - frags.push((label, mz)); - intens.push(*intensity); } Some(ArenaRow { @@ -583,7 +579,6 @@ fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option Date: Thu, 27 Aug 2026 09:16:29 -0700 Subject: [PATCH 21/27] perf(speclib_build_cli)!: stop writing four fields nobody reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ReferenceEG.id`, `.precursor_labels`, `.precursor_intensities` and `PrecursorEntry.decoy_group` were serialized and then dropped by the loader: `push_row` receives none of them, and the isotope argument is a literal `&[]`. The loader recomputes the envelope from composition (`IsotopeStrategy::FromComposition`) regardless. Producing `precursor_intensities` was not free — it was the only consumer of `count_cs_modified`, so every entry paid a second full ProForma parse plus a formula computation on top of the one `compute_precursor_mz` already does. That is roughly half the mzcore work in a Koina build, spent on bytes nobody reads. `count_cs_modified` is gone; `compute_precursor_mz` was already the malformed-sequence gate the deleted step claimed to be. Serde ignores unknown fields, so existing libraries still load. Also drops `speclib_build_cli`'s `strip_mods`, which had no callers outside its own test and handled only `[...]`, unlike the two real copies. --- rust/speclib_build_cli/src/entry.rs | 79 +++-------- rust/speclib_build_cli/src/pipeline.rs | 8 -- rust/timsseek/src/data_sources/speclib.rs | 154 ++++++++-------------- 3 files changed, 72 insertions(+), 169 deletions(-) diff --git a/rust/speclib_build_cli/src/entry.rs b/rust/speclib_build_cli/src/entry.rs index 68ef68a8..a611f87b 100644 --- a/rust/speclib_build_cli/src/entry.rs +++ b/rust/speclib_build_cli/src/entry.rs @@ -4,11 +4,7 @@ use timsseek::data_sources::speclib::{ ReferenceEG, SerSpeclibElement, }; -use timsseek::fragment_mass::elution_group_converter::{ - count_carbon_sulphur_in_sequence, - supersimpleprediction, -}; -use timsseek::isotopes::peptide_isotopes; +use timsseek::fragment_mass::elution_group_converter::supersimpleprediction; use crate::koina::models::{ FragmentPrediction, @@ -28,29 +24,13 @@ pub struct EntryFilters { // ── Helpers ─────────────────────────────────────────────────────────────────── -/// Strip bracket-enclosed modifications from a sequence. -/// -/// "PEPTC[U:4]IDEK" → "PEPTCIDEK" -pub fn strip_mods(seq: &str) -> String { - let mut out = String::with_capacity(seq.len()); - let mut depth = 0usize; - for ch in seq.chars() { - match ch { - '[' => depth += 1, - ']' => { - depth = depth.saturating_sub(1); - } - _ if depth == 0 => out.push(ch), - _ => {} - } - } - out -} - use timsseek::models::sequence::normalize_to_proforma; /// Compute the monoisotopic precursor m/z using mzcore. /// Input should be the modified sequence (mods included in mass). +/// +/// This is the only mzcore parse per library entry, and it doubles as the +/// malformed-sequence gate: anything it cannot parse is dropped here. fn compute_precursor_mz(modified_seq: &str, charge: u8) -> Option { use mzcore::prelude::*; let proforma = normalize_to_proforma(modified_seq); @@ -65,12 +45,6 @@ fn compute_precursor_mz(modified_seq: &str, charge: u8) -> Option { Some((mass + proton_mass * charge as f64) / charge as f64) } -/// Count carbon and sulphur from modified sequence (mods affect formula). -fn count_cs_modified(modified_seq: &str) -> Option<(u16, u16)> { - let proforma = normalize_to_proforma(modified_seq); - count_carbon_sulphur_in_sequence(&proforma).ok() -} - // ── Public API ──────────────────────────────────────────────────────────────── /// Convert Koina predictions + metadata into a [`SerSpeclibElement`]. @@ -83,28 +57,23 @@ pub fn build_entry( sequence: &str, charge: u8, decoy: bool, - decoy_group: u32, - id: u32, fragment: &FragmentPrediction, rt: &RtPrediction, filters: &EntryFilters, ) -> Option { - // 1. Carbon / sulphur count from modified sequence (includes mod contributions). - let (ncarbon, nsulphur) = count_cs_modified(sequence)?; - let iso = peptide_isotopes(ncarbon, nsulphur); - - // 2. Precursor m/z from modified sequence (includes mod masses). + // 1. Precursor m/z from modified sequence (includes mod masses). Also the + // malformed-sequence gate. let precursor_mz = compute_precursor_mz(sequence, charge)?; - // 4. Filter by precursor m/z range. + // 2. Filter by precursor m/z range. if precursor_mz < filters.min_mz as f64 || precursor_mz > filters.max_mz as f64 { return None; } - // 5. Ion mobility prediction. + // 3. Ion mobility prediction. let mobility = supersimpleprediction(precursor_mz, charge as i32) as f32; - // 6. Filter fragments: keep those within ion m/z bounds. + // 4. Filter fragments: keep those within ion m/z bounds. let min_ion = filters.min_ion_mz as f64; let max_ion = filters.max_ion_mz as f64; @@ -150,19 +119,16 @@ pub fn build_entry( return None; } - // 10. Build precursor labels and intensities from the isotope distribution. - let precursor_labels: Vec = vec![0i8, 1i8, 2i8]; - let precursor_intensities: Vec = vec![iso[0], iso[1], iso[2]]; - - // 11. Assemble the element. - let precursor = PrecursorEntry::new(sequence.to_owned(), charge, decoy, decoy_group); + // 8. Assemble the element. The precursor isotope envelope is NOT stored: + // the loader recomputes it from composition + // (`IsotopeStrategy::FromComposition`), so writing it here would cost a + // second mzcore parse per entry to produce bytes nobody reads — and + // would give isotopes two sources of truth. + let precursor = PrecursorEntry::new(sequence.to_owned(), charge, decoy); let elution_group = ReferenceEG::new( - id, precursor_mz, - precursor_labels, fragment_mzs, fragment_labels, - precursor_intensities, fragment_intensities, mobility, rt.irt, @@ -205,13 +171,6 @@ mod tests { } } - #[test] - fn test_strip_mods() { - assert_eq!(strip_mods("PEPTC[U:4]IDEK"), "PEPTCIDEK"); - assert_eq!(strip_mods("PEPTM[+15.995]IDEK"), "PEPTMIDEK"); - assert_eq!(strip_mods("PEPTIDEK"), "PEPTIDEK"); - } - #[test] fn test_compute_precursor_mz() { let mz = compute_precursor_mz("PEPTIDEK", 2).unwrap(); @@ -240,7 +199,7 @@ mod tests { let rt = RtPrediction { irt: 30.0 }; let filters = make_filters(3); - let result = build_entry("PEPTIDEK", 2, false, 0, 42, &fragment, &rt, &filters); + let result = build_entry("PEPTIDEK", 2, false, &fragment, &rt, &filters); assert!(result.is_some(), "Expected Some but got None"); } @@ -256,7 +215,7 @@ mod tests { let rt = RtPrediction { irt: 30.0 }; let filters = make_filters(3); - let result = build_entry("PEPTIDEK", 2, false, 0, 1, &fragment, &rt, &filters); + let result = build_entry("PEPTIDEK", 2, false, &fragment, &rt, &filters); assert!(result.is_none(), "Expected None but got Some"); } @@ -275,7 +234,7 @@ mod tests { min_ions: 3, }; - let result = build_entry("PEPTIDEK", 2, false, 0, 2, &fragment, &rt, &filters); + let result = build_entry("PEPTIDEK", 2, false, &fragment, &rt, &filters); assert!( result.is_none(), @@ -289,7 +248,7 @@ mod tests { let rt = RtPrediction { irt: 30.0 }; let filters = make_filters(3); - let result = build_entry("KEDITREP", 2, true, 99, 7, &fragment, &rt, &filters); + let result = build_entry("KEDITREP", 2, true, &fragment, &rt, &filters); // Decoy peptide should still build an entry if the mz/ions pass filters // (KEDITREP ~476 should pass default 400–2000 window). diff --git a/rust/speclib_build_cli/src/pipeline.rs b/rust/speclib_build_cli/src/pipeline.rs index 0c9a2639..85cbee9f 100644 --- a/rust/speclib_build_cli/src/pipeline.rs +++ b/rust/speclib_build_cli/src/pipeline.rs @@ -45,7 +45,6 @@ struct BatchItem { sequence: String, charge: u8, decoy: bool, - decoy_group: u32, } // ── flush_batch ───────────────────────────────────────────────────────────── @@ -93,8 +92,6 @@ async fn flush_batch( &item.sequence, item.charge, item.decoy, - item.decoy_group, - *entry_id, fragment, rt, filters, @@ -309,7 +306,6 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box = Vec::with_capacity(batch_size); let mut entry_id: u32 = 0; - let mut decoy_group: u32 = 0; for digest_slice in &base_peptides { let base_seq = digest_slice.as_str(); @@ -335,7 +331,6 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box Result<(), Box Result<(), Box Self { + pub fn new(sequence: String, charge: u8, decoy: bool) -> Self { Self { sequence, charge, decoy, - decoy_group, } } } +/// See [`PrecursorEntry`] for why the precursor-isotope fields are absent. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReferenceEG { - id: u32, precursor_mz: f64, - precursor_labels: Vec, #[serde(alias = "fragment_mz")] fragment_mzs: Vec, fragment_labels: Vec, - precursor_intensities: Vec, fragment_intensities: Vec, #[serde(alias = "mobility")] mobility_ook0: f32, @@ -81,25 +85,18 @@ pub struct ReferenceEG { } impl ReferenceEG { - #[allow(clippy::too_many_arguments)] pub fn new( - id: u32, precursor_mz: f64, - precursor_labels: Vec, fragment_mzs: Vec, fragment_labels: Vec, - precursor_intensities: Vec, fragment_intensities: Vec, mobility_ook0: f32, rt_seconds: f32, ) -> Self { Self { - id, precursor_mz, - precursor_labels, fragment_mzs, fragment_labels, - precursor_intensities, fragment_intensities, mobility_ook0, rt_seconds, @@ -534,35 +531,48 @@ mod tests { RefQuery, }; + /// One native-format element. Fragment m/z are positional stand-ins; no + /// test here asserts on them, only on labels, intensities and the row + /// metadata. + fn element( + sequence: &str, + decoy: bool, + precursor_mz: f64, + labels: &[&str], + intensities: &[f32], + ) -> SerSpeclibElement { + assert_eq!(labels.len(), intensities.len()); + SerSpeclibElement::new( + PrecursorEntry::new(sequence.to_string(), 2, decoy), + ReferenceEG::new( + precursor_mz, + (0..labels.len()) + .map(|i| 300.0 + 100.0 * i as f64) + .collect(), + labels + .iter() + .map(|l| IonAnnot::try_from(*l).expect("valid annotation")) + .collect(), + intensities.to_vec(), + 0.75, + 120.0, + ), + ) + } + /// `speclib_build_cli` writes with [`SpeclibWriter`] and timsseek reads /// with [`SpeclibReader`]; nothing else checks that the two agree, and a /// mismatch only shows up as an unreadable library at the end of a long /// Koina run. #[test] fn writer_output_reads_back_through_the_reader() { - let element = SerSpeclibElement::new( - PrecursorEntry::new("PEPTIDEK".to_string(), 2, false, 0), - ReferenceEG::new( - 7, - 450.5, - vec![0, 1], - vec![175.1, 288.2], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("b3^2").unwrap(), - ], - vec![1.0, 0.5], - vec![0.9, 0.4], - 0.95, - 1234.5, - ), - ); + let record = element("PEPTIDEK", false, 450.5, &["y1", "b3^2"], &[0.9, 0.4]); let mut writer = SpeclibWriter::new_ndjson_zstd(Vec::new()).expect("encoder"); - writer.append(&element).expect("append"); + writer.append(&record).expect("append"); // Twice, so the newline separator is exercised rather than the file // happening to hold one record. - writer.append(&element).expect("append"); + writer.append(&record).expect("append"); let bytes = writer.finish().expect("finish"); let read: Vec = SpeclibReader::new(bytes.as_slice()) @@ -572,7 +582,13 @@ mod tests { assert_eq!(read.len(), 2); assert_eq!(read[0].precursor.sequence, "PEPTIDEK"); - assert_eq!(read[0].elution_group.fragment_mzs, vec![175.1, 288.2]); + assert_eq!( + read[0].elution_group.fragment_mzs, + element("PEPTIDEK", false, 450.5, &["y1", "b3^2"], &[0.9, 0.4]) + .elution_group + .fragment_mzs, + "fragment m/z must survive the round trip" + ); let labels: Vec = read[0] .elution_group .fragment_labels @@ -1079,40 +1095,8 @@ mod tests { fn native_ndjson_load_builds_lazy_arena() { use crate::data_sources::reference_library::ScoredIdentity; - let target = SerSpeclibElement::new( - PrecursorEntry::new("PEPTIDEK".to_string(), 2, false, 0), - ReferenceEG::new( - 0, - 500.0, - vec![0, 1, 2], - vec![300.0, 400.0], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - vec![1.0, 0.5, 0.2], - vec![0.8, 0.3], - 0.75, - 120.0, - ), - ); - let decoy = SerSpeclibElement::new( - PrecursorEntry::new("KEDITPEP".to_string(), 2, true, 0), - ReferenceEG::new( - 1, - 500.0, - vec![0, 1, 2], - vec![300.0, 400.0], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - vec![1.0, 0.5, 0.2], - vec![0.6, 0.4], - 0.75, - 120.0, - ), - ); + let target = element("PEPTIDEK", false, 500.0, &["y1", "y2"], &[0.8, 0.3]); + let decoy = element("KEDITPEP", true, 500.0, &["y1", "y2"], &[0.6, 0.4]); let mut ndjson = String::new(); ndjson.push_str(&serde_json::to_string(&target).unwrap()); @@ -1156,42 +1140,10 @@ mod tests { /// the AOS `test_parse_gate_off_on_poisoned_row` was removed in Task 9. #[test] fn from_file_native_ndjson_poisoned_row_disables_sequence_features() { - let good = SerSpeclibElement::new( - PrecursorEntry::new("PEPTIDEK".to_string(), 2, false, 0), - ReferenceEG::new( - 0, - 500.0, - vec![0, 1, 2], - vec![300.0, 400.0], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - vec![1.0, 0.5, 0.2], - vec![0.8, 0.3], - 0.75, - 120.0, - ), - ); + let good = element("PEPTIDEK", false, 500.0, &["y1", "y2"], &[0.8, 0.3]); // Unparseable modified sequence: `!` is rejected by parse_sequence_fast // (`_ => return None`) and by the mzcore pro_forma fallback. - let poisoned = SerSpeclibElement::new( - PrecursorEntry::new("GARBAGE!!!".to_string(), 2, false, 1), - ReferenceEG::new( - 1, - 600.0, - vec![0, 1, 2], - vec![300.0, 400.0], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - vec![1.0, 0.5, 0.2], - vec![0.7, 0.4], - 0.75, - 120.0, - ), - ); + let poisoned = element("GARBAGE!!!", false, 600.0, &["y1", "y2"], &[0.7, 0.4]); let mut ndjson = String::new(); ndjson.push_str(&serde_json::to_string(&good).unwrap()); From 1516bed8a56880f03bef7e7ff206d2dfea797042 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 09:22:00 -0700 Subject: [PATCH 22/27] test: pin the Carafe output contract, drop the duplicated tolerance tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contract test covered only the input direction, and its header claimed the output half needed the built binary and a real `.d`. That is false for the field names, the aggregator names and the format name. Demonstrated: adding `#[serde(rename)]` to a `SpectrumOutput` field — invariant 3, the one that silently NPEs on Carafe's side — left the whole suite green. The output assertions live inside `timsquery_cli` because it has no library target. They deserialize the contract's own payloads into the real types and compare the emitted key sets, so no constructor is needed and a rename fails. `results.json` is now a named const so the basename is assertable. `cli_template_tolerance_spellings_still_deserialize` was a byte-for-byte copy of `WIDE_TOLERANCE_TEMPLATE` duplicating a test that already deserializes the actual constant the CLI writes; deleted. So was the tolerance re-serialize round trip, which cannot fail — `Tolerance` is only ever read on the Carafe path and `#[serde(alias)]` is deserialize-only, so both sides move together under any rename. `mz_range`'s panic message asserted "low and high are positive". Carafe emits a negative low when the calibration offset exceeds the half-width; the real invariant is `low + high >= 0`. Corrected and covered. Also drops the `AIGear.java` line-number citations, which go stale on any Carafe commit above line 6660. --- docs/CARAFE_CONTRACT.md | 23 +++-- rust/timsquery/src/models/tolerance.rs | 24 +++-- rust/timsquery/tests/carafe_contract.rs | 51 ++++++---- rust/timsquery_cli/src/commands.rs | 126 +++++++++++++++++++++++- 4 files changed, 192 insertions(+), 32 deletions(-) diff --git a/docs/CARAFE_CONTRACT.md b/docs/CARAFE_CONTRACT.md index 10155ca2..4bfd47e2 100644 --- a/docs/CARAFE_CONTRACT.md +++ b/docs/CARAFE_CONTRACT.md @@ -1,12 +1,23 @@ # Carafe contract What Carafe assumes when calling `timsquery`. Break any of it → silent failure or NPE. -Refs: `util/CallTimsQuery.java`, `ai/AIGear.java` (~6660–6912), `dia/{PSMQuery,PSMQueryResult,XICQueryResult}.java`. +Refs: `util/CallTimsQuery.java`, `ai/AIGear.java`, `dia/{PSMQuery,PSMQueryResult,XICQueryResult}.java`. > Vendored from the Carafe repo so the assumptions live next to the code that -> has to honour them. `rust/timsquery/tests/carafe_contract.rs` pins the parts -> that are mechanically checkable — field names, aliases, units — against the -> literal JSON in this document. If you edit a payload here, edit it there. +> has to honour them. +> +> **Line numbers are deliberately omitted**: an earlier version of this file +> cited `AIGear.java` at `~6660–6912`, which is wrong the moment Carafe lands a +> commit above line 6660. Search for the symbol instead. If you re-verify this +> contract against Carafe, record the commit you checked below. +> +> Last verified against Carafe: _(unrecorded — add the SHA when you next check)_ +> +> Two test modules pin the mechanically checkable parts against the literal +> JSON in this document. If you edit a payload here, edit it there: +> - `rust/timsquery/tests/carafe_contract.rs` — input (targets, tolerances) +> - `timsquery_cli`'s `carafe_output_contract` module — output (result field +> names, `-a`/`-f` flag values, the `results.json` basename) ## CLI @@ -42,14 +53,14 @@ Binary path: `bin/timsquery/{windows,macos,linux}/timsquery_cli[.exe]`. "quad": { "absolute": [quad, quad] } } ``` -`ms` unit key is dynamic (`itolu`, currently `ppm`). Each value is `[low, high]`. Defaults: itol 15, mobility 3.0, quad 0.1, rt_win 0.1 (spectra) / `CParameter.rt_win` (xic). +`ms` unit key is dynamic on Carafe's side (`itolu`, currently `ppm`), but timsquery only accepts `ppm`/`da` (and the `Ppm`/`Absolute` spellings) — any other unit fails to deserialize. Each value is `[low, high]`. A negative `low` is valid and means the window sits entirely above the target mass. Defaults: itol 15, mobility 3.0, quad 0.1, rt_win 0.1 (spectra) / `CParameter.rt_win` (xic). ### `ms` window derivation (`itol` / `itol_shift`) The `ms` window is `[itol - itol_shift, itol + itol_shift]`, built from two independent inputs: - **`itol`** = `CParameter.itol` — the configured fragment-ion tolerance (default 15). Static per run. -- **`itol_shift`** = per-run **median observed m/z error**, a data-driven calibration offset measured before the query (`AIGear.java` ~6835–6855): +- **`itol_shift`** = per-run **median observed m/z error**, a data-driven calibration offset measured before the query (`AIGear.java`, search `error_shift`): - Carafe collects MS1 and MS2 mass errors from already-matched ions, takes the median of each (`ms1_error_shift`, `ms2_error_shift`). - If precursor and fragment units match (`CParameter.tolu` == `CParameter.itolu`): `itol_shift = max(ms1_error_shift, ms2_error_shift)`. - Else: `itol_shift = ms2_error_shift` (fragment only). diff --git a/rust/timsquery/src/models/tolerance.rs b/rust/timsquery/src/models/tolerance.rs index eb335103..a93dbfab 100644 --- a/rust/timsquery/src/models/tolerance.rs +++ b/rust/timsquery/src/models/tolerance.rs @@ -94,6 +94,18 @@ impl Tolerance { // M/Z Tolerance Methods // ============================================================================ + /// Panic message for an inverted m/z window. + /// + /// The range is `(mz - low, mz + high)`, so it is well-formed whenever + /// `low + high >= 0` — not only when both are positive. A negative `low` is + /// a supported input: Carafe encodes a systematic calibration offset as + /// `[itol - itol_shift, itol + itol_shift]` and emits e.g. + /// `{"ppm": [-2.0, 32.0]}` when the offset exceeds the half-width. Their + /// sum is `2 * itol`, so the window stays valid for any positive tolerance. + /// See `docs/CARAFE_CONTRACT.md`. + const MZ_RANGE_INVARIANT: &str = + "m/z tolerance produced an inverted range: low + high must be >= 0"; + /// Calculate m/z tolerance range (primary method, returns f64). /// /// This is the canonical m/z range method. All other m/z methods delegate to this. @@ -129,15 +141,15 @@ impl Tolerance { /// ``` pub fn mz_range(&self, mz: f64) -> TupleRange { match self.ms { - MzTolerance::Absolute((low, high)) => (mz - low, mz + high).try_into().expect( - "mz tolerance should never result in an invalid range, since low and high are positive", - ), + MzTolerance::Absolute((low, high)) => (mz - low, mz + high) + .try_into() + .expect(Self::MZ_RANGE_INVARIANT), MzTolerance::Ppm((low, high)) => { let low = mz * low / 1e6; let high = mz * high / 1e6; - (mz - low, mz + high).try_into().expect( - "mz tolerance should never result in an invalid range, since low and high are positive", - ) + (mz - low, mz + high) + .try_into() + .expect(Self::MZ_RANGE_INVARIANT) } } } diff --git a/rust/timsquery/tests/carafe_contract.rs b/rust/timsquery/tests/carafe_contract.rs index 2c63fa97..f4969ff4 100644 --- a/rust/timsquery/tests/carafe_contract.rs +++ b/rust/timsquery/tests/carafe_contract.rs @@ -12,8 +12,13 @@ //! than internal types, so a refactor that keeps the internals working but //! changes the boundary still fails here. //! -//! NOT covered, because it needs the built binary and a real `.d`: aggregator -//! names, the `-o` directory layout, and the `results.json` basename. +//! This file covers the INPUT direction. The output direction — result field +//! names, the `-a`/`-f` flag values and the `results.json` basename — is +//! pinned by `timsquery_cli`'s `carafe_output_contract` module, which has to +//! live inside that crate because it has no library target. +//! +//! NOT covered, because it needs the built binary and a real `.d`: the `-o` +//! directory layout and the process exit code. use std::io::Write; use timsquery::models::tolerance::{ @@ -118,24 +123,32 @@ fn carafe_tolerance_spellings_deserialize() { assert_eq!(tol.rt, RtTolerance::Minutes((0.1, 0.1))); assert_eq!(tol.mobility, MobilityTolerance::Pct((3.0, 3.0))); assert_eq!(tol.quad, QuadTolerance::Absolute((0.1, 0.1))); - - let round = serde_json::to_string(&tol).expect("tolerance must re-serialize"); - let back: Tolerance = serde_json::from_str(&round).expect("and deserialize again"); - assert_eq!( - back, tol, - "tolerance must survive a round trip through its own output" - ); } -/// The CLI-template spellings must keep working too, so the two callers stay -/// interchangeable. +/// Carafe encodes a systematic calibration offset as +/// `[itol - itol_shift, itol + itol_shift]`, so when the offset exceeds the +/// half-width the LOW edge is negative — a window that is entirely to the +/// heavy side of the target mass. That is a supported input, not a bug, and +/// `mz_range`'s invariant is `low + high >= 0` rather than "both positive". #[test] -fn cli_template_tolerance_spellings_still_deserialize() { - let cli_style = r#"{ - "ms": { "da": [0.04, 0.04] }, - "rt": "Unrestricted", - "mobility": { "pct": [20.0, 20.0] }, - "quad": { "da": [0.2, 0.2] } - }"#; - serde_json::from_str::(cli_style).expect("the CLI's own spellings must deserialize"); +fn a_negative_low_tolerance_edge_is_accepted() { + let tol: Tolerance = serde_json::from_str( + r#"{ + "ms": { "ppm": [-2.0, 32.0] }, + "rt": { "minutes": [0.1, 0.1] }, + "mobility": { "percent": [3.0, 3.0] }, + "quad": { "absolute": [0.1, 0.1] } + }"#, + ) + .expect("a negative low edge must deserialize"); + assert_eq!(tol.ms, MzTolerance::Ppm((-2.0, 32.0))); + + // Both edges land above the target mass, in ascending order. + let range = tol.mz_range(1000.0); + assert!( + range.start() > 1000.0 && range.end() > range.start(), + "expected a well-formed window above the target, got {:?}..{:?}", + range.start(), + range.end() + ); } diff --git a/rust/timsquery_cli/src/commands.rs b/rust/timsquery_cli/src/commands.rs index 5a0c25a7..c2b35a3c 100644 --- a/rust/timsquery_cli/src/commands.rs +++ b/rust/timsquery_cli/src/commands.rs @@ -37,6 +37,12 @@ use crate::error::CliError; use crate::processing::AggregatorContainer; use timsquery::serde::LibraryArena; +/// Basename Carafe looks for inside the `-o` directory. Part of the contract +/// (`docs/CARAFE_CONTRACT.md`, invariant 5), so it is named rather than +/// inlined — a renamed output file is a silent failure on Carafe's side. +/// Despite the extension, the contents are ndjson. +pub const CARAFE_RESULTS_BASENAME: &str = "results.json"; + /// Main function for the 'query-index' subcommand. #[instrument] pub fn main_query_index(args: QueryIndexArgs) -> Result<(), CliError> { @@ -71,7 +77,7 @@ pub fn main_query_index(args: QueryIndexArgs) -> Result<(), CliError> { let batch_size = args.batch_size; std::fs::create_dir_all(&output_path)?; - let put_path = output_path.join("results.json"); + let put_path = output_path.join(CARAFE_RESULTS_BASENAME); // Every format funnels into one of the two label-typed arenas; extraction // is generic over the label, so both arms call the same driver over the @@ -516,3 +522,121 @@ mod tests { } } } + +/// The OUTPUT half of `docs/CARAFE_CONTRACT.md`. +/// +/// `rust/timsquery/tests/carafe_contract.rs` pins the input direction. These +/// live here rather than in `tests/` because `timsquery_cli` has no library +/// target, and they assert on the boundary types Carafe parses with fastjson: +/// no remap, no schema negotiation, so a renamed field is a null on their side +/// and an NPE somewhere unrelated. +#[cfg(test)] +mod carafe_output_contract { + use super::*; + use crate::cli::{ + PossibleAggregator, + SerializationFormat, + }; + use crate::processing::SpectrumOutput; + use clap::ValueEnum; + use timsquery::serde::chromatogram_output::ChromatogramOutput; + + /// Round-trip a contract payload through the real type and hand back the + /// key set it serializes to. Deserializing first means a renamed field + /// fails here without needing a constructor for these types. + fn key_set(json: &str) -> Vec { + let parsed: T = serde_json::from_str(json) + .unwrap_or_else(|e| panic!("the contract's payload must deserialize: {e}")); + let value = serde_json::to_value(&parsed).expect("serializable"); + let mut keys: Vec = value + .as_object() + .expect("an object") + .keys() + .cloned() + .collect(); + keys.sort(); + keys + } + + /// Verbatim from the contract's "spectrum-aggregator -> PSMQueryResult". + const CARAFE_SPECTRUM_RESULT: &str = r#"{ + "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, "precursor_mz":650.32, + "precursor_charge":2, "precursor_intensities":[1200,800,300], "precursor_labels":[0,1,2], + "fragment_mzs":[175.1,288.2], "fragment_intensities":[500,0] + }"#; + + /// Verbatim from the contract's "chromatogram-aggregator -> XICQueryResult". + const CARAFE_CHROMATOGRAM_RESULT: &str = r#"{ + "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, + "precursor_mzs":[650.32,650.82], "precursor_intensities":[[1.0],[2.0]], + "fragment_mzs":[175.1,288.2], "fragment_labels":["y1","b2"], + "fragment_intensities":[[3.0],[4.0]], "retention_time_results_seconds":[1230,1231] + }"#; + + /// Contract invariant 3. Note the deliberate singular/plural split between + /// the two modes (`precursor_mz` vs `precursor_mzs`): they are two schemas, + /// and "unifying" them would break Carafe without failing anything else. + #[test] + fn spectrum_output_emits_exactly_the_contract_field_names() { + assert_eq!( + key_set::(CARAFE_SPECTRUM_RESULT), + [ + "fragment_intensities", + "fragment_mzs", + "id", + "mobility_ook0", + "precursor_charge", + "precursor_intensities", + "precursor_labels", + "precursor_mz", + "rt_seconds", + ] + ); + } + + #[test] + fn chromatogram_output_emits_exactly_the_contract_field_names() { + assert_eq!( + key_set::(CARAFE_CHROMATOGRAM_RESULT), + [ + "fragment_intensities", + "fragment_labels", + "fragment_mzs", + "id", + "mobility_ook0", + "precursor_intensities", + "precursor_mzs", + "retention_time_results_seconds", + "rt_seconds", + ] + ); + } + + /// The `-a` and `-f` values Carafe passes on the command line. clap derives + /// these from the variant names, so a rename silently changes the CLI. + #[test] + fn aggregator_and_format_flag_values_match_the_contract() { + fn name(v: T) -> String { + v.to_possible_value() + .expect("not skipped") + .get_name() + .to_string() + } + + assert_eq!( + name(PossibleAggregator::SpectrumAggregator), + "spectrum-aggregator" + ); + assert_eq!( + name(PossibleAggregator::ChromatogramAggregator), + "chromatogram-aggregator" + ); + assert_eq!(name(SerializationFormat::Ndjson), "ndjson"); + } + + /// Contract invariant 5. + #[test] + fn the_results_basename_is_what_carafe_looks_for() { + assert_eq!(CARAFE_RESULTS_BASENAME, "results.json"); + } +} From e866df94bf0bd2e21c73ca01edf9c8a195b38b0d Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 09:37:59 -0700 Subject: [PATCH 23/27] test: assert the Carafe output contract on the bytes, not on the type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous version round-tripped the boundary types through serde and compared key sets. That pins the type's shape, not what Carafe reads: the whole path from `-f ndjson` through `JsonStreamSerializer` to the file was untested, and contract invariant 2 — one complete object per line, no array wrapper, no pretty-print — was not asserted anywhere. Records now go through the same serializer `stream_process_batches` uses and the assertions are on the resulting text. Verified by mutation that all three of these now fail and previously would not have: dropping the ndjson newline separator, swapping ndjson to `to_writer_pretty`, and renaming a field via `#[serde(rename)]`. Also pins that the non-ndjson formats DO wrap in an array and that the default format is not ndjson, so `-f ndjson` is demonstrably load-bearing rather than incidentally correct, and that an empty result set is an empty file rather than a truncated one. --- rust/timsquery_cli/src/commands.rs | 140 +++++++++++++++++++++-------- 1 file changed, 104 insertions(+), 36 deletions(-) diff --git a/rust/timsquery_cli/src/commands.rs b/rust/timsquery_cli/src/commands.rs index c2b35a3c..4478434f 100644 --- a/rust/timsquery_cli/src/commands.rs +++ b/rust/timsquery_cli/src/commands.rs @@ -541,23 +541,6 @@ mod carafe_output_contract { use clap::ValueEnum; use timsquery::serde::chromatogram_output::ChromatogramOutput; - /// Round-trip a contract payload through the real type and hand back the - /// key set it serializes to. Deserializing first means a renamed field - /// fails here without needing a constructor for these types. - fn key_set(json: &str) -> Vec { - let parsed: T = serde_json::from_str(json) - .unwrap_or_else(|e| panic!("the contract's payload must deserialize: {e}")); - let value = serde_json::to_value(&parsed).expect("serializable"); - let mut keys: Vec = value - .as_object() - .expect("an object") - .keys() - .cloned() - .collect(); - keys.sort(); - keys - } - /// Verbatim from the contract's "spectrum-aggregator -> PSMQueryResult". const CARAFE_SPECTRUM_RESULT: &str = r#"{ "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, "precursor_mz":650.32, @@ -573,31 +556,90 @@ mod carafe_output_contract { "fragment_intensities":[[3.0],[4.0]], "retention_time_results_seconds":[1230,1231] }"#; - /// Contract invariant 3. Note the deliberate singular/plural split between - /// the two modes (`precursor_mz` vs `precursor_mzs`): they are two schemas, - /// and "unifying" them would break Carafe without failing anything else. + /// Parse a contract payload into the real boundary type. + /// + /// Deserializing (rather than constructing) is what lets these tests exist + /// at all: both types are only ever built from an aggregator, which needs a + /// real `.d`. A renamed or dropped field fails here. + fn parse(json: &str) -> T { + serde_json::from_str(json) + .unwrap_or_else(|e| panic!("the contract's payload must deserialize: {e}")) + } + + /// Write records through the SAME serializer `stream_process_batches` uses, + /// and hand back the exact bytes Carafe would read out of `results.json`. + fn write_results(records: &[T], format: SerializationFormat) -> String { + let mut buf = Vec::new(); + let mut seq = JsonStreamSerializer::new(&mut buf, format); + for r in records { + seq.serialize(r).expect("serialize"); + } + seq.finish().expect("finish"); + String::from_utf8(buf).expect("utf-8") + } + + fn keys_of(line: &str) -> Vec { + let value: serde_json::Value = serde_json::from_str(line).expect("one object per line"); + let mut keys: Vec = value + .as_object() + .expect("an object, not an array or scalar") + .keys() + .cloned() + .collect(); + keys.sort(); + keys + } + + /// Contract invariants 2 and 3, asserted on the bytes rather than on the + /// type: ndjson means one complete object per line, no array wrapper and no + /// pretty-print, and the field names are exact (fastjson, no remap). + /// + /// Note the deliberate singular/plural split between the two modes + /// (`precursor_mz` vs `precursor_mzs`): they are two schemas, and + /// "unifying" them would break Carafe without failing anything else. #[test] - fn spectrum_output_emits_exactly_the_contract_field_names() { - assert_eq!( - key_set::(CARAFE_SPECTRUM_RESULT), - [ - "fragment_intensities", - "fragment_mzs", - "id", - "mobility_ook0", - "precursor_charge", - "precursor_intensities", - "precursor_labels", - "precursor_mz", - "rt_seconds", - ] + fn spectrum_results_are_ndjson_with_the_contract_field_names() { + let records: Vec = + vec![parse(CARAFE_SPECTRUM_RESULT), parse(CARAFE_SPECTRUM_RESULT)]; + let out = write_results(&records, SerializationFormat::Ndjson); + + assert!(!out.starts_with('['), "no array wrapper: {out}"); + assert!(out.ends_with('\n'), "every record is newline-terminated"); + let lines: Vec<&str> = out.lines().collect(); + assert_eq!(lines.len(), 2, "one line per record"); + assert!( + !out.contains("\n "), + "ndjson must not be pretty-printed: {out}" ); + + for line in lines { + assert_eq!( + keys_of(line), + [ + "fragment_intensities", + "fragment_mzs", + "id", + "mobility_ook0", + "precursor_charge", + "precursor_intensities", + "precursor_labels", + "precursor_mz", + "rt_seconds", + ] + ); + } } #[test] - fn chromatogram_output_emits_exactly_the_contract_field_names() { + fn chromatogram_results_are_ndjson_with_the_contract_field_names() { + let records: Vec = vec![parse(CARAFE_CHROMATOGRAM_RESULT)]; + let out = write_results(&records, SerializationFormat::Ndjson); + + assert!(!out.starts_with('['), "no array wrapper: {out}"); + let lines: Vec<&str> = out.lines().collect(); + assert_eq!(lines.len(), 1); assert_eq!( - key_set::(CARAFE_CHROMATOGRAM_RESULT), + keys_of(lines[0]), [ "fragment_intensities", "fragment_labels", @@ -612,6 +654,32 @@ mod carafe_output_contract { ); } + /// The ndjson/array distinction is real, not an accident of the writer + /// happening to emit one object. Carafe parses line-by-line, so an array + /// wrapper is a parse failure on their side — this pins that the OTHER + /// formats are the ones that wrap, and therefore that `-f ndjson` matters. + #[test] + fn the_non_ndjson_formats_do_wrap_in_an_array() { + let records: Vec = vec![parse(CARAFE_SPECTRUM_RESULT)]; + for format in [SerializationFormat::Json, SerializationFormat::PrettyJson] { + let out = write_results(&records, format); + assert!( + out.starts_with('[') && out.ends_with(']'), + "{format:?} must wrap, else `-f ndjson` is not load-bearing: {out}" + ); + } + // And the default is NOT ndjson, so Carafe passing `-f` is required. + assert_ne!(SerializationFormat::default(), SerializationFormat::Ndjson); + } + + /// An empty result set must still be one parseable file, not a truncated + /// one. Carafe reads every line; zero lines is a valid empty result. + #[test] + fn an_empty_ndjson_result_is_empty_not_malformed() { + let out = write_results::(&[], SerializationFormat::Ndjson); + assert_eq!(out, "", "no records means no lines, and no array wrapper"); + } + /// The `-a` and `-f` values Carafe passes on the command line. clap derives /// these from the variant names, so a rename silently changes the CLI. #[test] From 621f554e87ac81485c5918dbc55df74e955472dd Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 09:45:28 -0700 Subject: [PATCH 24/27] refactor: one strip_mods, trim dead serde exports, name the parse-gate offender MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three near-identical `strip_mods` copies differed only in which bracket pairs they recognised — DIA-NN's and timsseek's missed `{...}`, so a Spectronaut- style mod would survive into a "stripped" sequence. One implementation now handles all three pairs and is covered for nesting and unbalanced closers. The mzSpecLib reader took its stripped sequence from `MS:1000888` and fell back to `""`, which is not a peptide with no mods, it is a zero-residue peptide: the row silently took the averagine isotope path and was tallied two layers away, where the cause was no longer visible. That term is optional in mzSpecLib, so the stripped form is derived from the proforma and counted. `ElutionGroupCollection`, `FileReadingExtras`, the per-format `*PrecursorExtras` and `LibrarySniffError` were `pub use`d from `serde` with no consumers anywhere. Un-exporting them surfaced dead members the `pub` had been masking: an unused `len`, an extras slot on `StringLabels` no reader fills, and three sniff-error payloads that were formatted nowhere. `LibrarySniffError` now has a `Display` and the Spectronaut sniff logs it — `MissingColumns` names the columns a near-miss export lacks, which is the difference between "wrong format" and "wrong export settings". One unparsable row anywhere disables sequence-derived scoring for the entire library; the only record was an `info!` with no offending sequence. It is now a `warn!` that names the row and the usual cause. --- rust/timsquery/src/serde/diann_speclib_io.rs | 18 +------- rust/timsquery/src/serde/library_file.rs | 28 +++++++----- rust/timsquery/src/serde/mod.rs | 10 ++--- rust/timsquery/src/serde/mzspeclib_io.rs | 38 +++++++++------- rust/timsquery/src/serde/skyline_io.rs | 30 ++++--------- rust/timsquery/src/serde/spectronaut_io.rs | 14 ++++++ rust/timsquery/src/utils/mod.rs | 1 + rust/timsquery/src/utils/sequence.rs | 46 ++++++++++++++++++++ rust/timsseek/src/data_sources/speclib.rs | 43 ++++++++---------- 9 files changed, 130 insertions(+), 98 deletions(-) create mode 100644 rust/timsquery/src/utils/sequence.rs diff --git a/rust/timsquery/src/serde/diann_speclib_io.rs b/rust/timsquery/src/serde/diann_speclib_io.rs index 6ca4babc..0574286e 100644 --- a/rust/timsquery/src/serde/diann_speclib_io.rs +++ b/rust/timsquery/src/serde/diann_speclib_io.rs @@ -604,22 +604,6 @@ impl SpeclibDecodeStats { } } -/// Strip DIA-NN mod annotations — anything inside `(...)` or `[...]` — leaving -/// the bare residue string. -fn strip_mods(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - let mut depth: i32 = 0; - for c in s.chars() { - match c { - '(' | '[' => depth += 1, - ')' | ']' => depth = (depth - 1).max(0), - _ if depth == 0 => out.push(c), - _ => {} - } - } - out -} - /// Residue count of a mod-stripped sequence. fn residue_count(stripped: &str) -> usize { stripped.chars().filter(|c| c.is_ascii_alphabetic()).count() @@ -718,7 +702,7 @@ fn map_entry( name.clone() } }; - let stripped_peptide = strip_mods(&modified_peptide); + let stripped_peptide = crate::utils::sequence::strip_mods(&modified_peptide); // `Peptide.length` may be 0 in some libraries; recover it from the sequence // when needed (b/a/c series don't need it, y/x/z series do). diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index 925130ef..65bf8b33 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -75,18 +75,13 @@ pub enum FileReadingExtras { #[derive(Debug)] pub enum ElutionGroupCollection { - StringLabels(Vec>, Option), + /// No reader supplies extras for string labels: they carry no ion + /// chemistry, so there are no reference intensities to thread through. + StringLabels(Vec>), MzpafLabels(Vec>, Option), } impl ElutionGroupCollection { - pub fn len(&self) -> usize { - match self { - ElutionGroupCollection::StringLabels(egs, _) => egs.len(), - ElutionGroupCollection::MzpafLabels(egs, _) => egs.len(), - } - } - fn try_read_json(path: &Path) -> Result { let file_content = std::fs::read_to_string(path).map_err(LibraryReadingError::IoError)?; info!("Read file content from {}", path.display()); @@ -117,7 +112,7 @@ impl ElutionGroupCollection { if let Ok(eg_inputs) = serde_json::from_str::>>(content) { let out: Result>, ElutionGroupInputError> = eg_inputs.into_iter().map(|x| x.try_into()).collect(); - return Ok(ElutionGroupCollection::StringLabels(out?, None)); + return Ok(ElutionGroupCollection::StringLabels(out?)); } Err(LibraryReadingError::UnableToParseElutionGroups) } @@ -131,7 +126,7 @@ impl ElutionGroupCollection { } debug!("Attempting to deserialize elution groups with string labels"); if let Ok(egs) = serde_json::from_str::>>(content) { - return Ok(ElutionGroupCollection::StringLabels(egs, None)); + return Ok(ElutionGroupCollection::StringLabels(egs)); } Err(LibraryReadingError::UnableToParseElutionGroups) } @@ -324,7 +319,7 @@ impl LibraryArena { frag_intens: None, }) } - ElutionGroupCollection::StringLabels(egs, _) => { + ElutionGroupCollection::StringLabels(egs) => { // String-labelled arenas carry no ion chemistry and ship no // decoys: sequence/fragment features unavailable, decoys off. let mut geom = @@ -557,7 +552,16 @@ impl LibraryReader for SpectronautReader { } fn sniff(&self, path: &Path) -> bool { - sniff_spectronaut_library_file(path).is_ok() + // Logged rather than discarded: `MissingColumns` names the columns a + // near-miss Spectronaut export lacks, which is the difference between + // "wrong format" and "right format, wrong export settings". + match sniff_spectronaut_library_file(path) { + Ok(()) => true, + Err(e) => { + debug!("not a Spectronaut TSV: {e}"); + false + } + } } fn read(&self, path: &Path) -> Result { diff --git a/rust/timsquery/src/serde/mod.rs b/rust/timsquery/src/serde/mod.rs index 903eb6d5..7de878cd 100644 --- a/rust/timsquery/src/serde/mod.rs +++ b/rust/timsquery/src/serde/mod.rs @@ -10,14 +10,12 @@ mod spectronaut_io; pub use chromatogram_output::*; pub use index_serde::*; +// The reader-internal types (`ElutionGroupCollection`, `FileReadingExtras`, the +// per-format `*PrecursorExtras`, `LibrarySniffError`) are deliberately NOT +// re-exported: they had no consumers outside this module, and every format +// already funnels into `LibraryArena`, which is the boundary worth supporting. pub use library_file::{ - DiannPrecursorExtras, - ElutionGroupCollection, - FileReadingExtras, LibraryArena, LibraryReadingError, - SkylinePrecursorExtras, - SpectronautPrecursorExtras, read_library_file, }; -pub use spectronaut_io::LibrarySniffError; diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs index 6a251c35..083c0400 100644 --- a/rust/timsquery/src/serde/mzspeclib_io.rs +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -181,6 +181,9 @@ anomaly_counters! { /// Peaks dropped because the precursor had already spent all 255 unknown /// labels, so no distinct one was left. dropped_unknown_over_capacity => "dropped with the unknown labels exhausted", + /// Spectra with no `MS:1000888` stripped sequence, whose bare residues + /// were derived from the proforma instead. Not required by mzSpecLib. + stripped_sequence_derived => "spectra with a derived stripped sequence", /// Spectra with no retention-time term at all. spectra_without_rt => "spectra without an RT", /// Spectra whose mobility came from a drift time rather than 1/K0. @@ -498,23 +501,28 @@ fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option a.value.clone(), + None => { + stats.stripped_sequence_derived += 1; + crate::utils::sequence::strip_mods(modified.as_deref()?) + } + }; + let modified = modified.unwrap_or_else(|| stripped.clone()); if stripped.is_empty() && modified.is_empty() { return None; } diff --git a/rust/timsquery/src/serde/skyline_io.rs b/rust/timsquery/src/serde/skyline_io.rs index 696ea2cd..2340bfc9 100644 --- a/rust/timsquery/src/serde/skyline_io.rs +++ b/rust/timsquery/src/serde/skyline_io.rs @@ -155,23 +155,6 @@ impl SkylineLibraryRow { } } -/// Remove bracketed modification annotations, e.g. `C[+57.02]AM` -> `CAM`. -fn strip_modifications(modified_seq: &str) -> String { - let mut out = String::with_capacity(modified_seq.len()); - let mut depth: i32 = 0; - for ch in modified_seq.chars() { - match ch { - '[' | '(' | '{' => depth += 1, - ']' | ')' | '}' if depth > 0 => { - depth -= 1; - } - _ if depth == 0 => out.push(ch), - _ => {} - } - } - out -} - /// Check if a file is a Skyline Peptide Transition List CSV. pub fn sniff_skyline_library_file>(file: T) -> Result<(), SkylineSniffError> { let file_handle = std::fs::File::open(file.as_ref()).map_err(|e| { @@ -362,7 +345,7 @@ fn parse_precursor_group( } let modified_peptide = first_row.peptide_modified_sequence.clone(); - let stripped_peptide = strip_modifications(&modified_peptide); + let stripped_peptide = crate::utils::sequence::strip_mods(&modified_peptide); let precursor_extras = SkylinePrecursorExtras { modified_peptide, @@ -407,10 +390,13 @@ mod tests { #[test] fn test_strip_modifications() { - assert_eq!(strip_modifications("PEPTIDE"), "PEPTIDE"); - assert_eq!(strip_modifications("C[+57.021]AM"), "CAM"); - assert_eq!(strip_modifications("P[UniMod:35]IDE"), "PIDE"); - assert_eq!(strip_modifications("[+42]AB"), "AB"); + assert_eq!(crate::utils::sequence::strip_mods("PEPTIDE"), "PEPTIDE"); + assert_eq!(crate::utils::sequence::strip_mods("C[+57.021]AM"), "CAM"); + assert_eq!( + crate::utils::sequence::strip_mods("P[UniMod:35]IDE"), + "PIDE" + ); + assert_eq!(crate::utils::sequence::strip_mods("[+42]AB"), "AB"); } #[test] diff --git a/rust/timsquery/src/serde/spectronaut_io.rs b/rust/timsquery/src/serde/spectronaut_io.rs index 2c44ec4b..20a37f8d 100644 --- a/rust/timsquery/src/serde/spectronaut_io.rs +++ b/rust/timsquery/src/serde/spectronaut_io.rs @@ -31,6 +31,20 @@ pub enum LibrarySniffError { MissingColumns(Vec), } +impl std::fmt::Display for LibrarySniffError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::IoError(e) => write!(f, "could not be read: {e}"), + Self::InvalidFormat(e) => write!(f, "headers did not parse: {e}"), + // The columns are the actionable part: a near-miss export is a + // settings problem, not a wrong-format one. + Self::MissingColumns(cols) => { + write!(f, "missing required columns: {}", cols.join(", ")) + } + } + } +} + impl std::fmt::Display for SpectronautReadingError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/rust/timsquery/src/utils/mod.rs b/rust/timsquery/src/utils/mod.rs index 399f6db2..5bc22549 100644 --- a/rust/timsquery/src/utils/mod.rs +++ b/rust/timsquery/src/utils/mod.rs @@ -1,4 +1,5 @@ pub mod display; +pub mod sequence; pub mod sorting; pub mod streaming_calculators; pub mod tolerance_ranges; diff --git a/rust/timsquery/src/utils/sequence.rs b/rust/timsquery/src/utils/sequence.rs new file mode 100644 index 00000000..47fc8357 --- /dev/null +++ b/rust/timsquery/src/utils/sequence.rs @@ -0,0 +1,46 @@ +//! Sequence-string helpers shared by the library readers. + +/// Strip modification annotations — anything inside `(...)`, `[...]` or +/// `{...}` — leaving the bare residue string. +/// +/// One implementation for every reader: DIA-NN, Skyline and Spectronaut all +/// spell modifications differently but nest them the same way, and three +/// copies that differ only in which bracket pairs they recognise is how a +/// library ends up with a "stripped" sequence that still has a `{` in it. +/// Unbalanced closers are ignored rather than driving the depth negative, so a +/// malformed sequence degrades to a partial strip instead of dropping the tail. +pub fn strip_mods(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut depth: u32 = 0; + for c in s.chars() { + match c { + '(' | '[' | '{' => depth += 1, + ')' | ']' | '}' => depth = depth.saturating_sub(1), + _ if depth == 0 => out.push(c), + _ => {} + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_every_bracket_style() { + assert_eq!(strip_mods("PEPTC[UNIMOD:4]IDEK"), "PEPTCIDEK"); + assert_eq!(strip_mods("PEPTC(UniMod:4)IDEK"), "PEPTCIDEK"); + assert_eq!(strip_mods("PEPTC{57.02}IDEK"), "PEPTCIDEK"); + assert_eq!(strip_mods("C[+57.02]AM"), "CAM"); + assert_eq!(strip_mods("PEPTIDEK"), "PEPTIDEK"); + } + + #[test] + fn nesting_and_unbalanced_closers_do_not_lose_the_tail() { + assert_eq!(strip_mods("PE[a[b]c]PTIDE"), "PEPTIDE"); + // A stray closer must not drive the depth negative and swallow the + // rest of the sequence. + assert_eq!(strip_mods("PEP]TIDEK"), "PEPTIDEK"); + } +} diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 80f0456d..34cd5181 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -104,24 +104,6 @@ impl ReferenceEG { } } -/// Strip mod annotations — anything inside `(...)` or `[...]` — from a -/// sequence, leaving the bare residue string. The native format ships one -/// (modified) sequence per precursor; the arena's composition-isotope path -/// needs the stripped residues. -fn strip_mods(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - let mut depth: i32 = 0; - for c in s.chars() { - match c { - '(' | '[' => depth += 1, - ')' | ']' => depth = (depth - 1).max(0), - _ if depth == 0 => out.push(c), - _ => {} - } - } - out -} - /// Summary of a [`finalize_reference_library`] call, for load-time logging. #[derive(Debug, Clone, Copy)] pub struct LoadReport { @@ -189,14 +171,17 @@ fn finalize_reference_library( } } } - let mut all_parsable = true; + // The first row that fails to parse, kept so the warning can name it. One + // bad row anywhere disables sequence-derived scoring for the WHOLE library, + // so "which row" is the only actionable part of that news. + let mut first_unparsable: Option = None; let mut n_averagine_fallback = 0usize; for tgt in 0..n_rows { - if all_parsable { + if first_unparsable.is_none() { let modified = &geom.seq_mod_blob[geom.seq_mod_range(tgt)]; let normalized = normalize_to_proforma(modified); if parse_sequence(&normalized).is_none() { - all_parsable = false; + first_unparsable = Some(modified.to_string()); } } let stripped = &geom.seq_strip_blob[geom.seq_strip_range(tgt)]; @@ -208,10 +193,16 @@ fn finalize_reference_library( } } - let sequence_features = if all_parsable { - SeqFeatureState::Available - } else { - SeqFeatureState::Unavailable + let sequence_features = match &first_unparsable { + None => SeqFeatureState::Available, + Some(sequence) => { + tracing::warn!( + "Sequence-derived scoring features are DISABLED for this entire library: \ + {sequence:?} could not be parsed. Any non-Unimod modification \ + (PSI-MOD, RESID, XL-MOD, cross-links) has this effect." + ); + SeqFeatureState::Unavailable + } }; geom.caps.sequence_features = sequence_features; @@ -492,7 +483,7 @@ impl Speclib { // The native format ships a single (modified) sequence; strip mod // annotations for the composition-isotope path. let modified = &elem.precursor.sequence; - let stripped = strip_mods(modified); + let stripped = timsquery::utils::sequence::strip_mods(modified); geom.push_row( eg.precursor_mz, elem.precursor.charge, From 6d0c09c87870ae9f5ca22fb44b030664c1e9e81e Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 09:49:54 -0700 Subject: [PATCH 25/27] refactor(timsquery): one adapter for the four tabular readers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `LibraryReader::read` was four bodies differing only in the function called, the log string and the `FileReadingExtras` variant — and since `read` started returning `LibraryArena`, all four built an `ElutionGroupCollection` only to destructure it in the same call. Each is now one line through `arena_from_pairs`. That deletes `FileReadingExtras`, whose three variants existed to be immediately flattened into `PrecursorExtrasRow`, and lets both `ElutionGroupCollection` variants drop their extras slot. Those four `map_err`s also flattened every reader-specific failure into `UnableToParseElutionGroups`, which made `read_library_file`'s keep-the-first-error rule preserve nothing — the variable was still called `last_err`. Failures now carry the reader name and its own error, and the variable matches the rule. --- rust/timsquery/src/serde/library_file.rs | 115 ++++++++++------------- rust/timsquery_viewer/src/error.rs | 3 + 2 files changed, 54 insertions(+), 64 deletions(-) diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index 65bf8b33..6e6deaf4 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -47,6 +47,15 @@ pub enum LibraryReadingError { SerdeJsonError(serde_json::Error), ElutionGroupInputError(ElutionGroupInputError), UnableToParseElutionGroups, + /// A reader that sniffed positively then failed to parse. Carries which + /// reader and its own error, so `read_library_file`'s keep-the-first-error + /// rule has something specific to keep — previously every reader flattened + /// its failure into `UnableToParseElutionGroups` and the rule preserved no + /// information. + ReaderFailed { + reader: &'static str, + detail: String, + }, /// A `.speclib` whose version is newer (more negative) than this reader /// supports. UnsupportedSpeclibVersion(i32), @@ -66,19 +75,16 @@ impl From for LibraryReadingError { } } -#[derive(Debug)] -pub enum FileReadingExtras { - Diann(Vec), - Spectronaut(Vec), - Skyline(Vec), -} - +/// Elution groups with no per-precursor extras — the JSON path. +/// +/// The TSV/parquet readers do not go through this type: they hand their +/// `(group, extras)` pairs straight to [`arena_from_pairs`], which is what +/// removed the three-variant `FileReadingExtras` enum that used to exist only +/// to be immediately flattened into `PrecursorExtrasRow`. #[derive(Debug)] pub enum ElutionGroupCollection { - /// No reader supplies extras for string labels: they carry no ion - /// chemistry, so there are no reference intensities to thread through. StringLabels(Vec>), - MzpafLabels(Vec>, Option), + MzpafLabels(Vec>), } impl ElutionGroupCollection { @@ -106,7 +112,7 @@ impl ElutionGroupCollection { if let Ok(eg_inputs) = serde_json::from_str::>>(content) { let out: Result>, ElutionGroupInputError> = eg_inputs.into_iter().map(|x| x.try_into()).collect(); - return Ok(ElutionGroupCollection::MzpafLabels(out?, None)); + return Ok(ElutionGroupCollection::MzpafLabels(out?)); } debug!("Attempting to deserialize elution group inputs with string labels"); if let Ok(eg_inputs) = serde_json::from_str::>>(content) { @@ -122,7 +128,7 @@ impl ElutionGroupCollection { debug!("Attempting direct deserialization of elution groups"); debug!("Attempting to deserialize elution groups with mzpaf labels"); if let Ok(egs) = serde_json::from_str::>>(content) { - return Ok(ElutionGroupCollection::MzpafLabels(egs, None)); + return Ok(ElutionGroupCollection::MzpafLabels(egs)); } debug!("Attempting to deserialize elution groups with string labels"); if let Ok(egs) = serde_json::from_str::>>(content) { @@ -212,16 +218,8 @@ impl LibraryArena { /// modified sequence. fn mzpaf_with_intensities( egs: Vec>, - extras: FileReadingExtras, + rows: Vec, ) -> Result { - let rows: Vec = match extras { - FileReadingExtras::Diann(v) => v.into_iter().map(PrecursorExtrasRow::from).collect(), - FileReadingExtras::Skyline(v) => v.into_iter().map(PrecursorExtrasRow::from).collect(), - FileReadingExtras::Spectronaut(v) => { - v.into_iter().map(PrecursorExtrasRow::from).collect() - } - }; - if egs.len() != rows.len() { return Err(LibraryReadingError::SpeclibParse(format!( "elution groups ({}) and reader extras ({}) length mismatch", @@ -293,10 +291,7 @@ impl LibraryArena { /// historical behavior where timsseek rejected that shape. fn from_elution_groups(egc: ElutionGroupCollection) -> Result { match egc { - ElutionGroupCollection::MzpafLabels(egs, Some(extras)) => { - Self::mzpaf_with_intensities(egs, extras) - } - ElutionGroupCollection::MzpafLabels(egs, None) => { + ElutionGroupCollection::MzpafLabels(egs) => { let mut geom = QueryCollection::with_capabilities(LibCapabilities::default_diann_no_decoys()); for eg in &egs { @@ -424,6 +419,30 @@ impl FragmentSet { } } +/// Adapt a `(elution group, per-precursor extras)` reader into the arena. +/// +/// The four tabular readers (DIA-NN TSV/parquet, Spectronaut, Skyline) differ +/// only in which function they call and which `*PrecursorExtras` they return, +/// so this is the whole of each one's `read`. +fn arena_from_pairs( + result: Result, E)>, Err>, + reader: &'static str, +) -> Result +where + PrecursorExtrasRow: From, + Err: std::fmt::Debug, +{ + let pairs = result.map_err(|e| { + warn!("{reader}: failed to read library file: {e:?}"); + LibraryReadingError::ReaderFailed { + reader, + detail: format!("{e:?}"), + } + })?; + let (egs, extras): (Vec<_>, Vec<_>) = pairs.into_iter().unzip(); + LibraryArena::mzpaf_with_intensities(egs, extras.into_iter().map(Into::into).collect()) +} + /// Seal a directly-built mzpaf arena together with its reference-intensity /// sidecar. /// @@ -512,15 +531,7 @@ impl LibraryReader for DiannParquetReader { } fn read(&self, path: &Path) -> Result { - let egs = read_diann_parquet(path).map_err(|e| { - warn!("Failed to read DIA-NN parquet library file: {:?}", e); - LibraryReadingError::UnableToParseElutionGroups - })?; - let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - LibraryArena::from_elution_groups(ElutionGroupCollection::MzpafLabels( - egs, - Some(FileReadingExtras::Diann(extras)), - )) + arena_from_pairs(read_diann_parquet(path), self.name()) } } @@ -534,15 +545,7 @@ impl LibraryReader for DiannTsvReader { } fn read(&self, path: &Path) -> Result { - let egs = read_diann_tsv(path).map_err(|e| { - warn!("Failed to read DIA-NN TSV library file: {:?}", e); - LibraryReadingError::UnableToParseElutionGroups - })?; - let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - LibraryArena::from_elution_groups(ElutionGroupCollection::MzpafLabels( - egs, - Some(FileReadingExtras::Diann(extras)), - )) + arena_from_pairs(read_diann_tsv(path), self.name()) } } @@ -565,15 +568,7 @@ impl LibraryReader for SpectronautReader { } fn read(&self, path: &Path) -> Result { - let egs = read_spectronaut_tsv(path).map_err(|e| { - warn!("Failed to read Spectronaut TSV library file: {:?}", e); - LibraryReadingError::UnableToParseElutionGroups - })?; - let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - LibraryArena::from_elution_groups(ElutionGroupCollection::MzpafLabels( - egs, - Some(FileReadingExtras::Spectronaut(extras)), - )) + arena_from_pairs(read_spectronaut_tsv(path), self.name()) } } @@ -587,15 +582,7 @@ impl LibraryReader for SkylineReader { } fn read(&self, path: &Path) -> Result { - let egs = read_skyline_csv(path).map_err(|e| { - warn!("Failed to read Skyline transition list: {:?}", e); - LibraryReadingError::UnableToParseElutionGroups - })?; - let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - LibraryArena::from_elution_groups(ElutionGroupCollection::MzpafLabels( - egs, - Some(FileReadingExtras::Skyline(extras)), - )) + arena_from_pairs(read_skyline_csv(path), self.name()) } } @@ -636,7 +623,7 @@ fn registry() -> &'static [&'static dyn LibraryReader] { pub fn read_library_file>(path: T) -> Result { let path = path.as_ref(); - let mut last_err = None; + let mut first_err = None; for reader in registry() { if reader.sniff(path) { info!("Dispatching library read to {}", reader.name()); @@ -651,14 +638,14 @@ pub fn read_library_file>(path: T) -> Result { warn!("{} sniffed but failed to read: {:?}", reader.name(), e); - last_err.get_or_insert(e); + first_err.get_or_insert(e); } } } } // Dead default in practice (JsonReader always sniffs true) — a harmless // defensive fallback. - Err(last_err.unwrap_or(LibraryReadingError::UnableToParseElutionGroups)) + Err(first_err.unwrap_or(LibraryReadingError::UnableToParseElutionGroups)) } #[cfg(test)] diff --git a/rust/timsquery_viewer/src/error.rs b/rust/timsquery_viewer/src/error.rs index 56d65f0f..888721e4 100644 --- a/rust/timsquery_viewer/src/error.rs +++ b/rust/timsquery_viewer/src/error.rs @@ -27,6 +27,9 @@ impl From for ViewerError { timsquery::serde::LibraryReadingError::UnableToParseElutionGroups => { ViewerError::General("Unable to parse elution groups".to_string()) } + timsquery::serde::LibraryReadingError::ReaderFailed { reader, detail } => { + ViewerError::General(format!("{reader} could not read this file: {detail}")) + } timsquery::serde::LibraryReadingError::UnsupportedSpeclibVersion(v) => { ViewerError::General(format!("Unsupported .speclib version: {v}")) } From e7c445c9faa088ccf1a17623e4fc83775edb8922 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 09:58:21 -0700 Subject: [PATCH 26/27] chore: dependency notes, LazyLock, and trim narrative comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mzcore` takes `default-features = false`: its `flate2` default is reachable only through `CVIndex::init()`, which we never call. The `mzcv` entry now records why it is a direct dependency at all and what breaks if mzcore is bumped without checking — `cargo tree -d` confirms one shared mzcv today. `ontologies()` was a `OnceLock` behind a wrapper fn while the same file used `LazyLock` for `AA_COUNT_NAMES`; now a `LazyLock` static like its neighbour. Comment trims. The "Task 9"/"Task-4"/`speclib_data_flow.md` references point at planning documents that are not in version control, so they name nothing a reader can follow; replaced with what the code does. Also drops comments arguing for a test's own value, one recording which input the author tried first, and one explaining what deleted code would have done — except in `or_averagine_falls_back_on_nonstandard`, where "`X` does not work here" is a real trap for the next person to touch that fixture, so it stays, reworded. Test trims in `loss.rs`: `phospho_losses_resolve` asserted three TABLE rows the table round trip already covers, and `NeutralLoss::composition` was a `#[cfg(test)]` method whose only use asserted TABLE against itself. The formula test claimed to cover implicit counts but every count in it was a single digit; it now covers multi-digit counts and the `u8` boundary, which nothing exercised. --- Cargo.lock | 1 - Cargo.toml | 12 +++- rust/micromzpaf/src/lib.rs | 8 +-- rust/micromzpaf/src/loss.rs | 61 ++++++------------- rust/speclib_build_cli/src/pipeline.rs | 3 +- rust/timsquery/src/serde/mzspeclib_io.rs | 4 +- .../src/data_sources/reference_library.rs | 2 +- rust/timsseek/src/data_sources/speclib.rs | 15 +++-- rust/timsseek/src/fragment_mass/averagine.rs | 9 ++- rust/timsseek/src/models/sequence.rs | 56 +++++++++-------- rust/timsseek/src/scoring/pipeline.rs | 6 +- 11 files changed, 80 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3e2d9ae..53de20f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4162,7 +4162,6 @@ checksum = "bca74d59b7c73d9c705a25622c73a186a1bcbf62c0faaf2b97fac8968b4b8d45" dependencies = [ "bincode", "context_error", - "flate2", "itertools 0.14.0", "mzcv", "ordered-float 5.3.0", diff --git a/Cargo.toml b/Cargo.toml index fa8cc32e..578fdc25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,17 @@ insta = { version = "1.34.0" } bon = "3.8.1" tinyvec = { features = ["alloc", "serde"], version = "1.10.0" } smallvec = { version = "1.13", features = ["const_generics", "union"] } -mzcore = { version = "0.2.0" } +# `default = ["flate2"]` is only reachable through `CVIndex::init()`, which +# downloads and inflates ontologies at runtime. We use `init_static` +# exclusively (see `timsseek::models::sequence::ontologies`), so the default +# feature is dead weight. The `rustyms` entry this replaced was also +# `default-features = false`. +mzcore = { version = "0.2.0", default-features = false } +# Direct dependency only because mzcore does not re-export mzcv, yet its API +# takes mzcv types (`CVIndex`, `AccessionCode`). They interoperate only because +# this resolves to the SAME mzcv instance mzcore 0.2 depends on — bumping +# mzcore without checking can silently yield two mzcv versions and a type +# mismatch that reads as an unrelated trait error. `cargo tree -d` catches it. mzcv = { version = "0.3.0" } csv = "1.3" diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index eb2deb2f..7886de90 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -811,9 +811,7 @@ mod tests { } } - /// Every field must survive the pack/unpack at its extremes. The bit - /// fields truncate rather than wrap, so a missing range check corrupts - /// silently — this is the test that catches it. + /// Every field must survive the pack/unpack at its extremes. #[test] fn every_field_round_trips_at_its_extremes() { for charge in CHARGE_MIN..=CHARGE_MAX { @@ -873,7 +871,7 @@ mod tests { // Non-canonical spelling resolves to the same annotation, and renders // canonically -- so this pair is equal, which is the property that - // keeps per-precursor label uniqueness honest. + // upholds per-precursor label uniqueness. assert_eq!(ion("y5-CH3SOH"), ion("y5-CH4OS")); assert_eq!(format!("{}", ion("y5-CH3SOH")), "y5-CH4OS"); @@ -999,7 +997,7 @@ mod tests { } /// `Display`, `from_series_char` and the parser must agree with the - /// packing for every case, not just the handful the other tests spell out. + /// packing for every case. #[test] fn every_series_variant_round_trips_through_its_mzpaf_spelling() { for series in all_series() { diff --git a/rust/micromzpaf/src/loss.rs b/rust/micromzpaf/src/loss.rs index eb864b85..8645fed6 100644 --- a/rust/micromzpaf/src/loss.rs +++ b/rust/micromzpaf/src/loss.rs @@ -286,19 +286,6 @@ impl NeutralLoss { .map(|(_, l, _)| *l)) } - /// The composition this loss removes. - #[cfg(test)] - pub(crate) fn composition(self) -> Composition { - if self == NeutralLoss::None { - return Composition::default(); - } - TABLE - .iter() - .find(|(_, l, _)| *l == self) - .map(|(c, _, _)| *c) - .unwrap_or_default() - } - /// Canonical spelling, without the leading `-`. Empty for [`Self::None`]. pub(crate) fn canonical(self) -> &'static str { if self == NeutralLoss::None { @@ -325,19 +312,28 @@ impl Display for NeutralLoss { mod tests { use super::*; + /// The count parser: implicit 1, multi-digit counts, and a count past the + /// `u8` slot. Single-digit cases are covered by every TABLE row. #[test] - fn formula_parses_counts_and_implicit_ones() { + fn atom_counts_parse_and_reject_out_of_range() { assert_eq!( Composition::parse_expression("H2O").unwrap(), - Composition::new(0, 2, 0, 1, 0, 0) + Composition::new(0, 2, 0, 1, 0, 0), + "an element with no digits is one atom" ); assert_eq!( - Composition::parse_expression("NH3").unwrap(), - Composition::new(0, 3, 1, 0, 0, 0) + Composition::parse_expression("C10H12").unwrap(), + Composition::new(10, 12, 0, 0, 0, 0), + "counts are multi-digit, not one digit per element" ); assert_eq!( - Composition::parse_expression("C2H5NOS").unwrap(), - Composition::new(2, 5, 1, 1, 1, 0) + Composition::parse_expression("C255").unwrap(), + Composition::new(255, 0, 0, 0, 0, 0), + "the u8 slot is full at 255" + ); + assert!( + Composition::parse_expression("C256").is_err(), + "one past the slot must be an error, not a wrap to 0" ); } @@ -381,22 +377,6 @@ mod tests { ); } - #[test] - fn phospho_losses_resolve() { - assert_eq!( - NeutralLoss::from_expression("H3PO4").unwrap(), - Some(NeutralLoss::PhosphoricAcid) - ); - assert_eq!( - NeutralLoss::from_expression("HPO3").unwrap(), - Some(NeutralLoss::Metaphosphoric) - ); - assert_eq!( - NeutralLoss::from_expression("H3PO4-H2O").unwrap(), - Some(NeutralLoss::PhosphoricAcidWater) - ); - } - /// A well-formed composition outside the table is `Ok(None)` — "valid but /// not representable" — while malformed text is `Err`. Callers need to /// tell those apart to route one to an unknown label and the other to a @@ -415,14 +395,13 @@ mod tests { /// silently decode as [`NeutralLoss::None`] — this is what catches that. #[test] fn table_round_trips_through_canonical_spelling() { - for (comp, loss, canon) in TABLE { + for (_comp, loss, canon) in TABLE { assert_eq!( NeutralLoss::from_expression(canon).unwrap(), Some(*loss), "canonical spelling {canon} must resolve to its own loss" ); assert_eq!(loss.canonical(), *canon); - assert_eq!(loss.composition(), *comp); assert_eq!( NeutralLoss::from_discriminant(*loss as u8), *loss, @@ -452,13 +431,11 @@ mod tests { } } + /// The two things the TABLE round trip cannot see: the `-` prefix Display + /// adds, and that `None` renders as nothing at all rather than "-". #[test] - fn display_uses_canonical_spelling() { + fn display_adds_the_prefix_and_none_renders_empty() { assert_eq!(NeutralLoss::None.to_string(), ""); assert_eq!(NeutralLoss::Water.to_string(), "-H2O"); - // Non-canonical input renders canonically; byte-identical round-trip - // is intentionally not a property of this type. - let parsed = NeutralLoss::from_expression("CH3SOH").unwrap().unwrap(); - assert_eq!(parsed.to_string(), "-CH4OS"); } } diff --git a/rust/speclib_build_cli/src/pipeline.rs b/rust/speclib_build_cli/src/pipeline.rs index 85cbee9f..1f565a46 100644 --- a/rust/speclib_build_cli/src/pipeline.rs +++ b/rust/speclib_build_cli/src/pipeline.rs @@ -267,8 +267,7 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box = if remote_output { // Fixed rather than derived from the destination URI: the writer only // emits zstd-wrapped NDJSON, and the upload below carries the caller's - // own name anyway. (`Path::extension` would have yielded `zst` for - // `lib.ndjson.zst`, losing the part that identifies the format.) + // own name anyway. let tf = tempfile::Builder::new() .prefix("speclib-out-") .suffix(".ndjson.zst") diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs index 083c0400..6343d0ce 100644 --- a/rust/timsquery/src/serde/mzspeclib_io.rs +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -125,7 +125,7 @@ const UNIT_SECOND: &str = "UO:0000010"; /// /// Spelled as a macro because the alternative — a struct plus a hand-written /// `||` chain plus a hand-written `warn!` — is three places to update per -/// counter and the compiler checks none of them. That already went wrong once. +/// counter and the compiler checks none of them. macro_rules! anomaly_counters { ($( $(#[$doc:meta])* $field:ident => $label:literal, )+) => { /// Per-library tally of everything that did not land verbatim in the @@ -430,7 +430,7 @@ fn resolve_annotation(annotation: &str) -> Resolved { /// The counting lives here rather than inside [`spectrum_row`] so that every /// `?` in there lands on a tally. Without it a library that dropped half its /// spectra for a missing charge still reports "all annotated and -/// representable", which is the one thing this module exists to prevent. +/// representable". fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { // A malformed spectrum bails out of `spectrum_row` via `?`, possibly after // incrementing RT/mobility counters on the way. Those are held aside and diff --git a/rust/timsseek/src/data_sources/reference_library.rs b/rust/timsseek/src/data_sources/reference_library.rs index fdd5d1f7..4e134087 100644 --- a/rust/timsseek/src/data_sources/reference_library.rs +++ b/rust/timsseek/src/data_sources/reference_library.rs @@ -345,7 +345,7 @@ mod tests { #[test] fn item_at_scores_reference_library() { - // Task 9 collapsed `Speclib` to the single `ReferenceLibrary` arena; + // `Speclib` is a type alias for `ReferenceLibrary`; // scoring reads `RefQuery` flyweights via `item_at` (no materialized // arm). Variant 0 is the target. let lib = tiny_ref_lib(); diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 34cd5181..8688db82 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -116,14 +116,13 @@ pub struct LoadReport { /// Finalize a freshly-narrowed lazy `ReferenceLibrary` arena: apply the decoy /// strategy, seal, run the whole-library parse gate + averagine tally, and set /// `caps.sequence_features`. This is the single shared tail of the DEFAULT -/// `.speclib` load (see `speclib_data_flow.md`) — the memory-optimized path -/// that avoids the 9 GB peak RSS of the fully-materialized target+2-decoy -/// expansion. +/// `.speclib` load — the memory-optimized path that avoids the 9 GB peak RSS +/// of a fully-materialized target+2-decoy expansion. /// /// `policy` is the raw CLI decoy policy: this is the single place it is resolved /// (via `map_decoy_strategy`, keyed on whether the arena already ships decoys) /// and stamped onto `caps.decoys` BEFORE `seal()`, so the seal's -/// `LazyMassShift -> Passthrough` downgrade (the Task-4 gate) sees it. The parse +/// `LazyMassShift -> Passthrough` downgrade sees it. The parse /// gate walks the MODIFIED sequence blob (the form /// `RefQuery::materialize_peptide_in_group` parses) and, if any row fails, /// disables sequence-derived features library-wide. The same pass counts @@ -230,7 +229,7 @@ fn finalize_reference_library( /// The spectral library store. Collapsed to the single columnar /// `ReferenceLibrary` arena representation (the materialized AOS path was -/// deleted in Task 9): both load paths produce a lazy arena, and scoring +/// since deleted): both load paths produce a lazy arena, and scoring /// iterates `RefQuery` flyweights via [`ReferenceLibrary::item_at`]. pub type Speclib = ReferenceLibrary; @@ -657,7 +656,7 @@ mod tests { } } - /// `Speclib` is now a type alias for `ReferenceLibrary` (Task 9 collapsed + /// `Speclib` is now a type alias for `ReferenceLibrary` (which collapsed /// the enum), so a loaded library is already the lazy arena. This identity /// helper is kept so the fixture assertions below read as /// "get the arena" without churning every call site. @@ -1078,7 +1077,7 @@ mod tests { /// Native `SerSpeclibElement` reader (ndjson) builds the lazy arena /// directly. The fixture ships one target + one stored decoy, so the - /// Task-4 seal gate downgrades `LazyMassShift -> Passthrough`: the arena is + /// seal gate downgrades `LazyMassShift -> Passthrough`: the arena is /// 1:1 with the stored rows (no synthetic mass-shift expansion). Proves the /// native path produces a lazy `ReferenceLibrary` with the right length, target/ /// decoy flags, and per-fragment reference intensities. @@ -1128,7 +1127,7 @@ mod tests { /// rejected by both the fast byte-walk parser and the mzcore fallback), so /// the gate must report `!parsable_sequences()`. This is the inverse of /// `test_diann_tsv_parsable_gate`, and the only test of the OFF branch after - /// the AOS `test_parse_gate_off_on_poisoned_row` was removed in Task 9. + /// the materialized `test_parse_gate_off_on_poisoned_row` was removed. #[test] fn from_file_native_ndjson_poisoned_row_disables_sequence_features() { let good = element("PEPTIDEK", false, 500.0, &["y1", "y2"], &[0.8, 0.3]); diff --git a/rust/timsseek/src/fragment_mass/averagine.rs b/rust/timsseek/src/fragment_mass/averagine.rs index 5904521c..e67032cc 100644 --- a/rust/timsseek/src/fragment_mass/averagine.rs +++ b/rust/timsseek/src/fragment_mass/averagine.rs @@ -74,11 +74,10 @@ mod tests { #[test] fn or_averagine_falls_back_on_nonstandard() { - // `B` (Asx) is genuinely ambiguous between Asp/Asn in mzcore and - // resolves to more than one formula, which is the real trigger for - // the mzcore-backed count path to error today. (`X` was tried first - // but mzcore resolves it to a defined zero-C/S formula rather than - // erroring, so it does not exercise the fallback.) + // `B` (Asx) is ambiguous between Asp/Asn in mzcore and resolves to + // more than one formula, which is what makes the mzcore-backed count + // path error. `X` does NOT work here: mzcore gives it a defined + // zero-C/S formula, so it never reaches the fallback. let (src, env) = isotope_dist_or_averagine("PEPBK", 600.0); assert_eq!(src, IsotopeSource::Averagine); let max = env.iter().copied().fold(f32::MIN, f32::max); diff --git a/rust/timsseek/src/models/sequence.rs b/rust/timsseek/src/models/sequence.rs index d9ec1f1e..b51f1310 100644 --- a/rust/timsseek/src/models/sequence.rs +++ b/rust/timsseek/src/models/sequence.rs @@ -7,15 +7,12 @@ use serde::Serialize; use smallvec::SmallVec; use std::sync::{ Arc, - OnceLock, + LazyLock, }; /// Modification ontologies for ProForma parsing: everything mzcore ships -/// except GNOme. -/// -/// mzcore's own `STATIC_ONTOLOGIES` loads all six, and GNOme is 191_529 entries -/// / 26.4 MB of the 27.8 MB total. Skipping it takes the build from ~2.6 s to -/// ~48 ms. +/// except GNOme, which is 26.4 MB of the 27.8 MB total and is decoded on first +/// use. /// /// Dropping an ontology normally costs you the sequences that reference it, but /// not here. A GNO-accession glycopeptide is already unusable: every mod goes @@ -28,34 +25,38 @@ use std::sync::{ /// [`count_carbon_sulphur_in_sequence`](crate::fragment_mass::elution_group_converter::count_carbon_sulphur_in_sequence): /// a `[GNO:...]` sequence no longer yields a composition, so its isotope /// envelope comes from averagine instead — the documented fallback, already -/// tallied as `n_averagine_fallback`. PSI-MOD, XL-MOD and RESID stay loaded -/// (~1.4 MB combined) so that path is unchanged for them. -fn ontologies() -> &'static mzcore::ontology::Ontologies { - static ONTOLOGIES: OnceLock = OnceLock::new(); - ONTOLOGIES.get_or_init(|| { - let mut ontologies = mzcore::ontology::Ontologies::empty(); - *ontologies.unimod_mut() = mzcv::CVIndex::init_static(); - *ontologies.psimod_mut() = mzcv::CVIndex::init_static(); - *ontologies.xlmod_mut() = mzcv::CVIndex::init_static(); - *ontologies.resid_mut() = mzcv::CVIndex::init_static(); - ontologies - }) -} - -/// Parse a ProForma string against [`ontologies`]. +/// tallied as `n_averagine_fallback`. PSI-MOD, XL-MOD and RESID stay loaded so +/// that path is unchanged for them. +static ONTOLOGIES: LazyLock = LazyLock::new(|| { + let mut ontologies = mzcore::ontology::Ontologies::empty(); + *ontologies.unimod_mut() = mzcv::CVIndex::init_static(); + *ontologies.psimod_mut() = mzcv::CVIndex::init_static(); + *ontologies.xlmod_mut() = mzcv::CVIndex::init_static(); + *ontologies.resid_mut() = mzcv::CVIndex::init_static(); + ontologies +}); + +/// Parse a ProForma string against [`ONTOLOGIES`]. /// -/// Built on first use, and this is the fallback *past* the byte-walk fast path -/// in [`parse_sequence`] — so a library whose sequences all match the fast -/// grammar never pays for it at all. A real DIA-NN `.speclib` load peaks at -/// ~10 MB and never gets here. +/// This is the fallback *past* the byte-walk fast path in [`parse_sequence`], +/// so a library whose sequences all match the fast grammar never decodes an +/// ontology at all. /// /// mzcore also returns non-fatal parse warnings alongside the peptidoform; none /// of the callers can act on them, so they are dropped in one place rather than /// at each site. +/// +/// The error is a rendered `String` rather than mzcore's own +/// `Vec>`. Returning the latter would be cheaper on +/// the path that discards it, but `BoxedError` comes from `context_error`, +/// which mzcore does not re-export — naming it means a second direct +/// dependency version-coupled to mzcore's, the same hazard documented on +/// `mzcv` in the workspace manifest. Not worth one allocation on an error +/// path. pub fn parse_proforma( sequence: &str, ) -> Result, String> { - mzcore::sequence::Peptidoform::pro_forma(sequence, ontologies()) + mzcore::sequence::Peptidoform::pro_forma(sequence, &ONTOLOGIES) .map(|(peptidoform, _warnings)| peptidoform) .map_err(|errors| { errors @@ -766,7 +767,8 @@ mod tests { ); } - // The one casualty. It failed before this change too, just later. + // The only sequence whose parse verdict GNOme affected, and it was + // unusable either way. assert!( parse_sequence("PEPTN[GNO:G59626AS]IDEK").is_none(), "a GNO glycopeptide was never usable" diff --git a/rust/timsseek/src/scoring/pipeline.rs b/rust/timsseek/src/scoring/pipeline.rs index f974db55..0f0520ce 100644 --- a/rust/timsseek/src/scoring/pipeline.rs +++ b/rust/timsseek/src/scoring/pipeline.rs @@ -256,7 +256,7 @@ fn gate_expected_fragments(expected: &ExpectedIntensities) -> Result<( } /// Fill the per-worker scratch elution group in place from a `RefQuery` -/// flyweight (Task 9). `reset_from` copies the per-variant geometry — for a +/// flyweight. `reset_from` copies the per-variant geometry — for a /// decoy the fragment m/z values are ALREADY shifted by value, so no extra /// work is needed. It also sets the precursor labels to the isotope-envelope /// indices via the flyweight's `iter_precursors` (`0..n_isotopes`), which match @@ -692,7 +692,7 @@ impl Scorer { flat_range: std::ops::Range, calibration: &CalibrationResult, ) -> (Vec, ScoreTimings, SkipCounts) { - // Single columnar store (Task 9 deleted the materialized arm): the + // Single columnar store: the // flyweight is always a `RefQuery` from the arena, so the loop is // monomorphized over one concrete type — statically dispatched, no // per-item heap allocation on the scoring hot path. @@ -839,7 +839,7 @@ impl Scorer { config: &CalibrationConfig, timings: &mut PrescoreTimings, ) -> CalibrantHeap { - // Single columnar store (Task 9): iterate `RefQuery` flyweights from + // Single columnar store: iterate `RefQuery` flyweights from // the arena directly — monomorphized, no per-item heap alloc on the // prescore hot path (see `score_calibrated_batch`). self.prescore_batch_impl(|f| lib.item_at(f), flat_range, config, timings) From 7fd8ed9c76f49105fdda7895dce9828839b6ed9f Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 27 Aug 2026 10:02:15 -0700 Subject: [PATCH 27/27] test(timsseek): share the fixture setup, drop the identity helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine tests hand-rolled the same six-line path to the sibling crate's test data, and `expect_lazy` was an identity function documented as existing to avoid churning ten call sites — churning them is the smaller cost. The two native-NDJSON tests wrote to `env::temp_dir()` under a PID-derived name and cleaned up with a `remove_file` placed after the assertions, so a failing assertion leaked the file and two concurrent runs of the same test binary collided. `tempfile` was already a dependency. --- rust/timsseek/src/data_sources/speclib.rs | 141 ++++++++-------------- 1 file changed, 48 insertions(+), 93 deletions(-) diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 8688db82..31ee0036 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -656,12 +656,29 @@ mod tests { } } - /// `Speclib` is now a type alias for `ReferenceLibrary` (which collapsed - /// the enum), so a loaded library is already the lazy arena. This identity - /// helper is kept so the fixture assertions below read as - /// "get the arena" without churning every call site. - fn expect_lazy(speclib: &Speclib) -> &ReferenceLibrary { - speclib + /// An NDJSON library written to a temp file that is removed when the + /// returned handle drops — including on panic, unlike an explicit + /// `remove_file` after the assertions. + fn write_ndjson_fixture(ndjson: &str) -> tempfile::NamedTempFile { + use std::io::Write as _; + let mut f = tempfile::Builder::new() + .suffix(".ndjson") + .tempfile() + .expect("tempfile"); + f.write_all(ndjson.as_bytes()).expect("write fixture"); + f.flush().expect("flush"); + f + } + + /// A reader fixture from the sibling `timsquery` crate's test data. + fn fixture(dir: &str, name: &str) -> std::path::PathBuf { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crate dir has a parent") + .join("timsquery") + .join("tests") + .join(dir) + .join(name) } #[test] @@ -671,13 +688,7 @@ mod tests { // Use the test file from timsquery tests // Note: sample_lib.tsv is in Skyline format and won't load as DIA-NN // So we test with sample_lib.txt which is in DIA-NN TSV format - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); assert!( test_file.exists(), @@ -696,7 +707,7 @@ mod tests { "Expected 6 entries (2 targets + 4 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; // Verify first target entry structure (variant 0 == target) let first_target = lib @@ -723,13 +734,7 @@ mod tests { #[test] fn test_diann_tsv_parsable_gate() { - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); @@ -744,13 +749,7 @@ mod tests { fn test_load_skyline_csv_library() { use timsquery::traits::QueryGeom; - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("skyline_io_files") - .join("sample_transition_list.csv"); + let test_file = fixture("skyline_io_files", "sample_transition_list.csv"); assert!( test_file.exists(), @@ -772,7 +771,7 @@ mod tests { "Expected 42 entries (14 targets + 28 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; let n_rows = lib.iter().filter(|q| q.geom().variant() == 0).count(); let n_decoys = lib.iter().filter(|q| q.geom().variant() != 0).count(); assert_eq!(n_rows, 14, "Should have 14 targets"); @@ -794,13 +793,7 @@ mod tests { #[test] fn test_load_diann_txt_library() { - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); assert!( test_file.exists(), @@ -819,7 +812,7 @@ mod tests { "Expected 6 entries (2 targets + 4 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; let n_rows = lib.iter().filter(|q| q.geom().variant() == 0).count(); let n_decoys = lib.iter().filter(|q| q.geom().variant() != 0).count(); @@ -829,13 +822,7 @@ mod tests { #[test] fn test_load_diann_parquet_library() { - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_pq_speclib.parquet"); + let test_file = fixture("diann_io_files", "sample_pq_speclib.parquet"); assert!( test_file.exists(), @@ -854,7 +841,7 @@ mod tests { "Expected 9 entries (3 targets + 6 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; let n_rows = lib.iter().filter(|q| q.geom().variant() == 0).count(); let n_decoys = lib.iter().filter(|q| q.geom().variant() != 0).count(); @@ -874,18 +861,12 @@ mod tests { #[test] fn test_isotope_envelope_calculation() { // Use the DIA-NN TSV test file - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); - let lib = expect_lazy(&speclib); + let lib = &speclib; // Check that isotope intensities are normalized (M0 should be 1.0), // for every flat entry (targets AND decoy variants — the envelope is @@ -911,13 +892,7 @@ mod tests { #[test] fn test_decoy_generation_for_library_without_decoys() { - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); @@ -930,7 +905,7 @@ mod tests { "Should have 6 entries (2 targets + 4 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; let n_rows = lib.iter().filter(|q| q.geom().variant() == 0).count(); let n_decoys = lib.iter().filter(|q| q.geom().variant() != 0).count(); @@ -959,18 +934,12 @@ mod tests { fn test_mass_shift_decoys() { use timsquery::traits::QueryGeom; - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); - let lib = expect_lazy(&speclib); + let lib = &speclib; // Unified CH2 offset (see `map_decoy_strategy`), replacing the old // 12.0 (materialized `IfMissing`) / 14.0 (materialized `Force`) split. @@ -1019,18 +988,12 @@ mod tests { fn test_fragment_intensities_preserved() { use timsquery::traits::QueryGeom; - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); - let lib = expect_lazy(&speclib); + let lib = &speclib; for q in lib.iter() { let fragments: Vec<_> = q.iter_expected_fragments().collect(); assert_eq!( @@ -1061,7 +1024,7 @@ mod tests { let speclib = Speclib::from_file(path, crate::models::DecoyPolicy::default()) .expect("from_file should load the .speclib fixture"); - let lib = expect_lazy(&speclib); + let lib = &speclib; assert!(!lib.is_empty(), "library should have entries"); let first = lib.item_at(0); @@ -1094,17 +1057,13 @@ mod tests { ndjson.push_str(&serde_json::to_string(&decoy).unwrap()); ndjson.push('\n'); - let path = std::env::temp_dir().join(format!( - "timsseek_native_fixture_{}.ndjson", - std::process::id() - )); - std::fs::write(&path, ndjson).unwrap(); + let file = write_ndjson_fixture(&ndjson); + let path = file.path(); - let speclib = Speclib::from_file(&path, crate::models::DecoyPolicy::default()) + let speclib = Speclib::from_file(path, crate::models::DecoyPolicy::default()) .expect("native ndjson should load"); - std::fs::remove_file(&path).ok(); - let lib = expect_lazy(&speclib); + let lib = &speclib; // Ships a decoy -> Passthrough -> 1 variant/row -> flat len == n_rows. assert_eq!(lib.geom.variants_per_row(), 1, "downgraded to Passthrough"); assert_eq!(lib.len(), 2, "one target + one stored decoy, 1:1"); @@ -1141,15 +1100,11 @@ mod tests { ndjson.push_str(&serde_json::to_string(&poisoned).unwrap()); ndjson.push('\n'); - let path = std::env::temp_dir().join(format!( - "timsseek_poisoned_fixture_{}.ndjson", - std::process::id() - )); - std::fs::write(&path, ndjson).unwrap(); + let file = write_ndjson_fixture(&ndjson); + let path = file.path(); - let speclib = Speclib::from_file(&path, crate::models::DecoyPolicy::default()) + let speclib = Speclib::from_file(path, crate::models::DecoyPolicy::default()) .expect("native ndjson should load even with an unparseable sequence"); - std::fs::remove_file(&path).ok(); // The poisoned row flips the whole-library gate OFF. assert!(