diff --git a/.gitignore b/.gitignore index 9443e54f..001bd603 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,6 @@ scratchpad_ab/ ab_bench.out # Local bench outputs, logs, feature dumps (repo-local, never tracked) bench_out/ + +# In-place-edit artifacts +*.bak diff --git a/Cargo.lock b/Cargo.lock index 51e2c068..53de20f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -237,7 +237,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -248,7 +248,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -961,7 +961,7 @@ dependencies = [ "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] @@ -1246,6 +1246,15 @@ dependencies = [ "generic-array 0.14.7", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.5.1" @@ -1656,6 +1665,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1678,9 +1693,9 @@ dependencies = [ [[package]] name = "context_error" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7e1b8dc6f4cdc4f6b897d6aa1b7eaec6d95331bdb765d2a51cdd948e157ee0" +checksum = "c9bc4a1eacadc98da5e0c4f02884efd5c5ad4a267e5e6175b3a53a9918ecb458" dependencies = [ "serde", ] @@ -1856,6 +1871,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csscolorparser" version = "0.6.2" @@ -1970,11 +1994,43 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", +] + +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "dispatch" version = "0.2.0" @@ -2368,7 +2424,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3135,7 +3191,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -3214,6 +3270,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -3967,7 +4032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -4019,8 +4084,8 @@ dependencies = [ name = "micromzpaf" version = "0.33.0" dependencies = [ - "rustyms", "serde", + "serde_json", "thiserror 2.0.18", ] @@ -4089,6 +4154,40 @@ dependencies = [ "pxfm", ] +[[package]] +name = "mzcore" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca74d59b7c73d9c705a25622c73a186a1bcbf62c0faaf2b97fac8968b4b8d45" +dependencies = [ + "bincode", + "context_error", + "itertools 0.14.0", + "mzcv", + "ordered-float 5.3.0", + "roxmltree", + "serde", + "serde_json", + "sha2 0.11.0", + "thin-vec", + "uom", +] + +[[package]] +name = "mzcv" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97dae61b4d9d35973769f286a138dc2ec55fc57ef06e5692c3ffc771427c5f7" +dependencies = [ + "bincode", + "chrono", + "context_error", + "directories", + "flate2", + "serde", + "sha2 0.11.0", +] + [[package]] name = "mzdata" version = "0.65.4" @@ -4228,7 +4327,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4695,6 +4794,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "orbclient" version = "0.3.51" @@ -5568,6 +5673,17 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + [[package]] name = "regex" version = "1.12.3" @@ -5699,25 +5815,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rmp" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" -dependencies = [ - "num-traits", -] - -[[package]] -name = "rmp-serde" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" -dependencies = [ - "rmp", - "serde", -] - [[package]] name = "robust" version = "1.2.0" @@ -5751,6 +5848,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rstar" version = "0.8.4" @@ -5870,7 +5976,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5928,25 +6034,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "rustyms" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "011d3d672ae44d5e07db0488d855f2b5ed178e3d6bb7ef5b18c6415c20bbd61e" -dependencies = [ - "bincode", - "context_error", - "flate2", - "itertools 0.14.0", - "ordered-float 5.3.0", - "regex", - "serde", - "serde_json", - "similar", - "thin-vec", - "uom", -] - [[package]] name = "ryu" version = "1.0.23" @@ -6114,7 +6201,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -6125,7 +6212,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -6314,7 +6412,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6337,8 +6435,8 @@ dependencies = [ "clap", "indicatif", "micromzpaf", + "mzcore", "reqwest", - "rustyms", "serde", "serde_json", "tempfile", @@ -6477,7 +6575,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6499,7 +6597,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6548,7 +6646,7 @@ dependencies = [ "pest", "pest_derive", "phf", - "sha2", + "sha2 0.10.9", "signal-hook", "siphasher", "terminfo", @@ -6826,13 +6924,13 @@ dependencies = [ "forust-ml", "matrixmultiply", "micromzpaf", + "mzcore", + "mzcv", "parquet", "rand 0.9.3", "rayon", "regex", - "rmp-serde", "rusqlite", - "rustyms", "serde", "serde_json", "smallvec", @@ -7230,9 +7328,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -7248,7 +7346,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7756,7 +7854,7 @@ checksum = "692daff6d93d94e29e4114544ef6d5c942a7ed998b37abdc19b17136ea428eb7" dependencies = [ "getrandom 0.3.4", "mac_address", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "uuid", ] @@ -7983,7 +8081,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7cd1bee9..578fdc25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,18 @@ insta = { version = "1.34.0" } bon = "3.8.1" tinyvec = { features = ["alloc", "serde"], version = "1.10.0" } smallvec = { version = "1.13", features = ["const_generics", "union"] } -rustyms = { version = "0.11.0", default-features = false } +# `default = ["flate2"]` is only reachable through `CVIndex::init()`, which +# downloads and inflates ontologies at runtime. We use `init_static` +# exclusively (see `timsseek::models::sequence::ontologies`), so the default +# feature is dead weight. The `rustyms` entry this replaced was also +# `default-features = false`. +mzcore = { version = "0.2.0", default-features = false } +# Direct dependency only because mzcore does not re-export mzcv, yet its API +# takes mzcv types (`CVIndex`, `AccessionCode`). They interoperate only because +# this resolves to the SAME mzcv instance mzcore 0.2 depends on — bumping +# mzcore without checking can silently yield two mzcv versions and a type +# mismatch that reads as an unrelated trait error. `cargo tree -d` catches it. +mzcv = { version = "0.3.0" } csv = "1.3" tempfile = "3.23.0" diff --git a/README.md b/README.md index 20a03915..6beb14b7 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ integrate other sources of predictions for it.) ```bash DOTD_FILE="$HOME/data/my_data.d" FASTA_FILE="$HOME/fasta/VIMENTIN.fasta" -SPECLIB_NAME="vimentin.ndjson" +SPECLIB_NAME="vimentin.ndjson.zst" RESULTS_DIR="vimentin_search_results" # Build the spectral lib using Koina (Prosit) for fragment/RT prediction. @@ -85,11 +85,11 @@ Both CLIs accept `s3://` URIs anywhere a path is accepted (AWS / MinIO / R2). `. ```bash timsseek --raw-inputs s3://bkt/sample.d.tar \ - --speclib-uri s3://bkt/lib.msgpack.zst \ + --speclib-uri s3://bkt/lib.ndjson.zst \ --output-uri s3://bkt/runs/out speclib_build_cli --fasta s3://bkt/proteome.fasta \ - --output s3://bkt/lib.msgpack.zst + --output s3://bkt/lib.ndjson.zst ``` Auth via AWS default chain. MinIO/R2: set `AWS_ENDPOINT_URL`. See `docs/development.md` for `[staging]` config + env var list. @@ -98,6 +98,11 @@ Auth via AWS default chain. MinIO/R2: set `AWS_ENDPOINT_URL`. See `docs/developm See [docs/development.md](docs/development.md) for dev utilities, compile flags, env vars, Taskfile targets, and scripts. +[docs/CARAFE_CONTRACT.md](docs/CARAFE_CONTRACT.md) pins the CLI surface +[Carafe](https://github.com/Noble-Lab/Carafe) drives as a subprocess — read it +before renaming a JSON field or an output path. It is enforced by +`rust/timsquery/tests/carafe_contract.rs`. + ## License This project is licensed under the Apache License, Version 2.0. diff --git a/bench/wandb_bench.py b/bench/wandb_bench.py index d9b17ef0..1c6b6ddf 100644 --- a/bench/wandb_bench.py +++ b/bench/wandb_bench.py @@ -225,7 +225,7 @@ def wandb_context(config_dict: dict[str, Any], wandb_kwargs=None): def main(wandb_kwargs: dict | None = None, koina_url: str | None = None): fasta_file = Path.home() / "fasta/hela_gt20peps.fasta" - speclib_path = Path.home() / "fasta/asdad.msgpack.zstd" + speclib_path = Path.home() / "fasta/asdad.ndjson.zst" prefix = Path.home() / "data/decompressed_timstof/" dotd_files = [ diff --git a/docs/CARAFE_CONTRACT.md b/docs/CARAFE_CONTRACT.md new file mode 100644 index 00000000..4bfd47e2 --- /dev/null +++ b/docs/CARAFE_CONTRACT.md @@ -0,0 +1,114 @@ +# Carafe contract + +What Carafe assumes when calling `timsquery`. Break any of it → silent failure or NPE. +Refs: `util/CallTimsQuery.java`, `ai/AIGear.java`, `dia/{PSMQuery,PSMQueryResult,XICQueryResult}.java`. + +> Vendored from the Carafe repo so the assumptions live next to the code that +> has to honour them. +> +> **Line numbers are deliberately omitted**: an earlier version of this file +> cited `AIGear.java` at `~6660–6912`, which is wrong the moment Carafe lands a +> commit above line 6660. Search for the symbol instead. If you re-verify this +> contract against Carafe, record the commit you checked below. +> +> Last verified against Carafe: _(unrecorded — add the SHA when you next check)_ +> +> Two test modules pin the mechanically checkable parts against the literal +> JSON in this document. If you edit a payload here, edit it there: +> - `rust/timsquery/tests/carafe_contract.rs` — input (targets, tolerances) +> - `timsquery_cli`'s `carafe_output_contract` module — output (result field +> names, `-a`/`-f` flag values, the `results.json` basename) + +## CLI + +``` + query-index -a -r -t -e -f ndjson -o +``` + +| Flag | Value | +|---|---| +| subcmd | `query-index` | +| `-a` | `spectrum-aggregator` \| `chromatogram-aggregator` (selects output schema) | +| `-r` | raw `.d` input | +| `-t` | tolerance JSON (§tolerances) | +| `-e` | query targets JSON (§targets) | +| `-f` | `ndjson` | +| `-o` | output **directory** (not file) | + +Binary path: `bin/timsquery/{windows,macos,linux}/timsquery_cli[.exe]`. + +**Fragile couplings** (most likely to break on a timsquery change): +- Aggregator names are literals — rename breaks Carafe, no fallback. +- `-o` is a dir; Carafe reads `/results.json` — basename must be exactly `results.json`. +- Output parsed line-by-line — must be ndjson, one object per line. +- Exit 0 = success; nonzero logged, no retry. + +## Tolerances (`-t`) + +```json +{ + "ms": { "ppm": [itol-itol_shift, itol+itol_shift] }, + "rt": { "minutes": [rt_win, rt_win] }, + "mobility": { "percent": [mobility, mobility] }, + "quad": { "absolute": [quad, quad] } +} +``` +`ms` unit key is dynamic on Carafe's side (`itolu`, currently `ppm`), but timsquery only accepts `ppm`/`da` (and the `Ppm`/`Absolute` spellings) — any other unit fails to deserialize. Each value is `[low, high]`. A negative `low` is valid and means the window sits entirely above the target mass. Defaults: itol 15, mobility 3.0, quad 0.1, rt_win 0.1 (spectra) / `CParameter.rt_win` (xic). + +### `ms` window derivation (`itol` / `itol_shift`) + +The `ms` window is `[itol - itol_shift, itol + itol_shift]`, built from two independent inputs: + +- **`itol`** = `CParameter.itol` — the configured fragment-ion tolerance (default 15). Static per run. +- **`itol_shift`** = per-run **median observed m/z error**, a data-driven calibration offset measured before the query (`AIGear.java`, search `error_shift`): + - Carafe collects MS1 and MS2 mass errors from already-matched ions, takes the median of each (`ms1_error_shift`, `ms2_error_shift`). + - If precursor and fragment units match (`CParameter.tolu` == `CParameter.itolu`): `itol_shift = max(ms1_error_shift, ms2_error_shift)`. + - Else: `itol_shift = ms2_error_shift` (fragment only). + - No matched ions → `itol_shift = 0`, so the window collapses to `[itol, itol]`. + +timsquery interprets the pair as `[light_magnitude, heavy_magnitude]`: `[2, 7]` = the ion may be up to 2 ppm light **or** up to 7 ppm heavy (signed error range `[-2, +7]`). Under that convention Carafe's `[itol - itol_shift, itol + itol_shift]` is a `±itol` window recentered on the calibration offset `itol_shift`: + +- signed error window = `[itol_shift - itol, itol_shift + itol]` +- light magnitude = `itol - itol_shift`, heavy magnitude = `itol + itol_shift` + +Example: `itol` 15, `itol_shift` +2 → `[13, 17]` = "13 ppm light to 17 ppm heavy" = a ±15 ppm window centered on the +2 ppm systematic offset. This is the intended behavior — the offset shifts the window's center, `itol` sets its half-width. + +## Targets (`-e`, `psm_query.json`) + +JSON **array** of: +```json +{ "id": 0, "mobility": 0.95, "rt_seconds": 1234.5, "precursor": 650.32, + "precursor_charge": 2, "precursor_isotopes": [0,1,2], + "fragments": [175.1, 288.2], "fragment_labels": ["y1","y3^2"] } +``` +`id` = row index, echoed back in results. `rt_seconds` = RT_min × 60. `fragments`↔`fragment_labels` positional. Labels: charge 1 → `y3`, else `y3^2`; precursor → `p`/`p^z`. + +## Results (`-o`, `/results.json`) + +**ndjson** — one object per line, one line per `id`. Parsed `JSON.parseObject(line, …)`, keyed by `id`. +(`.json` name is misleading; it is not a JSON array.) + +### spectrum-aggregator → `PSMQueryResult` +```json +{ "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, "precursor_mz":650.32, + "precursor_charge":2, "precursor_intensities":[1200,800,300], "precursor_labels":[0,1,2], + "fragment_mzs":[175.1,288.2], "fragment_intensities":[500,0] } +``` +Scalar intensity per ion (no RT axis). `precursor_intensities`/`fragment_intensities` are **1-D**, positionally paired with their m/z arrays. + +### chromatogram-aggregator → `XICQueryResult` +```json +{ "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, + "precursor_mzs":[650.32,650.82], "precursor_intensities":[[…],[…]], + "fragment_mzs":[175.1,288.2], "fragment_labels":["y1","b2"], + "fragment_intensities":[[…],[…]], "retention_time_results_seconds":[1230,1231] } +``` +2-D matrices: `[ion][rt_point]`. Every row length == `retention_time_results_seconds.length`. + +## Invariants + +1. `id` echoed from input; unique + present. Dup → overwrite; missing → NPE downstream. +2. ndjson: one complete object per line, no array wrapper, no pretty-print. +3. Exact field names (fastjson, no remap). Note **singular vs plural** across modes: spectrum `precursor_mz`+`precursor_labels`(int[]); chromatogram `precursor_mzs`+`precursor_intensities`(2-D). Two distinct schemas. +4. m/z ↔ intensity arrays positionally paired (spectrum 1-D, chromatogram row-per-ion). +5. Output basename exactly `results.json` in `-o` dir. Exit 0 on success. diff --git a/example_speclib_config.toml b/example_speclib_config.toml index 717afdc6..ef3faf2e 100644 --- a/example_speclib_config.toml +++ b/example_speclib_config.toml @@ -5,8 +5,8 @@ # All fields are optional — omit a section or key to use the compiled-in default. # ── Output ──────────────────────────────────────────────────────────────────── -# Path for the output spectral library (msgpack + zstd). -output = "library.msgpack.zst" +# Path for the output spectral library (zstd-compressed NDJSON). +output = "library.ndjson.zst" # ── Digestion ───────────────────────────────────────────────────────────────── [digestion] diff --git a/run.bash b/run.bash index 868d2cf3..7b25a56b 100644 --- a/run.bash +++ b/run.bash @@ -7,7 +7,7 @@ if [ -n "${FULL_RUN}" ]; then echo "Full run" sleep 2 FASTA_FILE="$HOME/fasta/20231030_LINEARIZED_UP000005640_9606.fasta" - SPECLIB_NAME="data_ignore/20231030_LINEARIZED_UP000005640_9606.msgpack.zstd" + SPECLIB_NAME="data_ignore/20231030_LINEARIZED_UP000005640_9606.ndjson.zst" DOTD_FILE="/Users/sebastianpaez/git/ionmesh/benchmark/240402_PRTC_01_S1-A1_1_11342.d" RESULTS_DIR="data_ignore/hela_search_results" SUMMARY_DIR="data_ignore/hela_search_summary" @@ -16,7 +16,7 @@ elif [ -n "${FULL_MCCOSS}" ]; then sleep 2 DOTD_FILE="$HOME/data/bo_maccoss/N20211212chenc_WOSP00101_DIA_60min_K562_rep1_1_Slot2-37_1_9898.d" FASTA_FILE="$HOME/fasta/20231030_LINEARIZED_UP000005640_9606.fasta" - SPECLIB_NAME="data_ignore/20231030_LINEARIZED_UP000005640_9606.msgpack.zstd" + SPECLIB_NAME="data_ignore/20231030_LINEARIZED_UP000005640_9606.ndjson.zst" RESULTS_DIR="data_ignore/mccoss_search_results" SUMMARY_DIR="data_ignore/mccoss_search_summary" elif [ -n "${VIMENTIN_ONLY}" ]; then @@ -24,7 +24,7 @@ elif [ -n "${VIMENTIN_ONLY}" ]; then sleep 2 DOTD_FILE="$HOME/git/ionmesh/benchmark/240402_PRTC_01_S1-A1_1_11342.d" FASTA_FILE="$HOME/fasta/VIMENTIN.fasta" - SPECLIB_NAME="data_ignore/vimentin.msgpack.zstd" + SPECLIB_NAME="data_ignore/vimentin.ndjson.zst" RESULTS_DIR="data_ignore/vimentin_search_results" SUMMARY_DIR="data_ignore/vimentin_search_summary" else @@ -32,7 +32,7 @@ else echo "Quick run" sleep 2 FASTA_FILE="$HOME/fasta/hela_gt20peps.fasta" - SPECLIB_NAME="data_ignore/asdad.msgpack.zstd" + SPECLIB_NAME="data_ignore/asdad.ndjson.zst" DOTD_FILE="$HOME/git.bkp/ionmesh/benchmark/240402_PRTC_01_S1-A1_1_11342.d" RESULTS_DIR="data_ignore/top_proteins_hela" SUMMARY_DIR="data_ignore/top_proteins_hela_summary" diff --git a/rust/calib_dash/src/app.rs b/rust/calib_dash/src/app.rs index 44bba7ae..4bc0b1b0 100644 --- a/rust/calib_dash/src/app.rs +++ b/rust/calib_dash/src/app.rs @@ -729,7 +729,6 @@ impl CalibDash { None => self.app.clear_scrub(), } } - } /// Pauses the batch loop to render one interactive frame and block until the user's diff --git a/rust/calibrt/src/lib.rs b/rust/calibrt/src/lib.rs index d56c1b05..ee48222f 100644 --- a/rust/calibrt/src/lib.rs +++ b/rust/calibrt/src/lib.rs @@ -697,9 +697,7 @@ pub type GridRanges = ((f64, f64), (f64, f64)); /// comes out empty or inverted on either axis is `Err(ZeroRange)` here rather /// than later out of `Grid::new`, so a caller that only wants to know whether /// a grid is configurable never has to build one. -fn point_ranges( - points: impl IntoIterator, -) -> Result { +fn point_ranges(points: impl IntoIterator) -> Result { let mut x = (f64::INFINITY, f64::NEG_INFINITY); let mut y = x; for (px, py) in points { diff --git a/rust/micromzpaf/Cargo.toml b/rust/micromzpaf/Cargo.toml index 83c47f4c..d23bec52 100644 --- a/rust/micromzpaf/Cargo.toml +++ b/rust/micromzpaf/Cargo.toml @@ -4,9 +4,10 @@ version.workspace = true edition.workspace = true license.workspace = true +[dev-dependencies] +serde_json = { workspace = true } + [dependencies] serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } -# Workspace-inherited deps -rustyms = { workspace = true } diff --git a/rust/micromzpaf/src/lib.rs b/rust/micromzpaf/src/lib.rs index 551e4b24..7886de90 100644 --- a/rust/micromzpaf/src/lib.rs +++ b/rust/micromzpaf/src/lib.rs @@ -1,19 +1,52 @@ //! Compact representation of fragment ion annotations for mass spectrometry. //! -//! This crate provides a minimal, memory-efficient encoding of fragment ion annotations -//! that pack ion series, charge state, ordinal position, and isotope offset into a -//! compact structure suitable for high-throughput proteomics workflows. +//! A spectral library carries one annotation per fragment, so this type is +//! replicated millions of times in a loaded arena and compared on the scoring +//! hot path. It is therefore a packed `u32` rather than a struct of fields: +//! equality is a single word compare, and `(IonAnnot, f32)` stays 8 bytes with +//! no padding. //! -//! # mzPAF Format Compliance +//! # Bit layout //! -//! This implementation supports a subset of the mzPAF annotation format: -//! - Common ion series (a, b, c, d, v, w, x, y, z, precursor) -//! - Charge states (^N notation) -//! - Positive isotope offsets (+Ni notation) +//! ```text +//! bit: 31 30 29 18 17 12 11 8 7 4 3 0 +//! ┌───────┬────────────────┬──────────┬────────┬────────┬───────┐ +//! │ spare │ payload │ loss │isotope │ charge │ kind │ +//! │ 2b │ 12b │ 6b │ 4b zz │ 4b zz │ 4b │ +//! └───────┴────────────────┴──────────┴────────┴────────┴───────┘ +//! ``` +//! +//! `payload` is reinterpreted per `kind` — a tagged union inside the word: +//! +//! | kind | payload | +//! |---|---| +//! | backbone (a/b/c/d/v/w/x/y/z), unknown | ordinal, 8b (0..=255) | +//! | internal | start 6b │ end 6b (peptides to 63 residues) | +//! | immonium | residue index 5b | +//! | precursor | unused | +//! +//! `unknown` is discriminant 0 and `charge` is stored biased by one, so the +//! all-zero word is the valid annotation `?0` at charge 1. That matters +//! because `IonAnnot: Default` is forced by `tinyvec::Array` and a default can +//! reach any serde path. +//! +//! `charge` and `isotope` are zigzag-encoded so they stay signed in 4 bits. +//! Their ranges (±7) are far wider than anything observed: the HUPO-PSI corpus +//! tops out at charge 3 and isotope 3, with no negative charges at all. Because +//! the field truncates rather than wrapping loudly, every constructor +//! range-checks — see [`IonAnnot::try_new`]. +//! +//! # mzPAF compliance +//! +//! Supported: the a/b/c/d/v/w/x/y/z series, precursor (`p`), unknown (`?`), +//! internal fragments (`m:`), bare immonium (`IA`), charge (`^N`), +//! positive isotopes (`+Ni`), neutral losses from [`NeutralLoss`], and the +//! mass-error suffix (`/-0.0003`, `/1.2ppm`). //! -//! NOTABLY boes not support: -//! - Negative isotope offsets (not yet implemented) -//! - Complex neutral losses or modifications +//! Not supported: negative isotope offsets, modified immonium +//! (`IC[Carbamidomethyl]` carries an arbitrary mod string), and losses outside +//! the [`NeutralLoss`] table. These are reported as errors, never coerced into +//! a nearby representable ion. //! //! # Examples //! @@ -29,44 +62,79 @@ //! assert_eq!(ion.get_charge(), 3); //! ``` -use rustyms::fragment::FragmentType; +pub mod loss; + +pub use loss::NeutralLoss; use serde::{ Deserialize, Serialize, }; use std::fmt::Display; use std::hash::Hash; -use std::str::FromStr; use thiserror::Error; -/// Compact representation of fragment annotations. -/// -/// This is a very compressed representation of a fragment -/// ion annotation. Essentially we are packing in 32 bytes -/// the ion series (b, y, ...), charge (+1 / -1 ...), -/// ordinal (12 in the ion series) and isotope. -/// -/// It is not meant to represent all possible ions but rather have -/// a very compact representation of the common ones. -/// -/// # Invariants -/// -/// - **charge**: Must be non-zero (±1 to ±127). Zero charge is invalid and rejected by constructors. -/// - **ordinal**: Limited to u8 range (1-255). Peptides with >255 residues cannot be represented. -/// - **isotope**: Isotope offset relative to monoisotopic peak (M+0), range -128 to +127. +// ── Bit layout ─────────────────────────────────────────────────────────────── + +const KIND_SHIFT: u32 = 0; +const KIND_BITS: u32 = 4; +const CHARGE_SHIFT: u32 = 4; +const CHARGE_BITS: u32 = 4; +const ISOTOPE_SHIFT: u32 = 8; +const ISOTOPE_BITS: u32 = 4; +const LOSS_SHIFT: u32 = 12; +const LOSS_BITS: u32 = 6; +const PAYLOAD_SHIFT: u32 = 18; +const PAYLOAD_BITS: u32 = 12; + +/// Widest charge the 4-bit zigzag field holds. Observed maximum is 3. +pub const CHARGE_MIN: i8 = -7; +pub const CHARGE_MAX: i8 = 7; +/// Widest isotope offset the 4-bit zigzag field holds. Observed maximum is 3. +pub const ISOTOPE_MIN: i8 = -7; +pub const ISOTOPE_MAX: i8 = 7; +/// Width of each internal-fragment endpoint inside `payload`. +const INTERNAL_POS_BITS: u32 = 6; +/// Width of the immonium residue index inside `payload`. +const IMMONIUM_BITS: u32 = 5; +/// Widest residue index an internal fragment endpoint holds. +pub(crate) const INTERNAL_POS_MAX: u8 = mask(INTERNAL_POS_BITS) as u8; + +#[inline] +const fn mask(bits: u32) -> u32 { + (1u32 << bits) - 1 +} +/// Zigzag: map a small signed value onto an unsigned one without losing the +/// sign bit to the field width. +#[inline] +const fn zigzag(v: i8) -> u32 { + (((v as i32) << 1) ^ ((v as i32) >> 31)) as u32 +} +#[inline] +const fn unzigzag(u: u32) -> i8 { + (((u >> 1) as i32) ^ -((u & 1) as i32)) as i8 +} + +/// Charge is stored biased by one, so the zero field decodes to charge 1. /// -/// # Memory Layout +/// Charge 0 is rejected by every constructor, so it is not a value the field +/// needs to represent — and spending the zero word on it would make +/// `IonAnnot::default()` render an annotation that cannot be parsed back. +/// `CHARGE_MIN..=CHARGE_MAX` minus one still zigzags inside 4 bits. +#[inline] +const fn zigzag_charge(charge: i8) -> u32 { + zigzag(charge - 1) +} +#[inline] +const fn unzigzag_charge(u: u32) -> i8 { + unzigzag(u) + 1 +} + +/// Compact representation of fragment annotations. /// -/// The struct fits in 32 bytes with optimal packing: -/// - `IonSeriesOrdinal`: 2 bytes (enum discriminant + u8 ordinal) -/// - `charge`: 1 byte (i8) -/// - `isotope`: 1 byte (i8) +/// A packed `u32`; see the crate docs for the bit layout. Ordering is by the +/// packed word, not field-by-field. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] -pub struct IonAnnot { - series_ordinal: IonSeriesOrdinal, - charge: i8, - isotope: i8, -} +pub struct IonAnnot(u32); impl Serialize for IonAnnot { fn serialize(&self, serializer: S) -> Result @@ -83,6 +151,9 @@ impl Serialize for IonAnnot { /// b12+i^3 -> b12 charge 3 isotope 1 /// b12+3i^3 -> b12 charge 3 isotope 2 /// b13 -> b13 (implicit charge 1 and isotope 0) +/// +/// The wire format is the mzPAF string, not the packed word, so the bit layout +/// can change without breaking existing files. impl<'de> Deserialize<'de> for IonAnnot { fn deserialize(deserializer: D) -> Result where @@ -94,66 +165,241 @@ impl<'de> Deserialize<'de> for IonAnnot { } impl IonAnnot { + /// Build a backbone / precursor / unknown annotation. + /// + /// Errors when `charge` is zero or either of `charge`/`isotope` falls + /// outside the packed field's range. That range check is load-bearing: the + /// bit field truncates silently, so an unchecked value would corrupt the + /// annotation rather than fail. pub fn try_new( ion_type: char, ordinal: Option, charge: i8, isotope: i8, ) -> Result { - Ok(Self { - series_ordinal: IonSeriesOrdinal::try_new(ion_type, ordinal)?, + Self::try_new_with_loss(ion_type, ordinal, charge, isotope, NeutralLoss::None) + } + + /// As [`Self::try_new`], carrying a neutral loss. + pub fn try_new_with_loss( + ion_type: char, + ordinal: Option, + charge: i8, + isotope: i8, + loss: NeutralLoss, + ) -> Result { + Self::pack( + IonSeriesOrdinal::from_series_char(ion_type, ordinal)?, + loss, charge, isotope, - }) + ) } - pub fn from_fragment( - frag: FragmentType, + /// Build an internal fragment spanning residues `start..=end`. + /// + /// Endpoints are capped at [`INTERNAL_POS_MAX`], narrower than a backbone + /// ordinal, because two of them share the 12-bit payload. Internal + /// fragments are bounded by peptide length, so 63 is well past tryptic. + pub fn try_new_internal( + start: u8, + end: u8, charge: i8, isotope: i8, + loss: NeutralLoss, ) -> Result { - Ok(Self { - series_ordinal: IonSeriesOrdinal::try_from(frag)?, + if start > INTERNAL_POS_MAX || end > INTERNAL_POS_MAX { + return Err(IonParsingError::OrdinalOutOfRange { + ordinal: start.max(end), + series: 'm', + }); + } + Self::pack( + IonSeriesOrdinal::internal { start, end }, + loss, charge, isotope, - }) + ) } - pub fn terminality(&self) -> IonSeriesTerminality { - self.series_ordinal.terminality() + /// Build a bare immonium ion for an uppercase residue code. + /// + /// Modified immonium (`IC[Carbamidomethyl]`) is not representable at any + /// field width and is rejected by the parser rather than silently losing + /// the modification. + pub fn try_new_immonium( + residue: char, + charge: i8, + isotope: i8, + loss: NeutralLoss, + ) -> Result { + if !residue.is_ascii_uppercase() { + return Err(IonParsingError::UnsupportedFragmentType { + fragment_type: residue, + }); + } + Self::pack( + IonSeriesOrdinal::immonium { residue }, + loss, + charge, + isotope, + ) } - pub fn try_with_offset_neutrons(&self, offset_neutrons: i8) -> Result { - let new_isotope = - self.isotope - .checked_add(offset_neutrons) - .ok_or_else(|| IonParsingError::Custom { - error: format!( - "Isotope offset overflow: {} + {} exceeds i8 range", - self.isotope, offset_neutrons - ), - })?; + fn pack( + series: IonSeriesOrdinal, + loss: NeutralLoss, + charge: i8, + isotope: i8, + ) -> Result { + if charge == 0 { + return Err(IonParsingError::ChargeCannotBeZero); + } + if !(CHARGE_MIN..=CHARGE_MAX).contains(&charge) { + return Err(IonParsingError::ChargeOutOfRange { charge }); + } + if !(ISOTOPE_MIN..=ISOTOPE_MAX).contains(&isotope) { + return Err(IonParsingError::IsotopeOutOfRange { isotope }); + } + let (kind, payload) = series.to_parts(); + debug_assert!(payload <= mask(PAYLOAD_BITS), "payload overflows its field"); + Ok(IonAnnot( + (kind << KIND_SHIFT) + | ((zigzag_charge(charge) & mask(CHARGE_BITS)) << CHARGE_SHIFT) + | ((zigzag(isotope) & mask(ISOTOPE_BITS)) << ISOTOPE_SHIFT) + | ((loss as u32 & mask(LOSS_BITS)) << LOSS_SHIFT) + | ((payload & mask(PAYLOAD_BITS)) << PAYLOAD_SHIFT), + )) + } - Ok(Self { - series_ordinal: self.series_ordinal, - charge: self.charge, - isotope: new_isotope, - }) + #[inline] + fn payload(self) -> u32 { + (self.0 >> PAYLOAD_SHIFT) & mask(PAYLOAD_BITS) } + #[inline] pub fn get_charge(&self) -> i8 { - self.charge + unzigzag_charge((self.0 >> CHARGE_SHIFT) & mask(CHARGE_BITS)) + } + + #[inline] + pub fn get_isotope(&self) -> i8 { + unzigzag((self.0 >> ISOTOPE_SHIFT) & mask(ISOTOPE_BITS)) + } + + /// The neutral loss this ion carries, [`NeutralLoss::None`] if it carries + /// none. + #[inline] + pub fn loss(&self) -> NeutralLoss { + NeutralLoss::from_discriminant(((self.0 >> LOSS_SHIFT) & mask(LOSS_BITS)) as u8) } + /// Shift the isotope by `offset_neutrons`. + /// + /// Errors when the result leaves [`ISOTOPE_MIN`]..=[`ISOTOPE_MAX`]. That + /// bound is an order of magnitude past any observed isotope offset. + pub fn try_with_offset_neutrons(&self, offset_neutrons: i8) -> Result { + // Saturating rather than checked: the field is far narrower than `i8`, + // so the range check below is the one that matters and it reports the + // bound that actually applies. + let new_isotope = self.get_isotope().saturating_add(offset_neutrons); + if !(ISOTOPE_MIN..=ISOTOPE_MAX).contains(&new_isotope) { + return Err(IonParsingError::IsotopeOutOfRange { + isotope: new_isotope, + }); + } + Ok(IonAnnot( + (self.0 & !(mask(ISOTOPE_BITS) << ISOTOPE_SHIFT)) + | ((zigzag(new_isotope) & mask(ISOTOPE_BITS)) << ISOTOPE_SHIFT), + )) + } + + /// The series ordinal, for the kinds that have one. + /// + /// `None` for precursor, unknown, internal and immonium: `?1` is a + /// uniqueness counter, not a position in a ladder. pub fn try_get_ordinal(&self) -> Option { - self.series_ordinal.try_get_ordinal() + use IonSeriesOrdinal as S; + match self.series_ordinal() { + S::backbone { ordinal, .. } => Some(ordinal), + S::unknown { .. } | S::precursor | S::internal { .. } | S::immonium { .. } => None, + } + } + + /// The backbone series this annotation belongs to, if any. + /// + /// `None` for precursor, unknown, internal and immonium ions, none of + /// which sit on a backbone ladder. + pub fn try_get_series(&self) -> Option { + match self.series_ordinal() { + IonSeriesOrdinal::backbone { series, .. } => Some(series), + _ => None, + } + } + + /// The logical series-and-payload view of this annotation. + pub fn series_ordinal(&self) -> IonSeriesOrdinal { + IonSeriesOrdinal::from_parts((self.0 >> KIND_SHIFT) & mask(KIND_BITS), self.payload()) } } -impl TryFrom<&str> for IonAnnot { - type Error = IonParsingError; +/// A mass-error suffix on an mzPAF annotation: observed minus theoretical. +/// +/// mzSpecLib peak lists carry *observed* m/z, so the theoretical value a +/// library reader wants is `observed - error`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum MassError { + /// Absolute, in daltons. + Da(f64), + /// Relative, in parts per million. + Ppm(f64), +} - fn try_from(value: &str) -> Result { +impl MassError { + /// Recover the theoretical m/z from the observed one. + pub fn theoretical_from_observed(&self, observed: f64) -> f64 { + match self { + MassError::Da(d) => observed - d, + MassError::Ppm(p) => observed / (1.0 + p * 1e-6), + } + } +} + +/// Split the trailing `/[ppm]` off an annotation, if present. +/// +/// Public because a library reader needs the error even when the ion itself is +/// unrepresentable: the error is what recovers theoretical m/z, so a peak that +/// ends up with an unknown label still gets a correct mass. +pub fn split_mass_error(s: &str) -> Result<(&str, Option), IonParsingError> { + let Some((head, tail)) = s.rsplit_once('/') else { + return Ok((s, None)); + }; + let (num, is_ppm) = match tail.strip_suffix("ppm") { + Some(n) => (n, true), + None => (tail, false), + }; + let v: f64 = num.parse().map_err(|_| IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Unable to parse the mass-error suffix"), + })?; + Ok(( + head, + Some(if is_ppm { + MassError::Ppm(v) + } else { + MassError::Da(v) + }), + )) +} + +impl IonAnnot { + /// Parse the ion itself, with no mass-error suffix. + /// + /// A comma-separated list of alternatives is NOT handled here — that is + /// ambiguity, and resolving it needs the caller's policy. Split on `,` and + /// parse each alternative. + fn parse_ion(value: &str) -> Result { + // charge: trailing ^N let (rest, charge) = match value.split_once('^') { Some((rest, charge)) => { let charge = charge @@ -167,20 +413,15 @@ impl TryFrom<&str> for IonAnnot { None => (value, 1), }; - // TODO: Implement 'negative isotopes' parsing ... right now I dont use them - // for serialization ... - // Note that this is not 100% compliant with mzPAF + // isotope: +Ni. Negative isotope offsets are not supported. let (rest, isotope) = match rest.split_once('+') { Some((rest, adducts)) => { - // Make sure the adduct is only +{number}?i let adducts = adducts .strip_suffix('i') .ok_or(IonParsingError::ParsingError { error: adducts.to_string(), context: Some("Unsupported adduct found"), })?; - // If its empty its an implicit 1 isotope - // Since we stripped the 'i' from '+i' let isotope = if adducts.is_empty() { 1 } else { @@ -195,33 +436,99 @@ impl TryFrom<&str> for IonAnnot { } None => (rest, 0), }; - let series_ord = IonSeriesOrdinal::from_str(rest)?; - if charge == 0 { - return Err(IonParsingError::ParsingError { + + // neutral loss: everything from the first '-'. Internal fragments use + // 'm:' and never contain '-' before the loss, so splitting + // at the first '-' is unambiguous. + let (core, loss) = match rest.split_once('-') { + Some((core, loss_expr)) => { + let loss = NeutralLoss::from_expression(loss_expr)?.ok_or_else(|| { + IonParsingError::UnsupportedNeutralLoss { + loss: loss_expr.to_string(), + } + })?; + (core, loss) + } + None => (rest, NeutralLoss::None), + }; + + // internal fragment: m: + if let Some(spans) = core.strip_prefix('m') + && let Some((a, b)) = spans.split_once(':') + { + let start = a.parse::().map_err(|_| IonParsingError::ParsingError { error: value.to_string(), - context: Some("Charge cannot be 0"), - }); + context: Some("Unable to parse internal-fragment start"), + })?; + let end = b.parse::().map_err(|_| IonParsingError::ParsingError { + error: value.to_string(), + context: Some("Unable to parse internal-fragment end"), + })?; + return Self::try_new_internal(start, end, charge, isotope, loss); } - Ok(Self { - series_ordinal: series_ord, - charge, - isotope, - }) + + // immonium: I, bare only. + if let Some(res) = core.strip_prefix('I') { + let mut ch = res.chars(); + return match (ch.next(), ch.next()) { + (Some(r), None) => Self::try_new_immonium(r, charge, isotope, loss), + // `IC[Carbamidomethyl]` and friends carry a mod string that no + // fixed-width field can hold. + _ => Err(IonParsingError::UnsupportedModifiedImmonium { + annotation: value.to_string(), + }), + }; + } + + // Backbone / precursor / unknown: a series char then an ordinal. + let mut chars = core.chars(); + let series = chars.next().ok_or(IonParsingError::ParsingError { + error: value.to_string(), + context: Some("Empty string"), + })?; + let rest = chars.as_str(); + let ordinal = if rest.is_empty() { + None + } else { + Some( + rest.parse::() + .map_err(|e| IonParsingError::ParsingError { + error: format!("{rest} -> {e:?}"), + context: Some("Unable to parse the ordinal number"), + })?, + ) + }; + Self::try_new_with_loss(series, ordinal, charge, isotope, loss) + } +} + +impl TryFrom<&str> for IonAnnot { + type Error = IonParsingError; + + /// Parses an annotation, discarding any mass-error suffix. Use + /// [`split_mass_error`] first to keep it. + fn try_from(value: &str) -> Result { + Self::parse_ion(split_mass_error(value)?.0) } } impl Display for IonAnnot { + /// Renders the canonical mzPAF spelling. Not byte-inverse to parsing: a + /// non-canonical loss spelling (`-CH3SOH`) renders canonically (`-CH4OS`). + /// The mass-error suffix is not part of the annotation and is not rendered. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.series_ordinal)?; + write!(f, "{}", self.series_ordinal())?; + write!(f, "{}", self.loss())?; - match self.isotope { + match self.get_isotope() { 0 => {} 1 => write!(f, "+i")?, i => write!(f, "+{}i", i)?, } - if self.charge != 1 { - write!(f, "^{}", self.charge)?; + let charge = self.get_charge(); + if charge != 1 { + write!(f, "^{}", charge)?; } Ok(()) @@ -230,322 +537,498 @@ impl Display for IonAnnot { #[derive(Debug, Error)] pub enum IonParsingError { - #[error("Ordinal {ordinal} out of range for series '{series:?}'")] - OrdinalOutOfRange { ordinal: i32, series: Option }, + #[error("Ordinal {ordinal} out of range for series '{series}'")] + OrdinalOutOfRange { ordinal: u8, series: char }, + #[error("Series '{series}' requires an ordinal")] + MissingOrdinal { series: char }, #[error("Unsupported fragment type: '{fragment_type}'")] UnsupportedFragmentType { fragment_type: char }, + #[error("Charge cannot be 0")] + ChargeCannotBeZero, + #[error("Charge {charge} outside the representable range")] + ChargeOutOfRange { charge: i8 }, + #[error("Isotope offset {isotope} outside the representable range")] + IsotopeOutOfRange { isotope: i8 }, + #[error("Neutral loss '{loss}' is not representable")] + UnsupportedNeutralLoss { loss: String }, + #[error("Modified immonium ions are not representable: '{annotation}'")] + UnsupportedModifiedImmonium { annotation: String }, + #[error("Ran out of distinct unknown-ion labels: the 8-bit ordinal is exhausted")] + UnknownIonsExhausted, #[error("Parsing error: {error}{}", .context.map(|c| format!(" ({})", c)).unwrap_or_default())] ParsingError { error: String, context: Option<&'static str>, }, - #[error("{error}")] - Custom { error: String }, } -/// Refers to what terminus of the original peptide retains the -/// charge after a fragmentation. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum IonSeriesTerminality { - NTerm, - CTerm, - None, +/// Hands out `?1`, `?2`, ... for peaks whose annotation this crate cannot +/// represent. +/// +/// Fragment labels must be unique within a precursor — lookup is by first +/// match, so a repeated label makes every later peak carrying it unreachable. +/// A monotonic counter makes that uniqueness structural, and returning an +/// error once the 8-bit ordinal is spent keeps the overflow from being +/// something each reader has to remember to check. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnknownIonCounter(u8); + +impl UnknownIonCounter { + pub fn new() -> Self { + Self::default() + } + + /// The next unused unknown label at `charge`. + pub fn next(&mut self, charge: i8) -> Result { + let ordinal = self + .0 + .checked_add(1) + .ok_or(IonParsingError::UnknownIonsExhausted)?; + let annot = IonAnnot::try_new('?', Some(ordinal), charge, 0)?; + self.0 = ordinal; + Ok(annot) + } } -#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Default)] +/// A backbone fragment ion series. +/// +/// The nine mzPAF backbone series differ only by their letter, so they are one +/// enum with one letter table rather than nine variants repeated across every +/// match in this module. +#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] +#[allow(non_camel_case_types)] +#[repr(u8)] +pub enum Series { + a = 1, + b, + c, + d, + v, + w, + x, + y, + z, +} + +impl Series { + /// Every series, in discriminant order. Parallel to [`Self::CHARS`]. + pub const ALL: [Self; 9] = [ + Self::a, + Self::b, + Self::c, + Self::d, + Self::v, + Self::w, + Self::x, + Self::y, + Self::z, + ]; + /// The mzPAF letters, in discriminant order. The single place the + /// letter↔discriminant pairing lives. + const CHARS: &'static [u8; 9] = b"abcdvwxyz"; + + /// The mzPAF letter for this series. + pub const fn as_char(self) -> char { + Self::CHARS[self as usize - 1] as char + } + + /// The series for an mzPAF letter, or `None` if it names no backbone series. + fn from_char(c: char) -> Option { + let idx = Self::CHARS.iter().position(|&b| b as char == c)?; + Some(Self::ALL[idx]) + } +} + +impl Display for Series { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_char()) + } +} + +/// The logical series-and-payload view of an [`IonAnnot`]. +/// +/// This is a *view*: `IonAnnot` stores a packed word and reconstructs this on +/// demand. Constructing one directly does not allocate an annotation. +#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy)] #[allow(non_camel_case_types)] pub enum IonSeriesOrdinal { - a { - ordinal: u8, - }, - b { - ordinal: u8, - }, - c { - ordinal: u8, - }, - d { - ordinal: u8, - }, - v { - ordinal: u8, - }, - w { - ordinal: u8, - }, - x { - ordinal: u8, - }, - y { - ordinal: u8, - }, - z { + /// One of the nine backbone series, at position `ordinal` in the ladder. + backbone { + series: Series, ordinal: u8, }, + /// An unannotated peak. `ordinal` is a uniqueness counter, not a position. unknown { ordinal: u8, }, precursor, - - /// This variant should not be used directly ... its mainly added to satisfy trait constraints by TinyVec - #[default] - None, + /// An internal fragment spanning residues `start..=end`. + internal { + start: u8, + end: u8, + }, + /// A bare immonium ion for an uppercase residue code. + immonium { + residue: char, + }, } impl IonSeriesOrdinal { - pub fn try_new(series: char, ordinal: Option) -> Result { - let tmp = match (series, ordinal) { - ('a', Some(ordinal)) => Self::a { ordinal }, - ('b', Some(ordinal)) => Self::b { ordinal }, - ('c', Some(ordinal)) => Self::c { ordinal }, - ('d', Some(ordinal)) => Self::d { ordinal }, - ('v', Some(ordinal)) => Self::v { ordinal }, - ('w', Some(ordinal)) => Self::w { ordinal }, - ('x', Some(ordinal)) => Self::x { ordinal }, - ('y', Some(ordinal)) => Self::y { ordinal }, - ('z', Some(ordinal)) => Self::z { ordinal }, - ('?', Some(ordinal)) => Self::unknown { ordinal }, - ('p', None) => Self::precursor, - ('p', Some(ordinal)) => { - return Err(IonParsingError::OrdinalOutOfRange { - ordinal: ordinal as i32, - series: Some(series), - }); - } - _ => { - return Err(IonParsingError::UnsupportedFragmentType { - fragment_type: series, - }); + /// Split into the `kind` discriminant and its `payload`, the two fields + /// [`IonAnnot`] packs. + /// + /// This and [`Self::from_parts`] are the only place the numbering lives. + /// The nine backbone series share one arm, so their discriminants come + /// from [`Series`] itself and cannot drift out of step with their letters. + const fn to_parts(self) -> (u32, u32) { + match self { + Self::backbone { series, ordinal } => (series as u32, ordinal as u32), + // Discriminant 0, so the all-zero word — `IonAnnot::default()` — + // is the unknown ion `?0` rather than an undecodable value. + Self::unknown { ordinal } => (0, ordinal as u32), + Self::precursor => (10, 0), + Self::internal { start, end } => { + (12, (start as u32) | ((end as u32) << INTERNAL_POS_BITS)) } - }; - - Ok(tmp) + Self::immonium { residue } => (13, (residue as u8 - b'A') as u32), + } } - pub fn terminality(&self) -> IonSeriesTerminality { - match self { - IonSeriesOrdinal::a { ordinal: _ } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::b { ordinal: _ } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::c { ordinal: _ } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::d { ordinal: _ } => IonSeriesTerminality::NTerm, - IonSeriesOrdinal::v { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::w { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::x { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::y { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::z { ordinal: _ } => IonSeriesTerminality::CTerm, - IonSeriesOrdinal::unknown { ordinal: _ } => IonSeriesTerminality::None, - IonSeriesOrdinal::precursor => IonSeriesTerminality::None, - IonSeriesOrdinal::None => panic!("IonSeriesOrdinal::None should not be used directly"), + /// Inverse of [`Self::to_parts`]. Total by construction: `unknown` is the + /// catch-all discriminant, so a value this build does not recognise — only + /// reachable from a corrupted word — decodes as an unknown ion instead of + /// panicking on a path `Display` (and therefore `Serialize`) reaches. + const fn from_parts(kind: u32, payload: u32) -> Self { + let ordinal = payload as u8; + match kind { + 1..=9 => Self::backbone { + // `kind` is in range, so this is the inverse of `series as u32`. + series: Series::ALL[kind as usize - 1], + ordinal, + }, + 10 => Self::precursor, + 12 => Self::internal { + start: (payload & mask(INTERNAL_POS_BITS)) as u8, + end: ((payload >> INTERNAL_POS_BITS) & mask(INTERNAL_POS_BITS)) as u8, + }, + 13 => Self::immonium { + residue: (b'A' + (payload & mask(IMMONIUM_BITS)) as u8) as char, + }, + _ => Self::unknown { ordinal }, } } - pub fn try_get_ordinal(&self) -> Option { - match self { - IonSeriesOrdinal::a { ordinal } => Some(*ordinal), - IonSeriesOrdinal::b { ordinal } => Some(*ordinal), - IonSeriesOrdinal::c { ordinal } => Some(*ordinal), - IonSeriesOrdinal::d { ordinal } => Some(*ordinal), - IonSeriesOrdinal::v { ordinal } => Some(*ordinal), - IonSeriesOrdinal::w { ordinal } => Some(*ordinal), - IonSeriesOrdinal::x { ordinal } => Some(*ordinal), - IonSeriesOrdinal::y { ordinal } => Some(*ordinal), - IonSeriesOrdinal::z { ordinal } => Some(*ordinal), - IonSeriesOrdinal::unknown { .. } => None, - // ?1 does not mean its an ordinal, just a placeholder - IonSeriesOrdinal::precursor => None, - IonSeriesOrdinal::None => None, + /// Build the series view from an mzPAF series letter and its ordinal. + /// + /// `p` is the only letter that takes no ordinal; every other one requires + /// one. Internal fragments and immonium ions are spelled differently and + /// have their own constructors. + fn from_series_char(c: char, ordinal: Option) -> Result { + if c == 'p' { + return match ordinal { + None => Ok(Self::precursor), + Some(ordinal) => Err(IonParsingError::OrdinalOutOfRange { ordinal, series: c }), + }; + } + let ordinal = ordinal.ok_or(IonParsingError::MissingOrdinal { series: c })?; + if c == '?' { + return Ok(Self::unknown { ordinal }); } + Series::from_char(c) + .map(|series| Self::backbone { series, ordinal }) + .ok_or(IonParsingError::UnsupportedFragmentType { fragment_type: c }) } } impl Display for IonSeriesOrdinal { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - IonSeriesOrdinal::a { ordinal } => write!(f, "a{}", ordinal), - IonSeriesOrdinal::b { ordinal } => write!(f, "b{}", ordinal), - IonSeriesOrdinal::c { ordinal } => write!(f, "c{}", ordinal), - IonSeriesOrdinal::d { ordinal } => write!(f, "d{}", ordinal), - IonSeriesOrdinal::v { ordinal } => write!(f, "v{}", ordinal), - IonSeriesOrdinal::w { ordinal } => write!(f, "w{}", ordinal), - IonSeriesOrdinal::x { ordinal } => write!(f, "x{}", ordinal), - IonSeriesOrdinal::y { ordinal } => write!(f, "y{}", ordinal), - IonSeriesOrdinal::z { ordinal } => write!(f, "z{}", ordinal), - IonSeriesOrdinal::unknown { ordinal } => write!(f, "?{}", ordinal), - IonSeriesOrdinal::precursor => write!(f, "p"), - IonSeriesOrdinal::None => panic!("IonSeriesOrdinal::None should not be used directly"), + Self::backbone { series, ordinal } => write!(f, "{}{}", series.as_char(), ordinal), + Self::unknown { ordinal } => write!(f, "?{}", ordinal), + Self::precursor => write!(f, "p"), + Self::internal { start, end } => write!(f, "m{}:{}", start, end), + Self::immonium { residue } => write!(f, "I{}", residue), } } } +#[cfg(test)] +mod tests { + use super::*; -impl FromStr for IonSeriesOrdinal { - type Err = IonParsingError; - - fn from_str(s: &str) -> Result { - // "b12" split into "b" and "12" - let (series_chunk, ordinal_chunk) = s.split_at(1); - let series_id = series_chunk.chars().next(); - let series_ordinal = ordinal_chunk.parse::(); - - match (series_id, series_ordinal) { - (None, _) => Err(IonParsingError::ParsingError { - error: s.to_string(), - context: Some("Empty string"), - }), - (Some(x), Ok(y)) => IonSeriesOrdinal::try_new(x, Some(y)), - (Some('p'), Err(_)) => Ok(IonSeriesOrdinal::precursor), - (Some(_), Err(err)) => Err(IonParsingError::ParsingError { - error: format!("{ordinal_chunk} -> {err:?}"), - context: Some("Unable to parse the ordinal number"), - }), - } + fn ion(s: &str) -> IonAnnot { + IonAnnot::try_from(s).unwrap_or_else(|e| panic!("{s:?} must parse: {e}")) } -} -impl TryFrom for IonSeriesOrdinal { - type Error = IonParsingError; + /// The whole point of the packed representation. + #[test] + fn packs_into_one_word() { + assert_eq!(size_of::(), 4); + // Paired with an intensity on the scoring hot path; padding here would + // grow the inline TinyVec storage in `ExpectedIntensities`. + assert_eq!(size_of::<(IonAnnot, f32)>(), 8); + } + + /// `IonAnnot: Default` is not optional — `tinyvec::Array` requires + /// `Item: Default`, `TimsElutionGroup` stores labels in a `TinyVec`, and + /// timsquery's `KeyLike` propagates the bound. So a default can reach any + /// serde path, and the zero word has to mean something. + /// + /// It means `?0`: charge is stored as `zigzag(charge - 1)`, so the zero + /// field is charge 1 rather than the impossible charge 0. The default is + /// therefore a real annotation and `Serialize`/`Deserialize` are inverses + /// on it, instead of rendering a value no constructor accepts. + #[test] + fn the_default_annotation_is_a_real_annotation() { + let d = IonAnnot::default(); + assert_eq!(d.to_string(), "?0"); + assert_eq!(d.get_charge(), 1); + assert_eq!(serde_json::to_string(&d).unwrap(), "\"?0\""); + assert_eq!(IonAnnot::try_from("?0").expect("the default parses"), d); + } + + #[test] + fn test_deserialize() { + let cases = [ + ("b12", 'b', Some(12u8), 1i8, 0i8), + ("b12^3", 'b', Some(12), 3, 0), + ("y12^3", 'y', Some(12), 3, 0), + ("b12+i^3", 'b', Some(12), 3, 1), + ("b12+3i^3", 'b', Some(12), 3, 3), + ("b13", 'b', Some(13), 1, 0), + ("p^2", 'p', None, 2, 0), + ("p", 'p', None, 1, 0), + ("?12^2", '?', Some(12), 2, 0), + ]; - fn try_from(value: FragmentType) -> Result { - fn try_convert_ordinal(ordinal: usize, series: char) -> Result { - ordinal - .try_into() - .map_err(|_| IonParsingError::OrdinalOutOfRange { - ordinal: ordinal as i32, - series: Some(series), - }) + for (input, series, ordinal, charge, isotope) in cases { + let annot = ion(input); + let expected = IonAnnot::try_new(series, ordinal, charge, isotope).unwrap(); + assert_eq!(annot, expected, "{input}"); + assert_eq!(annot.get_charge(), charge, "{input}"); + assert_eq!(annot.get_isotope(), isotope, "{input}"); + // Round-trips byte-identically when no loss is involved. + assert_eq!(format!("{}", annot), input); } - let tmp = match value { - FragmentType::a(ordinal, _) => IonSeriesOrdinal::a { - ordinal: try_convert_ordinal(ordinal.series_number, 'a')?, - }, - FragmentType::b(ordinal, _) => IonSeriesOrdinal::b { - ordinal: try_convert_ordinal(ordinal.series_number, 'b')?, - }, - FragmentType::c(ordinal, _) => IonSeriesOrdinal::c { - ordinal: try_convert_ordinal(ordinal.series_number, 'c')?, - }, - FragmentType::d(ordinal, _, _, _, _) => IonSeriesOrdinal::d { - ordinal: try_convert_ordinal(ordinal.series_number, 'd')?, - }, - FragmentType::v(ordinal, _, _, _) => IonSeriesOrdinal::v { - ordinal: try_convert_ordinal(ordinal.series_number, 'v')?, - }, - FragmentType::w(ordinal, _, _, _, _) => IonSeriesOrdinal::w { - ordinal: try_convert_ordinal(ordinal.series_number, 'w')?, - }, - FragmentType::x(ordinal, _) => IonSeriesOrdinal::x { - ordinal: try_convert_ordinal(ordinal.series_number, 'x')?, - }, - FragmentType::y(ordinal, _) => IonSeriesOrdinal::y { - ordinal: try_convert_ordinal(ordinal.series_number, 'y')?, - }, - FragmentType::z(ordinal, _) => IonSeriesOrdinal::z { - ordinal: try_convert_ordinal(ordinal.series_number, 'z')?, - }, - FragmentType::Precursor => IonSeriesOrdinal::precursor, - _ => { - return Err(IonParsingError::Custom { - error: format!("Unsupported fragment type: {value:?}"), - }); + } + + /// Every field must survive the pack/unpack at its extremes. + #[test] + fn every_field_round_trips_at_its_extremes() { + for charge in CHARGE_MIN..=CHARGE_MAX { + if charge == 0 { + continue; } - }; - Ok(tmp) + for isotope in ISOTOPE_MIN..=ISOTOPE_MAX { + for ordinal in [1u8, 2, 127, 255] { + for loss in [ + NeutralLoss::None, + NeutralLoss::Water, + NeutralLoss::PhosphoricAcidWater, + ] { + let a = + IonAnnot::try_new_with_loss('y', Some(ordinal), charge, isotope, loss) + .expect("in range"); + assert_eq!(a.get_charge(), charge); + assert_eq!(a.get_isotope(), isotope); + assert_eq!(a.try_get_ordinal(), Some(ordinal)); + assert_eq!(a.loss(), loss); + } + } + } + } } -} -#[cfg(test)] -mod tests { - use super::*; + #[test] + fn out_of_range_is_rejected_not_truncated() { + assert!(matches!( + IonAnnot::try_new('y', Some(1), CHARGE_MAX + 1, 0), + Err(IonParsingError::ChargeOutOfRange { .. }) + )); + assert!(matches!( + IonAnnot::try_new('y', Some(1), 1, ISOTOPE_MAX + 1), + Err(IonParsingError::IsotopeOutOfRange { .. }) + )); + assert!(IonAnnot::try_new('y', Some(1), 0, 0).is_err()); + assert!(matches!( + IonAnnot::try_new_internal(INTERNAL_POS_MAX + 1, 1, 1, 0, NeutralLoss::None), + Err(IonParsingError::OrdinalOutOfRange { .. }) + )); + } #[test] - fn test_ion_series_ord_from_str() { - let ion: IonSeriesOrdinal = IonSeriesOrdinal::from_str("b12").unwrap(); - assert_eq!(ion, IonSeriesOrdinal::b { ordinal: 12 }); + fn isotope_offset_respects_the_field_bound() { + let a = ion("y5"); + assert_eq!(a.try_with_offset_neutrons(2).unwrap().get_isotope(), 2); + assert!(a.try_with_offset_neutrons(ISOTOPE_MAX + 1).is_err()); } #[test] - fn test_deserialize() { - let serde_pairs = vec![ - ( - "b12", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 12 }, - charge: 1, - isotope: 0, - }, - ), - ( - "b12^3", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 12 }, - charge: 3, - isotope: 0, - }, - ), - ( - "y12^3", - IonAnnot { - series_ordinal: IonSeriesOrdinal::y { ordinal: 12 }, - charge: 3, - isotope: 0, - }, - ), - ( - "b12+i^3", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 12 }, - charge: 3, - isotope: 1, - }, - ), - ( - "b12+3i^3", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 12 }, - charge: 3, - isotope: 3, - }, - ), - ( - "b13", - IonAnnot { - series_ordinal: IonSeriesOrdinal::b { ordinal: 13 }, - charge: 1, - isotope: 0, - }, - ), - ( - "p^2", - IonAnnot { - series_ordinal: IonSeriesOrdinal::precursor, - charge: 2, - isotope: 0, - }, - ), - ( - "p", - IonAnnot { - series_ordinal: IonSeriesOrdinal::precursor, - charge: 1, - isotope: 0, - }, - ), - ( - "?12^2", - IonAnnot { - series_ordinal: IonSeriesOrdinal::unknown { ordinal: 12 }, - charge: 2, - isotope: 0, - }, - ), - ]; + fn parses_neutral_losses() { + let a = ion("y5-H2O"); + assert_eq!(a.loss(), NeutralLoss::Water); + assert_eq!(a.try_get_ordinal(), Some(5)); + assert_eq!(format!("{}", a), "y5-H2O"); - for (input, expected) in serde_pairs { - let annot = IonAnnot::try_from(input).unwrap(); - assert_eq!(annot, expected); + // Non-canonical spelling resolves to the same annotation, and renders + // canonically -- so this pair is equal, which is the property that + // upholds per-precursor label uniqueness. + assert_eq!(ion("y5-CH3SOH"), ion("y5-CH4OS")); + assert_eq!(format!("{}", ion("y5-CH3SOH")), "y5-CH4OS"); + + // Loss combines with charge and isotope. + let b = ion("y10-NH3+i^2"); + assert_eq!(b.loss(), NeutralLoss::Ammonia); + assert_eq!(b.get_isotope(), 1); + assert_eq!(b.get_charge(), 2); + } - // Re-serialize and check that its the same - let serialized = format!("{}", annot); - assert_eq!(serialized, input); + #[test] + fn parses_internal_fragments() { + let a = ion("m2:11"); + assert_eq!( + a.series_ordinal(), + IonSeriesOrdinal::internal { start: 2, end: 11 } + ); + // An internal fragment has no ladder position. + assert_eq!(a.try_get_ordinal(), None); + assert_eq!(format!("{}", a), "m2:11"); + + let b = ion("m11:12-CO"); + assert_eq!(b.loss(), NeutralLoss::CarbonMonoxide); + assert_eq!(format!("{}", b), "m11:12-CO"); + } + + #[test] + fn parses_bare_immonium_and_rejects_modified() { + let a = ion("IA"); + assert_eq!( + a.series_ordinal(), + IonSeriesOrdinal::immonium { residue: 'A' } + ); + assert_eq!(format!("{}", a), "IA"); + + // Carries an arbitrary mod string -- no field width represents it, so + // it must fail rather than silently degrade to a bare immonium. + assert!(matches!( + IonAnnot::try_from("IC[Carbamidomethyl]"), + Err(IonParsingError::UnsupportedModifiedImmonium { .. }) + )); + } + + #[test] + fn parses_and_applies_the_mass_error_suffix() { + let (rest, err) = split_mass_error("y1/-0.0005").unwrap(); + assert_eq!(ion(rest), ion("y1")); + assert_eq!(err, Some(MassError::Da(-0.0005))); + // theoretical = observed - error; verified against a real SpectraST + // peak: y1 for C-terminal R, observed 175.1184, theoretical 175.11895. + let theo = err.unwrap().theoretical_from_observed(175.1184); + assert!((theo - 175.1189).abs() < 1e-9, "got {theo}"); + + let (_, err) = split_mass_error("y6/1.2ppm").unwrap(); + assert_eq!(err, Some(MassError::Ppm(1.2))); + let theo = err.unwrap().theoretical_from_observed(700.0); + assert!((theo - 699.99916).abs() < 1e-4, "got {theo}"); + + // Absent suffix is not an error. + assert_eq!(split_mass_error("y6").unwrap(), ("y6", None)); + // TryFrom discards it rather than failing. + assert_eq!(IonAnnot::try_from("y1/-0.0005").unwrap(), ion("y1")); + } + + /// One representative of every `IonSeriesOrdinal` case: all nine backbone + /// series (each with a distinct ordinal, so a transposition in + /// `to_parts`/`from_parts` cannot cancel out) plus the four others, at + /// their field boundaries. + fn all_series() -> Vec { + let mut out: Vec = Series::ALL + .iter() + .enumerate() + .map(|(i, &series)| IonSeriesOrdinal::backbone { + series, + ordinal: i as u8 + 1, + }) + .collect(); + out.extend([ + IonSeriesOrdinal::precursor, + IonSeriesOrdinal::unknown { ordinal: 10 }, + IonSeriesOrdinal::internal { start: 2, end: 11 }, + // Both endpoints at their 6-bit ceiling: the widest payload the + // 12 bits hold, and the case a narrowed field would silently clip. + IonSeriesOrdinal::internal { + start: INTERNAL_POS_MAX, + end: INTERNAL_POS_MAX, + }, + IonSeriesOrdinal::immonium { residue: 'A' }, + // Highest residue index the 5-bit immonium field must hold. + IonSeriesOrdinal::immonium { residue: 'Z' }, + ]); + out + } + + /// `to_parts` and `from_parts` are hand-written inverses. Without this, + /// swapping two arms (`v` encoding as `w`) mislabels a whole ion series + /// and every other test still passes. + #[test] + fn every_series_variant_round_trips_through_the_packed_word() { + for series in all_series() { + let (kind, payload) = series.to_parts(); + assert_eq!(IonSeriesOrdinal::from_parts(kind, payload), series); + assert!(kind <= mask(KIND_BITS), "{series:?} kind overflows"); + assert!( + payload <= mask(PAYLOAD_BITS), + "{series:?} payload overflows" + ); } + + // Every backbone series must land on its own discriminant; the other + // four are singletons and are covered by the round trip above. + let mut kinds: Vec = Series::ALL + .iter() + .map(|&series| { + IonSeriesOrdinal::backbone { series, ordinal: 1 } + .to_parts() + .0 + }) + .collect(); + kinds.sort_unstable(); + kinds.dedup(); + assert_eq!(kinds.len(), Series::ALL.len(), "two series share a kind"); + } + + /// `Display`, `from_series_char` and the parser must agree with the + /// packing for every case. + #[test] + fn every_series_variant_round_trips_through_its_mzpaf_spelling() { + for series in all_series() { + let annot = IonAnnot::pack(series, NeutralLoss::None, 1, 0).expect("valid"); + assert_eq!(annot.series_ordinal(), series); + let text = annot.to_string(); + assert_eq!(ion(&text).series_ordinal(), series, "{text}"); + } + } + + /// The letters are the mzPAF spelling of the discriminants, and + /// `from_char`/`as_char` are hand-written inverses of each other. + #[test] + fn series_letters_and_discriminants_agree() { + assert_eq!(Series::ALL.len(), Series::CHARS.len()); + for &series in &Series::ALL { + assert_eq!(Series::from_char(series.as_char()), Some(series)); + } + for c in ['p', '?', 'm', 'I', 'q', 'A'] { + assert_eq!(Series::from_char(c), None, "{c} is not a backbone series"); + } + } + + /// An unrepresentable loss must fail loudly. Parsing `y1-HCOOH` as plain + /// `y1` would put a loss peak's m/z on the `y1` label and collide with the + /// real `y1`. + #[test] + fn unrepresentable_loss_is_rejected_not_stripped() { + assert!(matches!( + IonAnnot::try_from("y1-HCOOH"), + Err(IonParsingError::UnsupportedNeutralLoss { .. }) + )); } } diff --git a/rust/micromzpaf/src/loss.rs b/rust/micromzpaf/src/loss.rs new file mode 100644 index 00000000..8645fed6 --- /dev/null +++ b/rust/micromzpaf/src/loss.rs @@ -0,0 +1,441 @@ +//! Neutral losses, keyed by atomic composition rather than by spelling. +//! +//! Libraries write the same chemical loss different ways. Two real examples +//! from the HUPO-PSI mzSpecLib corpus: +//! +//! | written | library | composition | +//! |---|---|---| +//! | `-CH3SOH` | NIST | C1H4O1S1 | +//! | `-CH4OS` | SpectraST | C1H4O1S1 | +//! | `-NH2-CO-CH2SH` | NIST | C2H5N1O1S1 | +//! | `-C2H5NOS` | SpectraST | C2H5N1O1S1 | +//! +//! Keying on the string would make `y5-CH4OS` and `y5-CH3SOH` distinct labels +//! for one ion. Since fragment labels must be unique within a precursor (see +//! `ExpectedIntensities::try_from_pairs` in timsseek), that is exactly the +//! wrong direction: it hides a genuine duplicate behind two spellings. +//! +//! So parsing goes `text -> composition -> discriminant`, and the composition +//! is a *parse-time* concept only. What gets stored on an `IonAnnot` is the +//! discriminant, so there is no per-annotation cost at runtime. + +use std::fmt::Display; + +use crate::IonParsingError; + +/// Slots in [`Composition`], in the order [`Composition::new`] takes them. +const C: usize = 0; +const H: usize = 1; +const N: usize = 2; +const O: usize = 3; +const S: usize = 4; +const P: usize = 5; + +/// Atom counts for the elements that appear in peptide neutral losses. +/// +/// Deliberately not a general chemical formula: these losses only ever draw +/// from C/H/N/O/S/P, and keeping it to six `u8`s makes equality a single +/// 6-byte compare during the parse-time table lookup. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Composition([u8; 6]); + +impl Composition { + pub(crate) const fn new(c: u8, h: u8, n: u8, o: u8, s: u8, p: u8) -> Self { + Self([c, h, n, o, s, p]) + } + + /// Multiply every count, saturating. Used for the `2H2O` multiplier form. + fn scaled(self, k: u8) -> Self { + Self(self.0.map(|n| n.saturating_mul(k))) + } + + fn plus(self, other: Self) -> Self { + Self(std::array::from_fn(|i| { + self.0[i].saturating_add(other.0[i]) + })) + } + + /// Parse a bare formula like `H2O`, `CH4OS`, `C2H5NOS`. + /// + /// Only single-letter C/H/N/O/S/P are recognized; anything else is an + /// error rather than a silent skip, so an unsupported loss surfaces as + /// "not representable" instead of being mistaken for a smaller one. + fn parse_formula(s: &str) -> Result { + if s.is_empty() { + return Err(IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Empty neutral-loss formula"), + }); + } + let mut out = Composition::default(); + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() { + let elem = b[i]; + i += 1; + let start = i; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + let count: u8 = if start == i { + 1 + } else { + s[start..i] + .parse() + .map_err(|_| IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Neutral-loss atom count out of range"), + })? + }; + let slot = match elem { + b'C' => C, + b'H' => H, + b'N' => N, + b'O' => O, + b'S' => S, + b'P' => P, + _ => { + return Err(IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Unsupported element in neutral loss"), + }); + } + }; + out.0[slot] = out.0[slot].saturating_add(count); + } + Ok(out) + } + + /// Parse a full loss expression: `-` separated terms, each optionally + /// prefixed by a repeat count. `2H2O`, `H2O-NH3`, `NH2-CO-CH2SH`. + /// + /// Because terms are summed, ordering and multiplier spelling collapse for + /// free: `H2O-NH3` == `NH3-H2O`, and `2H2O` == `H2O-H2O`. + pub(crate) fn parse_expression(s: &str) -> Result { + let mut total = Composition::default(); + for term in s.split('-') { + let term = term.trim(); + if term.is_empty() { + return Err(IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Empty term in neutral-loss expression"), + }); + } + // Leading digits are a repeat count for the whole term. + let digits = term + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(term.len()); + let (mult, formula) = if digits > 0 { + let m: u8 = term[..digits] + .parse() + .map_err(|_| IonParsingError::ParsingError { + error: s.to_string(), + context: Some("Neutral-loss repeat count out of range"), + })?; + (m, &term[digits..]) + } else { + (1, term) + }; + total = total.plus(Composition::parse_formula(formula)?.scaled(mult)); + } + Ok(total) + } +} + +/// The neutral losses this crate can represent, as a packed discriminant. +/// +/// Scoped deliberately: DIA-NN emits none, Spectronaut two (`-H2O`, `-NH3`), +/// NIST eight, plus the phospho losses that no non-phospho corpus can show. +/// SpectraST's wider combinatorics are NOT here — an unlisted loss parses to a +/// composition that misses the table and is reported as unrepresentable, which +/// routes the peak to an unknown label rather than silently mislabelling it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +#[repr(u8)] +pub enum NeutralLoss { + #[default] + None = 0, + /// H2O + Water = 1, + /// NH3 + Ammonia = 2, + /// CO + CarbonMonoxide = 3, + /// CO2 + CarbonDioxide = 4, + /// 2 H2O + WaterX2 = 5, + /// 2 NH3 + AmmoniaX2 = 6, + /// H2O + NH3 + WaterAmmonia = 7, + /// CH4OS — methanesulfenic acid, off oxidized Met. Also spelled `CH3SOH`. + Methanesulfenic = 8, + /// C2H5NOS — also spelled `NH2-CO-CH2SH`. + Carbamidomethylthiol = 9, + /// H3PO4 — phospho-Ser/Thr. + PhosphoricAcid = 10, + /// HPO3 — phospho-Tyr, and phospho-Ser/Thr. + Metaphosphoric = 11, + /// H3PO4 + H2O + PhosphoricAcidWater = 12, +} + +/// `(composition, discriminant, canonical spelling)`. +/// +/// The canonical spelling is what `Display` emits, so a non-canonical input +/// (`-CH3SOH`) round-trips to the canonical form (`-CH4OS`). Round-trip tests +/// must therefore compare parsed values, not bytes. +const TABLE: &[(Composition, NeutralLoss, &str)] = &[ + ( + Composition::new(0, 2, 0, 1, 0, 0), + NeutralLoss::Water, + "H2O", + ), + ( + Composition::new(0, 3, 1, 0, 0, 0), + NeutralLoss::Ammonia, + "NH3", + ), + ( + Composition::new(1, 0, 0, 1, 0, 0), + NeutralLoss::CarbonMonoxide, + "CO", + ), + ( + Composition::new(1, 0, 0, 2, 0, 0), + NeutralLoss::CarbonDioxide, + "CO2", + ), + ( + Composition::new(0, 4, 0, 2, 0, 0), + NeutralLoss::WaterX2, + "2H2O", + ), + ( + Composition::new(0, 6, 2, 0, 0, 0), + NeutralLoss::AmmoniaX2, + "2NH3", + ), + ( + Composition::new(0, 5, 1, 1, 0, 0), + NeutralLoss::WaterAmmonia, + "H2O-NH3", + ), + ( + Composition::new(1, 4, 0, 1, 1, 0), + NeutralLoss::Methanesulfenic, + "CH4OS", + ), + ( + Composition::new(2, 5, 1, 1, 1, 0), + NeutralLoss::Carbamidomethylthiol, + "C2H5NOS", + ), + ( + Composition::new(0, 3, 0, 4, 0, 1), + NeutralLoss::PhosphoricAcid, + "H3PO4", + ), + ( + Composition::new(0, 1, 0, 3, 0, 1), + NeutralLoss::Metaphosphoric, + "HPO3", + ), + ( + Composition::new(0, 5, 0, 5, 0, 1), + NeutralLoss::PhosphoricAcidWater, + "H3PO4-H2O", + ), +]; + +impl NeutralLoss { + /// Inverse of the `#[repr(u8)]` discriminant, for unpacking out of a bit + /// field. Lives next to the enum so the two cannot drift apart. + /// + /// An unrecognized value maps to [`Self::None`]: the only way to produce + /// one is a reserved discriminant, which no constructor emits. + pub(crate) fn from_discriminant(d: u8) -> Self { + match d { + 1 => Self::Water, + 2 => Self::Ammonia, + 3 => Self::CarbonMonoxide, + 4 => Self::CarbonDioxide, + 5 => Self::WaterX2, + 6 => Self::AmmoniaX2, + 7 => Self::WaterAmmonia, + 8 => Self::Methanesulfenic, + 9 => Self::Carbamidomethylthiol, + 10 => Self::PhosphoricAcid, + 11 => Self::Metaphosphoric, + 12 => Self::PhosphoricAcidWater, + _ => Self::None, + } + } + + /// Resolve a loss expression (without the leading `-`) to a discriminant. + /// + /// `None` means "parsed as a valid composition, but not one we represent" — + /// distinct from `Err`, which means the text was not a loss expression at + /// all. Callers route the former to an unknown label and the latter to a + /// parse failure. + pub(crate) fn from_expression(s: &str) -> Result, IonParsingError> { + let comp = Composition::parse_expression(s)?; + Ok(TABLE + .iter() + .find(|(c, _, _)| *c == comp) + .map(|(_, l, _)| *l)) + } + + /// Canonical spelling, without the leading `-`. Empty for [`Self::None`]. + pub(crate) fn canonical(self) -> &'static str { + if self == NeutralLoss::None { + return ""; + } + TABLE + .iter() + .find(|(_, l, _)| *l == self) + .map(|(_, _, s)| *s) + .unwrap_or("") + } +} + +impl Display for NeutralLoss { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if *self == NeutralLoss::None { + return Ok(()); + } + write!(f, "-{}", self.canonical()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The count parser: implicit 1, multi-digit counts, and a count past the + /// `u8` slot. Single-digit cases are covered by every TABLE row. + #[test] + fn atom_counts_parse_and_reject_out_of_range() { + assert_eq!( + Composition::parse_expression("H2O").unwrap(), + Composition::new(0, 2, 0, 1, 0, 0), + "an element with no digits is one atom" + ); + assert_eq!( + Composition::parse_expression("C10H12").unwrap(), + Composition::new(10, 12, 0, 0, 0, 0), + "counts are multi-digit, not one digit per element" + ); + assert_eq!( + Composition::parse_expression("C255").unwrap(), + Composition::new(255, 0, 0, 0, 0, 0), + "the u8 slot is full at 255" + ); + assert!( + Composition::parse_expression("C256").is_err(), + "one past the slot must be an error, not a wrap to 0" + ); + } + + /// The two cross-library spelling collisions this module exists for. + #[test] + fn different_spellings_resolve_to_one_loss() { + // NIST vs SpectraST, methanesulfenic acid. + assert_eq!( + NeutralLoss::from_expression("CH3SOH").unwrap(), + Some(NeutralLoss::Methanesulfenic) + ); + assert_eq!( + NeutralLoss::from_expression("CH4OS").unwrap(), + Some(NeutralLoss::Methanesulfenic) + ); + // NIST structural notation vs SpectraST molecular notation. + assert_eq!( + NeutralLoss::from_expression("NH2-CO-CH2SH").unwrap(), + Some(NeutralLoss::Carbamidomethylthiol) + ); + assert_eq!( + NeutralLoss::from_expression("C2H5NOS").unwrap(), + Some(NeutralLoss::Carbamidomethylthiol) + ); + } + + /// Summing terms collapses ordering and multiplier spelling for free. + #[test] + fn ordering_and_multipliers_normalize() { + assert_eq!( + NeutralLoss::from_expression("H2O-NH3").unwrap(), + NeutralLoss::from_expression("NH3-H2O").unwrap() + ); + assert_eq!( + NeutralLoss::from_expression("2H2O").unwrap(), + NeutralLoss::from_expression("H2O-H2O").unwrap() + ); + assert_eq!( + NeutralLoss::from_expression("2H2O").unwrap(), + Some(NeutralLoss::WaterX2) + ); + } + + /// A well-formed composition outside the table is `Ok(None)` — "valid but + /// not representable" — while malformed text is `Err`. Callers need to + /// tell those apart to route one to an unknown label and the other to a + /// parse failure. + #[test] + fn unrepresentable_is_distinct_from_malformed() { + assert_eq!(NeutralLoss::from_expression("HCOOH").unwrap(), None); + assert!(NeutralLoss::from_expression("Xe2").is_err()); + assert!(NeutralLoss::from_expression("").is_err()); + assert!(NeutralLoss::from_expression("H2O-").is_err()); + } + + /// Every table entry must survive canonical -> composition -> discriminant, + /// and back out through the bit field. `from_discriminant` hand-mirrors the + /// `#[repr(u8)]` values, so adding a loss without updating it would + /// silently decode as [`NeutralLoss::None`] — this is what catches that. + #[test] + fn table_round_trips_through_canonical_spelling() { + for (_comp, loss, canon) in TABLE { + assert_eq!( + NeutralLoss::from_expression(canon).unwrap(), + Some(*loss), + "canonical spelling {canon} must resolve to its own loss" + ); + assert_eq!(loss.canonical(), *canon); + assert_eq!( + NeutralLoss::from_discriminant(*loss as u8), + *loss, + "{canon} does not survive the discriminant round trip" + ); + } + assert_eq!(NeutralLoss::from_discriminant(0), NeutralLoss::None); + // Every non-None variant must be in TABLE, or it has no spelling and no + // composition and could never be produced by parsing. + assert_eq!( + TABLE.len(), + (1..=u8::MAX) + .filter(|d| NeutralLoss::from_discriminant(*d) != NeutralLoss::None) + .count(), + "a variant is decodable but missing from TABLE" + ); + } + + /// Compositions must be unique: two entries sharing one would make the + /// table lookup order-dependent. + #[test] + fn table_compositions_are_unique() { + for (i, (a, _, sa)) in TABLE.iter().enumerate() { + for (b, _, sb) in TABLE.iter().skip(i + 1) { + assert_ne!(a, b, "{sa} and {sb} share a composition"); + } + } + } + + /// The two things the TABLE round trip cannot see: the `-` prefix Display + /// adds, and that `None` renders as nothing at all rather than "-". + #[test] + fn display_adds_the_prefix_and_none_renders_empty() { + assert_eq!(NeutralLoss::None.to_string(), ""); + assert_eq!(NeutralLoss::Water.to_string(), "-H2O"); + } +} diff --git a/rust/speclib_build_cli/Cargo.toml b/rust/speclib_build_cli/Cargo.toml index 456fe6f7..9486fdf8 100644 --- a/rust/speclib_build_cli/Cargo.toml +++ b/rust/speclib_build_cli/Cargo.toml @@ -16,7 +16,7 @@ path = "src/main.rs" timsseek = { path = "../timsseek" } micromzpaf = { path = "../micromzpaf" } tims_stage = { path = "../tims_stage" } -rustyms = { workspace = true } +mzcore = { workspace = true } clap = { workspace = true } serde = { workspace = true } diff --git a/rust/speclib_build_cli/src/cli.rs b/rust/speclib_build_cli/src/cli.rs index ce27fd62..016e13a4 100644 --- a/rust/speclib_build_cli/src/cli.rs +++ b/rust/speclib_build_cli/src/cli.rs @@ -21,7 +21,7 @@ pub struct Cli { // ── Output ───────────────────────────────────────────────────────────── /// Output URI for the spectral library (local path or s3://...; default: - /// library.msgpack.zst). + /// library.ndjson.zst). #[arg(long, short = 'o')] pub output: Option, diff --git a/rust/speclib_build_cli/src/config.rs b/rust/speclib_build_cli/src/config.rs index 0c24bff1..2ab496b3 100644 --- a/rust/speclib_build_cli/src/config.rs +++ b/rust/speclib_build_cli/src/config.rs @@ -62,7 +62,7 @@ fn default_min_ions() -> usize { 3 } fn default_output() -> String { - "library.msgpack.zst".to_string() + "library.ndjson.zst".to_string() } // ── Sub-structs ────────────────────────────────────────────────────────────── diff --git a/rust/speclib_build_cli/src/entry.rs b/rust/speclib_build_cli/src/entry.rs index e5686aa6..a611f87b 100644 --- a/rust/speclib_build_cli/src/entry.rs +++ b/rust/speclib_build_cli/src/entry.rs @@ -4,11 +4,7 @@ use timsseek::data_sources::speclib::{ ReferenceEG, SerSpeclibElement, }; -use timsseek::fragment_mass::elution_group_converter::{ - count_carbon_sulphur_in_sequence, - supersimpleprediction, -}; -use timsseek::isotopes::peptide_isotopes; +use timsseek::fragment_mass::elution_group_converter::supersimpleprediction; use crate::koina::models::{ FragmentPrediction, @@ -28,33 +24,17 @@ pub struct EntryFilters { // ── Helpers ─────────────────────────────────────────────────────────────────── -/// Strip bracket-enclosed modifications from a sequence. -/// -/// "PEPTC[U:4]IDEK" → "PEPTCIDEK" -pub fn strip_mods(seq: &str) -> String { - let mut out = String::with_capacity(seq.len()); - let mut depth = 0usize; - for ch in seq.chars() { - match ch { - '[' => depth += 1, - ']' => { - depth = depth.saturating_sub(1); - } - _ if depth == 0 => out.push(ch), - _ => {} - } - } - out -} - use timsseek::models::sequence::normalize_to_proforma; -/// Compute the monoisotopic precursor m/z using rustyms. +/// Compute the monoisotopic precursor m/z using mzcore. /// Input should be the modified sequence (mods included in mass). +/// +/// This is the only mzcore parse per library entry, and it doubles as the +/// malformed-sequence gate: anything it cannot parse is dropped here. fn compute_precursor_mz(modified_seq: &str, charge: u8) -> Option { - use rustyms::prelude::*; + use mzcore::prelude::*; let proforma = normalize_to_proforma(modified_seq); - let peptide = Peptidoform::pro_forma(&proforma, None).ok()?; + let peptide = timsseek::models::sequence::parse_proforma(&proforma).ok()?; let linear = peptide.as_linear()?; let formulas = linear.formulas(); if formulas.is_empty() { @@ -65,12 +45,6 @@ fn compute_precursor_mz(modified_seq: &str, charge: u8) -> Option { Some((mass + proton_mass * charge as f64) / charge as f64) } -/// Count carbon and sulphur from modified sequence (mods affect formula). -fn count_cs_modified(modified_seq: &str) -> Option<(u16, u16)> { - let proforma = normalize_to_proforma(modified_seq); - count_carbon_sulphur_in_sequence(&proforma).ok() -} - // ── Public API ──────────────────────────────────────────────────────────────── /// Convert Koina predictions + metadata into a [`SerSpeclibElement`]. @@ -83,28 +57,23 @@ pub fn build_entry( sequence: &str, charge: u8, decoy: bool, - decoy_group: u32, - id: u32, fragment: &FragmentPrediction, rt: &RtPrediction, filters: &EntryFilters, ) -> Option { - // 1. Carbon / sulphur count from modified sequence (includes mod contributions). - let (ncarbon, nsulphur) = count_cs_modified(sequence)?; - let iso = peptide_isotopes(ncarbon, nsulphur); - - // 2. Precursor m/z from modified sequence (includes mod masses). + // 1. Precursor m/z from modified sequence (includes mod masses). Also the + // malformed-sequence gate. let precursor_mz = compute_precursor_mz(sequence, charge)?; - // 4. Filter by precursor m/z range. + // 2. Filter by precursor m/z range. if precursor_mz < filters.min_mz as f64 || precursor_mz > filters.max_mz as f64 { return None; } - // 5. Ion mobility prediction. + // 3. Ion mobility prediction. let mobility = supersimpleprediction(precursor_mz, charge as i32) as f32; - // 6. Filter fragments: keep those within ion m/z bounds. + // 4. Filter fragments: keep those within ion m/z bounds. let min_ion = filters.min_ion_mz as f64; let max_ion = filters.max_ion_mz as f64; @@ -150,19 +119,16 @@ pub fn build_entry( return None; } - // 10. Build precursor labels and intensities from the isotope distribution. - let precursor_labels: Vec = vec![0i8, 1i8, 2i8]; - let precursor_intensities: Vec = vec![iso[0], iso[1], iso[2]]; - - // 11. Assemble the element. - let precursor = PrecursorEntry::new(sequence.to_owned(), charge, decoy, decoy_group); + // 8. Assemble the element. The precursor isotope envelope is NOT stored: + // the loader recomputes it from composition + // (`IsotopeStrategy::FromComposition`), so writing it here would cost a + // second mzcore parse per entry to produce bytes nobody reads — and + // would give isotopes two sources of truth. + let precursor = PrecursorEntry::new(sequence.to_owned(), charge, decoy); let elution_group = ReferenceEG::new( - id, precursor_mz, - precursor_labels, fragment_mzs, fragment_labels, - precursor_intensities, fragment_intensities, mobility, rt.irt, @@ -205,13 +171,6 @@ mod tests { } } - #[test] - fn test_strip_mods() { - assert_eq!(strip_mods("PEPTC[U:4]IDEK"), "PEPTCIDEK"); - assert_eq!(strip_mods("PEPTM[+15.995]IDEK"), "PEPTMIDEK"); - assert_eq!(strip_mods("PEPTIDEK"), "PEPTIDEK"); - } - #[test] fn test_compute_precursor_mz() { let mz = compute_precursor_mz("PEPTIDEK", 2).unwrap(); @@ -223,7 +182,7 @@ mod tests { #[test] fn test_precursor_mz_includes_mod_mass() { - // to_proforma converts [U:4] → [UNIMOD:4] before rustyms + // to_proforma converts [U:4] → [UNIMOD:4] before mzcore let mz_unmod = compute_precursor_mz("PEPTCIDEK", 2).unwrap(); let mz_mod = compute_precursor_mz("PEPTC[U:4]IDEK", 2).unwrap(); let diff = mz_mod - mz_unmod; @@ -240,7 +199,7 @@ mod tests { let rt = RtPrediction { irt: 30.0 }; let filters = make_filters(3); - let result = build_entry("PEPTIDEK", 2, false, 0, 42, &fragment, &rt, &filters); + let result = build_entry("PEPTIDEK", 2, false, &fragment, &rt, &filters); assert!(result.is_some(), "Expected Some but got None"); } @@ -256,7 +215,7 @@ mod tests { let rt = RtPrediction { irt: 30.0 }; let filters = make_filters(3); - let result = build_entry("PEPTIDEK", 2, false, 0, 1, &fragment, &rt, &filters); + let result = build_entry("PEPTIDEK", 2, false, &fragment, &rt, &filters); assert!(result.is_none(), "Expected None but got Some"); } @@ -275,7 +234,7 @@ mod tests { min_ions: 3, }; - let result = build_entry("PEPTIDEK", 2, false, 0, 2, &fragment, &rt, &filters); + let result = build_entry("PEPTIDEK", 2, false, &fragment, &rt, &filters); assert!( result.is_none(), @@ -289,7 +248,7 @@ mod tests { let rt = RtPrediction { irt: 30.0 }; let filters = make_filters(3); - let result = build_entry("KEDITREP", 2, true, 99, 7, &fragment, &rt, &filters); + let result = build_entry("KEDITREP", 2, true, &fragment, &rt, &filters); // Decoy peptide should still build an entry if the mz/ions pass filters // (KEDITREP ~476 should pass default 400–2000 window). diff --git a/rust/speclib_build_cli/src/pipeline.rs b/rust/speclib_build_cli/src/pipeline.rs index 01537c50..1f565a46 100644 --- a/rust/speclib_build_cli/src/pipeline.rs +++ b/rust/speclib_build_cli/src/pipeline.rs @@ -45,7 +45,6 @@ struct BatchItem { sequence: String, charge: u8, decoy: bool, - decoy_group: u32, } // ── flush_batch ───────────────────────────────────────────────────────────── @@ -93,8 +92,6 @@ async fn flush_batch( &item.sequence, item.charge, item.decoy, - item.decoy_group, - *entry_id, fragment, rt, filters, @@ -268,13 +265,12 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box = if remote_output { - let ext = std::path::Path::new(output_uri.trim_end_matches('/')) - .extension() - .and_then(|s| s.to_str()) - .unwrap_or("msgpack.zst"); + // Fixed rather than derived from the destination URI: the writer only + // emits zstd-wrapped NDJSON, and the upload below carries the caller's + // own name anyway. let tf = tempfile::Builder::new() .prefix("speclib-out-") - .suffix(&format!(".{ext}")) + .suffix(".ndjson.zst") .tempfile()?; Some(tf) } else { @@ -293,7 +289,7 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box Result<(), Box = Vec::with_capacity(batch_size); let mut entry_id: u32 = 0; - let mut decoy_group: u32 = 0; for digest_slice in &base_peptides { let base_seq = digest_slice.as_str(); @@ -335,7 +330,6 @@ pub async fn run(config: &SpeclibBuildConfig) -> Result<(), Box Result<(), Box Result<(), Box= 0` — not only when both are positive. A negative `low` is + /// a supported input: Carafe encodes a systematic calibration offset as + /// `[itol - itol_shift, itol + itol_shift]` and emits e.g. + /// `{"ppm": [-2.0, 32.0]}` when the offset exceeds the half-width. Their + /// sum is `2 * itol`, so the window stays valid for any positive tolerance. + /// See `docs/CARAFE_CONTRACT.md`. + const MZ_RANGE_INVARIANT: &str = + "m/z tolerance produced an inverted range: low + high must be >= 0"; + /// Calculate m/z tolerance range (primary method, returns f64). /// /// This is the canonical m/z range method. All other m/z methods delegate to this. @@ -129,15 +141,15 @@ impl Tolerance { /// ``` pub fn mz_range(&self, mz: f64) -> TupleRange { match self.ms { - MzTolerance::Absolute((low, high)) => (mz - low, mz + high).try_into().expect( - "mz tolerance should never result in an invalid range, since low and high are positive", - ), + MzTolerance::Absolute((low, high)) => (mz - low, mz + high) + .try_into() + .expect(Self::MZ_RANGE_INVARIANT), MzTolerance::Ppm((low, high)) => { let low = mz * low / 1e6; let high = mz * high / 1e6; - (mz - low, mz + high).try_into().expect( - "mz tolerance should never result in an invalid range, since low and high are positive", - ) + (mz - low, mz + high) + .try_into() + .expect(Self::MZ_RANGE_INVARIANT) } } } diff --git a/rust/timsquery/src/serde/diann_io.rs b/rust/timsquery/src/serde/diann_io.rs index 291a04b4..a654a4ce 100644 --- a/rust/timsquery/src/serde/diann_io.rs +++ b/rust/timsquery/src/serde/diann_io.rs @@ -2,6 +2,7 @@ use crate::TimsElutionGroup; use crate::ion::{ IonAnnot, IonParsingError, + UnknownIonCounter, }; use arrow::array::{ Float32Array, @@ -344,7 +345,7 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(rows.len()); - let mut num_unknown_losses = 0; + let mut unknown_ions = UnknownIonCounter::new(); for (i, row) in rows.iter().enumerate() { let fragment_mz = row.fragment_mz; @@ -364,8 +365,7 @@ fn parse_precursor_group( row.fragment_loss_type, i ); - num_unknown_losses += 1; - let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; + let ion_annot = unknown_ions.next(frag_charge as i8)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); relative_intensities.push((ion_annot, rel_intensity)); @@ -658,7 +658,7 @@ fn parse_precursor_group_from_parquet( let mut fragment_mzs = Vec::with_capacity(indices.len()); buffers.fragment_labels.clear(); let mut rel_intensities = Vec::with_capacity(indices.len()); - let mut num_unknown_losses = 0; + let mut unknown_ions = UnknownIonCounter::new(); for (i, &idx) in indices.iter().enumerate() { let fragment_mz = columns.product_mzs[idx] as f64; @@ -680,8 +680,7 @@ fn parse_precursor_group_from_parquet( columns.fragment_loss_types[idx], i ); - num_unknown_losses += 1; - let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; + let ion_annot = unknown_ions.next(frag_charge as i8)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); rel_intensities.push((ion_annot, rel_intensity)); diff --git a/rust/timsquery/src/serde/diann_speclib_io.rs b/rust/timsquery/src/serde/diann_speclib_io.rs index 785f2b53..0574286e 100644 --- a/rust/timsquery/src/serde/diann_speclib_io.rs +++ b/rust/timsquery/src/serde/diann_speclib_io.rs @@ -26,8 +26,11 @@ //! than exposed as a random-access view. use super::library_file::{ + FragmentSet, + Inserted, LibraryArena, LibraryReadingError, + finish_mzpaf_arena, }; use crate::ion::IonAnnot; use crate::models::{ @@ -486,7 +489,7 @@ impl SpecLib { ), Vec::::new(), SpeclibDecodeStats::default(), - Vec::new(), + FragmentSet::default(), ) }, |(mut geom, mut frag_intens, mut stats, mut scratch), @@ -507,7 +510,7 @@ impl SpecLib { ), Vec::::new(), SpeclibDecodeStats::default(), - Vec::new(), + FragmentSet::default(), ) }, |(mut a_geom, mut a_int, sa, scratch), (b_geom, b_int, sb, _)| { @@ -601,22 +604,6 @@ impl SpeclibDecodeStats { } } -/// Strip DIA-NN mod annotations — anything inside `(...)` or `[...]` — leaving -/// the bare residue string. -fn strip_mods(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - let mut depth: i32 = 0; - for c in s.chars() { - match c { - '(' | '[' => depth += 1, - ')' | ']' => depth = (depth - 1).max(0), - _ if depth == 0 => out.push(c), - _ => {} - } - } - out -} - /// Residue count of a mod-stripped sequence. fn residue_count(stripped: &str) -> usize { stripped.chars().filter(|c| c.is_ascii_alphabetic()).count() @@ -687,7 +674,7 @@ fn map_entry( geom: &mut QueryCollection, frag_intens: &mut Vec, stats: &mut SpeclibDecodeStats, - scratch: &mut Vec<(IonAnnot, f64, f32)>, + scratch: &mut FragmentSet, ) -> Result<(), LibraryReadingError> { let pep = &entry.peptide; let name = entry.name; @@ -715,7 +702,7 @@ fn map_entry( name.clone() } }; - let stripped_peptide = strip_mods(&modified_peptide); + let stripped_peptide = crate::utils::sequence::strip_mods(&modified_peptide); // `Peptide.length` may be 0 in some libraries; recover it from the sequence // when needed (b/a/c series don't need it, y/x/z series do). @@ -725,9 +712,9 @@ fn map_entry( residue_count(&stripped_peptide) }; - // (IonAnnot, fragment mz as f64, height). Dedup by IonAnnot keeping max - // height so a duplicate label can't fail the whole load via - // `ExpectedIntensities::try_from_pairs`. Reuses the caller's scratch buffer. + // `FragmentSet` owns the per-precursor label-uniqueness invariant that + // `ExpectedIntensities::try_from_pairs` depends on. Reuses the caller's + // scratch buffer so the allocation is per worker, not per entry. scratch.clear(); let kept = scratch; @@ -794,25 +781,15 @@ fn map_entry( } }; - if let Some(slot) = kept.iter_mut().find(|(k, _, _)| *k == ion) { + if kept.insert(ion, f.mz() as f64, f.height()) == Inserted::Collapsed { stats.dedup_dropped += 1; - if f.height() > slot.2 { - slot.1 = f.mz() as f64; - slot.2 = f.height(); - } - } else { - kept.push((ion, f.mz() as f64, f.height())); } } - // (label, mz) pairs for the arena, with the parallel reference-intensity - // sidecar filled in the same order — dedup already collapsed duplicate - // labels, so this order is what lands in `geom.frag_labels`. - let mut frags: Vec<(IonAnnot, f64)> = Vec::with_capacity(kept.len()); - for &(ion, mz, height) in kept.iter() { - frags.push((ion, mz)); - frag_intens.push(height); - } + // The reference-intensity sidecar is filled in the same order the labels + // land in `geom.frag_labels`. + kept.extend_sidecar(frag_intens); + let frags = kept.frags(); // Record charge is i32; `push_target` wants u8. Charge was range-checked to // 1..=255 above, so the cast is safe. @@ -891,7 +868,7 @@ pub fn read_diann_speclib_library_file>( // mmap the file (not an owned read) — pages fault in on demand, no // file-sized resident buffer. - let (mut geom, frag_intens, stats, at_eof) = SpecLib::open_mmap(path)?.parse_parallel()?; + let (geom, frag_intens, stats, at_eof) = SpecLib::open_mmap(path)?.parse_parallel()?; if !at_eof { // A parse that doesn't land on EOF means the entries were misaligned @@ -925,16 +902,7 @@ pub fn read_diann_speclib_library_file>( path.display() ); - assert_eq!( - frag_intens.len(), - geom.frag_labels.len(), - "reference-intensity sidecar must stay parallel to the fragment-label arena" - ); - geom.seal(); - Ok(LibraryArena::Mzpaf { - geom, - frag_intens: Some(frag_intens), - }) + finish_mzpaf_arena(geom, frag_intens) } #[cfg(test)] diff --git a/rust/timsquery/src/serde/elution_group_inputs.rs b/rust/timsquery/src/serde/elution_group_inputs.rs index 05142133..e172b975 100644 --- a/rust/timsquery/src/serde/elution_group_inputs.rs +++ b/rust/timsquery/src/serde/elution_group_inputs.rs @@ -1,4 +1,3 @@ -use crate::ion::IonAnnot; use crate::tinyvec::{ TinyVec, tiny_vec, @@ -10,9 +9,16 @@ use crate::{ #[derive(Debug)] pub enum ElutionGroupInputError { - MismatchedFragmentLabelsLength { expected: usize, found: usize }, - AlreadyHasFragmentLabels, - IonConversionError { inner: String }, + MismatchedFragmentLabelsLength { + expected: usize, + found: usize, + }, + IonConversionError { + inner: String, + }, + /// The input shipped no `fragment_labels`. There is no way to synthesize + /// them: a label names an ion series and ordinal, and a positional index + /// carries no chemistry. MissingFragmentLabels, } @@ -34,53 +40,6 @@ pub struct ElutionGroupInput { pub fragment_labels: Option>, } -impl ElutionGroupInput { - pub fn needs_fragment_labels(&self) -> bool { - self.fragment_labels.is_none() - } - - pub fn try_fill_labels_u8(self) -> Result, ElutionGroupInputError> { - let num_fragments = self.fragments.len(); - if self.fragment_labels.is_some() { - return Err(ElutionGroupInputError::AlreadyHasFragmentLabels); - } - let fragment_labels: Vec = (0..num_fragments).map(|i| i as u8).collect(); - - Ok(ElutionGroupInput { - id: self.id, - mobility: self.mobility, - rt_seconds: self.rt_seconds, - precursor: self.precursor, - precursor_charge: self.precursor_charge, - precursor_isotopes: self.precursor_isotopes, - fragments: self.fragments, - fragment_labels: Some(fragment_labels), - }) - } - - pub fn try_fill_labels_annot( - self, - ) -> Result, ElutionGroupInputError> { - let tmp = self.try_fill_labels_u8()?; - let new_frags = tmp - .fragment_labels - .unwrap() - .into_iter() - .map(|lbl| IonAnnot::try_new('?', Some(lbl), 1, 1).unwrap()) - .collect(); - Ok(ElutionGroupInput { - id: tmp.id, - mobility: tmp.mobility, - rt_seconds: tmp.rt_seconds, - precursor: tmp.precursor, - precursor_charge: tmp.precursor_charge, - precursor_isotopes: tmp.precursor_isotopes, - fragments: tmp.fragments, - fragment_labels: Some(new_frags), - }) - } -} - impl + KeyLike> TryFrom> for TimsElutionGroup { type Error = ElutionGroupInputError; @@ -124,3 +83,51 @@ impl + KeyLike> TryFrom> for Tims Ok(builder.try_build().expect("I checked the sizes!")) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ion::IonAnnot; + + fn input(fragment_labels: Option>) -> ElutionGroupInput { + ElutionGroupInput { + id: 0, + mobility: 0.8, + rt_seconds: 100.0, + precursor: 500.0, + precursor_charge: 2, + precursor_isotopes: None, + fragments: vec![100.0, 200.0], + fragment_labels, + } + } + + /// Unlabelled input is rejected by name rather than by whatever fails + /// first downstream. Nothing can stand in for a missing label. + #[test] + fn missing_fragment_labels_is_its_own_error() { + let err = TimsElutionGroup::::try_from(input(None)) + .expect_err("no labels means no elution group"); + assert!( + matches!(err, ElutionGroupInputError::MissingFragmentLabels), + "expected MissingFragmentLabels, got {err:?}" + ); + } + + #[test] + fn label_count_must_match_fragment_count() { + let one_label = vec![IonAnnot::try_from("y1").unwrap()]; + let err = TimsElutionGroup::::try_from(input(Some(one_label))) + .expect_err("1 label for 2 fragments"); + assert!( + matches!( + err, + ElutionGroupInputError::MismatchedFragmentLabelsLength { + expected: 2, + found: 1 + } + ), + "expected MismatchedFragmentLabelsLength, got {err:?}" + ); + } +} diff --git a/rust/timsquery/src/serde/library_file.rs b/rust/timsquery/src/serde/library_file.rs index b9c8885a..6e6deaf4 100644 --- a/rust/timsquery/src/serde/library_file.rs +++ b/rust/timsquery/src/serde/library_file.rs @@ -13,6 +13,10 @@ use super::elution_group_inputs::{ ElutionGroupInput, ElutionGroupInputError, }; +use super::mzspeclib_io::{ + read_mzspeclib_library_file, + sniff_mzspeclib_library_file, +}; pub use super::skyline_io::SkylinePrecursorExtras; use super::skyline_io::{ read_library_file as read_skyline_csv, @@ -43,6 +47,15 @@ pub enum LibraryReadingError { SerdeJsonError(serde_json::Error), ElutionGroupInputError(ElutionGroupInputError), UnableToParseElutionGroups, + /// A reader that sniffed positively then failed to parse. Carries which + /// reader and its own error, so `read_library_file`'s keep-the-first-error + /// rule has something specific to keep — previously every reader flattened + /// its failure into `UnableToParseElutionGroups` and the rule preserved no + /// information. + ReaderFailed { + reader: &'static str, + detail: String, + }, /// A `.speclib` whose version is newer (more negative) than this reader /// supports. UnsupportedSpeclibVersion(i32), @@ -62,31 +75,19 @@ impl From for LibraryReadingError { } } -#[derive(Debug)] -pub enum FileReadingExtras { - Diann(Vec), - Spectronaut(Vec), - Skyline(Vec), -} - +/// Elution groups with no per-precursor extras — the JSON path. +/// +/// The TSV/parquet readers do not go through this type: they hand their +/// `(group, extras)` pairs straight to [`arena_from_pairs`], which is what +/// removed the three-variant `FileReadingExtras` enum that used to exist only +/// to be immediately flattened into `PrecursorExtrasRow`. #[derive(Debug)] pub enum ElutionGroupCollection { - StringLabels(Vec>, Option), - MzpafLabels(Vec>, Option), - TinyIntLabels(Vec>, Option), - IntLabels(Vec>, Option), + StringLabels(Vec>), + MzpafLabels(Vec>), } impl ElutionGroupCollection { - pub fn len(&self) -> usize { - match self { - ElutionGroupCollection::StringLabels(egs, _) => egs.len(), - ElutionGroupCollection::MzpafLabels(egs, _) => egs.len(), - ElutionGroupCollection::TinyIntLabels(egs, _) => egs.len(), - ElutionGroupCollection::IntLabels(egs, _) => egs.len(), - } - } - fn try_read_json(path: &Path) -> Result { let file_content = std::fs::read_to_string(path).map_err(LibraryReadingError::IoError)?; info!("Read file content from {}", path.display()); @@ -95,80 +96,43 @@ impl ElutionGroupCollection { info!("Successfully deserialized elution groups directly"); return Ok(egs); } - // Next try deserialization via inputed format - match Self::try_deser_inputed(&file_content) { - Ok(egs) => { - info!("Successfully deserialized elution groups via inputed format"); - Ok(egs) - } - Err(_) => Err(LibraryReadingError::UnableToParseElutionGroups), - } + // Next try deserialization via inputed format. Its error is returned + // as-is: it names the field that is wrong, which + // `UnableToParseElutionGroups` does not. + let egs = Self::try_deser_inputed(&file_content)?; + info!("Successfully deserialized elution groups via inputed format"); + Ok(egs) } fn try_deser_inputed(content: &str) -> Result { - // We can try from smallest to largest overhead - // Here we try to do the deser into ElutionGroupInput variants first + // mzpaf before string: `"y1"` deserializes as either, and only the + // mzpaf form carries ion chemistry. debug!("Attempting deserialization of elution group inputs"); - debug!("Attempting to deserialize elution group inputs with tiny int labels"); - if let Ok(eg_inputs) = serde_json::from_str::>>(content) { - // Here we can handle filling the inputs if they are needed... - let eg_inputs = if eg_inputs.first().is_some_and(|x| x.needs_fragment_labels()) { - debug!("Filling missing fragment labels with tiny int labels"); - eg_inputs - .into_iter() - .map(|x| x.try_fill_labels_u8()) - .collect::>() - } else { - Ok(eg_inputs) - }; - - let out: Result>, ElutionGroupInputError> = eg_inputs? - .into_iter() - .map( as TryInto>>::try_into) - .collect(); - return Ok(ElutionGroupCollection::TinyIntLabels(out?, None)); - } - debug!("Attempting to deserialize elution group inputs with int labels"); - if let Ok(eg_inputs) = serde_json::from_str::>>(content) { - let out: Result>, ElutionGroupInputError> = - eg_inputs.into_iter().map(|x| x.try_into()).collect(); - return Ok(ElutionGroupCollection::IntLabels(out?, None)); - } debug!("Attempting to deserialize elution group inputs with mzpaf labels"); if let Ok(eg_inputs) = serde_json::from_str::>>(content) { let out: Result>, ElutionGroupInputError> = eg_inputs.into_iter().map(|x| x.try_into()).collect(); - return Ok(ElutionGroupCollection::MzpafLabels(out?, None)); + return Ok(ElutionGroupCollection::MzpafLabels(out?)); } debug!("Attempting to deserialize elution group inputs with string labels"); if let Ok(eg_inputs) = serde_json::from_str::>>(content) { let out: Result>, ElutionGroupInputError> = eg_inputs.into_iter().map(|x| x.try_into()).collect(); - return Ok(ElutionGroupCollection::StringLabels(out?, None)); + return Ok(ElutionGroupCollection::StringLabels(out?)); } Err(LibraryReadingError::UnableToParseElutionGroups) } fn try_deser_direct(content: &str) -> Result { - // We can try from smallest to largest overhead - // Here we try to do the direct deser into ElutionGroupCollection variants - // u8 -> u32 -> IonAnnot -> String + // mzpaf before string, for the same reason as `try_deser_inputed`. debug!("Attempting direct deserialization of elution groups"); - debug!("Attempting to deserialize elution groups with tiny int labels"); - if let Ok(egs) = serde_json::from_str::>>(content) { - return Ok(ElutionGroupCollection::TinyIntLabels(egs, None)); - } - debug!("Attempting to deserialize elution groups with int labels"); - if let Ok(egs) = serde_json::from_str::>>(content) { - return Ok(ElutionGroupCollection::IntLabels(egs, None)); - } debug!("Attempting to deserialize elution groups with mzpaf labels"); if let Ok(egs) = serde_json::from_str::>>(content) { - return Ok(ElutionGroupCollection::MzpafLabels(egs, None)); + return Ok(ElutionGroupCollection::MzpafLabels(egs)); } debug!("Attempting to deserialize elution groups with string labels"); if let Ok(egs) = serde_json::from_str::>>(content) { - return Ok(ElutionGroupCollection::StringLabels(egs, None)); + return Ok(ElutionGroupCollection::StringLabels(egs)); } Err(LibraryReadingError::UnableToParseElutionGroups) } @@ -254,16 +218,8 @@ impl LibraryArena { /// modified sequence. fn mzpaf_with_intensities( egs: Vec>, - extras: FileReadingExtras, + rows: Vec, ) -> Result { - let rows: Vec = match extras { - FileReadingExtras::Diann(v) => v.into_iter().map(PrecursorExtrasRow::from).collect(), - FileReadingExtras::Skyline(v) => v.into_iter().map(PrecursorExtrasRow::from).collect(), - FileReadingExtras::Spectronaut(v) => { - v.into_iter().map(PrecursorExtrasRow::from).collect() - } - }; - if egs.len() != rows.len() { return Err(LibraryReadingError::SpeclibParse(format!( "elution groups ({}) and reader extras ({}) length mismatch", @@ -275,27 +231,36 @@ impl LibraryArena { let mut geom = QueryCollection::with_capabilities(LibCapabilities::default_diann_no_decoys()); let mut frag_intens: Vec = Vec::new(); + let mut n_collapsed = 0usize; for (eg, row) in egs.iter().zip(rows) { // Reference intensities keyed by fragment label (see fn docs). let lookup: std::collections::HashMap = row.relative_intensities.into_iter().collect(); - let frags: Vec<(IonAnnot, f64)> = eg.iter_fragments().map(|(l, mz)| (*l, mz)).collect(); - for (label, _) in &frags { + + // Through `FragmentSet` rather than straight into a Vec: two TSV + // rows with the same series + ordinal + charge for one precursor + // produce two identical labels, which panics in scoring. + let mut set = FragmentSet::with_capacity(eg.iter_fragments().count()); + for (label, mz) in eg.iter_fragments() { let intensity = lookup.get(label).ok_or_else(|| { LibraryReadingError::SpeclibParse(format!( "fragment {label:?} of precursor {:?} has no reference intensity", row.modified )) })?; - frag_intens.push(*intensity); + if set.insert(*label, mz, *intensity) == Inserted::Collapsed { + n_collapsed += 1; + } } + + set.extend_sidecar(&mut frag_intens); geom.push_row( eg.precursor_mz(), eg.precursor_charge(), eg.rt_seconds(), eg.mobility_ook0(), - &frags, + set.frags(), &row.stripped, &row.modified, &[], @@ -303,19 +268,11 @@ impl LibraryArena { ); } - if frag_intens.len() != geom.frag_labels.len() { - return Err(LibraryReadingError::SpeclibParse(format!( - "reference-intensity sidecar ({}) must stay parallel to the fragment-label arena ({})", - frag_intens.len(), - geom.frag_labels.len(), - ))); + if n_collapsed > 0 { + warn!("{n_collapsed} fragments collapsed onto a duplicate label"); } - geom.seal(); - Ok(LibraryArena::Mzpaf { - geom, - frag_intens: Some(frag_intens), - }) + finish_mzpaf_arena(geom, frag_intens) } /// Adapt the legacy [`ElutionGroupCollection`] (produced by the non-speclib @@ -334,10 +291,7 @@ impl LibraryArena { /// historical behavior where timsseek rejected that shape. fn from_elution_groups(egc: ElutionGroupCollection) -> Result { match egc { - ElutionGroupCollection::MzpafLabels(egs, Some(extras)) => { - Self::mzpaf_with_intensities(egs, extras) - } - ElutionGroupCollection::MzpafLabels(egs, None) => { + ElutionGroupCollection::MzpafLabels(egs) => { let mut geom = QueryCollection::with_capabilities(LibCapabilities::default_diann_no_decoys()); for eg in &egs { @@ -360,7 +314,7 @@ impl LibraryArena { frag_intens: None, }) } - ElutionGroupCollection::StringLabels(egs, _) => { + ElutionGroupCollection::StringLabels(egs) => { // String-labelled arenas carry no ion chemistry and ship no // decoys: sequence/fragment features unavailable, decoys off. let mut geom = @@ -384,12 +338,134 @@ impl LibraryArena { geom.seal(); Ok(LibraryArena::Str { geom }) } - ElutionGroupCollection::TinyIntLabels(..) | ElutionGroupCollection::IntLabels(..) => { - warn!("integer-labelled libraries have no LibraryArena variant; rejecting"); - Err(LibraryReadingError::UnableToParseElutionGroups) + } + } +} + +/// What [`FragmentSet::insert`] did with a fragment. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Inserted { + /// A label not yet in this precursor; stored. + Added, + /// The label was already present, so the two peaks were collapsed onto the + /// more intense one. The caller counts this under its own name. + Collapsed, +} + +/// The fragments of one precursor, with labels unique by construction. +/// +/// Fragment labels MUST be unique within a precursor. `linear_get` is +/// first-match, so a duplicate silently shadows one peak — and +/// `ExpectedIntensities::try_from_pairs` rejects duplicates outright, which +/// timsseek's scoring pipeline `.expect()`s. A duplicate reaching the arena is +/// therefore a panic mid-search, per candidate. +/// +/// Every reader that builds an mzpaf arena goes through this type, so the +/// invariant holds in one place instead of being re-derived (or, in the TSV +/// readers' case, forgotten) at each site. Collisions collapse onto the more +/// intense peak rather than keeping whichever came first: intensity is the +/// signal being scored, and file order is not meaningful. +#[derive(Debug, Default)] +pub(super) struct FragmentSet { + frags: Vec<(IonAnnot, f64)>, + intensities: Vec, +} + +impl FragmentSet { + pub(super) fn with_capacity(n: usize) -> Self { + Self { + frags: Vec::with_capacity(n), + intensities: Vec::with_capacity(n), + } + } + + pub(super) fn insert(&mut self, label: IonAnnot, mz: f64, intensity: f32) -> Inserted { + if let Some(idx) = self.frags.iter().position(|(l, _)| *l == label) { + if intensity > self.intensities[idx] { + self.frags[idx].1 = mz; + self.intensities[idx] = intensity; } + return Inserted::Collapsed; + } + self.frags.push((label, mz)); + self.intensities.push(intensity); + Inserted::Added + } + + pub(super) fn is_empty(&self) -> bool { + self.frags.is_empty() + } + + /// Empty without releasing the allocation, so one set can be reused across + /// entries (the DIA-NN `.speclib` reader keeps one per rayon worker). + pub(super) fn clear(&mut self) { + self.frags.clear(); + self.intensities.clear(); + } + + #[cfg(test)] + pub(super) fn len(&self) -> usize { + self.frags.len() + } + + pub(super) fn frags(&self) -> &[(IonAnnot, f64)] { + &self.frags + } + + /// Append this precursor's intensities to a whole-library sidecar, in the + /// same order `frags()` will be pushed. + pub(super) fn extend_sidecar(&self, sidecar: &mut Vec) { + sidecar.extend_from_slice(&self.intensities); + } +} + +/// Adapt a `(elution group, per-precursor extras)` reader into the arena. +/// +/// The four tabular readers (DIA-NN TSV/parquet, Spectronaut, Skyline) differ +/// only in which function they call and which `*PrecursorExtras` they return, +/// so this is the whole of each one's `read`. +fn arena_from_pairs( + result: Result, E)>, Err>, + reader: &'static str, +) -> Result +where + PrecursorExtrasRow: From, + Err: std::fmt::Debug, +{ + let pairs = result.map_err(|e| { + warn!("{reader}: failed to read library file: {e:?}"); + LibraryReadingError::ReaderFailed { + reader, + detail: format!("{e:?}"), } + })?; + let (egs, extras): (Vec<_>, Vec<_>) = pairs.into_iter().unzip(); + LibraryArena::mzpaf_with_intensities(egs, extras.into_iter().map(Into::into).collect()) +} + +/// Seal a directly-built mzpaf arena together with its reference-intensity +/// sidecar. +/// +/// The sidecar is indexed by the same offsets as `frag_labels`, so a length +/// mismatch means some path pushed a label without an intensity (or the +/// reverse) and every downstream fragment lookup is off by that much. Returned +/// as an error rather than asserted: it is a property of the file being read. +pub(super) fn finish_mzpaf_arena( + mut geom: QueryCollection, + frag_intens: Vec, +) -> Result { + if frag_intens.len() != geom.frag_labels.len() { + return Err(LibraryReadingError::SpeclibParse(format!( + "reference-intensity sidecar ({}) must stay parallel to the fragment-label arena ({})", + frag_intens.len(), + geom.frag_labels.len(), + ))); } + geom.seal(); + Ok(LibraryArena::Mzpaf { + geom, + frag_intens: Some(frag_intens), + }) } /// A single spectral-library format reader. Adding a format = one struct + one @@ -400,15 +476,51 @@ pub trait LibraryReader: Send + Sync { /// Cheap probe: header bytes / extension / first data row. Must not read the /// whole file. fn sniff(&self, path: &Path) -> bool; - fn read(&self, path: &Path) -> Result; + /// Read the whole library into the arena. + /// + /// Formats that go through [`ElutionGroupCollection`] adapt it with + /// [`LibraryArena::from_elution_groups`]; the binary `.speclib` and + /// mzSpecLib readers build the arena directly, because only that path + /// carries the reference-intensity sidecar. + fn read(&self, path: &Path) -> Result; } +struct MzSpecLibReader; +struct DiannSpeclibReader; struct DiannParquetReader; struct DiannTsvReader; struct SpectronautReader; struct SkylineReader; struct JsonReader; +impl LibraryReader for MzSpecLibReader { + fn name(&self) -> &'static str { + "mzspeclib" + } + + fn sniff(&self, path: &Path) -> bool { + sniff_mzspeclib_library_file(path) + } + + fn read(&self, path: &Path) -> Result { + read_mzspeclib_library_file(path) + } +} + +impl LibraryReader for DiannSpeclibReader { + fn name(&self) -> &'static str { + "diann-speclib" + } + + fn sniff(&self, path: &Path) -> bool { + sniff_diann_speclib_library_file(path) + } + + fn read(&self, path: &Path) -> Result { + read_diann_speclib_library_file(path) + } +} + impl LibraryReader for DiannParquetReader { fn name(&self) -> &'static str { "diann-parquet" @@ -418,16 +530,8 @@ impl LibraryReader for DiannParquetReader { sniff_diann_parquet_library_file(path) } - fn read(&self, path: &Path) -> Result { - let egs = read_diann_parquet(path).map_err(|e| { - warn!("Failed to read DIA-NN parquet library file: {:?}", e); - LibraryReadingError::UnableToParseElutionGroups - })?; - let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - Ok(ElutionGroupCollection::MzpafLabels( - egs, - Some(FileReadingExtras::Diann(extras)), - )) + fn read(&self, path: &Path) -> Result { + arena_from_pairs(read_diann_parquet(path), self.name()) } } @@ -440,16 +544,8 @@ impl LibraryReader for DiannTsvReader { sniff_diann_library_file(path) } - fn read(&self, path: &Path) -> Result { - let egs = read_diann_tsv(path).map_err(|e| { - warn!("Failed to read DIA-NN TSV library file: {:?}", e); - LibraryReadingError::UnableToParseElutionGroups - })?; - let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - Ok(ElutionGroupCollection::MzpafLabels( - egs, - Some(FileReadingExtras::Diann(extras)), - )) + fn read(&self, path: &Path) -> Result { + arena_from_pairs(read_diann_tsv(path), self.name()) } } @@ -459,19 +555,20 @@ impl LibraryReader for SpectronautReader { } fn sniff(&self, path: &Path) -> bool { - sniff_spectronaut_library_file(path).is_ok() + // Logged rather than discarded: `MissingColumns` names the columns a + // near-miss Spectronaut export lacks, which is the difference between + // "wrong format" and "right format, wrong export settings". + match sniff_spectronaut_library_file(path) { + Ok(()) => true, + Err(e) => { + debug!("not a Spectronaut TSV: {e}"); + false + } + } } - fn read(&self, path: &Path) -> Result { - let egs = read_spectronaut_tsv(path).map_err(|e| { - warn!("Failed to read Spectronaut TSV library file: {:?}", e); - LibraryReadingError::UnableToParseElutionGroups - })?; - let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - Ok(ElutionGroupCollection::MzpafLabels( - egs, - Some(FileReadingExtras::Spectronaut(extras)), - )) + fn read(&self, path: &Path) -> Result { + arena_from_pairs(read_spectronaut_tsv(path), self.name()) } } @@ -484,16 +581,8 @@ impl LibraryReader for SkylineReader { sniff_skyline_library_file(path).is_ok() } - fn read(&self, path: &Path) -> Result { - let egs = read_skyline_csv(path).map_err(|e| { - warn!("Failed to read Skyline transition list: {:?}", e); - LibraryReadingError::UnableToParseElutionGroups - })?; - let (egs, extras): (Vec<_>, Vec<_>) = egs.into_iter().unzip(); - Ok(ElutionGroupCollection::MzpafLabels( - egs, - Some(FileReadingExtras::Skyline(extras)), - )) + fn read(&self, path: &Path) -> Result { + arena_from_pairs(read_skyline_csv(path), self.name()) } } @@ -508,13 +597,22 @@ impl LibraryReader for JsonReader { true } - fn read(&self, path: &Path) -> Result { - ElutionGroupCollection::try_read_json(path) + fn read(&self, path: &Path) -> Result { + LibraryArena::from_elution_groups(ElutionGroupCollection::try_read_json(path)?) } } +/// Readers in dispatch order: most specific first, ending with the +/// always-sniffs-true JSON fallback. +/// +/// mzSpecLib and `.speclib` lead because their probes are exact (a magic first +/// line and a version-gated header), so they cannot steal another format's +/// file — and `.speclib`'s read is the only one that surfaces an +/// `UnsupportedSpeclibVersion` diagnostic, which a later reader would mask. fn registry() -> &'static [&'static dyn LibraryReader] { &[ + &MzSpecLibReader, + &DiannSpeclibReader, &DiannParquetReader, &DiannTsvReader, &SpectronautReader, @@ -525,21 +623,12 @@ fn registry() -> &'static [&'static dyn LibraryReader] { pub fn read_library_file>(path: T) -> Result { let path = path.as_ref(); - // The DIA-NN `.speclib` reader builds the columnar arena directly (with the - // reference-intensity sidecar); every other format still produces the legacy - // `ElutionGroupCollection`, adapted into the arena here. `.speclib` is - // sniffed first because its `read` path is the only one that can surface an - // `UnsupportedSpeclibVersion` diagnostic (the sniff has no version gate). - if sniff_diann_speclib_library_file(path) { - info!("Dispatching library read to diann-speclib (direct arena build)"); - return read_diann_speclib_library_file(path); - } - let mut last_err = None; + let mut first_err = None; for reader in registry() { if reader.sniff(path) { info!("Dispatching library read to {}", reader.name()); match reader.read(path) { - Ok(egs) => return LibraryArena::from_elution_groups(egs), + Ok(arena) => return Ok(arena), // A sniff can fire on a file the reader then fails to parse // (overlapping sniffs). Fall through to the next candidate // instead of committing to the first sniff. Keep the FIRST @@ -549,12 +638,63 @@ pub fn read_library_file>(path: T) -> Result { warn!("{} sniffed but failed to read: {:?}", reader.name(), e); - last_err.get_or_insert(e); + first_err.get_or_insert(e); } } } } // Dead default in practice (JsonReader always sniffs true) — a harmless // defensive fallback. - Err(last_err.unwrap_or(LibraryReadingError::UnableToParseElutionGroups)) + Err(first_err.unwrap_or(LibraryReadingError::UnableToParseElutionGroups)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ion(s: &str) -> IonAnnot { + IonAnnot::try_from(s).expect("valid annotation") + } + + /// Fragment labels must be unique within a precursor: `linear_get` is + /// first-match, and `ExpectedIntensities::try_from_pairs` rejects + /// duplicates outright — which timsseek's scoring pipeline `.expect()`s. + /// A duplicate reaching the arena is a panic mid-search, so this is the + /// invariant that stops it. + #[test] + fn duplicate_labels_collapse_onto_the_more_intense_peak() { + let mut set = FragmentSet::with_capacity(3); + assert_eq!(set.insert(ion("y1"), 175.1, 0.5), Inserted::Added); + assert_eq!(set.insert(ion("b3^2"), 200.0, 0.9), Inserted::Added); + + // Weaker duplicate: kept peak is unchanged. + assert_eq!(set.insert(ion("y1"), 999.9, 0.1), Inserted::Collapsed); + assert_eq!(set.frags()[0], (ion("y1"), 175.1)); + + // Stronger duplicate: takes over both m/z and intensity. + assert_eq!(set.insert(ion("y1"), 175.2, 0.8), Inserted::Collapsed); + assert_eq!(set.frags()[0], (ion("y1"), 175.2)); + + assert_eq!(set.len(), 2, "a collision must not grow the set"); + let mut sidecar = vec![0.0]; + set.extend_sidecar(&mut sidecar); + assert_eq!( + sidecar, + vec![0.0, 0.8, 0.9], + "the sidecar stays parallel to frags(), appended in order" + ); + + // Charge is part of the label, so these do not collide. + assert_eq!(set.insert(ion("y1^2"), 88.0, 0.3), Inserted::Added); + assert_eq!(set.len(), 3); + } + + #[test] + fn clear_keeps_the_set_reusable() { + let mut set = FragmentSet::with_capacity(2); + set.insert(ion("y1"), 175.1, 1.0); + set.clear(); + assert!(set.is_empty()); + assert_eq!(set.insert(ion("y1"), 175.1, 1.0), Inserted::Added); + } } diff --git a/rust/timsquery/src/serde/mod.rs b/rust/timsquery/src/serde/mod.rs index 4c07f6b1..7de878cd 100644 --- a/rust/timsquery/src/serde/mod.rs +++ b/rust/timsquery/src/serde/mod.rs @@ -4,19 +4,18 @@ pub mod diann_speclib_io; mod elution_group_inputs; pub mod index_serde; mod library_file; +mod mzspeclib_io; mod skyline_io; mod spectronaut_io; pub use chromatogram_output::*; pub use index_serde::*; +// The reader-internal types (`ElutionGroupCollection`, `FileReadingExtras`, the +// per-format `*PrecursorExtras`, `LibrarySniffError`) are deliberately NOT +// re-exported: they had no consumers outside this module, and every format +// already funnels into `LibraryArena`, which is the boundary worth supporting. pub use library_file::{ - DiannPrecursorExtras, - ElutionGroupCollection, - FileReadingExtras, LibraryArena, LibraryReadingError, - SkylinePrecursorExtras, - SpectronautPrecursorExtras, read_library_file, }; -pub use spectronaut_io::LibrarySniffError; diff --git a/rust/timsquery/src/serde/mzspeclib_io.rs b/rust/timsquery/src/serde/mzspeclib_io.rs new file mode 100644 index 00000000..6343d0ce --- /dev/null +++ b/rust/timsquery/src/serde/mzspeclib_io.rs @@ -0,0 +1,1079 @@ +//! Reader for the mzSpecLib text format (HUPO-PSI). +//! +//! # Why every field needs a fallback ladder +//! +//! mzSpecLib says *how* to write a controlled-vocabulary term, not *which* +//! term a writer must use. The two reference exports disagree on nearly +//! everything this reader needs: +//! +//! | field | DIA-NN writes | Spectronaut writes | +//! |---|---|---| +//! | precursor m/z | `MS:1000744` selected ion m/z | `MS:1003208` experimental precursor monoisotopic m/z | +//! | retention time | *nothing at all* | `MS:1000896` normalized retention time, in minutes | +//! | ion mobility | `MS:1002476` (written as `0.0`) | `MS:1002476` | +//! +//! So each field is resolved by trying terms in priority order, and RT carries +//! a unit that must be honoured rather than assumed. +//! +//! # Peak resolution +//! +//! The arena wants *theoretical* m/z. A peak list may carry either: whether it +//! does is declared by `MS:1003072|spectrum origin type`, which this reader +//! does not consult (see below). What it uses instead is the annotation's +//! mass-error suffix, `theoretical = observed - error` — correct for an +//! observed list, and a no-op on a theoretical one, where the error is `0.0`. +//! +//! So a theoretical mass exists only once a single identity is pinned: +//! +//! | annotation | kept | m/z | +//! |---|---|---| +//! | resolved, representable | real label | theoretical | +//! | resolved, not representable (`y1-HCOOH`) | unknown label | theoretical | +//! | resolved, no `/error` suffix | as above | observed, and counted | +//! | unannotated (`?`), tied ambiguity, malformed suffix | no | — | +//! +//! A peak with no single identity is skipped rather than stored at observed +//! m/z: an arena mixing observed and theoretical masses would be invisible +//! downstream. A known-but-unspellable identity is kept because it still has an +//! exact mass — only the label is lost. The suffix is optional in mzPAF, so the +//! third row is possible and is the one case where the mixture does happen; +//! `kept_at_observed_mz` is how it shows up. +//! +//! Ambiguous (comma-separated) annotations take the alternative with the +//! smallest absolute mass error. If that one is unrepresentable the peak gets an +//! unknown label rather than falling back to a worse-matching representable +//! alternative, which would assign both a wrong identity and a wrong mass. +//! +//! # Not implemented +//! +//! - **`MS:1003072|spectrum origin type`.** It distinguishes `MS:1003073` +//! (observed), `MS:1003074` (predicted) and `MS:1003424` (theoretical m/z, +//! observed intensity), i.e. exactly whether the subtraction above is needed. +//! Both vendored fixtures write `/0.0` throughout, so the subtraction is a +//! no-op on them either way. +//! - **Decoys.** `` + `MS:1003212` marks decoy +//! spectra in SpectraST exports. Every row here is pushed as a target; see +//! `ignored_attribute_set_entries`. + +use crate::ion::{ + IonAnnot, + UnknownIonCounter, +}; +use crate::models::{ + LibCapabilities, + QueryCollection, +}; +use crate::serde::library_file::{ + FragmentSet, + Inserted, + LibraryArena, + LibraryReadingError, + finish_mzpaf_arena, +}; +use micromzpaf::{ + MassError, + split_mass_error, +}; +use std::io::{ + BufRead, + BufReader, +}; +use std::path::Path; +use tracing::{ + info, + warn, +}; + +/// First non-empty line of an mzSpecLib text file. +const MAGIC: &str = ""; + +// CV term ladders, ordered most- to least-specific; the first present wins. + +/// Precursor m/z. Experimental monoisotopic is preferred over `selected ion +/// m/z`, which on a quadrupole instrument is the isolation-window centre and +/// need not be the monoisotopic peak. +const PRECURSOR_MZ_TERMS: &[&str] = &[ + "MS:1003208", // experimental precursor monoisotopic m/z + "MS:1003053", // theoretical monoisotopic m/z + "MS:1000744", // selected ion m/z +]; +/// Retention time. `normalized retention time` is an iRT-style index rather +/// than a clock reading, but it is what Spectronaut exports and the only RT +/// signal available in those files. +const RT_TERMS: &[&str] = &[ + "MS:1000894", // retention time + "MS:1000896", // normalized retention time +]; +/// Ion mobility. Note these are different quantities, not spellings of one: +/// `MS:1002815` is inverse reduced ion mobility (1/K0), which is what the +/// arena wants, while `MS:1002476` is a drift time. They are tried in that +/// order and a drift-time-only library is counted, since treating a drift time +/// as 1/K0 is only valid for instruments that report it that way. +const MOBILITY_INVERSE_REDUCED: &str = "MS:1002815"; +const MOBILITY_DRIFT_TIME: &str = "MS:1002476"; + +const CHARGE_TERM: &str = "MS:1000041"; +const STRIPPED_SEQ_TERM: &str = "MS:1000888"; +const PROFORMA_TERM: &str = "MS:1003270"; +const UNIT_TERM: &str = "UO:0000000"; + +const UNIT_MINUTE: &str = "UO:0000031"; +const UNIT_SECOND: &str = "UO:0000010"; + +/// Declare the per-library tally so that the counters, the "is anything +/// wrong?" test and the log line all come from one list. +/// +/// Spelled as a macro because the alternative — a struct plus a hand-written +/// `||` chain plus a hand-written `warn!` — is three places to update per +/// counter and the compiler checks none of them. +macro_rules! anomaly_counters { + ($( $(#[$doc:meta])* $field:ident => $label:literal, )+) => { + /// Per-library tally of everything that did not land verbatim in the + /// arena. + /// + /// Reported once at the end of a load rather than per row: a consensus + /// library can carry thousands of unannotated peaks, and a line each + /// would bury the signal. + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] + pub(crate) struct MzSpecLibStats { + /// Peaks stored with their parsed annotation. The only counter + /// here that is not an anomaly. + pub kept_annotated: usize, + $( $(#[$doc])* pub $field: usize, )+ + } + + impl MzSpecLibStats { + /// Every anomaly counter paired with how to say it. + fn anomalies(&self) -> impl Iterator { + [ $( ($label, self.$field), )+ ].into_iter() + } + + /// Fold `other` in. Used to hold a spectrum's counts aside until + /// it is known to be kept, so a dropped spectrum does not also + /// report the peaks it would have contributed. + fn merge(&mut self, other: &Self) { + self.kept_annotated += other.kept_annotated; + $( self.$field += other.$field; )+ + } + } + }; +} + +anomaly_counters! { + /// Identity known but not representable (a loss outside the table, a + /// modified immonium). Stored with an unknown label and an exact mass. + kept_unknown_label => "kept with an unknown label", + /// `?` — no annotation, so no mass error, so no theoretical m/z. + skipped_unannotated => "skipped as unannotated", + /// Comma-separated alternatives that tied on absolute mass error. + skipped_ambiguous => "skipped as ambiguous", + /// An `/error` suffix that would not parse. The peak has no recoverable + /// mass, so it cannot be stored at any label. + skipped_malformed_mass_error => "skipped for a malformed mass error", + /// Peaks kept at their OBSERVED m/z because the annotation carried no + /// `/error` suffix (which mzPAF makes optional). Everything else in the + /// arena is theoretical, so this is the one counter that means the arena + /// mixes the two. + kept_at_observed_mz => "kept at observed m/z (no mass-error suffix)", + /// Peaks dropped because their label collided with one already in the + /// precursor. + dropped_duplicate_label => "dropped for a duplicate label", + /// Peaks dropped because the precursor had already spent all 255 unknown + /// labels, so no distinct one was left. + dropped_unknown_over_capacity => "dropped with the unknown labels exhausted", + /// Spectra with no `MS:1000888` stripped sequence, whose bare residues + /// were derived from the proforma instead. Not required by mzSpecLib. + stripped_sequence_derived => "spectra with a derived stripped sequence", + /// Spectra with no retention-time term at all. + spectra_without_rt => "spectra without an RT", + /// Spectra whose mobility came from a drift time rather than 1/K0. + spectra_with_drift_time_mobility => "spectra using a drift time as mobility", + /// Spectra whose retention time carried a unit this reader does not know. + spectra_with_unknown_rt_unit => "spectra with an unknown RT unit", + /// Spectra dropped for missing or unparseable precursor m/z, charge or + /// sequence. + dropped_malformed_spectrum => "spectra dropped as malformed", + /// Precursors dropped for having no usable peak left. + dropped_empty_precursors => "precursors dropped as empty", + /// Attributes inside an `` block, which this reader does + /// not apply. Both vendored fixtures have these blocks empty; a writer + /// that hoists an RT unit group or a DECOY marker into one would otherwise + /// lose it with no trace. + ignored_attribute_set_entries => "attribute-set entries ignored", +} + +/// One spectrum converted into the shape [`QueryCollection::push_row`] takes. +struct ArenaRow { + precursor_mz: f64, + charge: u8, + rt_seconds: f32, + mobility: f32, + frags: FragmentSet, + stripped: String, + modified: String, +} + +impl MzSpecLibStats { + fn report(&self, path: &Path) { + let flagged: Vec = self + .anomalies() + .filter(|(_, n)| *n > 0) + .map(|(label, n)| format!("{n} {label}")) + .collect(); + if flagged.is_empty() { + info!( + "mzSpecLib {}: {} peaks, all annotated and representable", + path.display(), + self.kept_annotated + ); + } else { + warn!( + "mzSpecLib {}: kept {} annotated peaks; {}", + path.display(), + self.kept_annotated, + flagged.join(", "), + ); + } + } +} + +/// Which `<...>` block an attribute was written in. +/// +/// `[n]` group ids are scoped to their block, so two blocks can both use `[2]` +/// for unrelated groups — `spectronaut.mzSpecLib.txt` already does. Carrying +/// the block makes a group id unique within a spectrum. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BlockId { + Spectrum, + Analyte(u32), + Interpretation(u32), +} + +impl BlockId { + /// Parse a ``-style header. `None` for a header that opens no + /// attribute block (``, ``). + fn parse(header: &str) -> Option { + let inner = header.strip_prefix('<')?.strip_suffix('>')?; + if inner == "Spectrum" || inner.starts_with("Spectrum=") { + return Some(Self::Spectrum); + } + let (kind, id) = inner.split_once('=')?; + let id = id.parse().ok()?; + match kind { + "Analyte" => Some(Self::Analyte(id)), + "Interpretation" => Some(Self::Interpretation(id)), + _ => None, + } + } +} + +/// One `ACC|name=value` attribute, with its `[n]` group tag scoped to the +/// block it was written in. +#[derive(Debug, Clone)] +struct Attr { + group: Option<(BlockId, u32)>, + accession: String, + value: String, +} + +impl Attr { + fn parse(line: &str, block: BlockId) -> Option { + let (group, rest) = match line.strip_prefix('[') { + Some(r) => { + let (g, r) = r.split_once(']')?; + (Some((block, g.parse().ok()?)), r) + } + None => (None, line), + }; + let (key, value) = rest.split_once('=')?; + let accession = key.split('|').next()?.to_string(); + Some(Attr { + group, + accession, + value: value.to_string(), + }) + } + + /// The accession out of a `ACC|name` *value* (as opposed to a key), for + /// terms whose value is itself a CV term — e.g. `unit=UO:0000031|minute`. + fn value_accession(&self) -> &str { + self.value.split('|').next().unwrap_or(&self.value) + } +} + +/// Attributes collected for one spectrum (its own plus its analyte's). +#[derive(Debug, Default)] +struct AttrBag(Vec); + +impl AttrBag { + /// Spectrum-block attributes win over Analyte/Interpretation ones. Without + /// this the answer would depend on which block the writer emitted first. + fn find(&self, accession: &str) -> Option<&Attr> { + let matching = || self.0.iter().filter(|a| a.accession == accession); + matching() + .find(|a| a.group.is_none_or(|(b, _)| b == BlockId::Spectrum)) + .or_else(|| matching().next()) + } + + fn first_of(&self, accessions: &[&str]) -> Option<&Attr> { + accessions.iter().find_map(|a| self.find(a)) + } + + fn f64_of(&self, accessions: &[&str]) -> Option { + self.first_of(accessions)?.value.parse().ok() + } + + /// The unit term attached to `attr` via its `[n]` group, if any. + /// + /// The group is matched on `(block, id)`, not `id` alone: an unrelated + /// `[2]` in the Analyte block must not supply the unit for a `[2]` in the + /// Spectrum block. Getting that wrong is a silent 60x RT error. + fn unit_for(&self, attr: &Attr) -> Option<&str> { + let group = attr.group?; + self.0 + .iter() + .find(|a| a.group == Some(group) && a.accession == UNIT_TERM) + .map(|a| a.value_accession()) + } +} + +/// A spectrum accumulated from the text stream, before conversion. +#[derive(Debug, Default)] +struct RawSpectrum { + attrs: AttrBag, + /// `(observed mz, intensity, annotation)` + peaks: Vec<(f64, f32, String)>, +} + +/// Why a peak cannot be stored at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SkipReason { + /// No annotation, so no mass error, so no theoretical m/z. + Unannotated, + /// Alternatives that pin no single identity, so likewise no theoretical + /// m/z. Storing the observed one would mix the two. + Ambiguous, + /// The `/error` suffix was present but unparseable. Distinct from + /// `Ambiguous`: one malformed annotation is malformed, not ambiguous. + MalformedMassError, +} + +/// What resolving one peak's annotation produced. +/// +/// The mass error rides along on the two variants that keep the peak, because +/// it is what recovers theoretical m/z from the observed value in the file. It +/// is absent from `Skip` because a skipped peak has no mass to recover. +enum Resolved { + /// Parsed cleanly; store with this label. + Annotated(IonAnnot, Option), + /// Identity known, not representable. Store with an unknown label and the + /// exact mass. + UnknownLabel(Option), + Skip(SkipReason), +} + +/// Resolve one annotation string into a storage decision. +fn resolve_annotation(annotation: &str) -> Resolved { + let annotation = annotation.trim(); + if annotation.is_empty() || annotation == "?" { + return Resolved::Skip(SkipReason::Unannotated); + } + + // Splitting the error off comes first: it works even when the ion will not + // parse, which is exactly the case that still needs an exact mass. A + // malformed suffix on ANY alternative makes the whole peak unresolvable — + // dropping just that one would silently turn an ambiguous peak into an + // unambiguous one. + let mut alternatives = Vec::new(); + for alt in annotation.split(',') { + let Ok(parsed) = split_mass_error(alt.trim()) else { + return Resolved::Skip(SkipReason::MalformedMassError); + }; + alternatives.push(parsed); + } + + let (ion_str, mass_error) = if let [single] = alternatives[..] { + single + } else { + // Closest by absolute mass error. Comparing a Da magnitude against a + // ppm one would be meaningless, but a library uses one unit + // throughout. Errors are parsed decimal literals, so equal ones + // compare exactly and a tie pins no identity. + let magnitude = |m: &Option| match m { + Some(MassError::Da(v) | MassError::Ppm(v)) => v.abs(), + None => f64::INFINITY, + }; + let best = alternatives + .iter() + .map(|(_, e)| magnitude(e)) + .fold(f64::INFINITY, f64::min); + if !best.is_finite() { + return Resolved::Skip(SkipReason::Ambiguous); + } + let mut winners = alternatives.iter().filter(|(_, e)| magnitude(e) == best); + let winner = *winners.next().expect("the minimum came from this iterator"); + if winners.next().is_some() { + return Resolved::Skip(SkipReason::Ambiguous); + } + winner + }; + match IonAnnot::try_from(ion_str) { + Ok(ion) => Resolved::Annotated(ion, mass_error), + // Keep the peak and its exact mass, lose only the label. + Err(_) => Resolved::UnknownLabel(mass_error), + } +} + +/// Convert one accumulated spectrum into an arena row, counting whichever way +/// it failed. +/// +/// The counting lives here rather than inside [`spectrum_row`] so that every +/// `?` in there lands on a tally. Without it a library that dropped half its +/// spectra for a missing charge still reports "all annotated and +/// representable". +fn convert_spectrum(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { + // A malformed spectrum bails out of `spectrum_row` via `?`, possibly after + // incrementing RT/mobility counters on the way. Those are held aside and + // discarded on the bail-out, so it is reported once as malformed rather + // than also as "without an RT" and "using a drift time". + // + // An *empty* precursor is structurally fine, so its per-peak counts are + // kept: they are the explanation for why it came out empty. + let mut local = MzSpecLibStats::default(); + let Some(row) = spectrum_row(raw, &mut local) else { + stats.dropped_malformed_spectrum += 1; + return None; + }; + stats.merge(&local); + + if row.frags.is_empty() { + stats.dropped_empty_precursors += 1; + return None; + } + Some(row) +} + +/// The conversion proper: `None` means the spectrum lacked something +/// structural (precursor m/z, charge, a parseable RT or mobility, a sequence). +fn spectrum_row(raw: &RawSpectrum, stats: &mut MzSpecLibStats) -> Option { + let precursor_mz = raw.attrs.f64_of(PRECURSOR_MZ_TERMS)?; + let charge: u8 = raw.attrs.find(CHARGE_TERM)?.value.parse().ok()?; + + let rt_seconds = match raw.attrs.first_of(RT_TERMS) { + Some(attr) => { + let v: f64 = attr.value.parse().ok()?; + // Honour the unit rather than assuming: Spectronaut writes minutes. + match raw.attrs.unit_for(attr) { + Some(UNIT_SECOND) => v, + Some(UNIT_MINUTE) | None => v * 60.0, + // An unrecognized unit is counted, not silently treated as + // minutes: guessing wrong here is a 60x error in the RT the + // whole extraction window is built around. + Some(_) => { + stats.spectra_with_unknown_rt_unit += 1; + v * 60.0 + } + } + } + None => { + stats.spectra_without_rt += 1; + 0.0 + } + }; + + let mobility = match raw.attrs.find(MOBILITY_INVERSE_REDUCED) { + Some(a) => a.value.parse().ok()?, + None => match raw.attrs.find(MOBILITY_DRIFT_TIME) { + Some(a) => { + let drift: f32 = a.value.parse().ok()?; + // DIA-NN writes `MS:1002476|ion mobility drift time=0.0` on + // every spectrum, meaning "unset". Counting that as a real + // drift time warns on every DIA-NN load and buries the case + // this counter exists for. + if drift != 0.0 { + stats.spectra_with_drift_time_mobility += 1; + } + drift + } + // Absent is fine — an unset mobility is 0.0, same as DIA-NN writes. + // A *present but malformed* one drops the spectrum instead. + None => 0.0, + }, + }; + + // The proforma term carries a trailing `/charge` that is not part of the + // peptidoform. + let modified = raw.attrs.find(PROFORMA_TERM).map(|a| { + a.value + .rsplit_once('/') + .map(|(p, _)| p.to_string()) + .unwrap_or_else(|| a.value.clone()) + }); + + // `MS:1000888` is not required by mzSpecLib. Deriving the stripped form + // from the proforma is better than defaulting to "", which reads as a + // zero-residue peptide and silently routes the row to averagine isotopes + // — tallied downstream as `n_averagine_fallback`, at a layer that can no + // longer say the sequence was simply absent. + let stripped = match raw.attrs.find(STRIPPED_SEQ_TERM) { + Some(a) => a.value.clone(), + None => { + stats.stripped_sequence_derived += 1; + crate::utils::sequence::strip_mods(modified.as_deref()?) + } + }; + let modified = modified.unwrap_or_else(|| stripped.clone()); + if stripped.is_empty() && modified.is_empty() { + return None; + } + + let mut frags = FragmentSet::with_capacity(raw.peaks.len()); + let mut unknown_ions = UnknownIonCounter::new(); + + for (observed_mz, intensity, annotation) in &raw.peaks { + let (label, mass_error, annotated) = match resolve_annotation(annotation) { + Resolved::Annotated(ion, error) => (ion, error, true), + Resolved::UnknownLabel(error) => match unknown_ions.next(1) { + Ok(ion) => (ion, error, false), + Err(_) => { + stats.dropped_unknown_over_capacity += 1; + continue; + } + }, + Resolved::Skip(SkipReason::Unannotated) => { + stats.skipped_unannotated += 1; + continue; + } + Resolved::Skip(SkipReason::Ambiguous) => { + stats.skipped_ambiguous += 1; + continue; + } + Resolved::Skip(SkipReason::MalformedMassError) => { + stats.skipped_malformed_mass_error += 1; + continue; + } + }; + + // The `/error` suffix is optional in mzPAF. Without it the observed + // m/z is the best available value, but it is NOT the theoretical one + // the rest of the arena holds. + let mz = match mass_error { + Some(e) => e.theoretical_from_observed(*observed_mz), + None => *observed_mz, + }; + + // `FragmentSet` owns the per-precursor uniqueness invariant; a + // collision collapses onto the more intense peak. Unknown labels come + // off a monotonic counter and cannot collide, so this only ever fires + // for annotated ones. Every kept counter is behind this check, so a + // collided peak is tallied once, as dropped. + if frags.insert(label, mz, *intensity) == Inserted::Collapsed { + stats.dropped_duplicate_label += 1; + continue; + } + if mass_error.is_none() { + stats.kept_at_observed_mz += 1; + } + if annotated { + stats.kept_annotated += 1; + } else { + stats.kept_unknown_label += 1; + } + } + + Some(ArenaRow { + precursor_mz, + charge, + rt_seconds: rt_seconds as f32, + mobility, + frags, + stripped, + modified, + }) +} + +/// Cheap probe: the format's magic first line. +pub fn sniff_mzspeclib_library_file>(path: T) -> bool { + let Ok(file) = std::fs::File::open(path.as_ref()) else { + return false; + }; + // Lazy, so only the first non-empty line is read: this stays O(1) on a + // multi-gigabyte library. + BufReader::new(file) + .lines() + .map_while(Result::ok) + .find(|l| !l.trim().is_empty()) + .is_some_and(|l| l.trim() == MAGIC) +} + +/// Which part of a spectrum block the line loop is inside. +enum Section { + Attributes, + Peaks, +} + +/// Convert an accumulated spectrum and append it to the arena. +fn flush( + current: Option, + geom: &mut QueryCollection, + frag_intens: &mut Vec, + stats: &mut MzSpecLibStats, +) { + let Some(raw) = current else { return }; + let Some(row) = convert_spectrum(&raw, stats) else { + return; + }; + row.frags.extend_sidecar(frag_intens); + geom.push_row( + row.precursor_mz, + row.charge, + row.rt_seconds, + row.mobility, + row.frags.frags(), + &row.stripped, + &row.modified, + &[], + false, + ); +} + +/// Read an mzSpecLib text file into the columnar arena. +pub fn read_mzspeclib_library_file>( + path: T, +) -> Result { + let path = path.as_ref(); + let file = std::fs::File::open(path).map_err(LibraryReadingError::IoError)?; + let reader = BufReader::new(file); + + let mut geom = QueryCollection::with_capabilities(LibCapabilities::default_diann_no_decoys()); + let mut frag_intens: Vec = Vec::new(); + let mut stats = MzSpecLibStats::default(); + + let mut current: Option = None; + let mut section = Section::Attributes; + // Which block the attributes now being read belong to. `[n]` group ids are + // scoped to it, so it has to be threaded into every `Attr::parse`. + let mut block = BlockId::Spectrum; + let mut in_attribute_set = false; + + for line in reader.lines() { + let line = line.map_err(LibraryReadingError::IoError)?; + let trimmed = line.trim_end(); + + // Any `<...>` header ends the peak list; only `` opens one. + // `` and `` attributes are folded into + // the spectrum's bag: this reader wants the union, not the hierarchy, + // but each keeps its own group scope. + if trimmed.starts_with('<') { + if trimmed.starts_with("" { + Section::Peaks + } else { + Section::Attributes + }; + // `` declares defaults applied to whole classes + // of spectra by name. Honouring them is not implemented, so a + // non-empty one is counted rather than silently dropped: it can + // carry the RT unit, or the DECOY marker. + in_attribute_set = trimmed.starts_with(" { + let mut cols = trimmed.split('\t'); + let (Some(mz), Some(intensity)) = (cols.next(), cols.next()) else { + continue; + }; + let (Ok(mz), Ok(intensity)) = + (mz.trim().parse::(), intensity.trim().parse::()) + else { + continue; + }; + let annotation = cols.next().unwrap_or("?").to_string(); + spec.peaks.push((mz, intensity, annotation)); + } + Section::Attributes => { + if let Some(attr) = Attr::parse(trimmed, block) { + spec.attrs.0.push(attr); + } + } + } + } + flush(current.take(), &mut geom, &mut frag_intens, &mut stats); + + stats.report(path); + + if geom.n_rows() == 0 { + return Err(LibraryReadingError::SpeclibParse(format!( + "mzSpecLib {} yielded no usable precursors", + path.display() + ))); + } + finish_mzpaf_arena(geom, frag_intens) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture(name: &str) -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/mzspeclib_io_files") + .join(name) + } + + #[test] + fn sniffs_only_mzspeclib() { + assert!(sniff_mzspeclib_library_file(fixture("diann.mzSpecLib.txt"))); + assert!(sniff_mzspeclib_library_file(fixture( + "spectronaut.mzSpecLib.txt" + ))); + // A DIA-NN TSV must not be claimed. + let tsv = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/diann_io_files/sample_lib.tsv"); + assert!(!sniff_mzspeclib_library_file(tsv)); + } + + #[test] + fn reads_diann_export() { + let arena = read_mzspeclib_library_file(fixture("diann.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, frag_intens } = arena else { + panic!("mzSpecLib must build an mzpaf arena"); + }; + assert!(geom.n_rows() > 0); + let intens = frag_intens.expect("reference intensities are populated"); + assert_eq!(intens.len(), geom.frag_labels.len()); + } + + /// DIA-NN's export writes every mass error as exactly `0.0`, so observed + /// and theoretical coincide and the m/z must pass through untouched. + #[test] + fn zero_mass_error_leaves_mz_unchanged() { + let arena = read_mzspeclib_library_file(fixture("diann.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + // First spectrum's first peak in the fixture: 427.22995, annotated b6/0.0 + let first = geom.frag_mzs[0]; + assert!( + (first - 427.22995).abs() < 1e-6, + "expected the observed m/z verbatim, got {first}" + ); + } + + /// Spectronaut's export carries `-H2O`/`-NH3` losses. `IonAnnot` + /// represents those, so they must keep real labels rather than degrading + /// to unknown. + #[test] + fn spectronaut_losses_keep_real_labels() { + let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + let has_loss = geom + .frag_labels + .iter() + .any(|l| l.loss() != micromzpaf::NeutralLoss::None); + assert!(has_loss, "expected at least one loss-bearing label"); + let unknowns = geom + .frag_labels + .iter() + .filter(|l| l.try_get_ordinal().is_none() && l.loss() == micromzpaf::NeutralLoss::None) + .count(); + assert_eq!(unknowns, 0, "no peak should need an unknown label here"); + } + + /// RT is unit-tagged; Spectronaut writes minutes and the arena wants + /// seconds. Getting this wrong is a silent 60x error. + #[test] + fn retention_time_honours_its_unit() { + let arena = read_mzspeclib_library_file(fixture("spectronaut.mzSpecLib.txt")).unwrap(); + let LibraryArena::Mzpaf { geom, .. } = arena else { + unreachable!() + }; + // Fixture's first spectrum: normalized retention time = 28.658491 min. + let rt = geom.rt_seconds[0]; + assert!( + (rt - 28.658491 * 60.0).abs() < 0.01, + "expected minutes converted to seconds, got {rt}" + ); + } + + /// `[n]` group ids are scoped to their block, so an Analyte `[2]` must not + /// supply the unit for a Spectrum `[2]`. + /// + /// `spectronaut.mzSpecLib.txt` already carries both: `[2]` is the RT + + /// unit pair in its Spectrum block and the NCBI TaxID in its Analyte + /// block. It survives today only because the Analyte's `[2]` happens to + /// carry no unit term. If it did, the RT would be read in the wrong unit — + /// a silent 60x error in the value the extraction window is built on. + #[test] + fn group_ids_do_not_leak_between_blocks() { + let grouped = |block: BlockId, id: u32, accession: &str, value: &str| Attr { + group: Some((block, id)), + accession: accession.to_string(), + value: value.to_string(), + }; + let rt = grouped(BlockId::Spectrum, 2, RT_TERMS[1], "10.0"); + let bag = AttrBag(vec![ + rt.clone(), + // Same group id, different block, and it does carry a unit. + grouped(BlockId::Analyte(1), 2, "MS:1001467", "9606"), + grouped(BlockId::Analyte(1), 2, UNIT_TERM, UNIT_SECOND), + ]); + + assert_eq!( + bag.unit_for(&rt), + None, + "the Analyte block's unit must not reach the Spectrum block's group" + ); + + // And when the unit is in the same block, it is found. + let mut same_block = bag; + same_block + .0 + .push(grouped(BlockId::Spectrum, 2, UNIT_TERM, UNIT_SECOND)); + assert_eq!(same_block.unit_for(&rt), Some(UNIT_SECOND)); + } + + /// The `/error` suffix is optional in mzPAF, so a bare `y5` is legal and + /// its m/z stays observed while the rest of the arena is theoretical. + /// Both vendored fixtures write `/0.0` everywhere, so nothing else covers + /// this. A malformed suffix is a different verdict from an ambiguous one. + #[test] + fn a_missing_mass_error_is_counted_and_a_malformed_one_is_skipped() { + assert!(matches!( + resolve_annotation("y5"), + Resolved::Annotated(_, None) + )); + assert!(matches!( + resolve_annotation("y5/not-a-number"), + Resolved::Skip(SkipReason::MalformedMassError) + )); + + let attr = |accession: &str, value: &str| Attr { + group: None, + accession: accession.to_string(), + value: value.to_string(), + }; + let raw = RawSpectrum { + attrs: AttrBag(vec![ + attr(PRECURSOR_MZ_TERMS[0], "500.25"), + attr(CHARGE_TERM, "2"), + attr(STRIPPED_SEQ_TERM, "PEPTIDEK"), + attr(RT_TERMS[0], "10.0"), + ]), + peaks: vec![ + (175.1, 1.0, "y1/0.0".to_string()), + (288.2, 1.0, "y2".to_string()), + ], + }; + + let mut stats = MzSpecLibStats::default(); + let row = convert_spectrum(&raw, &mut stats).expect("both peaks are usable"); + assert_eq!(row.frags.len(), 2); + assert_eq!(stats.kept_annotated, 2); + assert_eq!( + stats.kept_at_observed_mz, 1, + "only the suffix-less peak keeps its observed m/z" + ); + } + + /// A collided peak must be reported once, as dropped — not as kept AND + /// dropped, which is what counting before the uniqueness check produced. + #[test] + fn a_duplicate_label_is_counted_once_as_dropped() { + let attr = |accession: &str, value: &str| Attr { + group: None, + accession: accession.to_string(), + value: value.to_string(), + }; + let raw = RawSpectrum { + attrs: AttrBag(vec![ + attr(PRECURSOR_MZ_TERMS[0], "500.25"), + attr(CHARGE_TERM, "2"), + attr(STRIPPED_SEQ_TERM, "PEPTIDEK"), + attr(RT_TERMS[0], "10.0"), + ]), + peaks: vec![ + (175.1, 1.0, "y1/0.0".to_string()), + (175.2, 2.0, "y1/0.0".to_string()), + ], + }; + + let mut stats = MzSpecLibStats::default(); + let row = convert_spectrum(&raw, &mut stats).expect("the first peak is usable"); + assert_eq!(row.frags.len(), 1); + assert_eq!(stats.kept_annotated, 1); + assert_eq!(stats.dropped_duplicate_label, 1); + } + + /// DIA-NN writes `MS:1002476|ion mobility drift time=0.0` on every + /// spectrum, meaning "unset". Counting that warns on every DIA-NN load. + #[test] + fn a_zero_drift_time_is_absent_not_a_drift_time() { + let mut stats = MzSpecLibStats::default(); + let attr = |accession: &str, value: &str| Attr { + group: None, + accession: accession.to_string(), + value: value.to_string(), + }; + let with_drift = |drift: &str| RawSpectrum { + attrs: AttrBag(vec![ + attr(PRECURSOR_MZ_TERMS[0], "500.25"), + attr(CHARGE_TERM, "2"), + attr(STRIPPED_SEQ_TERM, "PEPTIDEK"), + attr(RT_TERMS[0], "10.0"), + attr(MOBILITY_DRIFT_TIME, drift), + ]), + peaks: vec![(175.1, 1.0, "y1/0.0".to_string())], + }; + + assert!(convert_spectrum(&with_drift("0.0"), &mut stats).is_some()); + assert_eq!(stats.spectra_with_drift_time_mobility, 0); + assert!(convert_spectrum(&with_drift("0.85"), &mut stats).is_some()); + assert_eq!(stats.spectra_with_drift_time_mobility, 1); + } + + /// A spectrum missing a structural field is dropped by `?` deep inside + /// `spectrum_row`. Nothing there increments a counter, so this is what + /// stops the load from reporting "all annotated and representable" while + /// having silently lost half the library. + #[test] + fn structurally_broken_spectra_are_counted_not_swallowed() { + let attr = |accession: &str, value: &str| Attr { + group: None, + accession: accession.to_string(), + value: value.to_string(), + }; + let good = |charge: &str| RawSpectrum { + attrs: AttrBag(vec![ + attr(PRECURSOR_MZ_TERMS[0], "500.25"), + attr(CHARGE_TERM, charge), + attr(STRIPPED_SEQ_TERM, "PEPTIDEK"), + attr(RT_TERMS[0], "10.0"), + ]), + peaks: vec![(175.1, 1.0, "y1/0.0".to_string())], + }; + + let mut stats = MzSpecLibStats::default(); + assert!(convert_spectrum(&good("2"), &mut stats).is_some()); + // Unparseable charge — the `?` that used to vanish. + assert!(convert_spectrum(&good("not-a-number"), &mut stats).is_none()); + // No peak survives resolution, which is a different failure. + let mut empty = good("2"); + empty.peaks = vec![(175.1, 1.0, "?".to_string())]; + assert!(convert_spectrum(&empty, &mut stats).is_none()); + + assert_eq!(stats.kept_annotated, 1); + assert_eq!(stats.dropped_malformed_spectrum, 1); + assert_eq!(stats.dropped_empty_precursors, 1); + assert_eq!(stats.skipped_unannotated, 1); + assert!( + stats.anomalies().any(|(_, n)| n > 0), + "the report must not claim a clean load" + ); + } + + #[test] + fn resolves_unambiguous_representable() { + assert!(matches!( + resolve_annotation("y5/-0.0005"), + Resolved::Annotated(_, Some(MassError::Da(-0.0005))) + )); + } + + /// Known identity, unrepresentable spelling: keep the peak and the exact + /// mass, erase only the label. The mass error must survive so theoretical + /// m/z stays exact. + #[test] + fn unrepresentable_loss_keeps_peak_with_unknown_label() { + assert!(matches!( + resolve_annotation("y1-HCOOH/0.0003"), + Resolved::UnknownLabel(Some(MassError::Da(0.0003))) + )); + } + + #[test] + fn unannotated_peak_is_skipped() { + assert!(matches!( + resolve_annotation("?"), + Resolved::Skip(SkipReason::Unannotated) + )); + } + + /// Closest-by-error wins; if that alternative is unrepresentable the peak + /// takes an unknown label rather than falling back to the representable + /// one, which would assign a wrong identity and a wrong mass. + #[test] + fn ambiguity_resolves_to_the_closest_not_the_representable() { + // a2 is representable and further; y2-CO2-NH3 is closer and is not. + assert!( + matches!( + resolve_annotation("a2/-0.0040,y2-CO2-NH3/-0.0001"), + Resolved::UnknownLabel(Some(MassError::Da(-0.0001))) + ), + "the closest alternative wins even when unrepresentable" + ); + + // When the closest one IS representable, it is used. + assert!(matches!( + resolve_annotation("a2/-0.0001,y2-CO2-NH3/-0.0040"), + Resolved::Annotated(..) + )); + } + + /// An exact tie pins no identity, so no theoretical m/z exists and the peak + /// cannot be stored without mixing observed and theoretical masses. + #[test] + fn tied_ambiguity_is_skipped() { + assert!(matches!( + resolve_annotation("a2/-0.0004,y2-CO2-NH3/-0.0004"), + Resolved::Skip(SkipReason::Ambiguous) + )); + } + + #[test] + fn mass_error_recovers_theoretical() { + // Real SpectraST peak: y1 for C-terminal R, observed 175.1184. + let e = MassError::Da(-0.0005); + assert!((e.theoretical_from_observed(175.1184) - 175.1189).abs() < 1e-9); + } + + /// The public entry point must dispatch mzSpecLib itself. It is the + /// registry's first entry precisely because the JSON reader at the end + /// accepts anything, so a regression in the sniff falls through to it and + /// surfaces as a generic parse error rather than as "wrong reader". + #[test] + fn public_read_library_file_dispatches_mzspeclib() { + use crate::serde::read_library_file; + for name in ["diann.mzSpecLib.txt", "spectronaut.mzSpecLib.txt"] { + let arena = read_library_file(fixture(name)) + .unwrap_or_else(|e| panic!("{name} must load through the registry: {e:?}")); + let LibraryArena::Mzpaf { geom, frag_intens } = arena else { + panic!("{name} must land in the mzpaf arena"); + }; + assert!(geom.n_rows() > 0, "{name} produced no precursors"); + assert!( + frag_intens.is_some(), + "{name} must populate the reference-intensity sidecar" + ); + } + } +} diff --git a/rust/timsquery/src/serde/skyline_io.rs b/rust/timsquery/src/serde/skyline_io.rs index 1f42c43e..2340bfc9 100644 --- a/rust/timsquery/src/serde/skyline_io.rs +++ b/rust/timsquery/src/serde/skyline_io.rs @@ -2,6 +2,7 @@ use crate::TimsElutionGroup; use crate::ion::{ IonAnnot, IonParsingError, + UnknownIonCounter, }; use serde::{ Deserialize, @@ -154,23 +155,6 @@ impl SkylineLibraryRow { } } -/// Remove bracketed modification annotations, e.g. `C[+57.02]AM` -> `CAM`. -fn strip_modifications(modified_seq: &str) -> String { - let mut out = String::with_capacity(modified_seq.len()); - let mut depth: i32 = 0; - for ch in modified_seq.chars() { - match ch { - '[' | '(' | '{' => depth += 1, - ']' | ')' | '}' if depth > 0 => { - depth -= 1; - } - _ if depth == 0 => out.push(ch), - _ => {} - } - } - out -} - /// Check if a file is a Skyline Peptide Transition List CSV. pub fn sniff_skyline_library_file>(file: T) -> Result<(), SkylineSniffError> { let file_handle = std::fs::File::open(file.as_ref()).map_err(|e| { @@ -312,7 +296,7 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(fragment_rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(fragment_rows.len()); - let mut num_unknown_losses = 0u8; + let mut unknown_ions = UnknownIonCounter::new(); for (i, row) in fragment_rows.iter().enumerate() { let fragment_mz = row.product_mz; @@ -351,8 +335,7 @@ fn parse_precursor_group( falling back to unknown ion", frag_type, i ); - num_unknown_losses = num_unknown_losses.saturating_add(1); - IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)? + unknown_ions.next(frag_charge as i8)? } }; @@ -362,7 +345,7 @@ fn parse_precursor_group( } let modified_peptide = first_row.peptide_modified_sequence.clone(); - let stripped_peptide = strip_modifications(&modified_peptide); + let stripped_peptide = crate::utils::sequence::strip_mods(&modified_peptide); let precursor_extras = SkylinePrecursorExtras { modified_peptide, @@ -407,10 +390,13 @@ mod tests { #[test] fn test_strip_modifications() { - assert_eq!(strip_modifications("PEPTIDE"), "PEPTIDE"); - assert_eq!(strip_modifications("C[+57.021]AM"), "CAM"); - assert_eq!(strip_modifications("P[UniMod:35]IDE"), "PIDE"); - assert_eq!(strip_modifications("[+42]AB"), "AB"); + assert_eq!(crate::utils::sequence::strip_mods("PEPTIDE"), "PEPTIDE"); + assert_eq!(crate::utils::sequence::strip_mods("C[+57.021]AM"), "CAM"); + assert_eq!( + crate::utils::sequence::strip_mods("P[UniMod:35]IDE"), + "PIDE" + ); + assert_eq!(crate::utils::sequence::strip_mods("[+42]AB"), "AB"); } #[test] diff --git a/rust/timsquery/src/serde/spectronaut_io.rs b/rust/timsquery/src/serde/spectronaut_io.rs index f5037486..20a37f8d 100644 --- a/rust/timsquery/src/serde/spectronaut_io.rs +++ b/rust/timsquery/src/serde/spectronaut_io.rs @@ -2,6 +2,7 @@ use crate::TimsElutionGroup; use crate::ion::{ IonAnnot, IonParsingError, + UnknownIonCounter, }; use serde::Deserialize; use std::path::Path; @@ -30,6 +31,20 @@ pub enum LibrarySniffError { MissingColumns(Vec), } +impl std::fmt::Display for LibrarySniffError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::IoError(e) => write!(f, "could not be read: {e}"), + Self::InvalidFormat(e) => write!(f, "headers did not parse: {e}"), + // The columns are the actionable part: a near-miss export is a + // settings problem, not a wrong-format one. + Self::MissingColumns(cols) => { + write!(f, "missing required columns: {}", cols.join(", ")) + } + } + } +} + impl std::fmt::Display for SpectronautReadingError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -286,7 +301,7 @@ fn parse_precursor_group( let mut fragment_mzs = Vec::with_capacity(included_rows.len()); buffers.fragment_labels.clear(); let mut relative_intensities = Vec::with_capacity(included_rows.len()); - let mut num_unknown_losses = 0; + let mut unknown_ions = UnknownIonCounter::new(); for (i, row) in included_rows.iter().enumerate() { let fragment_mz = row.fragment_mz; @@ -306,8 +321,7 @@ fn parse_precursor_group( row.fragment_loss_type, i ); - num_unknown_losses += 1; - let ion_annot = IonAnnot::try_new('?', Some(num_unknown_losses), frag_charge as i8, 0)?; + let ion_annot = unknown_ions.next(frag_charge as i8)?; buffers.fragment_labels.push(ion_annot); fragment_mzs.push(fragment_mz); relative_intensities.push((ion_annot, rel_intensity)); diff --git a/rust/timsquery/src/traits/fragment_label.rs b/rust/timsquery/src/traits/fragment_label.rs index 9abf2f0a..dca01e1d 100644 --- a/rust/timsquery/src/traits/fragment_label.rs +++ b/rust/timsquery/src/traits/fragment_label.rs @@ -1,5 +1,6 @@ use crate::IonAnnot; use crate::traits::KeyLike; +use micromzpaf::Series; use std::sync::Arc; /// Capability to apply a decoy m/z shift to a fragment label's m/z. @@ -20,11 +21,12 @@ pub trait DecoyShift { /// `FragmentLabel`. NOTE: decoy m/z shifting is gated on `DecoyShift`, not this /// trait — a decoy variant computes its shift through `DecoyShift::decoy_shift_mz`, /// which every label implements. -// TODO(fragment-features): add series() accessor once a standalone series -// enum exists (today series and ordinal are fused in `IonSeriesOrdinal`), and -// wire this bound into the score that consumes it. +// TODO(fragment-features): wire this bound into the score that consumes it. pub trait FragmentLabel: KeyLike + DecoyShift { fn try_get_ordinal(&self) -> Option; + /// The backbone series, or `None` for ions that sit on no ladder + /// (precursor, unknown, internal, immonium). + fn try_get_series(&self) -> Option; fn get_charge(&self) -> i8; } @@ -45,6 +47,10 @@ impl FragmentLabel for IonAnnot { IonAnnot::try_get_ordinal(self) } + fn try_get_series(&self) -> Option { + IonAnnot::try_get_series(self) + } + fn get_charge(&self) -> i8 { IonAnnot::get_charge(self) } diff --git a/rust/timsquery/src/utils/mod.rs b/rust/timsquery/src/utils/mod.rs index 399f6db2..5bc22549 100644 --- a/rust/timsquery/src/utils/mod.rs +++ b/rust/timsquery/src/utils/mod.rs @@ -1,4 +1,5 @@ pub mod display; +pub mod sequence; pub mod sorting; pub mod streaming_calculators; pub mod tolerance_ranges; diff --git a/rust/timsquery/src/utils/sequence.rs b/rust/timsquery/src/utils/sequence.rs new file mode 100644 index 00000000..47fc8357 --- /dev/null +++ b/rust/timsquery/src/utils/sequence.rs @@ -0,0 +1,46 @@ +//! Sequence-string helpers shared by the library readers. + +/// Strip modification annotations — anything inside `(...)`, `[...]` or +/// `{...}` — leaving the bare residue string. +/// +/// One implementation for every reader: DIA-NN, Skyline and Spectronaut all +/// spell modifications differently but nest them the same way, and three +/// copies that differ only in which bracket pairs they recognise is how a +/// library ends up with a "stripped" sequence that still has a `{` in it. +/// Unbalanced closers are ignored rather than driving the depth negative, so a +/// malformed sequence degrades to a partial strip instead of dropping the tail. +pub fn strip_mods(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut depth: u32 = 0; + for c in s.chars() { + match c { + '(' | '[' | '{' => depth += 1, + ')' | ']' | '}' => depth = depth.saturating_sub(1), + _ if depth == 0 => out.push(c), + _ => {} + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_every_bracket_style() { + assert_eq!(strip_mods("PEPTC[UNIMOD:4]IDEK"), "PEPTCIDEK"); + assert_eq!(strip_mods("PEPTC(UniMod:4)IDEK"), "PEPTCIDEK"); + assert_eq!(strip_mods("PEPTC{57.02}IDEK"), "PEPTCIDEK"); + assert_eq!(strip_mods("C[+57.02]AM"), "CAM"); + assert_eq!(strip_mods("PEPTIDEK"), "PEPTIDEK"); + } + + #[test] + fn nesting_and_unbalanced_closers_do_not_lose_the_tail() { + assert_eq!(strip_mods("PE[a[b]c]PTIDE"), "PEPTIDE"); + // A stray closer must not drive the depth negative and swallow the + // rest of the sequence. + assert_eq!(strip_mods("PEP]TIDEK"), "PEPTIDEK"); + } +} diff --git a/rust/timsquery/tests/carafe_contract.rs b/rust/timsquery/tests/carafe_contract.rs new file mode 100644 index 00000000..f4969ff4 --- /dev/null +++ b/rust/timsquery/tests/carafe_contract.rs @@ -0,0 +1,154 @@ +//! Executable form of `docs/CARAFE_CONTRACT.md`. +//! +//! Carafe drives `timsquery_cli` as a subprocess and parses its output with +//! fastjson, with no field remapping and no schema negotiation. A renamed +//! field or a dropped serde alias fails loudly on neither side — Carafe just +//! gets a null and NPEs somewhere unrelated. These tests pin the mechanically +//! checkable parts, using the literal payloads from the document so the two +//! cannot drift apart silently. +//! +//! Everything goes through the public entry points Carafe actually reaches +//! (a file path into `read_library_file`, a JSON blob into `Tolerance`) rather +//! than internal types, so a refactor that keeps the internals working but +//! changes the boundary still fails here. +//! +//! This file covers the INPUT direction. The output direction — result field +//! names, the `-a`/`-f` flag values and the `results.json` basename — is +//! pinned by `timsquery_cli`'s `carafe_output_contract` module, which has to +//! live inside that crate because it has no library target. +//! +//! NOT covered, because it needs the built binary and a real `.d`: the `-o` +//! directory layout and the process exit code. + +use std::io::Write; +use timsquery::models::tolerance::{ + MobilityTolerance, + MzTolerance, + QuadTolerance, + RtTolerance, + Tolerance, +}; +use timsquery::serde::{ + LibraryArena, + read_library_file, +}; + +/// Verbatim from the contract's "Targets (`-e`, `psm_query.json`)" section. +const CARAFE_TARGETS: &str = r#"[ + { "id": 0, "mobility": 0.95, "rt_seconds": 1234.5, "precursor": 650.32, + "precursor_charge": 2, "precursor_isotopes": [0,1,2], + "fragments": [175.1, 288.2], "fragment_labels": ["y1","y3^2"] } +]"#; + +/// Verbatim from the contract's "Tolerances (`-t`)" section. Note `percent` +/// and `absolute`, not the `pct`/`da` spellings the CLI's own templates use — +/// both must deserialize. +const CARAFE_TOLERANCES: &str = r#"{ + "ms": { "ppm": [13.0, 17.0] }, + "rt": { "minutes": [0.1, 0.1] }, + "mobility": { "percent": [3.0, 3.0] }, + "quad": { "absolute": [0.1, 0.1] } +}"#; + +fn write_targets(json: &str) -> tempfile::NamedTempFile { + let mut f = tempfile::Builder::new() + .suffix(".json") + .tempfile() + .expect("tempfile"); + f.write_all(json.as_bytes()).expect("write targets"); + f.flush().expect("flush"); + f +} + +/// Every field name Carafe emits must land somewhere. `precursor`, +/// `fragments` and `fragment_labels` are serde aliases of names used elsewhere +/// in the codebase, so a cleanup that "unifies" them would break Carafe +/// without failing any other test. +#[test] +fn carafe_target_payload_loads_through_the_public_reader() { + let f = write_targets(CARAFE_TARGETS); + let arena = read_library_file(f.path()).expect("Carafe's target JSON must load"); + + // String fragment labels must resolve to ion annotations, not to the + // string-labelled arena, which carries no ion chemistry. + let LibraryArena::Mzpaf { geom, .. } = arena else { + panic!("labelled targets must land in the ion-annotated arena"); + }; + assert_eq!(geom.n_rows(), 1); + assert_eq!(geom.precursor_mz[0], 650.32); + assert_eq!(geom.charge[0], 2); + assert_eq!(geom.rt_seconds[0], 1234.5); + assert_eq!(geom.mobility[0], 0.95); + + // Labels are positionally paired with their m/z values, and the `^N` + // charge suffix survives. + assert_eq!(geom.frag_mzs.len(), 2); + assert_eq!(geom.frag_mzs[0], 175.1); + assert_eq!(geom.frag_mzs[1], 288.2); + let labels: Vec = geom.frag_labels.iter().map(|l| l.to_string()).collect(); + assert_eq!(labels, vec!["y1".to_string(), "y3^2".to_string()]); +} + +/// `id` is echoed back and used as Carafe's map key. A missing `id` NPEs +/// downstream rather than erroring here, so it must not silently default. +#[test] +fn carafe_id_field_is_required() { + // Build the payload without `id` rather than string-surgering the const, + // so the test cannot pass because the edit produced malformed JSON. + let without_id = r#"[ + { "mobility": 0.95, "rt_seconds": 1234.5, "precursor": 650.32, + "precursor_charge": 2, "precursor_isotopes": [0,1,2], + "fragments": [175.1, 288.2], "fragment_labels": ["y1","y3^2"] } + ]"#; + serde_json::from_str::(without_id).expect("still valid JSON"); + let f = write_targets(without_id); + assert!( + read_library_file(f.path()).is_err(), + "a target without `id` must fail rather than default it" + ); +} + +/// Carafe writes `percent` and `absolute`; the CLI's own templates write `pct` +/// and `da`. Dropping either spelling silently breaks one caller. +#[test] +fn carafe_tolerance_spellings_deserialize() { + let tol: Tolerance = + serde_json::from_str(CARAFE_TOLERANCES).expect("Carafe's tolerance JSON must deserialize"); + + // The `ms` window is `[itol - itol_shift, itol + itol_shift]`: a +-itol + // window recentred on a measured calibration offset, read as "13 ppm light + // to 17 ppm heavy". Both edges must stay distinct — collapsing them to a + // symmetric tolerance would quietly recentre every extraction window. + assert_eq!(tol.ms, MzTolerance::Ppm((13.0, 17.0))); + assert_eq!(tol.rt, RtTolerance::Minutes((0.1, 0.1))); + assert_eq!(tol.mobility, MobilityTolerance::Pct((3.0, 3.0))); + assert_eq!(tol.quad, QuadTolerance::Absolute((0.1, 0.1))); +} + +/// Carafe encodes a systematic calibration offset as +/// `[itol - itol_shift, itol + itol_shift]`, so when the offset exceeds the +/// half-width the LOW edge is negative — a window that is entirely to the +/// heavy side of the target mass. That is a supported input, not a bug, and +/// `mz_range`'s invariant is `low + high >= 0` rather than "both positive". +#[test] +fn a_negative_low_tolerance_edge_is_accepted() { + let tol: Tolerance = serde_json::from_str( + r#"{ + "ms": { "ppm": [-2.0, 32.0] }, + "rt": { "minutes": [0.1, 0.1] }, + "mobility": { "percent": [3.0, 3.0] }, + "quad": { "absolute": [0.1, 0.1] } + }"#, + ) + .expect("a negative low edge must deserialize"); + assert_eq!(tol.ms, MzTolerance::Ppm((-2.0, 32.0))); + + // Both edges land above the target mass, in ascending order. + let range = tol.mz_range(1000.0); + assert!( + range.start() > 1000.0 && range.end() > range.start(), + "expected a well-formed window above the target, got {:?}..{:?}", + range.start(), + range.end() + ); +} diff --git a/rust/timsquery/tests/mzspeclib_io_files/README.md b/rust/timsquery/tests/mzspeclib_io_files/README.md new file mode 100644 index 00000000..30994f3e --- /dev/null +++ b/rust/timsquery/tests/mzspeclib_io_files/README.md @@ -0,0 +1,18 @@ +# mzSpecLib test fixtures + +Verbatim from [HUPO-PSI/mzSpecLib](https://github.com/HUPO-PSI/mzSpecLib) +`examples/`, Apache-2.0 (same licence as this project). + +| file | why | +|---|---| +| `diann.mzSpecLib.txt` | a DIA-NN export: all peaks annotated, every mass error exactly `0.0` | +| `spectronaut.mzSpecLib.txt` | carries `-H2O`/`-NH3` losses and a unit-tagged retention time in minutes | + +Both are fully representable, so neither exercises the unknown-label or +skip paths — those are covered by unit tests over `resolve_annotation`. + +Deliberately NOT vendored: the NIST and SpectraST examples. They are +dominated by internal fragments, immonium ions and unannotated (`?`) peaks, +and by consensus spectra whose observed m/z carries real calibration error. +Worth adding when the resolution policy needs end-to-end coverage; today +they would only make these two harder to read. diff --git a/rust/timsquery/tests/mzspeclib_io_files/diann.mzSpecLib.txt b/rust/timsquery/tests/mzspeclib_io_files/diann.mzSpecLib.txt new file mode 100644 index 00000000..b835867a --- /dev/null +++ b/rust/timsquery/tests/mzspeclib_io_files/diann.mzSpecLib.txt @@ -0,0 +1,358 @@ + +MS:1003186|library format version=1.0 +MS:1003188|library name=phl004_canonical_sall_pv_plasma.head.diann +MS:1003207|library creation software=MS:1003253|DIA-NN + + + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAAASAGGK2 +MS:1000744|selected ion m/z=778.41296 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=20 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAAASAGGK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAAASAGGK/2 +MS:1001117|theoretical mass=1554.8114178831797 +[1]MS:1000885|protein accession=P0CG40 +[1]MS:1000886|protein name=SP9_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P0CG40 + +427.22995 1.0 b6/0.0 +498.26706 0.88039231 b7/0.0 +569.3042 0.66275221 b8/0.0 +356.19284 0.58039355 b5/0.0 +640.34131 0.44118175 b9/0.0 +703.37335 0.40000239 y9/0.0 +916.48468 0.40000239 y12/0.0 +1129.5961 0.36078224 y15/0.0 +774.41046 0.34117815 y10/0.0 +845.44757 0.30195799 y11/0.0 +490.26199 0.301357 y6/0.0 +419.22488 0.29999879 y5/0.0 +1058.5588 0.25881943 y14/0.0 +632.33624 0.2572208 y8/0.0 +853.45264 0.18174933 b12/0.0 +987.52179 0.17647269 y13/0.0 +782.41553 0.13921174 b11/0.0 +924.48975 0.13725254 b13/0.0 +711.37842 0.11764847 b10/0.0 +1200.6332 0.10526822 y16/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAAASAGGK3 +MS:1000744|selected ion m/z=519.27777 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=3 +MS:1003059|number of peaks=14 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAAASAGGK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAAASAGGK/3 +MS:1001117|theoretical mass=1554.8114178831797 +[1]MS:1000885|protein accession=P0CG40 +[1]MS:1000886|protein name=SP9_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P0CG40 + +703.37335 1.0 y9/0.0 +419.22488 0.75598669 y5/0.0 +498.26706 0.67160469 b7/0.0 +640.34131 0.61734796 b9/0.0 +569.3042 0.56162828 b8/0.0 +356.19284 0.51522595 b5/0.0 +427.22995 0.51522595 b6/0.0 +774.41046 0.48234525 y10/0.0 +561.29907 0.41774848 y7/0.0 +490.26199 0.37849048 y6/0.0 +648.33875 0.37133196 b18^2/0.0 +632.33624 0.32955998 y8/0.0 +711.37842 0.27849901 b10/0.0 +782.41553 0.23208249 b11/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAAGAGAGAK2 +MS:1000744|selected ion m/z=798.92627 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=24 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAAGAGAGAK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAAGAGAGAK/2 +MS:1001117|theoretical mass=1595.8379669841897 +[1]MS:1000885|protein accession=P55011 +[1]MS:1000886|protein name=S12A2_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P55011 + +427.22995 1.0 b6/0.0 +356.19284 0.91602999 b5/0.0 +498.26706 0.83227003 b7/0.0 +569.3042 0.79017997 b8/0.0 +640.34131 0.59228998 b9/0.0 +815.43701 0.54980999 y11/0.0 +886.47412 0.51920998 y12/0.0 +531.28851 0.50638998 y7/0.0 +957.51123 0.49983999 y13/0.0 +673.36279 0.46117997 y9/0.0 +1028.5483 0.44073999 y14/0.0 +711.37842 0.42937002 b10/0.0 +744.3999 0.41159999 y10/0.0 +602.32562 0.40491998 y8/0.0 +782.41553 0.38191 b11/0.0 +403.22995 0.36974999 y5/0.0 +1099.5854 0.34740001 y15/0.0 +1170.6226 0.31896001 y16/0.0 +853.45264 0.27395001 b12/0.0 +474.26706 0.26207 y6/0.0 +995.52686 0.20731001 b14/0.0 +1241.6597 0.2052 y17/0.0 +924.48975 0.18191999 b13/0.0 +1312.6968 0.114 y18/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAAGAGAGAK3 +MS:1000744|selected ion m/z=532.95325 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=3 +MS:1003059|number of peaks=20 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAAGAGAGAK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAAGAGAGAK/3 +MS:1001117|theoretical mass=1595.8379669841897 +[1]MS:1000885|protein accession=P55011 +[1]MS:1000886|protein name=S12A2_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P55011 + +531.28851 1.0 y7/0.0 +498.26706 0.82642001 b7/0.0 +427.22995 0.75252002 b6/0.0 +356.19284 0.71425998 b5/0.0 +602.32562 0.66922998 y8/0.0 +569.3042 0.56159002 b8/0.0 +673.36279 0.52272004 y9/0.0 +640.34131 0.47526002 b9/0.0 +711.37842 0.40158999 b10/0.0 +744.3999 0.39754999 y10/0.0 +403.22995 0.31963 y5/0.0 +462.7485 0.27653 b13^2/0.0 +782.41553 0.27631998 b11/0.0 +815.43701 0.24988998 y11/0.0 +474.26706 0.24581002 y6/0.0 +391.7114 0.18824001 b11^2/0.0 +853.45264 0.17081 b12/0.0 +886.47412 0.16329999 y12/0.0 +924.48975 0.10528 b13/0.0 +957.51123 0.091109999 y13/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAASGFAYPGTSER3 +MS:1000744|selected ion m/z=746.36969 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=3 +MS:1003059|number of peaks=13 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAASGFAYPGTSER +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAASGFAYPGTSER/3 +MS:1001117|theoretical mass=2236.087258363109 +[1]MS:1000885|protein accession=P35453 +[1]MS:1000886|protein name=HXD13_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P35453 + +646.31549 1.0 y6/0.0 +880.41589 0.72346348 y8/0.0 +498.26706 0.66272509 b7/0.0 +569.3042 0.60243261 b8/0.0 +356.19284 0.58793229 b5/0.0 +809.37878 0.48742947 y7/0.0 +711.37842 0.48370829 b10/0.0 +427.22995 0.48304707 b6/0.0 +1027.4844 0.38692665 y9/0.0 +853.45264 0.32992482 b12/0.0 +1171.5378 0.31720817 y11/0.0 +782.41553 0.21106207 b11/0.0 +640.34131 0.21104671 b9/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAAAAAASGFAYPGTSER4 +MS:1000744|selected ion m/z=560.02911 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=4 +MS:1003059|number of peaks=11 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAAAAAASGFAYPGTSER +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAAAAAASGFAYPGTSER/4 +MS:1001117|theoretical mass=2236.087258363109 +[1]MS:1000885|protein accession=P35453 +[1]MS:1000886|protein name=HXD13_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P35453 + +646.31549 1.0 y6/0.0 +356.19284 0.66913706 b5/0.0 +492.24124 0.59304374 y4/0.0 +498.26706 0.4210082 b7/0.0 +809.37878 0.4210082 y7/0.0 +569.3042 0.36934987 b8/0.0 +711.37842 0.344071 b10/0.0 +514.24579 0.33137658 y9^2/0.0 +427.22995 0.33042264 b6/0.0 +640.34131 0.31578368 b9/0.0 +586.27252 0.27729672 y11^2/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAK2 +MS:1000744|selected ion m/z=429.24561 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=10 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAK +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAK/2 +MS:1001117|theoretical mass=856.4766655447999 +[1]MS:1000885|protein accession=P50914,P50458,A6NHT5,P15502,DECOY_Q9Y651,DECOY_O60341 +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P50914,P50458,A6NHT5,P15502,DECOY_Q9Y651,DECOY_O60341 + +573.33551 1.0 y7/0.0 +502.29837 0.84049195 y6/0.0 +644.37262 0.62802154 y8/0.0 +715.40973 0.53955686 y9/0.0 +356.19284 0.52399796 b5/0.0 +431.26126 0.46901798 y5/0.0 +427.22995 0.32801768 b6/0.0 +360.22415 0.30400032 y4/0.0 +498.26706 0.22399409 b7/0.0 +786.44684 0.19198385 y10/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAAR2 +MS:1000744|selected ion m/z=443.24869 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=9 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAAR +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAAR/2 +MS:1001117|theoretical mass=884.4828135543999 +[1]MS:1000885|protein accession=P47928,Q9Y651,DECOY_Q76L83,DECOY_Q8WXD9,DECOY_Q5VZB9,DECOY_P35453,DECOY_O14654,DECOY_P55011,DECOY_P0CG40 +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=P47928,Q9Y651,DECOY_Q76L83,DECOY_Q8WXD9,DECOY_Q5VZB9,DECOY_P35453,DECOY_O14654,DECOY_P55011,DECOY_P0CG40 + +601.34161 1.0 y7/0.0 +530.3045 0.90602791 y6/0.0 +672.37872 0.83756346 y8/0.0 +356.19284 0.69194162 b5/0.0 +388.23029 0.52842641 y4/0.0 +459.2674 0.47519037 y5/0.0 +743.41583 0.39619291 y9/0.0 +427.22995 0.36180204 b6/0.0 +569.3042 0.20031726 b8/0.0 + + +MS:1003061|library spectrum name=AAAAAAAAAASGAAIPPLIPPR2 +MS:1000744|selected ion m/z=950.04419 +MS:1003203|constituent spectrum file=file:///home/andrew/hc-storage/diabetes_study/speclib/phl004_canonical_sall_pv.csv +MS:1003072|spectrum origin type=MS:1003074|predicted spectrum +MS:1003065|spectrum aggregation type=MS:1003074|predicted spectrum +MS:1002476|ion mobility drift time=0.0 +MS:1000041|charge state=2 +MS:1003059|number of peaks=25 +[1]MS:1003275|other attribute name=ExcludeFromAssay +[1]MS:1003276|other attribute value=False + +MS:1000888|stripped peptide sequence=AAAAAAAAAASGAAIPPLIPPR +MS:1003270|proforma peptidoform ion notation=AAAAAAAAAASGAAIPPLIPPR/2 +MS:1001117|theoretical mass=1898.0737805754497 +[1]MS:1000885|protein accession=O14654 +[1]MS:1000886|protein name=IRS4_HUMAN +[2]MS:1003275|other attribute name=Proteotypic +[2]MS:1003276|other attribute value=1 +[3]MS:1003275|other attribute name=ProteinGroup +[3]MS:1003276|other attribute value=O14654 + +789.49811 1.0 y7/0.0 +369.22449 0.58921003 y3/0.0 +427.22995 0.31046999 b6/0.0 +498.26706 0.30821002 b7/0.0 +569.3042 0.26800001 b8/0.0 +356.19284 0.25783998 b5/0.0 +640.34131 0.20063001 b9/0.0 +1101.6779 0.15594999 y11/0.0 +1188.71 0.15020999 y12/0.0 +902.58221 0.13153 y8/0.0 +1259.7471 0.1284 y13/0.0 +973.61932 0.11756 y9/0.0 +997.50616 0.11383001 b14/0.0 +711.37842 0.10726 b10/0.0 +1330.7842 0.10523 y14/0.0 +1110.5902 0.10514 b15/0.0 +926.46899 0.10461 b13/0.0 +1401.8213 0.080120005 y15/0.0 +855.43188 0.074140005 b12/0.0 +1044.6564 0.072379999 y10/0.0 +482.30853 0.067829996 y4/0.0 +798.41046 0.05923 b11/0.0 +1472.8584 0.055300001 y16/0.0 +692.44537 0.044360001 y6/0.0 +1543.8955 0.036620002 y17/0.0 + diff --git a/rust/timsquery/tests/mzspeclib_io_files/spectronaut.mzSpecLib.txt b/rust/timsquery/tests/mzspeclib_io_files/spectronaut.mzSpecLib.txt new file mode 100644 index 00000000..6236f10a --- /dev/null +++ b/rust/timsquery/tests/mzspeclib_io_files/spectronaut.mzSpecLib.txt @@ -0,0 +1,520 @@ + +MS:1003186|library format version=1.0 +MS:1003188|library name=human_serum.head.spectronaut +MS:1003207|library creation software=MS:1001327|Spectronaut + + + + +MS:1003061|library spectrum name=AQIPILR/2 +MS:1003208|experimental precursor monoisotopic m/z=405.7634379 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_AQIPILR_ +MS:1002476|ion mobility drift time=0.7629655 +MS:1001581|FAIMS compensation voltage=-60.0 +[2]MS:1000896|normalized retention time=28.658491 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=False +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P04114 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=6 + +MS:1000888|stripped peptide sequence=AQIPILR +MS:1003270|proforma peptidoform ion notation=AQIPILR/2 +MS:1001117|theoretical mass=809.5123227775299 +[1]MS:1000885|protein accession=P04114 +[1]MS:1000886|protein name=APOB_HUMAN +[1]MS:1001088|protein description=Apolipoprotein B-100 +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P04114 + +313.1870317 3.564491 b3/0.0 +401.2870801 3.3441753 y3/0.0 +498.339844 100.0 y4/0.0 +249.6735602 11.528888 y4^2/0.0 +611.423908 36.853043 y5/0.0 +481.3132956 5.708535 y4-NH3/0.0 + + +MS:1003061|library spectrum name=AGVLFGMSDR/2 +MS:1003208|experimental precursor monoisotopic m/z=526.763309 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_AGVLFGMSDR_ +MS:1002476|ion mobility drift time=0.8644136 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=46.12812 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=False +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P09172 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=13 + +MS:1000888|stripped peptide sequence=AGVLFGMSDR +MS:1003270|proforma peptidoform ion notation=AGVLFGMSDR/2 +MS:1001117|theoretical mass=1051.5120650773501 +[1]MS:1000885|protein accession=P09172 +[1]MS:1000886|protein name=DOPO_HUMAN +[1]MS:1001088|protein description=Dopamine beta-hydroxylase +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P09172 + +228.1342679 48.582985 b3/0.0 +377.1779236 4.8015785 y3/0.0 +341.2183319 4.1744895 b4/0.0 +508.2184085 5.056169 y4/0.0 +565.2398722 69.96272 y5/0.0 +712.3082861 100.0 y6/0.0 +825.3923501 51.118713 y7/0.0 +924.460764 1.5956572 y8/0.0 +360.1513752 3.1122544 y3-NH3/0.0 +491.1918601 1.0637656 y4-NH3/0.0 +694.2977213 1.3498487 y6-H2O/0.0 +695.2817378 2.9444132 y6-NH3/0.0 +359.1673588 2.1920948 y3-H2O/0.0 + + +MS:1003061|library spectrum name=DEDNNLLTEK/2 +MS:1003208|experimental precursor monoisotopic m/z=595.7804071 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_DEDNNLLTEK_ +MS:1002476|ion mobility drift time=0.90230334 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=13.850179 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P09486 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=19 + +MS:1000888|stripped peptide sequence=DEDNNLLTEK +MS:1003270|proforma peptidoform ion notation=DEDNNLLTEK/2 +MS:1001117|theoretical mass=1189.54626122625 +[1]MS:1000885|protein accession=P09486 +[1]MS:1000886|protein name=SPRC_HUMAN +[1]MS:1001088|protein description=SPARC +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P09486 + +360.1037556 14.370086 b3/0.0 +377.2030757 69.91263 y3/0.0 +490.2871397 47.727154 y4/0.0 +588.1896105 3.50105 b5/0.0 +603.3712037 20.282309 y5/0.0 +701.2736745 4.035067 b6/0.0 +717.4141311 24.80533 y6/0.0 +831.4570586 21.473965 y7/0.0 +946.4840016 100.0 y8/0.0 +342.0931908 5.760026 b3-H2O/0.0 +359.1925109 5.090061 y3-H2O/0.0 +472.2765749 4.874231 y4-H2O/0.0 +571.1630621 4.1113725 b5-NH3/0.0 +684.2471261 4.05031 b6-NH3/0.0 +700.3875827 3.518 y6-NH3/0.0 +814.4305102 7.3916 y7-NH3/0.0 +928.4734368 5.1134515 y8-H2O/0.0 +929.4574532 10.1338825 y8-NH3/0.0 +1075.526595 14.727533 y9/0.0 + + +MS:1003061|library spectrum name=GWVTDGFSSLK/2 +MS:1003208|experimental precursor monoisotopic m/z=598.8009456 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_GWVTDGFSSLK_ +MS:1002476|ion mobility drift time=0.92365193 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=62.688942 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P02656 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=14 + +MS:1000888|stripped peptide sequence=GWVTDGFSSLK +MS:1003270|proforma peptidoform ion notation=GWVTDGFSSLK/2 +MS:1001117|theoretical mass=1195.5873381925899 +[1]MS:1000885|protein accession=P02656 +[1]MS:1000886|protein name=APOC3_HUMAN +[1]MS:1001088|protein description=Apolipoprotein C-III +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P02656 + +343.1764671 3.8687427 b3/0.0 +347.2288965 6.1732 y3/0.0 +444.2241455 1.421 b4/0.0 +434.260925 9.774844 y4/0.0 +581.3293389 2.66143 y5/0.0 +638.3508026 19.109653 y6/0.0 +753.3777456 20.841564 y7/0.0 +854.4254241 60.931602 y8/0.0 +953.493838 36.67316 y9/0.0 +416.2503601 2.702636 y4-H2O/0.0 +735.3671808 2.1958628 y7-H2O/0.0 +836.4148593 4.374068 y8-H2O/0.0 +935.4832732 4.7364564 y9-H2O/0.0 +468.2452748 1.662614 y9-H2O^2/0.0 + + +MS:1003061|library spectrum name=VTSIQDWVQK/2 +MS:1003208|experimental precursor monoisotopic m/z=602.3220451 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_VTSIQDWVQK_ +MS:1002476|ion mobility drift time=0.9102522 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=39.73143 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P00738 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=22 + +MS:1000888|stripped peptide sequence=VTSIQDWVQK +MS:1003270|proforma peptidoform ion notation=VTSIQDWVQK/2 +MS:1001117|theoretical mass=1202.62953735774 +[1]MS:1000885|protein accession=P00738 +[1]MS:1000886|protein name=HPT_HUMAN +[1]MS:1001088|protein description=Haptoglobin +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P00738 + +288.1553973 13.402388 b3/0.0 +374.2397956 35.40592 y3/0.0 +560.3191085 46.133404 y4/0.0 +675.3460516 49.92076 y5/0.0 +803.4046291 88.30317 y6/0.0 +916.488693 35.01724 y7/0.0 +458.7479848 8.279677 y7^2/0.0 +1003.520721 100.0 y8/0.0 +1104.5684 49.551434 y9/0.0 +270.1448324 95.16555 b3-H2O/0.0 +785.3940643 24.796286 y6-H2O/0.0 +393.2006704 13.1079035 y6-H2O^2/0.0 +786.3780807 46.45572 y6-NH3/0.0 +985.5101566 7.3205233 y8-H2O/0.0 +493.2587166 4.255348 y8-H2O^2/0.0 +1086.557835 7.42398 y9-H2O/0.0 +201.1233688 88.65463 b4^2/0.0 +401.2394612 4.14027 b4/0.0 +552.7878382 4.422511 y9^2/0.0 +383.2288964 12.326735 b4-H2O/0.0 +986.4941731 24.063145 y8-NH3/0.0 +357.2132472 60.34427 y3-NH3/0.0 + + +MS:1003061|library spectrum name=QELSEAEQATR/2 +MS:1003208|experimental precursor monoisotopic m/z=631.3045807 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_QELSEAEQATR_ +MS:1002476|ion mobility drift time=0.92576075 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=-14.638051 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P01024 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=15 + +MS:1000888|stripped peptide sequence=QELSEAEQATR +MS:1003270|proforma peptidoform ion notation=QELSEAEQATR/2 +MS:1001117|theoretical mass=1260.594608401 +[1]MS:1000885|protein accession=P01024 +[1]MS:1000886|protein name=CO3_HUMAN +[1]MS:1001088|protein description=Complement C3 +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P01024 + +371.192511 3.1688333 b3/0.0 +347.2037444 10.465045 y3/0.0 +475.2623219 23.276728 y4/0.0 +604.304915 37.09764 y5/0.0 +675.3420288 64.2233 y6/0.0 +804.3846219 37.812637 y7/0.0 +891.4166503 96.74726 y8/0.0 +1004.500714 100.0 y9/0.0 +353.1819462 4.8028316 b3-H2O/0.0 +354.1659627 5.46505 b3-NH3/0.0 +458.2357735 1.8879279 y4-NH3/0.0 +658.3154804 1.120537 y6-NH3/0.0 +786.3740571 7.649042 y7-H2O/0.0 +873.4060855 4.228747 y8-H2O/0.0 +986.4901495 1.3095266 y9-H2O/0.0 + + +MS:1003061|library spectrum name=EEGTDLEVTANR/2 +MS:1003208|experimental precursor monoisotopic m/z=667.3151454 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_EEGTDLEVTANR_ +MS:1002476|ion mobility drift time=0.94296074 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=5.3526073 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P20742 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=9 + +MS:1000888|stripped peptide sequence=EEGTDLEVTANR +MS:1003270|proforma peptidoform ion notation=EEGTDLEVTANR/2 +MS:1001117|theoretical mass=1332.6157377683999 +[1]MS:1000885|protein accession=P20742 +[1]MS:1000886|protein name=PZP_HUMAN +[1]MS:1001088|protein description=Pregnancy zone protein +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P20742 + +360.1989934 20.52395 y3/0.0 +461.2466719 85.759026 y4/0.0 +560.3150858 82.32931 y5/0.0 +689.3576789 100.0 y6/0.0 +802.4417428 79.06382 y7/0.0 +917.4686859 72.09271 y8/0.0 +1018.516364 21.396385 y9/0.0 +1075.537828 83.90465 y10/0.0 +671.3471141 8.983159 y6-H2O/0.0 + + +MS:1003061|library spectrum name=C[Carbamidomethyl (C)]EEDEEFTC[Carbamidomethyl (C)]R/2 +MS:1003208|experimental precursor monoisotopic m/z=687.7504667 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_C[Carbamidomethyl (C)]EEDEEFTC[Carbamidomethyl (C)]R_ +MS:1002476|ion mobility drift time=0.9011946 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=-1.2823446 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P00747 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=17 + +MS:1000888|stripped peptide sequence=CEEDEEFTCR +MS:1003270|proforma peptidoform ion notation=C[Carbamidomethyl]EEDEEFTC[Carbamidomethyl]R/2 +MS:1001117|theoretical mass=1373.4863810338297 +[1]MS:1000885|protein accession=P00747 +[1]MS:1000886|protein name=PLMN_HUMAN +[1]MS:1001088|protein description=Plasminogen +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P00747 + +419.1231111 14.944622 b3/0.0 +436.1972791 30.738253 y3/0.0 +534.1500542 9.713619 b4/0.0 +583.2656931 51.723152 y4/0.0 +712.3082861 50.78903 y5/0.0 +841.3508792 59.771255 y6/0.0 +956.3778223 75.695114 y7/0.0 +1085.420415 100.0 y8/0.0 +1214.463008 15.559877 y9/0.0 +401.1125463 9.39616 b3-H2O/0.0 +516.1394894 5.2136855 b4-H2O/0.0 +566.2391447 6.1200476 y4-NH3/0.0 +694.2977213 6.602107 y5-H2O/0.0 +823.3403144 10.601803 y6-H2O/0.0 +938.3672575 6.748549 y7-H2O/0.0 +1067.409851 26.568663 y8-H2O/0.0 +1196.452444 10.719879 y9-H2O/0.0 + + +MS:1003061|library spectrum name=KQELSEAEQATR/2 +MS:1003208|experimental precursor monoisotopic m/z=695.3520622 +MS:1000041|charge state=2 +MS:1003203|constituent spectrum file=file:///IK_221028_C19_lib2_01 +MS:1003072|spectrum origin type=MS:1003424|selected fragment theoretical m/z observed intensity spectrum +MS:1003065|spectrum aggregation type=MS:1003067|consensus spectrum +[1]MS:1003275|other attribute name=LabeledPeptide +[1]MS:1003276|other attribute value=_KQELSEAEQATR_ +MS:1002476|ion mobility drift time=0.9663669 +MS:1001581|FAIMS compensation voltage=-40.0 +[2]MS:1000896|normalized retention time=-26.474043 +[2]UO:0000000|unit=UO:0000031|minute +[3]MS:1003275|other attribute name=ExcludeFromAssay +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=BGSInferenceId +[4]MS:1003276|other attribute value=P01024 +[5]MS:1003275|other attribute name=AllowForNormalization +[5]MS:1003276|other attribute value=True +[6]MS:1003275|other attribute name=Workflow +[6]MS:1003276|other attribute value= +MS:1003059|number of peaks=29 + +MS:1000888|stripped peptide sequence=KQELSEAEQATR +MS:1003270|proforma peptidoform ion notation=KQELSEAEQATR/2 +MS:1001117|theoretical mass=1388.6895714149998 +[1]MS:1000885|protein accession=P01024 +[1]MS:1000886|protein name=CO3_HUMAN +[1]MS:1001088|protein description=Complement C3 +[2]MS:1001467|taxonomy: NCBI TaxID=9606 +[2]MS:1001469|taxonomy: scientific name=Homo sapiens +[3]MS:1003275|other attribute name=IsProteotypic +[3]MS:1003276|other attribute value=True +[4]MS:1003275|other attribute name=FASTAName +[4]MS:1003276|other attribute value=H_sapiens_uniprot_reviewed_cannonical_3AUP000005640_2-2022.08.12-15.33.51.77 +[5]MS:1003275|other attribute name=Database +[5]MS:1003276|other attribute value=sp +[6]MS:1003275|other attribute name=ProteinGroups +[6]MS:1003276|other attribute value=P01024 + +386.2034101 7.7662497 b3/0.0 +347.2037444 3.2926106 y3/0.0 +499.2874741 6.734219 b4/0.0 +475.2623219 14.221682 y4/0.0 +586.3195025 1.5547751 b5/0.0 +604.304915 12.158777 y5/0.0 +715.3620955 3.230758 b6/0.0 +675.3420288 25.758886 y6/0.0 +786.3992093 1.9753934 b7/0.0 +804.3846219 22.087849 y7/0.0 +915.4418024 1.8401791 b8/0.0 +458.2245394 1.6086894 b8^2/0.0 +891.4166503 74.71736 y8/0.0 +1004.500714 54.28254 y9/0.0 +1114.537494 1.0495659 b10/0.0 +1133.543307 100.0 y10/0.0 +1261.601885 24.588951 y11/0.0 +631.3045807 2.1570513 y11^2/0.0 +368.1928453 3.4925315 b3-H2O/0.0 +369.1768617 2.9452903 b3-NH3/0.0 +481.2769092 1.0290943 b4-H2O/0.0 +458.2357735 1.8126523 y4-NH3/0.0 +873.4060855 2.0101676 y8-H2O/0.0 +1115.532743 6.641937 y10-H2O/0.0 +622.2992983 6.5424037 y11-H2O^2/0.0 +1243.59132 3.3419182 y11-H2O/0.0 +1244.575336 10.56645 y11-NH3/0.0 +482.2609257 1.1637812 b4-NH3/0.0 +330.177196 1.6216956 y3-NH3/0.0 + diff --git a/rust/timsquery_cli/src/commands.rs b/rust/timsquery_cli/src/commands.rs index 5a0c25a7..4478434f 100644 --- a/rust/timsquery_cli/src/commands.rs +++ b/rust/timsquery_cli/src/commands.rs @@ -37,6 +37,12 @@ use crate::error::CliError; use crate::processing::AggregatorContainer; use timsquery::serde::LibraryArena; +/// Basename Carafe looks for inside the `-o` directory. Part of the contract +/// (`docs/CARAFE_CONTRACT.md`, invariant 5), so it is named rather than +/// inlined — a renamed output file is a silent failure on Carafe's side. +/// Despite the extension, the contents are ndjson. +pub const CARAFE_RESULTS_BASENAME: &str = "results.json"; + /// Main function for the 'query-index' subcommand. #[instrument] pub fn main_query_index(args: QueryIndexArgs) -> Result<(), CliError> { @@ -71,7 +77,7 @@ pub fn main_query_index(args: QueryIndexArgs) -> Result<(), CliError> { let batch_size = args.batch_size; std::fs::create_dir_all(&output_path)?; - let put_path = output_path.join("results.json"); + let put_path = output_path.join(CARAFE_RESULTS_BASENAME); // Every format funnels into one of the two label-typed arenas; extraction // is generic over the label, so both arms call the same driver over the @@ -516,3 +522,189 @@ mod tests { } } } + +/// The OUTPUT half of `docs/CARAFE_CONTRACT.md`. +/// +/// `rust/timsquery/tests/carafe_contract.rs` pins the input direction. These +/// live here rather than in `tests/` because `timsquery_cli` has no library +/// target, and they assert on the boundary types Carafe parses with fastjson: +/// no remap, no schema negotiation, so a renamed field is a null on their side +/// and an NPE somewhere unrelated. +#[cfg(test)] +mod carafe_output_contract { + use super::*; + use crate::cli::{ + PossibleAggregator, + SerializationFormat, + }; + use crate::processing::SpectrumOutput; + use clap::ValueEnum; + use timsquery::serde::chromatogram_output::ChromatogramOutput; + + /// Verbatim from the contract's "spectrum-aggregator -> PSMQueryResult". + const CARAFE_SPECTRUM_RESULT: &str = r#"{ + "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, "precursor_mz":650.32, + "precursor_charge":2, "precursor_intensities":[1200,800,300], "precursor_labels":[0,1,2], + "fragment_mzs":[175.1,288.2], "fragment_intensities":[500,0] + }"#; + + /// Verbatim from the contract's "chromatogram-aggregator -> XICQueryResult". + const CARAFE_CHROMATOGRAM_RESULT: &str = r#"{ + "id":0, "mobility_ook0":0.95, "rt_seconds":1234.5, + "precursor_mzs":[650.32,650.82], "precursor_intensities":[[1.0],[2.0]], + "fragment_mzs":[175.1,288.2], "fragment_labels":["y1","b2"], + "fragment_intensities":[[3.0],[4.0]], "retention_time_results_seconds":[1230,1231] + }"#; + + /// Parse a contract payload into the real boundary type. + /// + /// Deserializing (rather than constructing) is what lets these tests exist + /// at all: both types are only ever built from an aggregator, which needs a + /// real `.d`. A renamed or dropped field fails here. + fn parse(json: &str) -> T { + serde_json::from_str(json) + .unwrap_or_else(|e| panic!("the contract's payload must deserialize: {e}")) + } + + /// Write records through the SAME serializer `stream_process_batches` uses, + /// and hand back the exact bytes Carafe would read out of `results.json`. + fn write_results(records: &[T], format: SerializationFormat) -> String { + let mut buf = Vec::new(); + let mut seq = JsonStreamSerializer::new(&mut buf, format); + for r in records { + seq.serialize(r).expect("serialize"); + } + seq.finish().expect("finish"); + String::from_utf8(buf).expect("utf-8") + } + + fn keys_of(line: &str) -> Vec { + let value: serde_json::Value = serde_json::from_str(line).expect("one object per line"); + let mut keys: Vec = value + .as_object() + .expect("an object, not an array or scalar") + .keys() + .cloned() + .collect(); + keys.sort(); + keys + } + + /// Contract invariants 2 and 3, asserted on the bytes rather than on the + /// type: ndjson means one complete object per line, no array wrapper and no + /// pretty-print, and the field names are exact (fastjson, no remap). + /// + /// Note the deliberate singular/plural split between the two modes + /// (`precursor_mz` vs `precursor_mzs`): they are two schemas, and + /// "unifying" them would break Carafe without failing anything else. + #[test] + fn spectrum_results_are_ndjson_with_the_contract_field_names() { + let records: Vec = + vec![parse(CARAFE_SPECTRUM_RESULT), parse(CARAFE_SPECTRUM_RESULT)]; + let out = write_results(&records, SerializationFormat::Ndjson); + + assert!(!out.starts_with('['), "no array wrapper: {out}"); + assert!(out.ends_with('\n'), "every record is newline-terminated"); + let lines: Vec<&str> = out.lines().collect(); + assert_eq!(lines.len(), 2, "one line per record"); + assert!( + !out.contains("\n "), + "ndjson must not be pretty-printed: {out}" + ); + + for line in lines { + assert_eq!( + keys_of(line), + [ + "fragment_intensities", + "fragment_mzs", + "id", + "mobility_ook0", + "precursor_charge", + "precursor_intensities", + "precursor_labels", + "precursor_mz", + "rt_seconds", + ] + ); + } + } + + #[test] + fn chromatogram_results_are_ndjson_with_the_contract_field_names() { + let records: Vec = vec![parse(CARAFE_CHROMATOGRAM_RESULT)]; + let out = write_results(&records, SerializationFormat::Ndjson); + + assert!(!out.starts_with('['), "no array wrapper: {out}"); + let lines: Vec<&str> = out.lines().collect(); + assert_eq!(lines.len(), 1); + assert_eq!( + keys_of(lines[0]), + [ + "fragment_intensities", + "fragment_labels", + "fragment_mzs", + "id", + "mobility_ook0", + "precursor_intensities", + "precursor_mzs", + "retention_time_results_seconds", + "rt_seconds", + ] + ); + } + + /// The ndjson/array distinction is real, not an accident of the writer + /// happening to emit one object. Carafe parses line-by-line, so an array + /// wrapper is a parse failure on their side — this pins that the OTHER + /// formats are the ones that wrap, and therefore that `-f ndjson` matters. + #[test] + fn the_non_ndjson_formats_do_wrap_in_an_array() { + let records: Vec = vec![parse(CARAFE_SPECTRUM_RESULT)]; + for format in [SerializationFormat::Json, SerializationFormat::PrettyJson] { + let out = write_results(&records, format); + assert!( + out.starts_with('[') && out.ends_with(']'), + "{format:?} must wrap, else `-f ndjson` is not load-bearing: {out}" + ); + } + // And the default is NOT ndjson, so Carafe passing `-f` is required. + assert_ne!(SerializationFormat::default(), SerializationFormat::Ndjson); + } + + /// An empty result set must still be one parseable file, not a truncated + /// one. Carafe reads every line; zero lines is a valid empty result. + #[test] + fn an_empty_ndjson_result_is_empty_not_malformed() { + let out = write_results::(&[], SerializationFormat::Ndjson); + assert_eq!(out, "", "no records means no lines, and no array wrapper"); + } + + /// The `-a` and `-f` values Carafe passes on the command line. clap derives + /// these from the variant names, so a rename silently changes the CLI. + #[test] + fn aggregator_and_format_flag_values_match_the_contract() { + fn name(v: T) -> String { + v.to_possible_value() + .expect("not skipped") + .get_name() + .to_string() + } + + assert_eq!( + name(PossibleAggregator::SpectrumAggregator), + "spectrum-aggregator" + ); + assert_eq!( + name(PossibleAggregator::ChromatogramAggregator), + "chromatogram-aggregator" + ); + assert_eq!(name(SerializationFormat::Ndjson), "ndjson"); + } + + /// Contract invariant 5. + #[test] + fn the_results_basename_is_what_carafe_looks_for() { + assert_eq!(CARAFE_RESULTS_BASENAME, "results.json"); + } +} diff --git a/rust/timsquery_viewer/src/error.rs b/rust/timsquery_viewer/src/error.rs index 56d65f0f..888721e4 100644 --- a/rust/timsquery_viewer/src/error.rs +++ b/rust/timsquery_viewer/src/error.rs @@ -27,6 +27,9 @@ impl From for ViewerError { timsquery::serde::LibraryReadingError::UnableToParseElutionGroups => { ViewerError::General("Unable to parse elution groups".to_string()) } + timsquery::serde::LibraryReadingError::ReaderFailed { reader, detail } => { + ViewerError::General(format!("{reader} could not read this file: {detail}")) + } timsquery::serde::LibraryReadingError::UnsupportedSpeclibVersion(v) => { ViewerError::General(format!("Unsupported .speclib version: {v}")) } diff --git a/rust/timsseek/Cargo.toml b/rust/timsseek/Cargo.toml index 6834cb80..f774e856 100644 --- a/rust/timsseek/Cargo.toml +++ b/rust/timsseek/Cargo.toml @@ -6,7 +6,6 @@ license.workspace = true [dependencies] regex = "1.10.6" -rmp-serde = "1.1" zstd = "0.13" # Gradient boosted tree @@ -23,7 +22,8 @@ array2d = { path = "../array2d" } timsseek_macros = { path = "../timsseek_macros" } # Workspace-inherited deps -rustyms = { workspace = true } +mzcore = { workspace = true } +mzcv = { workspace = true } timsrust = { workspace = true } rusqlite = {workspace = true } serde = { workspace = true } diff --git a/rust/timsseek/examples/query_bench.rs b/rust/timsseek/examples/query_bench.rs index 892e4272..3fb17641 100644 --- a/rust/timsseek/examples/query_bench.rs +++ b/rust/timsseek/examples/query_bench.rs @@ -75,7 +75,7 @@ fn main() { ); let speclib_path = env( "BENCH_SPECLIB", - "/Users/sebastianpaez/fasta/asdad.msgpack.zstd", + "/Users/sebastianpaez/fasta/asdad.ndjson.zst", ); let n: usize = env("QB_N", "2000").parse().unwrap(); let iters: usize = env("QB_ITERS", "1").parse().unwrap(); diff --git a/rust/timsseek/src/data_sources/mod.rs b/rust/timsseek/src/data_sources/mod.rs index 15a139c4..cd26f59a 100644 --- a/rust/timsseek/src/data_sources/mod.rs +++ b/rust/timsseek/src/data_sources/mod.rs @@ -11,5 +11,4 @@ pub use speclib::{ ReferenceEG, SerSpeclibElement, Speclib, - SpeclibWriter, }; diff --git a/rust/timsseek/src/data_sources/reference_library.rs b/rust/timsseek/src/data_sources/reference_library.rs index fdd5d1f7..4e134087 100644 --- a/rust/timsseek/src/data_sources/reference_library.rs +++ b/rust/timsseek/src/data_sources/reference_library.rs @@ -345,7 +345,7 @@ mod tests { #[test] fn item_at_scores_reference_library() { - // Task 9 collapsed `Speclib` to the single `ReferenceLibrary` arena; + // `Speclib` is a type alias for `ReferenceLibrary`; // scoring reads `RefQuery` flyweights via `item_at` (no materialized // arm). Variant 0 is the target. let lib = tiny_ref_lib(); diff --git a/rust/timsseek/src/data_sources/speclib.rs b/rust/timsseek/src/data_sources/speclib.rs index 30e76ed0..31ee0036 100644 --- a/rust/timsseek/src/data_sources/speclib.rs +++ b/rust/timsseek/src/data_sources/speclib.rs @@ -17,6 +17,7 @@ use std::io::{ BufRead, BufReader, Read, + Write, }; use std::path::{ Path, @@ -29,7 +30,7 @@ use timsquery::utils::constants::PROTON_MASS; /// The serializable, on-disk form of a native speclib element. Kept backwards /// compatible; the load path builds the columnar `ReferenceLibrary` arena -/// directly from these elements (see `Speclib::from_file_with_format`). +/// directly from these elements (see `Speclib::from_native_file`). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SerSpeclibElement { precursor: PrecursorEntry, @@ -43,84 +44,40 @@ impl SerSpeclibElement { elution_group, } } - - pub fn sample() -> Self { - SerSpeclibElement { - precursor: PrecursorEntry { - sequence: "PEPTIDESEK".into(), - charge: 2, - decoy: false, - decoy_group: 32, - }, - elution_group: ReferenceEG { - id: 32, - precursor_mz: 512.2, - precursor_labels: vec![0, 2], - fragment_mzs: vec![312.2, 675.7], - fragment_labels: vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - precursor_intensities: vec![1.0, 0.5], - fragment_intensities: vec![0.8, 0.3], - mobility_ook0: 0.75, - rt_seconds: 120.0, - }, - } - } - - pub fn sample_json() -> &'static str { - r#"{ - "precursor": { - "sequence": "PEPTIDEPINK", - "charge": 2, - "decoy": false, - "decoy_group": 0 - }, - "elution_group": { - "id": 0, - "precursor_mz": 876.5432, - "precursor_labels": [ 0, 1 ], - "fragment_mzs": [ 123.0, 123.0, 123.0 ], - "fragment_labels": ["a1", "b1", "c1^2"], - "precursor_intensities": [1.0, 1.0], - "fragment_intensities": [1.0, 1.0, 1.0], - "precursor_charge": 2, - "mobility_ook0": 0.8, - "rt_seconds": 0.0 - } - }"# - } } +/// Only the fields the loader reads are in the format. +/// +/// `id`, `decoy_group`, `precursor_labels` and `precursor_intensities` used to +/// be written here and dropped at read time — the isotope envelope is +/// recomputed from composition (`IsotopeStrategy::FromComposition`), so storing +/// it cost the writer a full ProForma parse per entry and gave isotopes two +/// sources of truth. Serde ignores unknown fields, so libraries that still +/// carry them load unchanged. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecursorEntry { sequence: String, charge: u8, decoy: bool, - decoy_group: u32, } impl PrecursorEntry { - pub fn new(sequence: String, charge: u8, decoy: bool, decoy_group: u32) -> Self { + pub fn new(sequence: String, charge: u8, decoy: bool) -> Self { Self { sequence, charge, decoy, - decoy_group, } } } +/// See [`PrecursorEntry`] for why the precursor-isotope fields are absent. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReferenceEG { - id: u32, precursor_mz: f64, - precursor_labels: Vec, #[serde(alias = "fragment_mz")] fragment_mzs: Vec, fragment_labels: Vec, - precursor_intensities: Vec, fragment_intensities: Vec, #[serde(alias = "mobility")] mobility_ook0: f32, @@ -128,25 +85,18 @@ pub struct ReferenceEG { } impl ReferenceEG { - #[allow(clippy::too_many_arguments)] pub fn new( - id: u32, precursor_mz: f64, - precursor_labels: Vec, fragment_mzs: Vec, fragment_labels: Vec, - precursor_intensities: Vec, fragment_intensities: Vec, mobility_ook0: f32, rt_seconds: f32, ) -> Self { Self { - id, precursor_mz, - precursor_labels, fragment_mzs, fragment_labels, - precursor_intensities, fragment_intensities, mobility_ook0, rt_seconds, @@ -154,24 +104,6 @@ impl ReferenceEG { } } -/// Strip mod annotations — anything inside `(...)` or `[...]` — from a -/// sequence, leaving the bare residue string. The native format ships one -/// (modified) sequence per precursor; the arena's composition-isotope path -/// needs the stripped residues. -fn strip_mods(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - let mut depth: i32 = 0; - for c in s.chars() { - match c { - '(' | '[' => depth += 1, - ')' | ']' => depth = (depth - 1).max(0), - _ if depth == 0 => out.push(c), - _ => {} - } - } - out -} - /// Summary of a [`finalize_reference_library`] call, for load-time logging. #[derive(Debug, Clone, Copy)] pub struct LoadReport { @@ -184,14 +116,13 @@ pub struct LoadReport { /// Finalize a freshly-narrowed lazy `ReferenceLibrary` arena: apply the decoy /// strategy, seal, run the whole-library parse gate + averagine tally, and set /// `caps.sequence_features`. This is the single shared tail of the DEFAULT -/// `.speclib` load (see `speclib_data_flow.md`) — the memory-optimized path -/// that avoids the 9 GB peak RSS of the fully-materialized target+2-decoy -/// expansion. +/// `.speclib` load — the memory-optimized path that avoids the 9 GB peak RSS +/// of a fully-materialized target+2-decoy expansion. /// /// `policy` is the raw CLI decoy policy: this is the single place it is resolved /// (via `map_decoy_strategy`, keyed on whether the arena already ships decoys) /// and stamped onto `caps.decoys` BEFORE `seal()`, so the seal's -/// `LazyMassShift -> Passthrough` downgrade (the Task-4 gate) sees it. The parse +/// `LazyMassShift -> Passthrough` downgrade sees it. The parse /// gate walks the MODIFIED sequence blob (the form /// `RefQuery::materialize_peptide_in_group` parses) and, if any row fails, /// disables sequence-derived features library-wide. The same pass counts @@ -239,14 +170,17 @@ fn finalize_reference_library( } } } - let mut all_parsable = true; + // The first row that fails to parse, kept so the warning can name it. One + // bad row anywhere disables sequence-derived scoring for the WHOLE library, + // so "which row" is the only actionable part of that news. + let mut first_unparsable: Option = None; let mut n_averagine_fallback = 0usize; for tgt in 0..n_rows { - if all_parsable { + if first_unparsable.is_none() { let modified = &geom.seq_mod_blob[geom.seq_mod_range(tgt)]; let normalized = normalize_to_proforma(modified); if parse_sequence(&normalized).is_none() { - all_parsable = false; + first_unparsable = Some(modified.to_string()); } } let stripped = &geom.seq_strip_blob[geom.seq_strip_range(tgt)]; @@ -258,10 +192,16 @@ fn finalize_reference_library( } } - let sequence_features = if all_parsable { - SeqFeatureState::Available - } else { - SeqFeatureState::Unavailable + let sequence_features = match &first_unparsable { + None => SeqFeatureState::Available, + Some(sequence) => { + tracing::warn!( + "Sequence-derived scoring features are DISABLED for this entire library: \ + {sequence:?} could not be parsed. Any non-Unimod modification \ + (PSI-MOD, RESID, XL-MOD, cross-links) has this effect." + ); + SeqFeatureState::Unavailable + } }; geom.caps.sequence_features = sequence_features; @@ -289,88 +229,67 @@ fn finalize_reference_library( /// The spectral library store. Collapsed to the single columnar /// `ReferenceLibrary` arena representation (the materialized AOS path was -/// deleted in Task 9): both load paths produce a lazy arena, and scoring +/// since deleted): both load paths produce a lazy arena, and scoring /// iterates `RefQuery` flyweights via [`ReferenceLibrary::item_at`]. pub type Speclib = ReferenceLibrary; -#[derive(Debug, Clone, Copy)] -pub enum SpeclibFormat { - NdJson, - NdJsonZstd, - MessagePack, - MessagePackZstd, -} - -impl SpeclibFormat { - /// Detect a native timsseek format by EXTENSION ONLY. Returns `None` for - /// anything else (including `.speclib`), which routes to the timsquery - /// bridge. - /// - /// Extension-only is deliberate: msgpack has no reliable magic byte, so a - /// content sniff would misclaim raw binaries like `.speclib` as msgpack. - pub fn detect_from_extension(path: &Path) -> Option { - let path_str = path.to_string_lossy().to_lowercase(); - - // Accept both `.zst` and `.zstd` — DIA-NN/user pipelines use either. - if path_str.ends_with(".msgpack.zst") || path_str.ends_with(".msgpack.zstd") { - Some(SpeclibFormat::MessagePackZstd) - } else if path_str.ends_with(".msgpack") { - Some(SpeclibFormat::MessagePack) - } else if path_str.ends_with(".ndjson.zst") || path_str.ends_with(".ndjson.zstd") { - Some(SpeclibFormat::NdJsonZstd) - } else if path_str.ends_with(".ndjson") { - Some(SpeclibFormat::NdJson) - } else { - None - } - } +/// Whether `path` names a native timsseek library, by EXTENSION ONLY. +/// +/// This answers *which reader family*, not *which encoding*. Content sniffing +/// cannot answer it: a DIA-NN `.speclib` and a native library are both opaque +/// byte streams, and the point of the extension rule is that a native extension +/// commits to the native reader and surfaces its error rather than falling +/// through the timsquery registry to report some other reader's complaint. +/// Whether the bytes are zstd-wrapped is a separate question, and +/// [`SpeclibReader`] answers that one from the magic number. +fn is_native_extension(path: &Path) -> bool { + let path_str = path.to_string_lossy().to_lowercase(); + // `.zst` and `.zstd` are both in the wild. + let stem = path_str + .strip_suffix(".zst") + .or_else(|| path_str.strip_suffix(".zstd")) + .unwrap_or(path_str.as_str()); + stem.ends_with(".ndjson") } /// Streams raw `SerSpeclibElement`s out of a native timsseek library file. /// /// The native path builds the columnar arena directly from these elements (see -/// `Speclib::from_file_with_format`), so the reader stays at the serializable +/// `Speclib::from_native_file`), so the reader stays at the serializable /// element and does not eagerly build per-row scoring items. +/// +/// The payload is always NDJSON; zstd only adds a decoder underneath, so the +/// boxing is over the byte source rather than over the line parser. pub struct SpeclibReader<'a> { - inner: Box> + Send + 'a>, + reader: Box, } +/// Leading bytes of a zstd frame. +const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD]; + impl<'a> SpeclibReader<'a> { - pub fn new( - reader: R, - format: SpeclibFormat, - ) -> Result { - let inner: Box> + Send> = - match format { - SpeclibFormat::NdJson => Box::new(NdJsonReader::new(BufReader::new(reader))), - SpeclibFormat::NdJsonZstd => { - let decoder = zstd::Decoder::new(reader).map_err(|e| { - LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error creating ZSTD decoder", - } - })?; - Box::new(NdJsonReader::new(BufReader::new(decoder))) - } - SpeclibFormat::MessagePack => Box::new(MessagePackReader::new(reader)), - SpeclibFormat::MessagePackZstd => { - let decoder = zstd::Decoder::new(reader).map_err(|e| { - LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error creating ZSTD decoder", - } - })?; - Box::new(MessagePackReader::new(decoder)) - } - }; + /// Compression is detected from the first four bytes, not the file name, so + /// a mislabelled `.ndjson` that is really zstd (or the reverse) still reads. + pub fn new(reader: R) -> Result { + let mut buffered = BufReader::new(reader); + let compressed = buffered + .fill_buf() + .map_err(|source| LibraryReadingError::FileReadingError { + source, + context: "Error reading the start of the speclib", + path: None, + })? + .starts_with(&ZSTD_MAGIC); + + let reader: Box = if compressed { + let decoder = zstd::Decoder::with_buffer(buffered) + .map_err(|source| LibraryReadingError::Decompression { source })?; + Box::new(BufReader::new(decoder)) + } else { + Box::new(buffered) + }; - Ok(SpeclibReader { inner }) + Ok(SpeclibReader { reader }) } } @@ -378,91 +297,70 @@ impl Iterator for SpeclibReader<'_> { type Item = Result; fn next(&mut self) -> Option { - self.inner.next() - } -} - -struct NdJsonReader { - reader: R, -} - -impl NdJsonReader { - fn new(reader: R) -> Self { - Self { reader } - } -} - -impl Iterator for NdJsonReader { - type Item = Result; - - fn next(&mut self) -> Option { - let mut line = String::new(); - match self.reader.read_line(&mut line) { - Ok(0) => None, // EOF - Ok(_) => { - if line.trim().is_empty() { - return self.next(); // Skip empty lines - } - - let elem: SerSpeclibElement = match serde_json::from_str(&line) { - Ok(x) => x, - Err(e) => { - return Some(Err(LibraryReadingError::SpeclibParsingError { + // Looping rather than recursing: a file with a long run of blank lines + // would otherwise recurse once per line and blow the stack. + loop { + let mut line = String::new(); + match self.reader.read_line(&mut line) { + Ok(0) => return None, // EOF + Ok(_) => { + if line.trim().is_empty() { + continue; + } + return Some(serde_json::from_str(&line).map_err(|e| { + LibraryReadingError::SpeclibParsingError { source: e, context: "Error parsing NDJSON line", - })); - } - }; - - Some(Ok(elem)) + } + })); + } + Err(e) => { + return Some(Err(LibraryReadingError::FileReadingError { + source: e, + context: "Error reading line", + path: None, + })); + } } - Err(e) => Some(Err(LibraryReadingError::FileReadingError { - source: e, - context: "Error reading line", - path: PathBuf::new(), - })), } } } -struct MessagePackReader { - deserializer: rmp_serde::Deserializer>, +/// Writes a native timsseek library: one JSON object per line, zstd-wrapped. +/// +/// The exact inverse of [`SpeclibReader`], so +/// what `speclib_build_cli` emits is what `Speclib::from_file` reads back. +pub struct SpeclibWriter { + encoder: zstd::Encoder<'static, W>, } -impl MessagePackReader { - fn new(reader: R) -> Self { - Self { - deserializer: rmp_serde::Deserializer::new(reader), - } +impl SpeclibWriter { + pub fn new_ndjson_zstd(writer: W) -> Result { + Ok(Self { + encoder: zstd::Encoder::new(writer, 3)?, + }) } -} - -impl Iterator for MessagePackReader { - type Item = Result; - fn next(&mut self) -> Option { - use serde::Deserialize; + pub fn append(&mut self, elem: &SerSpeclibElement) -> Result<(), LibraryReadingError> { + let io_err = |e: std::io::Error| LibraryReadingError::FileReadingError { + source: e, + context: "Error writing NDJSON", + path: None, + }; + serde_json::to_writer(&mut self.encoder, elem).map_err(|e| { + LibraryReadingError::SpeclibParsingError { + source: e, + context: "Error serializing NDJSON line", + } + })?; + // The newline is the record separator; without it the whole library is + // one unreadable line. + self.encoder.write_all(b"\n").map_err(io_err) + } - match SerSpeclibElement::deserialize(&mut self.deserializer) { - Ok(elem) => Some(Ok(elem)), - Err(rmp_serde::decode::Error::InvalidMarkerRead(ref io_err)) - if io_err.kind() == std::io::ErrorKind::UnexpectedEof => - { - None - } // EOF - Err(rmp_serde::decode::Error::InvalidDataRead(ref io_err)) - if io_err.kind() == std::io::ErrorKind::UnexpectedEof => - { - None - } // EOF - Err(e) => Some(Err(LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error reading MessagePack", - })), - } + /// Flushes the zstd frame. Skipping this truncates the library. + pub fn finish(self) -> Result { + self.encoder.finish() } } @@ -489,17 +387,26 @@ impl Speclib { path: &Path, decoy_policy: crate::models::DecoyPolicy, ) -> Result { - // Native timsseek formats are matched by EXTENSION ONLY: a native - // extension commits to the native reader and surfaces its error. A - // `.speclib` matches no native extension and falls through to the - // bridge -> timsquery registry -> binary reader. - if let Some(format) = SpeclibFormat::detect_from_extension(path) { - tracing::info!( - "Loading native speclib format ({:?}) from {}", - format, - path.display() - ); - return Self::from_file_with_format(path, format, decoy_policy); + // msgpack was removed in the mzcore migration. Without this arm the + // file reaches the JSON reader and is reported as invalid UTF-8, which + // names neither the real problem nor the fix. + let path_str = path.to_string_lossy().to_lowercase(); + if path_str.contains(".msgpack") { + return Err(LibraryReadingError::UnsupportedFormat { + message: format!( + "{}: msgpack speclibs are no longer supported. Rebuild with \ + speclib_build_cli, which now emits .ndjson.zst", + path.display() + ), + }); + } + + // See `is_native_extension`: a native extension commits to the native + // reader and surfaces its error. A `.speclib` matches no native + // extension and falls through to the bridge -> timsquery registry. + if is_native_extension(path) { + tracing::info!("Loading native speclib from {}", path.display()); + return Self::from_native_file(path, decoy_policy); } // Terminal source: bridge to the timsquery reader registry (DIA-NN @@ -523,19 +430,20 @@ impl Speclib { Ok(lib) } - pub fn from_file_with_format( + /// Load a native timsseek library (NDJSON, optionally zstd-wrapped — + /// [`SpeclibReader`] sniffs which). + fn from_native_file( path: &Path, - format: SpeclibFormat, decoy_policy: crate::models::DecoyPolicy, ) -> Result { let file = std::fs::File::open(path).map_err(|e| LibraryReadingError::FileReadingError { source: e, context: "Error opening speclib file", - path: PathBuf::from(path), + path: Some(PathBuf::from(path)), })?; - let reader = SpeclibReader::new(file, format)?; + let reader = SpeclibReader::new(file)?; // Build the columnar arena directly from the streamed elements (same // lazy shape as the `.speclib` path), instead of collecting per-row @@ -574,7 +482,7 @@ impl Speclib { // The native format ships a single (modified) sequence; strip mod // annotations for the composition-isotope path. let modified = &elem.precursor.sequence; - let stripped = strip_mods(modified); + let stripped = timsquery::utils::sequence::strip_mods(modified); geom.push_row( eg.precursor_mz, elem.precursor.charge, @@ -605,47 +513,6 @@ impl Speclib { ); } } - -pub struct SpeclibWriter { - inner: SpeclibWriterInner, -} - -enum SpeclibWriterInner { - MsgpackZstd(zstd::Encoder<'static, W>), -} - -impl SpeclibWriter { - pub fn new_msgpack_zstd(writer: W) -> Result { - let encoder = zstd::Encoder::new(writer, 3)?; - Ok(Self { - inner: SpeclibWriterInner::MsgpackZstd(encoder), - }) - } - - pub fn append(&mut self, elem: &SerSpeclibElement) -> Result<(), LibraryReadingError> { - match &mut self.inner { - SpeclibWriterInner::MsgpackZstd(encoder) => { - rmp_serde::encode::write(encoder, elem).map_err(|e| { - LibraryReadingError::SpeclibParsingError { - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - )), - context: "Error writing MessagePack", - } - })?; - } - } - Ok(()) - } - - pub fn finish(self) -> Result { - match self.inner { - SpeclibWriterInner::MsgpackZstd(encoder) => encoder.finish(), - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -654,41 +521,164 @@ mod tests { RefQuery, }; + /// One native-format element. Fragment m/z are positional stand-ins; no + /// test here asserts on them, only on labels, intensities and the row + /// metadata. + fn element( + sequence: &str, + decoy: bool, + precursor_mz: f64, + labels: &[&str], + intensities: &[f32], + ) -> SerSpeclibElement { + assert_eq!(labels.len(), intensities.len()); + SerSpeclibElement::new( + PrecursorEntry::new(sequence.to_string(), 2, decoy), + ReferenceEG::new( + precursor_mz, + (0..labels.len()) + .map(|i| 300.0 + 100.0 * i as f64) + .collect(), + labels + .iter() + .map(|l| IonAnnot::try_from(*l).expect("valid annotation")) + .collect(), + intensities.to_vec(), + 0.75, + 120.0, + ), + ) + } + + /// `speclib_build_cli` writes with [`SpeclibWriter`] and timsseek reads + /// with [`SpeclibReader`]; nothing else checks that the two agree, and a + /// mismatch only shows up as an unreadable library at the end of a long + /// Koina run. + #[test] + fn writer_output_reads_back_through_the_reader() { + let record = element("PEPTIDEK", false, 450.5, &["y1", "b3^2"], &[0.9, 0.4]); + + let mut writer = SpeclibWriter::new_ndjson_zstd(Vec::new()).expect("encoder"); + writer.append(&record).expect("append"); + // Twice, so the newline separator is exercised rather than the file + // happening to hold one record. + writer.append(&record).expect("append"); + let bytes = writer.finish().expect("finish"); + + let read: Vec = SpeclibReader::new(bytes.as_slice()) + .expect("reader") + .collect::>() + .expect("every record must parse"); + + assert_eq!(read.len(), 2); + assert_eq!(read[0].precursor.sequence, "PEPTIDEK"); + assert_eq!( + read[0].elution_group.fragment_mzs, + element("PEPTIDEK", false, 450.5, &["y1", "b3^2"], &[0.9, 0.4]) + .elution_group + .fragment_mzs, + "fragment m/z must survive the round trip" + ); + let labels: Vec = read[0] + .elution_group + .fragment_labels + .iter() + .map(|l| l.to_string()) + .collect(); + assert_eq!(labels, vec!["y1".to_string(), "b3^2".to_string()]); + } + #[test] - fn test_detect_native_format_by_extension() { + fn native_extensions_route_to_the_native_reader() { use std::path::Path; - // Both .zst and .zstd must map to the native zstd readers. - for ext in ["lib.msgpack.zst", "lib.msgpack.zstd"] { - assert!(matches!( - SpeclibFormat::detect_from_extension(Path::new(ext)), - Some(SpeclibFormat::MessagePackZstd) - )); + for ext in [ + "lib.ndjson", + "lib.ndjson.zst", + "lib.ndjson.zstd", + "LIB.NDJSON", + ] { + assert!(is_native_extension(Path::new(ext)), "{ext} is native"); } - for ext in ["lib.ndjson.zst", "lib.ndjson.zstd"] { - assert!(matches!( - SpeclibFormat::detect_from_extension(Path::new(ext)), - Some(SpeclibFormat::NdJsonZstd) - )); + // Everything else routes to the timsquery bridge, which sniffs by + // content. Claiming one of these here would bypass that. + for ext in ["lib.speclib", "lib.mzSpecLib.txt", "lib.tsv", "lib.zst"] { + assert!( + !is_native_extension(Path::new(ext)), + "{ext} must not be claimed as a native format" + ); } - assert!(matches!( - SpeclibFormat::detect_from_extension(Path::new("lib.msgpack")), - Some(SpeclibFormat::MessagePack) - )); - assert!(matches!( - SpeclibFormat::detect_from_extension(Path::new("lib.ndjson")), - Some(SpeclibFormat::NdJson) - )); - // A .speclib must NOT be claimed as native -> routes to the bridge. - assert!(SpeclibFormat::detect_from_extension(Path::new("lib.speclib")).is_none()); - assert!(SpeclibFormat::detect_from_extension(Path::new("lib.tsv")).is_none()); } - /// `Speclib` is now a type alias for `ReferenceLibrary` (Task 9 collapsed - /// the enum), so a loaded library is already the lazy arena. This identity - /// helper is kept so the fixture assertions below read as - /// "get the arena" without churning every call site. - fn expect_lazy(speclib: &Speclib) -> &ReferenceLibrary { - speclib + /// Compression is decided by the magic number, so the two encodings are + /// interchangeable regardless of what the file is called. + #[test] + fn the_reader_sniffs_zstd_rather_than_trusting_the_name() { + let plain = b"{\"precursor\":{\"sequence\":\"PEPTIDEK\",\"charge\":2,\"decoy\":false,\ + \"decoy_group\":0},\"elution_group\":{\"id\":0,\"precursor_mz\":500.0,\ + \"precursor_labels\":[],\"fragment_mzs\":[175.1],\ + \"fragment_labels\":[\"y1\"],\"precursor_intensities\":[],\ + \"fragment_intensities\":[1.0],\"mobility_ook0\":0.9,\ + \"rt_seconds\":10.0}}\n"; + + let from_plain: Vec = SpeclibReader::new(&plain[..]) + .expect("uncompressed reader") + .collect::>() + .expect("plain NDJSON parses"); + + let compressed = zstd::encode_all(&plain[..], 3).expect("encode"); + assert!(compressed.starts_with(&ZSTD_MAGIC)); + let from_zstd: Vec = SpeclibReader::new(compressed.as_slice()) + .expect("compressed reader") + .collect::>() + .expect("zstd NDJSON parses"); + + assert_eq!(from_plain.len(), 1); + assert_eq!( + from_plain[0].precursor.sequence, + from_zstd[0].precursor.sequence + ); + } + + /// A leftover `.msgpack.zst` must say what happened, not "invalid UTF-8". + #[test] + fn msgpack_libraries_report_the_format_removal() { + let err = Speclib::from_file( + Path::new("/nonexistent/lib.msgpack.zst"), + crate::models::DecoyPolicy::default(), + ) + .expect_err("msgpack is no longer supported"); + match err { + LibraryReadingError::UnsupportedFormat { message } => { + assert!(message.contains("msgpack"), "{message}"); + assert!(message.contains("speclib_build_cli"), "{message}"); + } + other => panic!("expected UnsupportedFormat, got {other:?}"), + } + } + + /// An NDJSON library written to a temp file that is removed when the + /// returned handle drops — including on panic, unlike an explicit + /// `remove_file` after the assertions. + fn write_ndjson_fixture(ndjson: &str) -> tempfile::NamedTempFile { + use std::io::Write as _; + let mut f = tempfile::Builder::new() + .suffix(".ndjson") + .tempfile() + .expect("tempfile"); + f.write_all(ndjson.as_bytes()).expect("write fixture"); + f.flush().expect("flush"); + f + } + + /// A reader fixture from the sibling `timsquery` crate's test data. + fn fixture(dir: &str, name: &str) -> std::path::PathBuf { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crate dir has a parent") + .join("timsquery") + .join("tests") + .join(dir) + .join(name) } #[test] @@ -698,13 +688,7 @@ mod tests { // Use the test file from timsquery tests // Note: sample_lib.tsv is in Skyline format and won't load as DIA-NN // So we test with sample_lib.txt which is in DIA-NN TSV format - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); assert!( test_file.exists(), @@ -723,7 +707,7 @@ mod tests { "Expected 6 entries (2 targets + 4 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; // Verify first target entry structure (variant 0 == target) let first_target = lib @@ -750,13 +734,7 @@ mod tests { #[test] fn test_diann_tsv_parsable_gate() { - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); @@ -771,13 +749,7 @@ mod tests { fn test_load_skyline_csv_library() { use timsquery::traits::QueryGeom; - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("skyline_io_files") - .join("sample_transition_list.csv"); + let test_file = fixture("skyline_io_files", "sample_transition_list.csv"); assert!( test_file.exists(), @@ -799,7 +771,7 @@ mod tests { "Expected 42 entries (14 targets + 28 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; let n_rows = lib.iter().filter(|q| q.geom().variant() == 0).count(); let n_decoys = lib.iter().filter(|q| q.geom().variant() != 0).count(); assert_eq!(n_rows, 14, "Should have 14 targets"); @@ -821,13 +793,7 @@ mod tests { #[test] fn test_load_diann_txt_library() { - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); assert!( test_file.exists(), @@ -846,7 +812,7 @@ mod tests { "Expected 6 entries (2 targets + 4 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; let n_rows = lib.iter().filter(|q| q.geom().variant() == 0).count(); let n_decoys = lib.iter().filter(|q| q.geom().variant() != 0).count(); @@ -856,13 +822,7 @@ mod tests { #[test] fn test_load_diann_parquet_library() { - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_pq_speclib.parquet"); + let test_file = fixture("diann_io_files", "sample_pq_speclib.parquet"); assert!( test_file.exists(), @@ -881,7 +841,7 @@ mod tests { "Expected 9 entries (3 targets + 6 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; let n_rows = lib.iter().filter(|q| q.geom().variant() == 0).count(); let n_decoys = lib.iter().filter(|q| q.geom().variant() != 0).count(); @@ -901,18 +861,12 @@ mod tests { #[test] fn test_isotope_envelope_calculation() { // Use the DIA-NN TSV test file - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); - let lib = expect_lazy(&speclib); + let lib = &speclib; // Check that isotope intensities are normalized (M0 should be 1.0), // for every flat entry (targets AND decoy variants — the envelope is @@ -938,13 +892,7 @@ mod tests { #[test] fn test_decoy_generation_for_library_without_decoys() { - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); @@ -957,7 +905,7 @@ mod tests { "Should have 6 entries (2 targets + 4 decoys)" ); - let lib = expect_lazy(&speclib); + let lib = &speclib; let n_rows = lib.iter().filter(|q| q.geom().variant() == 0).count(); let n_decoys = lib.iter().filter(|q| q.geom().variant() != 0).count(); @@ -986,18 +934,12 @@ mod tests { fn test_mass_shift_decoys() { use timsquery::traits::QueryGeom; - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); - let lib = expect_lazy(&speclib); + let lib = &speclib; // Unified CH2 offset (see `map_decoy_strategy`), replacing the old // 12.0 (materialized `IfMissing`) / 14.0 (materialized `Force`) split. @@ -1046,18 +988,12 @@ mod tests { fn test_fragment_intensities_preserved() { use timsquery::traits::QueryGeom; - let test_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("timsquery") - .join("tests") - .join("diann_io_files") - .join("sample_lib.txt"); + let test_file = fixture("diann_io_files", "sample_lib.txt"); let speclib = Speclib::from_file(&test_file, crate::models::DecoyPolicy::default()) .expect("Failed to load DIA-NN TSV library"); - let lib = expect_lazy(&speclib); + let lib = &speclib; for q in lib.iter() { let fragments: Vec<_> = q.iter_expected_fragments().collect(); assert_eq!( @@ -1071,22 +1007,6 @@ mod tests { } } - #[test] - fn test_speclib_writer_roundtrip() { - let elem = SerSpeclibElement::sample(); - let mut buf = Vec::new(); - { - let mut writer = SpeclibWriter::new_msgpack_zstd(&mut buf).unwrap(); - writer.append(&elem).unwrap(); - writer.append(&elem).unwrap(); - writer.finish().unwrap(); - } - let reader = - SpeclibReader::new(std::io::Cursor::new(&buf), SpeclibFormat::MessagePackZstd).unwrap(); - let items: Vec<_> = reader.collect::, _>>().unwrap(); - assert_eq!(items.len(), 2); - } - /// End-to-end `Speclib::from_file` over the real DIA-NN HeLa `.speclib` /// fixture (the actual workload path). Proves: the arena narrows to a lazy /// library with targets, variant-0 is a target, and the intensity sidecar @@ -1104,7 +1024,7 @@ mod tests { let speclib = Speclib::from_file(path, crate::models::DecoyPolicy::default()) .expect("from_file should load the .speclib fixture"); - let lib = expect_lazy(&speclib); + let lib = &speclib; assert!(!lib.is_empty(), "library should have entries"); let first = lib.item_at(0); @@ -1120,48 +1040,16 @@ mod tests { /// Native `SerSpeclibElement` reader (ndjson) builds the lazy arena /// directly. The fixture ships one target + one stored decoy, so the - /// Task-4 seal gate downgrades `LazyMassShift -> Passthrough`: the arena is + /// seal gate downgrades `LazyMassShift -> Passthrough`: the arena is /// 1:1 with the stored rows (no synthetic mass-shift expansion). Proves the /// native path produces a lazy `ReferenceLibrary` with the right length, target/ /// decoy flags, and per-fragment reference intensities. #[test] - fn from_file_with_format_native_ndjson_builds_lazy_arena() { + fn native_ndjson_load_builds_lazy_arena() { use crate::data_sources::reference_library::ScoredIdentity; - let target = SerSpeclibElement::new( - PrecursorEntry::new("PEPTIDEK".to_string(), 2, false, 0), - ReferenceEG::new( - 0, - 500.0, - vec![0, 1, 2], - vec![300.0, 400.0], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - vec![1.0, 0.5, 0.2], - vec![0.8, 0.3], - 0.75, - 120.0, - ), - ); - let decoy = SerSpeclibElement::new( - PrecursorEntry::new("KEDITPEP".to_string(), 2, true, 0), - ReferenceEG::new( - 1, - 500.0, - vec![0, 1, 2], - vec![300.0, 400.0], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - vec![1.0, 0.5, 0.2], - vec![0.6, 0.4], - 0.75, - 120.0, - ), - ); + let target = element("PEPTIDEK", false, 500.0, &["y1", "y2"], &[0.8, 0.3]); + let decoy = element("KEDITPEP", true, 500.0, &["y1", "y2"], &[0.6, 0.4]); let mut ndjson = String::new(); ndjson.push_str(&serde_json::to_string(&target).unwrap()); @@ -1169,17 +1057,13 @@ mod tests { ndjson.push_str(&serde_json::to_string(&decoy).unwrap()); ndjson.push('\n'); - let path = std::env::temp_dir().join(format!( - "timsseek_native_fixture_{}.ndjson", - std::process::id() - )); - std::fs::write(&path, ndjson).unwrap(); + let file = write_ndjson_fixture(&ndjson); + let path = file.path(); - let speclib = Speclib::from_file(&path, crate::models::DecoyPolicy::default()) + let speclib = Speclib::from_file(path, crate::models::DecoyPolicy::default()) .expect("native ndjson should load"); - std::fs::remove_file(&path).ok(); - let lib = expect_lazy(&speclib); + let lib = &speclib; // Ships a decoy -> Passthrough -> 1 variant/row -> flat len == n_rows. assert_eq!(lib.geom.variants_per_row(), 1, "downgraded to Passthrough"); assert_eq!(lib.len(), 2, "one target + one stored decoy, 1:1"); @@ -1199,48 +1083,16 @@ mod tests { /// `parse_sequence(normalize_to_proforma(..))`, sequence-derived features are /// disabled library-wide (`SeqFeatureState::Unavailable`). Here one target /// parses (`PEPTIDEK`) and one is poisoned (`GARBAGE!!!`: the `!` bytes are - /// rejected by both the fast byte-walk parser and the rustyms fallback), so + /// rejected by both the fast byte-walk parser and the mzcore fallback), so /// the gate must report `!parsable_sequences()`. This is the inverse of /// `test_diann_tsv_parsable_gate`, and the only test of the OFF branch after - /// the AOS `test_parse_gate_off_on_poisoned_row` was removed in Task 9. + /// the materialized `test_parse_gate_off_on_poisoned_row` was removed. #[test] fn from_file_native_ndjson_poisoned_row_disables_sequence_features() { - let good = SerSpeclibElement::new( - PrecursorEntry::new("PEPTIDEK".to_string(), 2, false, 0), - ReferenceEG::new( - 0, - 500.0, - vec![0, 1, 2], - vec![300.0, 400.0], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - vec![1.0, 0.5, 0.2], - vec![0.8, 0.3], - 0.75, - 120.0, - ), - ); + let good = element("PEPTIDEK", false, 500.0, &["y1", "y2"], &[0.8, 0.3]); // Unparseable modified sequence: `!` is rejected by parse_sequence_fast - // (`_ => return None`) and by the rustyms pro_forma fallback. - let poisoned = SerSpeclibElement::new( - PrecursorEntry::new("GARBAGE!!!".to_string(), 2, false, 1), - ReferenceEG::new( - 1, - 600.0, - vec![0, 1, 2], - vec![300.0, 400.0], - vec![ - IonAnnot::try_from("y1").unwrap(), - IonAnnot::try_from("y2").unwrap(), - ], - vec![1.0, 0.5, 0.2], - vec![0.7, 0.4], - 0.75, - 120.0, - ), - ); + // (`_ => return None`) and by the mzcore pro_forma fallback. + let poisoned = element("GARBAGE!!!", false, 600.0, &["y1", "y2"], &[0.7, 0.4]); let mut ndjson = String::new(); ndjson.push_str(&serde_json::to_string(&good).unwrap()); @@ -1248,15 +1100,11 @@ mod tests { ndjson.push_str(&serde_json::to_string(&poisoned).unwrap()); ndjson.push('\n'); - let path = std::env::temp_dir().join(format!( - "timsseek_poisoned_fixture_{}.ndjson", - std::process::id() - )); - std::fs::write(&path, ndjson).unwrap(); + let file = write_ndjson_fixture(&ndjson); + let path = file.path(); - let speclib = Speclib::from_file(&path, crate::models::DecoyPolicy::default()) + let speclib = Speclib::from_file(path, crate::models::DecoyPolicy::default()) .expect("native ndjson should load even with an unparseable sequence"); - std::fs::remove_file(&path).ok(); // The poisoned row flips the whole-library gate OFF. assert!( diff --git a/rust/timsseek/src/errors.rs b/rust/timsseek/src/errors.rs index d37d2311..f638e72d 100644 --- a/rust/timsseek/src/errors.rs +++ b/rust/timsseek/src/errors.rs @@ -39,7 +39,14 @@ pub enum LibraryReadingError { FileReadingError { source: std::io::Error, context: &'static str, - path: PathBuf, + /// `None` where the failure happens on an already-open stream, which + /// no longer knows where it came from. + path: Option, + }, + /// The zstd frame could not be opened or decoded. Distinct from + /// [`Self::SpeclibParsingError`]: the bytes never became text. + Decompression { + source: std::io::Error, }, TimsQueryLibraryError { source: timsquery::serde::LibraryReadingError, diff --git a/rust/timsseek/src/fragment_mass/averagine.rs b/rust/timsseek/src/fragment_mass/averagine.rs index 53e6fb0d..e67032cc 100644 --- a/rust/timsseek/src/fragment_mass/averagine.rs +++ b/rust/timsseek/src/fragment_mass/averagine.rs @@ -74,11 +74,10 @@ mod tests { #[test] fn or_averagine_falls_back_on_nonstandard() { - // `B` (Asx) is genuinely ambiguous between Asp/Asn in rustyms and - // resolves to more than one formula, which is the real trigger for - // the rustyms-backed count path to error today. (`X` was tried first - // but rustyms resolves it to a defined zero-C/S formula rather than - // erroring, so it does not exercise the fallback.) + // `B` (Asx) is ambiguous between Asp/Asn in mzcore and resolves to + // more than one formula, which is what makes the mzcore-backed count + // path error. `X` does NOT work here: mzcore gives it a defined + // zero-C/S formula, so it never reaches the fallback. let (src, env) = isotope_dist_or_averagine("PEPBK", 600.0); assert_eq!(src, IsotopeSource::Averagine); let max = env.iter().copied().fold(f32::MIN, f32::max); diff --git a/rust/timsseek/src/fragment_mass/elution_group_converter.rs b/rust/timsseek/src/fragment_mass/elution_group_converter.rs index 6ff9a0d9..2a26d2e3 100644 --- a/rust/timsseek/src/fragment_mass/elution_group_converter.rs +++ b/rust/timsseek/src/fragment_mass/elution_group_converter.rs @@ -1,8 +1,8 @@ use crate::isotopes::peptide_isotopes; -use rustyms::prelude::{ +use mzcore::prelude::{ + AmbiguousMolecule, Element, MolecularFormula, - Peptidoform, }; /// Super simple 1/k0 prediction. @@ -54,11 +54,11 @@ fn count_carbon_sulphur(form: &MolecularFormula) -> (u16, u16) { } /// In-chain (C, S) atom counts per standard residue, indexed by `byte - b'A'`. -/// `None` = a non-standard code (B/J/O/U/X/Z) — defer to the rustyms path. +/// `None` = a non-standard code (B/J/O/U/X/Z) — defer to the mzcore path. /// /// A residue contributes the same carbon/sulfur as its free amino acid: forming /// a peptide bond removes one water per bond and the terminal water carries -/// neither C nor S, so a bare-sequence sum equals rustyms' formula exactly. +/// neither C nor S, so a bare-sequence sum equals mzcore's formula exactly. const RESIDUE_CS: [Option<(u16, u16)>; 26] = { // Alphabet offset of an uppercase residue byte (as a fn so `b'A'` maps to 0 // without a literal `b'A' - b'A'`, which clippy's eq_op denies). @@ -90,7 +90,7 @@ const RESIDUE_CS: [Option<(u16, u16)>; 26] = { }; /// Fast (C, S) tally over a bare amino-acid sequence via [`RESIDUE_CS`]. -/// `None` on an empty string or any non-standard residue, forcing the rustyms +/// `None` on an empty string or any non-standard residue, forcing the mzcore /// fallback so behavior (including the error path) is preserved. fn count_cs_fast(sequence: &str) -> Option<(u16, u16)> { if sequence.is_empty() { @@ -108,25 +108,18 @@ fn count_cs_fast(sequence: &str) -> Option<(u16, u16)> { } /// (C, S) counts for `sequence` (a bare, mod-stripped peptide on the hot path). -/// Tries the allocation-free table first; defers to the rustyms formula path for +/// Tries the allocation-free table first; defers to the mzcore formula path for /// empty / non-standard input, which stays the authority. pub fn count_carbon_sulphur_in_sequence(sequence: &str) -> Result<(u16, u16), String> { if let Some(cs) = count_cs_fast(sequence) { return Ok(cs); } - count_carbon_sulphur_in_sequence_rustyms(sequence) + count_carbon_sulphur_in_sequence_mzcore(sequence) } -fn count_carbon_sulphur_in_sequence_rustyms(sequence: &str) -> Result<(u16, u16), String> { - let peptide = match Peptidoform::pro_forma(sequence, None) { - Ok(pep) => pep, - Err(e) => { - return Err(format!( - "Error parsing peptide sequence {}: {:?}", - sequence, e - )); - } - }; +fn count_carbon_sulphur_in_sequence_mzcore(sequence: &str) -> Result<(u16, u16), String> { + let peptide = crate::models::sequence::parse_proforma(sequence) + .map_err(|e| format!("Error parsing peptide sequence {sequence}: {e}"))?; let peptide = match peptide.as_linear() { Some(pep) => pep, None => return Err("Peptide is not linear.".to_string()), @@ -151,22 +144,22 @@ mod tests { use super::*; #[test] - fn cs_table_matches_rustyms_per_residue() { - // Every standard residue: the table must equal rustyms' formula count. + fn cs_table_matches_mzcore_per_residue() { + // Every standard residue: the table must equal mzcore's formula count. for &aa in b"ACDEFGHIKLMNPQRSTVWY" { let seq = String::from_utf8(vec![aa, aa, aa]).unwrap(); // e.g. "AAA" let fast = count_cs_fast(&seq).expect("standard residue in table"); - let slow = count_carbon_sulphur_in_sequence_rustyms(&seq) - .unwrap_or_else(|e| panic!("rustyms failed on {seq}: {e}")); + let slow = count_carbon_sulphur_in_sequence_mzcore(&seq) + .unwrap_or_else(|e| panic!("mzcore failed on {seq}: {e}")); assert_eq!(fast, slow, "C/S mismatch for {seq}"); } } #[test] - fn cs_table_matches_rustyms_on_peptides() { + fn cs_table_matches_mzcore_on_peptides() { for seq in ["AAAGAAATHLEVAR", "LEGNSPQGSNQGVK", "MCMCMCK", "PEPTIDEK"] { let fast = count_cs_fast(seq).expect("standard peptide"); - let slow = count_carbon_sulphur_in_sequence_rustyms(seq).unwrap(); + let slow = count_carbon_sulphur_in_sequence_mzcore(seq).unwrap(); assert_eq!(fast, slow, "C/S mismatch for {seq}"); } } diff --git a/rust/timsseek/src/lib.rs b/rust/timsseek/src/lib.rs index 2ea374f6..88908d39 100644 --- a/rust/timsseek/src/lib.rs +++ b/rust/timsseek/src/lib.rs @@ -33,6 +33,5 @@ pub use scoring::{ pub use timsquery::ion::{ IonAnnot, IonParsingError, - IonSeriesTerminality, }; pub use traits::ScorerQueriable; diff --git a/rust/timsseek/src/models/query_item.rs b/rust/timsseek/src/models/query_item.rs index 4f6824d9..60291d6a 100644 --- a/rust/timsseek/src/models/query_item.rs +++ b/rust/timsseek/src/models/query_item.rs @@ -76,7 +76,7 @@ impl Default for ExpectedIntensities { impl ExpectedIntensities { /// Construct from fragment and precursor pair iterators, erroring on any /// repeated key in either input. Preferred entry point for all library - /// load paths (speclib ndjson/msgpack, DIA-NN/Spectronaut/Skyline TSV). + /// load paths (speclib ndjson, mzSpecLib, DIA-NN/Spectronaut/Skyline TSV). pub fn try_from_pairs(frags: FI, precs: PI) -> Result where FI: IntoIterator, diff --git a/rust/timsseek/src/models/sequence.rs b/rust/timsseek/src/models/sequence.rs index 827f3209..b51f1310 100644 --- a/rust/timsseek/src/models/sequence.rs +++ b/rust/timsseek/src/models/sequence.rs @@ -5,7 +5,67 @@ use crate::models::decoy::DecoyMarking; use serde::Serialize; use smallvec::SmallVec; -use std::sync::Arc; +use std::sync::{ + Arc, + LazyLock, +}; + +/// Modification ontologies for ProForma parsing: everything mzcore ships +/// except GNOme, which is 26.4 MB of the 27.8 MB total and is decoded on first +/// use. +/// +/// Dropping an ontology normally costs you the sequences that reference it, but +/// not here. A GNO-accession glycopeptide is already unusable: every mod goes +/// through [`modification_to_mod`], which returns `None` for anything +/// non-Unimod, and [`parse_sequence_mzcore`] propagates that `None` for the +/// whole peptide. So GNOme's only effect was to make such a peptide parse and +/// then be discarded one step later. +/// +/// The single behavioural difference is +/// [`count_carbon_sulphur_in_sequence`](crate::fragment_mass::elution_group_converter::count_carbon_sulphur_in_sequence): +/// a `[GNO:...]` sequence no longer yields a composition, so its isotope +/// envelope comes from averagine instead — the documented fallback, already +/// tallied as `n_averagine_fallback`. PSI-MOD, XL-MOD and RESID stay loaded so +/// that path is unchanged for them. +static ONTOLOGIES: LazyLock = LazyLock::new(|| { + let mut ontologies = mzcore::ontology::Ontologies::empty(); + *ontologies.unimod_mut() = mzcv::CVIndex::init_static(); + *ontologies.psimod_mut() = mzcv::CVIndex::init_static(); + *ontologies.xlmod_mut() = mzcv::CVIndex::init_static(); + *ontologies.resid_mut() = mzcv::CVIndex::init_static(); + ontologies +}); + +/// Parse a ProForma string against [`ONTOLOGIES`]. +/// +/// This is the fallback *past* the byte-walk fast path in [`parse_sequence`], +/// so a library whose sequences all match the fast grammar never decodes an +/// ontology at all. +/// +/// mzcore also returns non-fatal parse warnings alongside the peptidoform; none +/// of the callers can act on them, so they are dropped in one place rather than +/// at each site. +/// +/// The error is a rendered `String` rather than mzcore's own +/// `Vec>`. Returning the latter would be cheaper on +/// the path that discards it, but `BoxedError` comes from `context_error`, +/// which mzcore does not re-export — naming it means a second direct +/// dependency version-coupled to mzcore's, the same hazard documented on +/// `mzcv` in the workspace manifest. Not worth one allocation on an error +/// path. +pub fn parse_proforma( + sequence: &str, +) -> Result, String> { + mzcore::sequence::Peptidoform::pro_forma(sequence, &ONTOLOGIES) + .map(|(peptidoform, _warnings)| peptidoform) + .map_err(|errors| { + errors + .iter() + .map(|e| e.to_string()) + .collect::>() + .join("; ") + }) +} /// Amino acid stored as alphabet offset `c - b'A'` (0..=25). `u8::MAX` /// means "unrecognized / non-alpha". Unreachable slots in count buffers @@ -158,9 +218,9 @@ impl Serialize for Peptide { /// A hand-rolled byte walk handles the grammar `normalize_to_proforma` actually /// emits (bare residues, `[UNIMOD:n]`, `[+/-mass]`, N-/C-terminal forms). It /// returns `Some` ONLY for inputs it fully recognizes; anything else — named -/// mods, cross-links, unexpected bytes — yields `None` and defers to the rustyms +/// mods, cross-links, unexpected bytes — yields `None` and defers to the mzcore /// parser, which stays the authority for what is valid. So the fast path can -/// never accept something rustyms would reject, with one deliberate exception: +/// never accept something mzcore would reject, with one deliberate exception: /// it does not check that a `UNIMOD:n` id exists in the ontology (a /// syntactically valid id is accepted). Real DIA-NN output only carries real /// ids, so this never triggers in practice. @@ -168,15 +228,22 @@ pub fn parse_sequence(normalized: &str) -> Option { if let Some(parsed) = parse_sequence_fast(normalized) { return Some(parsed); } - parse_sequence_rustyms(normalized) + parse_sequence_mzcore(normalized) } /// Classify one bracket body (`UNIMOD:n` or a signed mass like `+15.995`) into a /// [`Mod`]. `None` for anything else — a named mod, an unsigned number, empty — -/// which forces the rustyms fallback in [`parse_sequence`]. +/// which forces the mzcore fallback in [`parse_sequence`]. fn classify_mod(body: &str) -> Option { let body = body.trim(); - if body.len() >= 7 && body[..7].eq_ignore_ascii_case("UNIMOD:") { + // Match on bytes: `body` is arbitrary text from the library, so `body[..7]` + // would panic on a multi-byte char straddling byte 7. Once the prefix + // matches it is ASCII, so byte 7 is a char boundary and `body[7..]` is safe. + if body + .as_bytes() + .get(..7) + .is_some_and(|p| p.eq_ignore_ascii_case(b"UNIMOD:")) + { return body[7..].trim().parse::().ok().map(Mod::Unimod); } match body.as_bytes().first() { @@ -186,7 +253,7 @@ fn classify_mod(body: &str) -> Option { } /// Byte-walk parser for the `normalize_to_proforma` output grammar. `None` means -/// "not recognized — defer to rustyms", never "definitively invalid" (that +/// "not recognized — defer to mzcore", never "definitively invalid" (that /// verdict is the fallback's). See [`parse_sequence`] for the contract. fn parse_sequence_fast(s: &str) -> Option { let b = s.as_bytes(); @@ -198,7 +265,7 @@ fn parse_sequence_fast(s: &str) -> Option { if b.first() == Some(&b'[') { let close = i + 1 + b[i + 1..].iter().position(|&c| c == b']')?; // A leading bracket not of the `[..]-` shape is something we do not - // model; let rustyms decide. + // model; let mzcore decide. if b.get(close + 1) != Some(&b'-') { return None; } @@ -240,7 +307,7 @@ fn parse_sequence_fast(s: &str) -> Option { }); i = close + 1; } - _ => return None, // anything unexpected -> rustyms fallback + _ => return None, // anything unexpected -> mzcore fallback } } @@ -250,14 +317,16 @@ fn parse_sequence_fast(s: &str) -> Option { Some(ParsedSequence { residues, mods }) } -/// The rustyms-backed parser. Authoritative fallback for [`parse_sequence`]: +/// The mzcore-backed parser. Authoritative fallback for [`parse_sequence`]: /// validates against the ontology, handles named mods, and rejects non-linear /// peptides. Off the hot path once the fast path covers the common grammar. -fn parse_sequence_rustyms(normalized: &str) -> Option { - use rustyms::prelude::IsAminoAcid; - use rustyms::sequence::Peptidoform; +/// +/// `pro_forma` also returns non-fatal parse warnings; they are dropped, since +/// this function's contract is a binary parsed/not-parsed verdict. +fn parse_sequence_mzcore(normalized: &str) -> Option { + use mzcore::prelude::IsAminoAcid; - let pf = Peptidoform::pro_forma(normalized, None).ok()?; + let pf = parse_proforma(normalized).ok()?; let linear = pf.into_linear()?; let mut residues: SmallVec<[AminoAcid; 32]> = SmallVec::new(); @@ -294,9 +363,9 @@ fn parse_sequence_rustyms(normalized: &str) -> Option { Some(ParsedSequence { residues, mods }) } -fn modification_to_mod(m: &rustyms::sequence::Modification) -> Option { - use rustyms::ontology::Ontology; - use rustyms::sequence::{ +fn modification_to_mod(m: &mzcore::sequence::Modification) -> Option { + use mzcore::ontology::Ontology; + use mzcore::sequence::{ Modification, SimpleModificationInner, }; @@ -305,12 +374,16 @@ fn modification_to_mod(m: &rustyms::sequence::Modification) -> Option { _ => return None, // Cross-link / ambiguous — out of v1 scope }; match simple.as_ref() { - SimpleModificationInner::Mass(mass) => Some(Mod::Mass(mass.value as f32)), + SimpleModificationInner::Mass(_tag, mass, _digits) => Some(Mod::Mass(mass.value as f32)), SimpleModificationInner::Database { id, .. } => { - if id.ontology == Ontology::Unimod { - Some(Mod::Unimod(id.id? as u16)) - } else { - None + if id.ontology != Ontology::Unimod { + return None; + } + // UNIMOD accessions are numeric; a non-numeric CURIE is not + // something `Mod::Unimod(u16)` can represent. + match id.id() { + mzcv::AccessionCode::Numeric(n) => u16::try_from(n).ok().map(Mod::Unimod), + _ => None, } } _ => None, @@ -385,7 +458,7 @@ fn convert_paren_unimod(s: &str) -> String { out } -/// Coerce DIA-NN / short-form modified-sequence strings into rustyms-parseable +/// Coerce DIA-NN / short-form modified-sequence strings into mzcore-parseable /// ProForma. Strips `_..._` wrapping used by DIA-NN, converts DIA-NN's /// parenthesised mods (`C(UniMod:4)`) to ProForma brackets (`C[UNIMOD:4]`), and /// normalizes UNIMOD tag casing (`[UniMod:`, `[Unimod:`, `[U:` → `[UNIMOD:`). @@ -623,7 +696,7 @@ mod tests { #[test] fn fast_path_takes_recognized_grammar() { // Bare and numeric-UNIMOD/mass inputs must be served by the fast path - // (never reach rustyms), else there's no speedup. + // (never reach mzcore), else there's no speedup. for s in [ "PEPTIDEK", "AAC[UNIMOD:4]DEK", @@ -642,7 +715,7 @@ mod tests { #[test] fn fast_path_defers_named_and_garbage() { // Named mods and unexpected bytes must defer (fast returns None) so the - // rustyms authority decides validity + resolves the name. + // mzcore authority decides validity + resolves the name. for s in [ "[Acetyl]-PEPTIDEK", "C[Carbamidomethyl (C)]PEPK", @@ -655,10 +728,115 @@ mod tests { } } + /// [`ontologies`] omits GNOme, which is 95% of what mzcore's own + /// `STATIC_ONTOLOGIES` loads. This pins the reasoning: every non-Unimod + /// mod already yields no parsed sequence, because `modification_to_mod` + /// returns `None` and `parse_sequence_mzcore` propagates it for the whole + /// peptide. Dropping GNOme moves where a `[GNO:...]` sequence fails, not + /// whether it fails. + #[test] + fn dropping_gnome_costs_no_sequence_that_was_usable() { + // Unimod, by name and by id, plus a bare mass: all still resolve. + for usable in [ + "PEPTIDEK", + "PEPTC[UNIMOD:4]IDEK", + "PEPTC[Carbamidomethyl]IDEK", + "PEPT[+79.966]IDEK", + ] { + assert!( + parse_sequence(usable).is_some(), + "{usable:?} must still parse" + ); + } + + // Ontologies still loaded: these reach mzcore, and are then rejected + // by `modification_to_mod` for not being Unimod. Kept loaded so the + // formula path (`count_carbon_sulphur_in_sequence`) still sees them. + for non_unimod in [ + "PEPTK[MOD:00046]IDEK", + "PEPTK[XLMOD:02001]IDEK", + "PEPTK[RESID:AA0038]IDEK", + ] { + assert!( + parse_proforma(non_unimod).is_ok(), + "{non_unimod:?} must still reach mzcore" + ); + assert!( + parse_sequence(non_unimod).is_none(), + "{non_unimod:?} yields no usable sequence either way" + ); + } + + // The only sequence whose parse verdict GNOme affected, and it was + // unusable either way. + assert!( + parse_sequence("PEPTN[GNO:G59626AS]IDEK").is_none(), + "a GNO glycopeptide was never usable" + ); + // A glycan *composition* needs no index, so it is unaffected. + assert!(parse_proforma("PEPTN[Glycan:HexNAc]IDEK").is_ok()); + } + + /// The fallback must not just accept a named mod — it must resolve it + /// through the UNIMOD ontology to the same numeric id the `[UNIMOD:n]` + /// spelling yields. This is the one behavior that has no fast-path + /// equivalent, so nothing else covers it. + #[test] + fn mzcore_fallback_resolves_named_mods_via_ontology() { + for (named, expected) in [ + ("[Acetyl]-PEPTIDEK", Mod::Unimod(1)), + ("PEPTC[Carbamidomethyl]IDEK", Mod::Unimod(4)), + ("PEPTM[Oxidation]IDEK", Mod::Unimod(35)), + ] { + let parsed = parse_sequence(named) + .unwrap_or_else(|| panic!("mzcore must resolve named mod in {named:?}")); + assert_eq!( + parsed.mods.len(), + 1, + "expected exactly one mod in {named:?}, got {:?}", + parsed.mods + ); + assert_eq!( + parsed.mods[0].kind, expected, + "ontology resolved {named:?} to the wrong id" + ); + } + } + + /// A named mod and its numeric spelling must land on the same + /// `ParsedSequence` — including residues, so the ontology path cannot + /// quietly disagree with the fast path about the peptide itself. + #[test] + fn named_and_numeric_mod_spellings_agree() { + let named = parse_sequence("PEPTC[Carbamidomethyl]IDEK").expect("named form parses"); + let numeric = parse_sequence("PEPTC[UNIMOD:4]IDEK").expect("numeric form parses"); + assert_eq!(named, numeric); + } + + #[test] + fn non_ascii_bracket_bodies_are_rejected_not_panicked_on() { + // Bracket bodies come straight from the library file, so they can hold + // any UTF-8. A verdict of `None` is fine; aborting the search is not. + for s in [ + "PEPT[abcdef√]IDEK", + "PEPT[√]IDEK", + "[abcdef√]-PEPTIDEK", + "PEPTIDEK-[abcdef√]", + "PEPT[unimod:√]IDEK", + ] { + assert!(parse_sequence_fast(s).is_none(), "{s:?} must defer"); + } + // The prefix match stays case-insensitive over the ASCII it accepts. + assert_eq!( + parse_sequence("PEPTC[unimod:4]IDEK"), + parse_sequence("PEPTC[UNIMOD:4]IDEK") + ); + } + #[test] - fn fast_matches_rustyms_on_recognized_grammar() { + fn fast_matches_mzcore_on_recognized_grammar() { // Differential test: wherever the fast path claims an input, it must - // produce the exact same ParsedSequence rustyms would. Guards against + // produce the exact same ParsedSequence mzcore would. Guards against // the fast path silently diverging on residue counts or mod mapping. let corpus = [ "PEPTIDEK", @@ -671,8 +849,8 @@ mod tests { ]; for s in corpus { if let Some(fast) = parse_sequence_fast(s) { - let slow = parse_sequence_rustyms(s) - .unwrap_or_else(|| panic!("rustyms must also parse {s:?}")); + let slow = parse_sequence_mzcore(s) + .unwrap_or_else(|| panic!("mzcore must also parse {s:?}")); assert_eq!(fast.residues, slow.residues, "residues mismatch for {s:?}"); assert_eq!( fast.mods.len(), diff --git a/rust/timsseek/src/scoring/pipeline.rs b/rust/timsseek/src/scoring/pipeline.rs index f974db55..0f0520ce 100644 --- a/rust/timsseek/src/scoring/pipeline.rs +++ b/rust/timsseek/src/scoring/pipeline.rs @@ -256,7 +256,7 @@ fn gate_expected_fragments(expected: &ExpectedIntensities) -> Result<( } /// Fill the per-worker scratch elution group in place from a `RefQuery` -/// flyweight (Task 9). `reset_from` copies the per-variant geometry — for a +/// flyweight. `reset_from` copies the per-variant geometry — for a /// decoy the fragment m/z values are ALREADY shifted by value, so no extra /// work is needed. It also sets the precursor labels to the isotope-envelope /// indices via the flyweight's `iter_precursors` (`0..n_isotopes`), which match @@ -692,7 +692,7 @@ impl Scorer { flat_range: std::ops::Range, calibration: &CalibrationResult, ) -> (Vec, ScoreTimings, SkipCounts) { - // Single columnar store (Task 9 deleted the materialized arm): the + // Single columnar store: the // flyweight is always a `RefQuery` from the arena, so the loop is // monomorphized over one concrete type — statically dispatched, no // per-item heap allocation on the scoring hot path. @@ -839,7 +839,7 @@ impl Scorer { config: &CalibrationConfig, timings: &mut PrescoreTimings, ) -> CalibrantHeap { - // Single columnar store (Task 9): iterate `RefQuery` flyweights from + // Single columnar store: iterate `RefQuery` flyweights from // the arena directly — monomorphized, no per-item heap alloc on the // prescore hot path (see `score_calibrated_batch`). self.prescore_batch_impl(|f| lib.item_at(f), flat_range, config, timings) diff --git a/rust/timsseek_cli/assets/default_config.toml b/rust/timsseek_cli/assets/default_config.toml index 3a09acdf..2fd3b94e 100644 --- a/rust/timsseek_cli/assets/default_config.toml +++ b/rust/timsseek_cli/assets/default_config.toml @@ -13,7 +13,7 @@ ## Input spectral library (optional — `--speclib-uri` overrides this). # [input] # type = "speclib" -# uri = "path/to/library.msgpack.zst" +# uri = "path/to/library.ndjson.zst" [analysis]