From 90b818870874dce0c303410bcad427b501342581 Mon Sep 17 00:00:00 2001 From: NingyuSun Date: Wed, 30 Sep 2026 16:30:36 +0900 Subject: [PATCH 1/5] Add generic identifier, table and reference grounders, and the feedback loop OntologyGrounder, GeneGrounder and VariantGrounder check the ontology terms, gene symbols and HGVS variants a record cites against pinned releases; TableGrounder checks cited rows and values of a pinned table; ReferenceGrounder sends a record that a curated reference contradicts to review. Messages name what to fix. New codes BEV023 (obsolete or superseded identifier), BEV024 (malformed or wrong kind) and BEV025 (reference conflict, review only). feedback.revise runs validate, feedback and revision around any proposer: fixable findings go back as reasons, conflicts and policy decisions go to a person unchanged, and verified evidence is carried through revisions. --- CHANGELOG.md | 11 + docs/ENGINEERING.md | 73 +++++ src/bioevidence_validator/cli.py | 22 ++ src/bioevidence_validator/crosscheck.py | 81 +++++ src/bioevidence_validator/feedback.py | 133 ++++++++ src/bioevidence_validator/grounding.py | 8 +- src/bioevidence_validator/identifiers.py | 374 +++++++++++++++++++++++ src/bioevidence_validator/tables.py | 118 +++++++ tests/test_feedback.py | 119 ++++++++ tests/test_reference_grounders.py | 318 +++++++++++++++++++ 10 files changed, 1255 insertions(+), 2 deletions(-) create mode 100644 src/bioevidence_validator/crosscheck.py create mode 100644 src/bioevidence_validator/feedback.py create mode 100644 src/bioevidence_validator/identifiers.py create mode 100644 src/bioevidence_validator/tables.py create mode 100644 tests/test_feedback.py create mode 100644 tests/test_reference_grounders.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b238b..3495ff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +- Add generic grounders that check what a record cites against pinned reference releases, with + messages that say what to fix: `OntologyGrounder` (ontology terms: existence, obsoletion, label, + kind), `GeneGrounder` (HGNC symbols, previous symbols and aliases), `VariantGrounder` (HGVS form, + RefSeq accession, genome build, position), `TableGrounder` (rows and values of a pinned table) and + `ReferenceGrounder` (a curated reference resource that contradicts the claim or its evidence). + New codes BEV023 (obsolete or superseded identifier), BEV024 (malformed or wrong kind) and BEV025 + (reference conflict, review only). `bioevidence validate` gains `--ontology`, `--term-root`, + `--genes` and `--assembly-report`. +- Add `feedback`, the validate–feedback–revise loop around any proposer: fixable findings go back as + reasons, conflicts and policy decisions go to a person unchanged, and verified evidence is carried + through revisions. - Benchmark scenario 2 (`agent_loop.py`): an agent searches PubMed, reads papers and submits a decision with citations; bioevidence checks each submission and returns its reasons, and the agent may revise. Five of six models cited only what they had read; Claude Haiku 4.5 misquoted in 8 of diff --git a/docs/ENGINEERING.md b/docs/ENGINEERING.md index 64d0196..6ae8932 100644 --- a/docs/ENGINEERING.md +++ b/docs/ENGINEERING.md @@ -44,6 +44,9 @@ evidence items), and duplicate source/item/line/adjudication IDs cannot be admit | BEV020 | A required type the profile lists under `verified_evidence_types` has no item a grounder verified | Review affected use | | BEV021 | The use requires independent review, and no independent non-human reviewer accepted it (or one deferred or rejected) | Review affected use | | BEV022 | Semantic cue: a quote supporting a positive claim is negated, hedged (optional), or only about animals or cells for a human-scoped item | Review | +| BEV023 | Grounding: a cited identifier is obsolete, withdrawn, or not its current name (a previous gene symbol or alias) | Reject | +| BEV024 | Grounding: a cited identifier is malformed, or of the wrong kind for its place (e.g. not a cell type) | Reject | +| BEV025 | Cross-check: a pinned reference resource contradicts the claim or its supporting evidence | Review | A human acceptance does not erase contradictions, missing evidence, source mismatches, or other human rejection/deferral. Low-strength extraction permissions are explicit @@ -126,6 +129,76 @@ it cannot tell whether that source is itself right, whether a verbatim quote rea the claim (a correctly quoted sentence read with the wrong polarity passes), or anything about sources without a pinned snapshot and a grounder. +### Identifiers, tables and reference resources + +These grounders are generic: they check what a record cites against a pinned release of a reference, +whatever the domain. Their messages say what to fix, so a model can act on them (see the feedback loop +below). Each grounder's name carries the release it used, and so does the report's `grounding` key. + +Identifiers are read from the statement's subject and object (`id`, `label`, `entity_type`), its scope +tokens, and evidence locators written as `key=value` pairs separated by `;` (`cluster=3;gene=CD8A`). + +- `OntologyGrounder` (`identifiers`) checks terms against pinned OBO releases (Cell Ontology, HPO, + MONDO, UBERON …): + - the term exists (BEV016); + - it is not obsolete (BEV023, naming its replacement); + - its label is the term's name or an exact synonym, ignoring case and punctuation (BEV017, naming the + term that label belongs to); + - it is of the right kind: `roots` maps an entity type or locator key to the terms it must descend + from, so a `cell_type` must be under CL:0000000 (BEV024). +- `GeneGrounder` checks human gene symbols and HGNC identifiers against a pinned HGNC complete set: + - an unknown symbol is BEV016; + - a previous symbol, an alias or a wrong case is BEV023, naming the approved symbol; + - a record scoped to another species is not checked (BEV015). +- `VariantGrounder` checks HGVS descriptions on RefSeq sequences against pinned NCBI assembly reports: + - the description must be well formed and versioned (BEV024); + - a chromosome accession must be a real sequence (BEV016), on the genome build of the record's scope + (`GRCh38`, `GRCh37`, `hg19`, `hg38`; BEV017), with the position inside the chromosome (BEV016); + - variants on more than one build in one record are BEV017. +- `TableGrounder` (`tables`) is the data counterpart of the quote check. + - An evidence item of a configured type names one row of a pinned table (TSV or CSV) by its key + columns in the locator, and may claim values from it in `extracted_text` (`logfc=2.4; pct_in=0.91`). + - A missing row is BEV016, naming the closest key values. Several matching rows are BEV015. + - A claimed value that differs from the table is BEV017; a number matches at the precision it is + written with. + - An item whose row exists and whose values match counts as verified. +- `ReferenceGrounder` (`crosscheck`) compares a record with a pinned reference resource it does not cite, + a table of curated `subject`, `predicate`, `object` assertions. A disagreement sends the record to review + (BEV025) and never rejects it: references are partial and can be out of date, and their silence is not + support. It raises BEV025 in two cases: + - the reference asserts the opposite predicate for the statement's subject and object; + - supporting evidence names an entity (e.g. `gene=CD19`) that the reference relates (e.g. `marker_of`) + only to objects unrelated to the statement's object. With an ontology, an ancestor or descendant + counts as related. + +```bash +bioevidence validate record.json --ontology cl-basic.obo --term-root cell_type=CL:0000000 --genes hgnc_complete_set.txt --assembly-report GRCh38_assembly_report.txt +``` + +`TableGrounder` and `ReferenceGrounder` need their configuration (evidence types, keys, relation) and are +used from Python. + +### The feedback loop + +`feedback.revise(propose, validator)` runs the loop around any proposer: a model call, an agent or a person. + +1. The proposer returns a record, and bioevidence validates it. +2. Every finding the proposer can fix goes back as a reason that points at the part of the record + concerned ("evidence 2 (cluster=3;gene=CD8B): No row of the pinned table has cluster=3, gene=CD8B. + Closest gene values for cluster=3: CD8A."). +3. The proposer may then revise, up to a set number of rounds. + +Two kinds of finding are not fed back. The record goes to a person as it is: + +- **Judgment:** its own evidence disagrees (BEV004), a reference contradicts it (BEV025), a semantic cue + flags it (BEV022), or it lacks an independent review (BEV021). +- **Policy:** the use needs a human decision whatever the proposer does (BEV008–BEV013). + +Evidence verified in one attempt is carried into the next, so a revision can fix or replace what failed +but cannot withdraw verified evidence against its answer. The literature benchmark's agent loop +(`evaluation/llm_benchmark/agent_loop.py`) showed why: before that rule, an agent whose record held a +misquote and a verified quote against its decision dropped the latter and was admitted. + ## Semantic checks Grounding proves that a quote is real; it cannot tell whether the quote supports the claim. diff --git a/src/bioevidence_validator/cli.py b/src/bioevidence_validator/cli.py index 7cddae0..edde33a 100644 --- a/src/bioevidence_validator/cli.py +++ b/src/bioevidence_validator/cli.py @@ -13,6 +13,7 @@ from .draft import build_record, draft_json_schema, load_draft from .engine import default_schema_path, generate_json_schema, list_profiles, profile_path, validate_record from .grounding import SnapshotStore, SourceBytesGrounder +from .identifiers import Assemblies, GeneGrounder, Genes, Ontology, OntologyGrounder, VariantGrounder from .literature import CATALOG, LiteratureGrounder, ground_record, http_fetch @@ -27,6 +28,14 @@ def parser() -> argparse.ArgumentParser: validate.add_argument("--snapshot-dir", type=Path, help="Directory of source snapshots named by SHA-256; recompute source hashes from them, " "and check cited publications if it holds a literature.json from `ground`") + validate.add_argument("--ontology", type=Path, action="append", default=[], + help="A pinned OBO release; check the ontology terms the record cites (repeatable)") + validate.add_argument("--term-root", action="append", default=[], metavar="KIND=CURIE", + help="Terms for this entity type or locator key must descend from CURIE, e.g. " + "cell_type=CL:0000000 (repeatable)") + validate.add_argument("--genes", type=Path, help="A pinned HGNC complete set (TSV); check human gene symbols") + validate.add_argument("--assembly-report", type=Path, action="append", default=[], + help="A pinned NCBI assembly report; check HGVS variants and genome builds (repeatable)") ground = commands.add_parser("ground", help="Resolve cited publications and pin their metadata and open-access " "full text (uses the network)") ground.add_argument("input", type=Path) @@ -175,6 +184,19 @@ def _run(args) -> int: grounders.append(SourceBytesGrounder(SnapshotStore.from_directory(args.snapshot_dir))) if (args.snapshot_dir / CATALOG).exists(): grounders.append(LiteratureGrounder.from_directory(args.snapshot_dir)) + if args.ontology: + roots: dict[str, list[str]] = {} + for rule in args.term_root: + kind, _, curie = rule.partition("=") + if not kind or not curie: + raise ValueError(f"--term-root must be KIND=CURIE, not {rule!r}") + roots.setdefault(kind, []).append(curie) + grounders.append(OntologyGrounder([Ontology.from_obo(path.read_bytes()) for path in args.ontology], roots, + locator_keys=list(roots))) + if args.genes: + grounders.append(GeneGrounder(Genes.from_hgnc(args.genes.read_bytes(), args.genes.name))) + if args.assembly_report: + grounders.append(VariantGrounder(Assemblies.from_reports(path.read_bytes() for path in args.assembly_report))) report = validate_record(record, schema_path=args.schema, profile=args.profile, grounders=grounders) rendered = json.dumps(report, indent=2) + "\n" if args.output: diff --git a/src/bioevidence_validator/crosscheck.py b/src/bioevidence_validator/crosscheck.py new file mode 100644 index 0000000..d2cfd8a --- /dev/null +++ b/src/bioevidence_validator/crosscheck.py @@ -0,0 +1,81 @@ +"""Cross-check against a pinned reference resource: what a curated reference contradicts goes to an expert. + +Grounding checks a record against the sources it cites. A reference resource is a second, independent +source the record does not cite: a knowledge base of curated assertions. Where it disagrees, the record is +not wrong on that ground alone (the reference can be out of date, and its coverage is partial), so the +finding sends the record to review (BEV025) and never rejects it. Silence of the reference is not support. + +Assertions are rows with `subject`, `predicate` and `object` (optionally `subject_label`, `object_label`), +e.g. a CIViC-style `BRAF V600E | does_not_support_sensitivity | vemurafenib in X`, or a marker table +`CD19 | marker_of | CL:0000236`. Two checks: + +- statement: the reference asserts the opposite predicate for the same subject and object (`opposites` + maps each predicate to its opposite); +- evidence: supporting evidence names an entity in its locator (`evidence_key`, e.g. `gene=CD19`) that the + reference relates (`relation`, e.g. `marker_of`) only to objects unrelated to the statement's object. With + an ontology, an ancestor or descendant of the object counts as related (a B cell marker supports "naive + B cell"); without one, only the same identifier does. +""" +from __future__ import annotations + +import csv +import io +from collections.abc import Iterable +from typing import Any + +from .engine import Finding +from .grounding import finding +from .identifiers import Ontology, pairs + + +class ReferenceGrounder: + def __init__(self, assertions: Iterable[dict[str, str]], *, label: str, opposites: dict[str, str] | None = None, + evidence_key: str | None = None, relation: str | None = None, ontology: Ontology | None = None): + self.assertions = list(assertions) + self.label, self.ontology = label, ontology + self.opposites = dict(opposites or {}) + self.opposites.update({v: k for k, v in self.opposites.items()}) + self.key, self.relation = evidence_key, relation + self.name = f"reference:{label}" + self._pairs: dict[tuple[str, str], list[dict[str, str]]] = {} + self._objects: dict[str, list[dict[str, str]]] = {} + for row in self.assertions: + self._pairs.setdefault((row["subject"], row["object"]), []).append(row) + if row["predicate"] == relation: + self._objects.setdefault(row["subject"], []).append(row) + + @classmethod + def from_table(cls, data: bytes, **options: Any) -> ReferenceGrounder: + text = data.decode("utf-8") + delimiter = "\t" if "\t" in text.partition("\n")[0] else "," + return cls(csv.DictReader(io.StringIO(text), delimiter=delimiter), **options) + + def _related(self, a: str, b: str) -> bool: + return a == b or bool(self.ontology and self.ontology.related(a, b)) + + def _name(self, row: dict[str, str]) -> str: + return f"{row['object']} ({row['object_label']})" if row.get("object_label") else row["object"] + + def check(self, record: dict[str, Any]) -> list[Finding]: + statement = record["statement"] + subject, predicate, target = statement["subject"]["id"], statement["predicate"], statement["object"]["id"] + out = [] + for row in self._pairs.get((subject, target), []): + if self.opposites.get(row["predicate"]) == predicate: + out.append(finding(record, "BEV025", f"The {self.label} reference records {subject} {row['predicate']} " + f"{target}, the opposite of this claim.", "$.statement.predicate")) + if not self.key: + return out + supporting = {i for line in statement["evidence_lines"] if line["direction"] == "supports" + for i in line["evidence_item_ids"]} + for index, item in enumerate(record["evidence_items"]): + entity = pairs(item["locator"]).get(self.key) + rows = self._objects.get(entity or "", []) + if item["id"] not in supporting or not rows or any(self._related(r["object"], target) for r in rows): + continue + names = sorted({self._name(r) for r in rows}) + more = f" and {len(names) - 3} more" if len(names) > 3 else "" + out.append(finding(record, "BEV025", f"The {self.label} reference lists {entity} as a {(self.relation or '').replace('_', ' ')} " + f"{'; '.join(names[:3])}{more}, not of {target} or a related term.", + f"$.evidence_items[{index}].locator")) + return out diff --git a/src/bioevidence_validator/feedback.py b/src/bioevidence_validator/feedback.py new file mode 100644 index 0000000..5421380 --- /dev/null +++ b/src/bioevidence_validator/feedback.py @@ -0,0 +1,133 @@ +"""The feedback loop: validate what an AI proposes, tell it what to fix, and keep what was verified. + +An agent proposes a record; bioevidence validates it with its grounders. Every finding the agent can fix +(an identifier that does not exist, a label that belongs to another term, a quote or table value that is +not in the source) goes back to it as a reason that points at the part of the record concerned, and it +may revise and propose again. Two kinds of finding are not fed back, and send the record to a person as it +is. Judgment: its own evidence disagrees (BEV004), a reference resource contradicts it (BEV025), a semantic +cue flags it (BEV022) or it lacks an independent review (BEV021); the agent is never asked to make the +evidence against its answer go away. Policy: the use needs a human decision whatever the agent does +(LLM-only or string-match-only evidence where the profile forbids it, BEV008/009/013; human acceptance +required, deferred or refused, BEV010/012/011). For the same reason, evidence verified in one attempt is carried +into the next: a revision may fix or replace what failed, but not withdraw what was verified. + +`revise` runs the loop around any proposer (a model call, an agent, a person); nothing here calls a model. +""" +from __future__ import annotations + +import copy +import re +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +from .engine import RecordValidator + +JUDGMENT = frozenset({"BEV004", "BEV021", "BEV022", "BEV025"}) +POLICY = frozenset({"BEV008", "BEV009", "BEV010", "BEV011", "BEV012", "BEV013"}) +EXPERT = JUDGMENT | POLICY + + +def to_expert(report: dict[str, Any]) -> bool: + """The record goes to a person as it is: it is not admitted, and only for findings the agent cannot fix.""" + codes = {f["rule_id"] for f in report["findings"]} + return report["overall_status"] != "admitted" and bool(codes) and codes <= EXPERT + + +def where(record: dict[str, Any], path: str) -> str: + """A short, human-readable name for the part of the record a finding's path points at.""" + if match := re.match(r"\$\.evidence_items\[(\d+)\]", path): + n = int(match.group(1)) + items = record.get("evidence_items") or [] + locator = items[n].get("locator") if n < len(items) else None + return f"evidence {n + 1}" + (f" ({locator})" if locator else "") + if match := re.match(r"\$\.source_artifacts\[(\d+)\]", path): + return f"source {int(match.group(1)) + 1}" + if match := re.match(r"\$\.statement\.(subject|object)(?:\.(id|label))?", path): + return match.group(1) + (f" {match.group(2)}" if match.group(2) else "") + return path.removeprefix("$.").replace("_", " ") or "record" + + +def reasons(record: dict[str, Any], report: dict[str, Any]) -> list[str]: + """What to fix, one line per finding, without the findings that go to an expert.""" + lines = [f"{where(record, f['field_path'])}: {f['message']}" for f in report["findings"] if f["rule_id"] not in EXPERT] + return list(dict.fromkeys(lines)) + + +def verified_items(validator: RecordValidator, record: dict[str, Any]) -> set[str]: + found: set[str] = set() + for grounder in validator.grounders: + if hasattr(grounder, "verified_items"): + found |= set(grounder.verified_items(record)) + return found + + +def _key(record: dict[str, Any], item: dict[str, Any]) -> tuple[str, str, str]: + sha = next((s["sha256"] for s in record["source_artifacts"] if s["id"] == item["source_artifact_id"]), "") + return sha, item["locator"], item.get("extracted_text") or "" + + +def carry(previous: dict[str, Any], verified: set[str], record: dict[str, Any]) -> dict[str, Any]: + """`record` plus the verified evidence of `previous` that it left out, in the evidence line it was in.""" + merged = copy.deepcopy(record) + present = {_key(merged, item) for item in merged["evidence_items"]} + directions = {i: line["direction"] for line in previous["statement"]["evidence_lines"] for i in line["evidence_item_ids"]} + ids = {x["id"] for x in merged["evidence_items"]} | {s["id"] for s in merged["source_artifacts"]} + for item in previous["evidence_items"]: + if item["id"] not in verified or _key(previous, item) in present: + continue + source = next(s for s in previous["source_artifacts"] if s["id"] == item["source_artifact_id"]) + same = next((s for s in merged["source_artifacts"] if s["sha256"] == source["sha256"]), None) + if same is None: + same = dict(source, id=_fresh(source["id"], ids)) + merged["source_artifacts"].append(same) + kept = dict(item, id=_fresh(item["id"], ids), source_artifact_id=same["id"]) + merged["evidence_items"].append(kept) + direction = directions.get(item["id"], "supports") + line = next((x for x in merged["statement"]["evidence_lines"] if x["direction"] == direction), None) + if line is None: + line = {"id": _fresh(f"carried-{direction}", ids), "direction": direction, "evidence_item_ids": []} + merged["statement"]["evidence_lines"].append(line) + line["evidence_item_ids"].append(kept["id"]) + present.add(_key(previous, item)) + return merged + + +def _fresh(base: str, taken: set[str]) -> str: + name, n = base, 1 + while name in taken: + n += 1 + name = f"{base}-{n}" + taken.add(name) + return name + + +@dataclass +class Attempt: + record: dict[str, Any] + report: dict[str, Any] + feedback: list[str] = field(default_factory=list) + + +def revise(propose: Callable[[list[str]], dict[str, Any] | None], validator: RecordValidator, + rounds: int = 3) -> list[Attempt]: + """Up to `rounds` attempts. `propose(feedback)` returns a record, or None to stop; the first call gets no + feedback. The loop ends when a record is admitted, goes to an expert, or the rounds run out.""" + attempts: list[Attempt] = [] + feedback: list[str] = [] + previous, verified = None, set() + for _ in range(rounds): + record = propose(feedback) + if record is None: + break + if previous is not None: + record = carry(previous, verified, record) + report = validator.validate(record) + attempts.append(Attempt(record, report)) + if report["overall_status"] == "admitted" or to_expert(report): + break + feedback = reasons(record, report) + attempts[-1].feedback = feedback + sound = not {f["rule_id"] for f in report["findings"]} & {"SCHEMA", "RECORD_INTEGRITY"} + previous, verified = record, verified_items(validator, record) if sound else set() + return attempts diff --git a/src/bioevidence_validator/grounding.py b/src/bioevidence_validator/grounding.py index ecb4a2a..89169b0 100644 --- a/src/bioevidence_validator/grounding.py +++ b/src/bioevidence_validator/grounding.py @@ -13,12 +13,16 @@ BEV018 review the source holds evidence the record leaves out BEV019 error the cited source is retracted BEV022 review a semantic cue suggests the quote is misread (see `semantic`) + BEV023 error a cited identifier is obsolete, withdrawn or not its current name (see `identifiers`) + BEV024 error a cited identifier is malformed, or of the wrong kind for its place (see `identifiers`) + BEV025 review a pinned reference resource contradicts the claim or its evidence (see `crosscheck`) A grounder may also offer `verified_items(record)`: the evidence items it confirmed against their source. Only those count for evidence types a profile lists under `verified_evidence_types` (BEV020 otherwise). Domain grounders (which fields to recompute for a given source format) live with their importers, -outside the engine; `SourceBytesGrounder` is generic. +outside the engine. Generic grounders: `SourceBytesGrounder` (here), `identifiers` (ontology terms, gene symbols, +variant nomenclature), `tables` (rows of a pinned table), `crosscheck` (a pinned reference resource), `literature`. """ from __future__ import annotations @@ -31,7 +35,7 @@ from .engine import Finding SEVERITY = {"BEV014": "error", "BEV015": "review", "BEV016": "error", "BEV017": "error", "BEV018": "review", - "BEV019": "error", "BEV022": "review"} + "BEV019": "error", "BEV022": "review", "BEV023": "error", "BEV024": "error", "BEV025": "review"} class Grounder(Protocol): diff --git a/src/bioevidence_validator/identifiers.py b/src/bioevidence_validator/identifiers.py new file mode 100644 index 0000000..1e91c30 --- /dev/null +++ b/src/bioevidence_validator/identifiers.py @@ -0,0 +1,374 @@ +"""Identifier grounders: check the identifiers a record cites against pinned reference releases. + +A model writes identifiers that look right: an ontology ID whose label belongs to another term, an obsolete +term, a gene alias, a variant on the wrong genome build. These grounders look each one up in a pinned +release (an OBO ontology, the HGNC complete set, NCBI assembly reports) and say what is wrong in words an +agent can act on ("'T cell' is the name of CL:0000084"). They are offline and deterministic, and each +grounder's name carries the release it used, so a report records it. + +Where identifiers are looked up: +- the statement's subject and object (`id`, `label`, `entity_type`); +- the statement's scope tokens (ontology terms only); +- evidence item locators written as `key=value` pairs separated by `;` (e.g. `cluster=3;gene=CD8A`), for + the keys a grounder is given. + +Rule codes (see `grounding`): + BEV015 review the reference cannot confirm the identifier (e.g. a gene outside the reference's species) + BEV016 error the identifier does not exist in the pinned release + BEV017 error the identifier and its label disagree, or a variant is on another genome build + BEV023 error the identifier is obsolete, withdrawn, or not its current name (previous symbol, alias) + BEV024 error the identifier is malformed, or of the wrong kind for its place (e.g. not a cell type) +""" +from __future__ import annotations + +import csv +import difflib +import io +import re +from collections.abc import Iterable, Iterator, Mapping +from dataclasses import dataclass, field +from functools import cache +from typing import Any + +from .engine import Finding +from .grounding import finding + +HUMAN = "NCBITaxon:9606" + + +def pairs(text: str | None) -> dict[str, str]: + """`key=value` pairs separated by `;`; empty when the text is not written that way.""" + out = {} + for part in (text or "").split(";"): + key, sep, value = part.partition("=") + if not sep or not key.strip(): + return {} + out[key.strip()] = value.strip() + return out + + +def _prefix(curie: str) -> str: + return curie.partition(":")[0] if ":" in curie else "" + + +def _label_key(text: str) -> str: + return " ".join(re.sub(r"[^0-9a-z]+", " ", text.casefold()).split()) + + +def _entities(record: dict[str, Any]) -> Iterator[tuple[str, dict[str, Any]]]: + statement = record["statement"] + for role in ("subject", "object"): + yield f"$.statement.{role}", statement[role] + + +# Ontologies (OBO format) + +@dataclass +class Term: + id: str + name: str = "" + synonyms: list[str] = field(default_factory=list) + parents: list[str] = field(default_factory=list) + obsolete: bool = False + replaced_by: list[str] = field(default_factory=list) + consider: list[str] = field(default_factory=list) + + +class Ontology: + """The terms of one pinned OBO release: names, exact synonyms, `is_a` parents and obsoletion.""" + + def __init__(self, terms: dict[str, Term], prefix: str, version: str): + self.terms, self.prefix, self.version = terms, prefix, version + self._labels: dict[str, list[str]] = {} + for term in terms.values(): + if not term.obsolete: + for key in dict.fromkeys(_label_key(text) for text in [term.name, *term.synonyms]): + self._labels.setdefault(key, []).append(term.id) + self._names = {_label_key(t.name): t.id for t in terms.values() if not t.obsolete and _prefix(t.id) == prefix} + + @classmethod + def from_obo(cls, data: bytes) -> Ontology: + terms: dict[str, Term] = {} + header: dict[str, str] = {} + term: Term | None = None + in_header = True + for line in data.decode("utf-8").splitlines(): + line = line.strip() + if line.startswith("["): + in_header, term = False, None + if line == "[Term]": + term = Term("") + continue + tag, _, value = line.partition(": ") + if in_header: + header.setdefault(tag, value) + elif term is not None and tag: + value = value.split(" ! ", 1)[0].strip() + if tag == "id": + term.id = value + terms[value] = term + elif tag == "name": + term.name = value + elif tag == "synonym" and (match := re.match(r'"((?:[^"\\]|\\.)*)" EXACT\b', value)): + term.synonyms.append(match.group(1).replace('\\"', '"')) + elif tag == "is_a": + term.parents.append(value.split()[0]) + elif tag == "is_obsolete": + term.obsolete = value == "true" + elif tag == "replaced_by": + term.replaced_by.append(value) + elif tag == "consider": + term.consider.append(value) + # `ontology: cl` or `ontology: cl/cl-basic`; a release file may also hold a few terms of other ontologies + prefix = header.get("ontology", "").split("/")[0].upper() + version = header.get("data-version", "") or header.get("date", "") + return cls(terms, prefix, version) + + def describe(self, curie: str) -> str: + term = self.terms.get(curie) + return f"{curie} ({term.name})" if term and term.name else curie + + def ancestors(self, curie: str) -> frozenset[str]: + """The term and every `is_a` ancestor.""" + return self._ancestors(curie) + + @cache # noqa: B019 - one instance per pinned release, kept for the process + def _ancestors(self, curie: str) -> frozenset[str]: + term = self.terms.get(curie) + found = {curie} + for parent in term.parents if term else []: + found |= self._ancestors(parent) + return frozenset(found) + + def related(self, a: str, b: str) -> bool: + """The same term, or one is an ancestor of the other.""" + return a in self.ancestors(b) or b in self.ancestors(a) + + def by_label(self, label: str) -> list[str]: + return self._labels.get(_label_key(label), []) + + def hint(self, label: str | None) -> str: + """A suggestion for a label: the term it names, else the closest names.""" + if not label: + return "" + exact = self.by_label(label) + if exact: + return f" {label!r} is the name of {self.describe(exact[0])}." + close = difflib.get_close_matches(_label_key(label), list(self._names), n=3, cutoff=0.75) + return (" Closest names: " + "; ".join(self.describe(self._names[c]) for c in close) + ".") if close else "" + + +class OntologyGrounder: + """Ontology terms cited by a record: they exist, are current, match their labels and are of the right kind. + + `roots` maps an entity type (or a locator key) to the terms its identifiers must descend from, e.g. + `{"cell_type": ["CL:0000000"]}`; an identifier from another ontology is then the wrong kind (BEV024). + """ + + def __init__(self, ontologies: Iterable[Ontology], roots: Mapping[str, Iterable[str]] | None = None, + locator_keys: Iterable[str] = ()): + self.ontologies = {o.prefix: o for o in ontologies} + self.roots = {kind: tuple(ids) for kind, ids in (roots or {}).items()} + self.keys = tuple(locator_keys) + self.name = "ontology:" + ",".join(f"{o.prefix}@{o.version}" for o in self.ontologies.values()) + + def check(self, record: dict[str, Any]) -> list[Finding]: + out = [] + for path, entity in _entities(record): + out += self._term(record, entity["id"], entity["label"], entity["entity_type"], path + ".id", path + ".label") + for index, token in enumerate(record["statement"]["scope"]): + out += self._term(record, token, None, None, f"$.statement.scope[{index}]") + for index, item in enumerate(record["evidence_items"]): + located = pairs(item["locator"]) + for key in self.keys: + if key in located: + out += self._term(record, located[key], None, key, f"$.evidence_items[{index}].locator") + return out + + def _term(self, record: dict[str, Any], curie: str, label: str | None, kind: str | None, path: str, + label_path: str = "") -> list[Finding]: + roots = self.roots.get(kind or "") + if roots and _prefix(curie) not in {_prefix(r) for r in roots}: + owner = self.ontologies.get(_prefix(roots[0])) + wanted = " or ".join(sorted({_prefix(r) for r in roots})) + return [finding(record, "BEV024", f"A {kind} must be a {wanted} term; {curie!r} is not." + f"{owner.hint(label) if owner else ''}", path)] + ontology = self.ontologies.get(_prefix(curie)) + if ontology is None: + return [] + term = ontology.terms.get(curie) + if term is None: + return [finding(record, "BEV016", f"{curie} is not in the pinned {ontology.prefix} release " + f"({ontology.version}).{ontology.hint(label)}", path)] + if term.obsolete: + replacement = term.replaced_by or term.consider + advice = (" Use " + " or ".join(ontology.describe(r) for r in replacement) + "." if replacement + else ontology.hint(label)) + return [finding(record, "BEV023", f"{ontology.describe(curie)} is obsolete in the pinned {ontology.prefix} " + f"release.{advice}", path)] + out = [] + if label is not None and curie not in ontology.by_label(label): + out.append(finding(record, "BEV017", f"The label {label!r} is not the name of {curie} ({term.name!r})." + f"{ontology.hint(label)}", label_path or path)) + if roots and not any(r in ontology.ancestors(curie) for r in roots): + under = " or ".join(ontology.describe(r) for r in roots) + out.append(finding(record, "BEV024", f"{ontology.describe(curie)} is not a {kind}: it is not under {under}.", + path)) + return out + + +# Genes (HGNC complete set) + +class Genes: + """Approved human gene symbols and identifiers from a pinned HGNC complete set (TSV).""" + + def __init__(self, rows: Iterable[dict[str, str]], version: str = ""): + self.version = version + self.by_id: dict[str, dict[str, str]] = {} + self.approved: dict[str, str] = {} + self.folded: dict[str, str] = {} + self.renamed: dict[str, list[str]] = {} + for row in rows: + symbol = row["symbol"] + self.by_id[row["hgnc_id"]] = row + self.approved[symbol] = row["hgnc_id"] + self.folded.setdefault(symbol.casefold(), symbol) + for column in ("prev_symbol", "alias_symbol"): + for old in filter(None, (row.get(column) or "").strip('"').split("|")): + self.renamed.setdefault(old, []).append(symbol) + + @classmethod + def from_hgnc(cls, data: bytes, version: str = "") -> Genes: + return cls(csv.DictReader(io.StringIO(data.decode("utf-8")), delimiter="\t"), version) + + def problem(self, symbol: str) -> tuple[str, str] | None: + """(rule code, message) for a symbol that is not an approved one.""" + if symbol in self.approved: + return None + current = sorted(set(self.renamed.get(symbol, []))) + if current: + return "BEV023", (f"{symbol} is not an approved HGNC symbol; it is a previous symbol or alias of " + + " / ".join(current) + ".") + if symbol.casefold() in self.folded: + return "BEV023", f"{symbol} is not an approved HGNC symbol; the approved symbol is {self.folded[symbol.casefold()]}." + close = difflib.get_close_matches(symbol, list(self.approved), n=3, cutoff=0.8) + return "BEV016", f"{symbol} is not an HGNC gene symbol." + (f" Closest: {', '.join(close)}." if close else "") + + +class GeneGrounder: + """Human gene symbols and HGNC identifiers cited by a record are current and consistent.""" + + def __init__(self, genes: Genes, locator_keys: Iterable[str] = ("gene",), entity_types: Iterable[str] = ("gene",)): + self.genes = genes + self.keys, self.types = tuple(locator_keys), set(entity_types) + self.name = f"genes:HGNC@{genes.version}" + + def check(self, record: dict[str, Any]) -> list[Finding]: + taxa = {t for t in record["statement"]["scope"] if t.startswith("NCBITaxon:")} + if taxa and HUMAN not in taxa: + return [finding(record, "BEV015", f"HGNC names human genes; this record is scoped to {', '.join(sorted(taxa))}, " + "so its gene symbols are not checked.", "$.statement.scope")] + out = [] + for path, entity in _entities(record): + if entity["entity_type"] in self.types or entity["id"].startswith("HGNC:"): + out += self._entity(record, entity, path) + for index, item in enumerate(record["evidence_items"]): + located = pairs(item["locator"]) + for key in self.keys: + if key in located and (problem := self.genes.problem(located[key])): + out.append(finding(record, problem[0], problem[1], f"$.evidence_items[{index}].locator")) + return out + + def _entity(self, record: dict[str, Any], entity: dict[str, Any], path: str) -> list[Finding]: + if entity["id"].startswith("HGNC:"): + row = self.genes.by_id.get(entity["id"]) + if row is None: + return [finding(record, "BEV016", f"{entity['id']} is not in the pinned HGNC set.", path + ".id")] + if entity["label"] != row["symbol"]: + return [finding(record, "BEV017", f"The label {entity['label']!r} is not the symbol of {entity['id']} " + f"({row['symbol']}).", path + ".label")] + return [] + problem = self.genes.problem(entity["label"]) + return [finding(record, problem[0], problem[1], path + ".label")] if problem else [] + + +# Variants (HGVS on RefSeq sequences, NCBI assembly reports) + +HGVS = re.compile(r"^(?P(?PN[CGMRPTW]|X[MRP])_\d+)(?:\.(?P\d+))?:(?P[cgmnpr])\.(?P\S+)$") +BUILD = re.compile(r"^(GRCh3[78]|GRCm3[89]|hg19|hg38)$") +ALIASES = {"hg19": "GRCh37", "hg38": "GRCh38"} + + +class Assemblies: + """RefSeq chromosome sequences (accession.version → build, name, length) from NCBI assembly reports.""" + + def __init__(self, sequences: dict[str, tuple[str, str, int]]): + self.sequences = sequences + self.version = "+".join(sorted({build for build, _, _ in sequences.values()})) + + @classmethod + def from_reports(cls, reports: Iterable[bytes]) -> Assemblies: + sequences = {} + for data in reports: + text = data.decode("utf-8") + match = re.search(r"^# Assembly name:\s*(\S+)", text, re.M) + build = (match.group(1) if match else "").split(".")[0] + for line in text.splitlines(): + cells = line.split("\t") + if line.startswith("#") or len(cells) < 9 or cells[6] == "na": + continue + sequences[cells[6]] = (build, cells[0], int(cells[8]) if cells[8].isdigit() else 0) + return cls(sequences) + + +class VariantGrounder: + """HGVS variant descriptions: well formed, versioned, on a real sequence of the record's genome build.""" + + def __init__(self, assemblies: Assemblies, locator_keys: Iterable[str] = ("variant", "hgvs"), + entity_types: Iterable[str] = ("variant",)): + self.assemblies = assemblies + self.keys, self.types = tuple(locator_keys), set(entity_types) + self.name = f"variants:{assemblies.version}" + + def check(self, record: dict[str, Any]) -> list[Finding]: + builds = {ALIASES.get(t, t) for t in record["statement"]["scope"] if BUILD.match(t)} + cited: list[tuple[str, str]] = [] + for path, entity in _entities(record): + if entity["entity_type"] in self.types: + text = entity["id"] if HGVS.match(entity["id"]) else entity["label"] + cited.append((path + (".id" if text == entity["id"] else ".label"), text)) + for index, item in enumerate(record["evidence_items"]): + located = pairs(item["locator"]) + cited += [(f"$.evidence_items[{index}].locator", located[k]) for k in self.keys if k in located] + out, seen = [], set() + for path, text in cited: + match = HGVS.match(text) + if match is None: + out.append(finding(record, "BEV024", f"{text!r} is not an HGVS description on a RefSeq sequence " + "(e.g. NC_000007.14:g.140753336A>T).", path)) + continue + if match["version"] is None: + out.append(finding(record, "BEV024", f"{match['ac']} has no version; HGVS needs a versioned accession.", + path)) + continue + if match["kind"] != "NC": + continue # transcripts and proteins: form checked only + accession = f"{match['ac']}.{match['version']}" + if accession not in self.assemblies.sequences: + known = sorted(a for a in self.assemblies.sequences if a.startswith(match["ac"] + ".")) + advice = f" Pinned versions: {', '.join(known)}." if known else "" + out.append(finding(record, "BEV016", f"{accession} is not a sequence of the pinned assemblies.{advice}", path)) + continue + build, name, length = self.assemblies.sequences[accession] + seen.add(build) + if builds and build not in builds: + out.append(finding(record, "BEV017", f"{accession} (chromosome {name}) is on {build}, but the record is " + f"scoped to {', '.join(sorted(builds))}.", path)) + position = re.match(r"(\d+)", match["rest"]) + if match["type"] == "g" and position and length and int(position.group(1)) > length: + out.append(finding(record, "BEV016", f"Position {position.group(1)} is beyond the end of chromosome " + f"{name} on {build} ({length} bp).", path)) + if len(seen) > 1 and not builds: + out.append(finding(record, "BEV017", "The record cites variants on more than one genome build: " + + ", ".join(sorted(seen)) + ".", "$.statement.scope")) + return out diff --git a/src/bioevidence_validator/tables.py b/src/bioevidence_validator/tables.py new file mode 100644 index 0000000..ac622e4 --- /dev/null +++ b/src/bioevidence_validator/tables.py @@ -0,0 +1,118 @@ +"""Table grounding: evidence that cites rows of a pinned table must be in that table, with the values it claims. + +The data counterpart of a quote check. An evidence item points at a table snapshot (TSV or CSV with a +header, pinned by SHA-256 in the snapshot store) and names one row by its key columns in the locator, +`key=value` pairs separated by `;` (e.g. `cluster=3;gene=CD8A`). Its `extracted_text` may claim values +from that row the same way (`logfc=2.4; pct_in=0.91`). A claimed number matches when the table's value +rounds to it at the precision the claim is written with; text matches ignoring case. + +Only items of the evidence types the grounder is given are checked, so tables used in other ways are left +alone. Rule codes (see `grounding`): BEV015 (bytes unavailable, or the locator names several rows), +BEV016 (no such row), BEV017 (a claimed value differs from the table), BEV024 (a locator or claim that is +not `key=value` pairs, or names a column the table does not have). +""" +from __future__ import annotations + +import csv +import difflib +import io +from collections.abc import Iterable +from typing import Any + +from .engine import Finding +from .grounding import SnapshotStore, finding +from .identifiers import pairs + + +def parse_table(data: bytes) -> tuple[list[str], list[dict[str, str]]]: + text = data.decode("utf-8") + delimiter = "\t" if "\t" in text.partition("\n")[0] else "," + reader = csv.DictReader(io.StringIO(text), delimiter=delimiter) + rows = [{k: (v or "").strip() for k, v in row.items()} for row in reader] + return list(reader.fieldnames or []), rows + + +def same_value(claimed: str, actual: str) -> bool: + try: + number, value = float(claimed), float(actual) + except ValueError: + return claimed.strip().casefold() == actual.strip().casefold() + decimals = len(claimed.strip().partition(".")[2].rstrip()) if "." in claimed else 0 + return abs(number - value) <= 0.5 * 10 ** -decimals + 1e-12 + + +class TableGrounder: + """Check evidence items of the given types against the rows of their pinned table.""" + + name = "tables" + + def __init__(self, store: SnapshotStore, evidence_types: Iterable[str]): + self.store = store + self.types = set(evidence_types) + self._tables: dict[str, tuple[list[str], list[dict[str, str]]]] = {} + + def _table(self, sha256: str) -> tuple[list[str], list[dict[str, str]]] | None: + if not self.store.verified(sha256): + return None + if sha256 not in self._tables: + self._tables[sha256] = parse_table(self.store.get(sha256) or b"") + return self._tables[sha256] + + def _evaluate(self, record: dict[str, Any]) -> tuple[list[Finding], set[str]]: + sources = {s["id"]: s for s in record["source_artifacts"]} + findings, verified = [], set() + for index, item in enumerate(record["evidence_items"]): + source = sources.get(item["source_artifact_id"]) + if item["evidence_type"] not in self.types or source is None: + continue + path = f"$.evidence_items[{index}]" + + def add(code: str, message: str, where: str = path) -> None: + findings.append(finding(record, code, message, where)) + + table = self._table(source["sha256"]) + if table is None: + add("BEV015", "The table's bytes for this frozen hash are not in the snapshot store.") + continue + columns, rows = table + key, claims = pairs(item["locator"]), pairs(item.get("extracted_text")) + if not key or (item.get("extracted_text") and not claims): + add("BEV024", "A table citation needs a locator, and any claimed values, written as key=value pairs.") + continue + unknown = sorted(set(key) - set(columns)) + sorted(set(claims) - set(columns)) + if unknown: + add("BEV024", f"The table has no column {', '.join(map(repr, unknown))}; columns: {', '.join(columns)}.") + continue + matches = [r for r in rows if all(same_value(v, r[k]) for k, v in key.items())] + if not matches: + add("BEV016", f"No row of the pinned table has {_render(key)}.{_closest(rows, key)}", path + ".locator") + continue + if len(matches) > 1: + add("BEV015", f"{len(matches)} rows have {_render(key)}; the locator must name one row.", path + ".locator") + continue + wrong = {k: matches[0][k] for k, v in claims.items() if not same_value(v, matches[0][k])} + if wrong: + add("BEV017", f"The row {_render(key)} has {_render(wrong)}, not {_render({k: claims[k] for k in wrong})}.", + path + ".extracted_text") + else: + verified.add(item["id"]) + return findings, verified + + def check(self, record: dict[str, Any]) -> list[Finding]: + return self._evaluate(record)[0] + + def verified_items(self, record: dict[str, Any]) -> set[str]: + return self._evaluate(record)[1] + + +def _render(values: dict[str, str]) -> str: + return ", ".join(f"{k}={v}" for k, v in values.items()) + + +def _closest(rows: list[dict[str, str]], key: dict[str, str]) -> str: + """Values of the last key column among rows that match the other keys, closest first.""" + *fixed, last = list(key) + candidates = sorted({r[last] for r in rows if all(same_value(key[k], r[k]) for k in fixed)}) + close = difflib.get_close_matches(key[last], candidates, n=3, cutoff=0.6) + return f" Closest {last} values for {_render({k: key[k] for k in fixed})}: {', '.join(close)}." if close and fixed \ + else (f" Closest {last} values: {', '.join(close)}." if close else "") diff --git a/tests/test_feedback.py b/tests/test_feedback.py new file mode 100644 index 0000000..de3fff5 --- /dev/null +++ b/tests/test_feedback.py @@ -0,0 +1,119 @@ +"""The feedback loop: what goes back to the proposer, what goes to an expert, and what is carried.""" +import copy +import hashlib + +from test_reference_grounders import CELLS, CL, MARKERS, TABLE, base + +from bioevidence_validator.crosscheck import ReferenceGrounder +from bioevidence_validator.engine import RecordValidator +from bioevidence_validator.feedback import EXPERT, carry, reasons, revise, to_expert, where +from bioevidence_validator.grounding import SnapshotStore +from bioevidence_validator.tables import TableGrounder + +SHA = hashlib.sha256(TABLE).hexdigest() + + +def annotation(target, genes, *, against=()): + """A cluster annotated as `target`, citing marker genes of cluster 3 (and some against it).""" + record = base(object={"id": target, "label": CL.terms[target].name if target in CL.terms else "T cell", + "entity_type": "cell_type"}) + record["source_artifacts"][0].update(sha256=SHA, observed_sha256=SHA) + items = [{"id": f"bioev:item-{n}", "source_artifact_id": "bioev:source-1", "locator": f"cluster=3;gene={gene}", + "extracted_text": text, "evidence_type": "marker_gene", "extraction_method": "deterministic_parser", + "scope": ["taxon:synthetic"]} for n, (gene, text) in enumerate([*genes, *against], start=1)] + record["evidence_items"] = items + lines = [{"id": "bioev:line-1", "direction": "supports", "evidence_item_ids": [i["id"] for i in items[:len(genes)]]}] + if against: + lines.append({"id": "bioev:line-2", "direction": "contradicts", + "evidence_item_ids": [i["id"] for i in items[len(genes):]]}) + record["statement"]["evidence_lines"] = lines + return record + + +def validator(*extra): + tables = TableGrounder(SnapshotStore({SHA: lambda: TABLE}), ["marker_gene"]) + return RecordValidator(grounders=[tables, CELLS, *extra]) + + +def test_reasons_point_at_what_to_fix(): + record = annotation("CL:0000084", [("CD8A", "logfc=3.0"), ("CD8B", "")]) + record["statement"]["object"]["label"] = "B cell" + report = validator().validate(record) + lines = reasons(record, report) + assert any(line.startswith("evidence 1 (cluster=3;gene=CD8A): The row") for line in lines) + assert any(line.startswith("evidence 2 (cluster=3;gene=CD8B): No row") for line in lines) + assert any(line.startswith("object label: The label 'B cell'") for line in lines) + assert where(record, "$.source_artifacts[0]") == "source 1" and where(record, "$.statement.scope") == "statement.scope" + + +def test_conflicts_go_to_an_expert_and_are_not_fed_back(): + record = annotation("CL:0000084", [("CD8A", "call=up")], against=[("CD19", "call=absent")]) + report = validator().validate(record) + assert {f["rule_id"] for f in report["findings"]} == {"BEV004"} and to_expert(report) + assert reasons(record, report) == [] + marker = ReferenceGrounder(MARKERS, label="ASCT+B", evidence_key="gene", relation="marker_of", ontology=CL) + record = annotation("CL:0000084", [("CD19", "")]) + report = validator(marker).validate(record) + assert to_expert(report) and reasons(record, report) == [] + assert "BEV025" in EXPERT and not to_expert({"overall_status": "admitted", "findings": [{"rule_id": "BEV004"}]}) + + +def test_policy_findings_go_to_a_person_not_back_to_the_agent(): + record = annotation("CL:0000625", [("CD8A", "logfc=2.41")]) + for item in record["evidence_items"]: + item["extraction_method"] = "llm_extraction" + report = validator().validate(record) + assert {f["rule_id"] for f in report["findings"]} == {"BEV008"} and to_expert(report) + assert reasons(record, report) == [] + record["evidence_items"][0]["locator"] = "cluster=3;gene=CD8B" # a fixable finding as well: fed back + report = validator().validate(record) + assert not to_expert(report) and [r.split(":")[0] for r in reasons(record, report)] == ["evidence 1 (cluster=3;gene=CD8B)"] + + +def test_carry_keeps_verified_evidence_in_its_line(): + previous = annotation("CL:0000084", [("CD8A", "call=up")], against=[("CD19", "call=absent")]) + new = annotation("CL:0000084", [("CD8A", "call=up")]) + merged = carry(previous, {"bioev:item-1", "bioev:item-2"}, new) + assert [i["locator"] for i in merged["evidence_items"]] == ["cluster=3;gene=CD8A", "cluster=3;gene=CD19"] + lines = {line["direction"]: line["evidence_item_ids"] for line in merged["statement"]["evidence_lines"]} + assert lines == {"supports": ["bioev:item-1"], "contradicts": ["bioev:item-2"]} + assert new["evidence_items"][0]["id"] == "bioev:item-1" and len(new["evidence_items"]) == 1 # not modified + other = copy.deepcopy(new) + other["source_artifacts"][0].update(id="bioev:other", sha256="c" * 64, observed_sha256="c" * 64) + other["evidence_items"][0]["source_artifact_id"] = "bioev:other" + merged = carry(previous, {"bioev:item-1"}, other) + assert [s["id"] for s in merged["source_artifacts"]] == ["bioev:other", "bioev:source-1"] + assert merged["evidence_items"][1]["id"] == "bioev:item-1-2" # a fresh id, no collision + + +def test_revise_fixes_then_admits(): + attempts, seen = [], [] + wrong = annotation("CL:0000084", [("CD8A", "logfc=3.0")]) + right = annotation("CL:0000625", [("CD8A", "logfc=2.41")]) + + def propose(feedback): + seen.append(feedback) + return [wrong, right][len(seen) - 1] + + attempts = revise(propose, validator()) + assert [a.report["overall_status"] for a in attempts] == ["rejected", "admitted"] + assert seen[0] == [] and seen[1] and seen[1] == attempts[0].feedback + + +def test_revise_cannot_withdraw_verified_evidence_against_it(): + first = annotation("CL:0000084", [("CD8A", "logfc=3.0")], against=[("CD19", "call=absent")]) + second = annotation("CL:0000084", [("CD8A", "logfc=2.41")]) # fixed the value, dropped the evidence against + proposals = iter([first, second]) + attempts = revise(lambda feedback: next(proposals), validator()) + assert [a.report["overall_status"] for a in attempts] == ["rejected", "review_required"] + assert to_expert(attempts[-1].report) and len(attempts[-1].record["evidence_items"]) == 2 + + +def test_revise_stops_when_the_proposer_stops_or_rounds_run_out(): + assert revise(lambda feedback: None, validator()) == [] + wrong = annotation("CL:9999999", [("CD8A", "")]) + attempts = revise(lambda feedback: copy.deepcopy(wrong), validator(), rounds=2) + assert len(attempts) == 2 and all(a.report["overall_status"] == "rejected" for a in attempts) + broken = {"record_id": "x"} + attempts = revise(lambda feedback: broken, validator(), rounds=1) + assert attempts[0].report["schema_valid"] is False diff --git a/tests/test_reference_grounders.py b/tests/test_reference_grounders.py new file mode 100644 index 0000000..8ecc0bc --- /dev/null +++ b/tests/test_reference_grounders.py @@ -0,0 +1,318 @@ +"""Generic grounders on hand-made fixtures: ontology terms, gene symbols, variants, tables, a reference resource.""" +import copy +import hashlib +import json +from pathlib import Path + +import pytest + +from bioevidence_validator.crosscheck import ReferenceGrounder +from bioevidence_validator.engine import validate_record +from bioevidence_validator.grounding import SnapshotStore +from bioevidence_validator.identifiers import ( + Assemblies, + GeneGrounder, + Genes, + Ontology, + OntologyGrounder, + VariantGrounder, + pairs, +) +from bioevidence_validator.tables import TableGrounder, parse_table, same_value + +ROOT = Path(__file__).resolve().parents[1] +OBO = b"""format-version: 1.2 +data-version: cl/releases/2026-01-01/cl-basic.obo +ontology: cl + +[Term] +id: CL:0000000 +name: cell + +[Term] +id: CL:0000084 +name: T cell +synonym: "T-lymphocyte" EXACT [] +synonym: "T lymphocyte" RELATED [] +is_a: CL:0000000 ! cell + +[Term] +id: CL:0000625 +name: CD8-positive, alpha-beta T cell +is_a: CL:0000084 ! T cell + +[Term] +id: CL:0000236 +name: B cell +is_a: CL:0000000 ! cell + +[Term] +id: CL:0000788 +name: naive B cell +is_a: CL:0000236 ! B cell + +[Term] +id: CL:0000999 +name: obsolete lymphocyte of B lineage +is_obsolete: true +replaced_by: CL:0000236 + +[Term] +id: CL:0000998 +name: obsolete lymphocyte +is_obsolete: true +consider: CL:0000084 + +[Term] +id: CL:0001000 +name: cell part +is_a: CL:0009999 ! something that is not a cell + +[Typedef] +id: part_of +name: part of +""" +HGNC = (b"hgnc_id\tsymbol\tname\tstatus\tprev_symbol\talias_symbol\n" + b"HGNC:1706\tCD8A\tCD8 subunit alpha\tApproved\t\tCD8|Leu2\n" + b"HGNC:1633\tCD19\tCD19 molecule\tApproved\t\t\n" + b"HGNC:3430\tERBB2\terb-b2 receptor tyrosine kinase 2\tApproved\tNGL\tHER-2|HER2\n") +REPORT_38 = b"""# Assembly name: GRCh38.p14 +# Sequence-Name\tSequence-Role\tAssigned-Molecule\tAssigned-Molecule-Location/Type\tGenBank-Accn\tRelationship\tRefSeq-Accn\tAssembly-Unit\tSequence-Length\tUCSC-style-name +7\tassembled-molecule\t7\tChromosome\tCM000669.2\t=\tNC_000007.14\tPrimary Assembly\t159345973\tchr7 +""" +REPORT_37 = b"""# Assembly name: GRCh37.p13 +7\tassembled-molecule\t7\tChromosome\tCM000669.1\t=\tNC_000007.13\tPrimary Assembly\t159138663\tchr7 +""" +TABLE = b"cluster\tgene\tlogfc\tpct_in\tcall\n3\tCD8A\t2.4137\t0.91\tup\n3\tCD19\t-1.2\t0.02\tabsent\n5\tCD19\t3.1\t0.88\tup\n" + + +def base(**statement): + record = json.loads((ROOT / "examples/general/curated_assertion.json").read_text(encoding="utf-8")) + record["statement"].update(statement) + return record + + +def codes(record, grounder): + return [f.rule_id for f in grounder.check(record)] + + +def messages(record, grounder): + return " | ".join(f.message for f in grounder.check(record)) + + +CL = Ontology.from_obo(OBO) +CELLS = OntologyGrounder([CL], roots={"cell_type": ["CL:0000000"]}, locator_keys=["cell_type"]) + + +def cell(curie, label): + return base(object={"id": curie, "label": label, "entity_type": "cell_type"}) + + +def test_ontology_parsing_and_hierarchy(): + assert CL.prefix == "CL" and CL.version == "cl/releases/2026-01-01/cl-basic.obo" + assert Ontology.from_obo(OBO.replace(b"ontology: cl", b"ontology: cl/cl-basic")).prefix == "CL" + assert CL.by_label("T lymphocyte") == CL.by_label("t-lymphocyte") == ["CL:0000084"] # no duplicates + assert CL.terms["CL:0000084"].synonyms == ["T-lymphocyte"] # only exact synonyms name a term + assert CL.ancestors("CL:0000625") == {"CL:0000625", "CL:0000084", "CL:0000000"} + assert CL.related("CL:0000084", "CL:0000625") and not CL.related("CL:0000236", "CL:0000625") + assert CELLS.name == "ontology:CL@cl/releases/2026-01-01/cl-basic.obo" + + +def test_ontology_grounder_accepts_a_current_matching_term(): + assert codes(cell("CL:0000625", "CD8-positive, alpha-beta T cell"), CELLS) == [] + assert codes(cell("CL:0000625", "cd8 positive alpha beta t cell"), CELLS) == [] # case and punctuation + assert codes(cell("CL:0000084", "T-lymphocyte"), CELLS) == [] # exact synonym + + +def test_ontology_grounder_explains_what_to_fix(): + record = cell("CL:0000084", "B cell") + assert codes(record, CELLS) == ["BEV017"] + assert "'B cell' is the name of CL:0000236 (B cell)" in messages(record, CELLS) + record = cell("CL:9999999", "T cell") + assert codes(record, CELLS) == ["BEV016"] and "CL:0000084 (T cell)" in messages(record, CELLS) + record = cell("CL:0000999", "B cell") + assert codes(record, CELLS) == ["BEV023"] and "Use CL:0000236 (B cell)" in messages(record, CELLS) + assert "Closest names" in messages(cell("CL:9999999", "naive B-cells"), CELLS) + assert "Use CL:0000084 (T cell)" in messages(cell("CL:0000998", "T cell"), CELLS) + + +def test_ontology_grounder_checks_the_kind_of_term(): + record = cell("UBERON:0002371", "T cell") + assert codes(record, CELLS) == ["BEV024"] and "must be a CL term" in messages(record, CELLS) + assert codes(cell("CL:0001000", "cell part"), CELLS) == ["BEV024"] + other = base(object={"id": "GO:0008150", "label": "biological_process", "entity_type": "biological_process"}) + assert codes(other, CELLS) == [] # no loaded ontology and no root for this kind: nothing to say + + +def test_ontology_grounder_checks_scope_and_locators(): + record = cell("CL:0000084", "T cell") + record["statement"]["scope"] = ["CL:9999999"] + record["evidence_items"][0]["locator"] = "cell_type=UBERON:1" + assert sorted(codes(record, CELLS)) == ["BEV016", "BEV024"] + + +GENES = GeneGrounder(Genes.from_hgnc(HGNC, "2026-09-29")) + + +def with_gene(symbol, scope=("NCBITaxon:9606",)): + record = base(scope=list(scope)) + record["evidence_items"][0].update(locator=f"cluster=3;gene={symbol}") + return record + + +@pytest.mark.parametrize("symbol,code,text", [ + ("CD8A", None, ""), ("HER2", "BEV023", "alias of ERBB2"), ("NGL", "BEV023", "previous symbol or alias of ERBB2"), + ("Cd8a", "BEV023", "the approved symbol is CD8A"), ("CD8AA", "BEV016", "Closest: CD8A"), + ("XYZ1", "BEV016", "not an HGNC gene symbol")]) +def test_gene_grounder_symbols(symbol, code, text): + record = with_gene(symbol) + assert codes(record, GENES) == ([code] if code else []) and text in messages(record, GENES) + + +def test_gene_grounder_entities_and_species(): + record = base(subject={"id": "HGNC:1706", "label": "CD8A", "entity_type": "gene"}) + assert codes(record, GENES) == [] + record["statement"]["subject"]["label"] = "CD19" + assert codes(record, GENES) == ["BEV017"] + record["statement"]["subject"] = {"id": "HGNC:0", "label": "X", "entity_type": "gene"} + assert codes(record, GENES) == ["BEV016"] + record["statement"]["subject"] = {"id": "gene:her2", "label": "HER2", "entity_type": "gene"} + assert codes(record, GENES) == ["BEV023"] + assert codes(with_gene("Cd8a", scope=["NCBITaxon:10090"]), GENES) == ["BEV015"] + assert GENES.name == "genes:HGNC@2026-09-29" + + +VARIANTS = VariantGrounder(Assemblies.from_reports([REPORT_38, REPORT_37])) + + +def variant(text, scope=("GRCh38",)): + return base(subject={"id": "var:1", "label": text, "entity_type": "variant"}, scope=list(scope)) + + +@pytest.mark.parametrize("text,scope,code", [ + ("NC_000007.14:g.140753336A>T", ["GRCh38"], None), + ("NC_000007.13:g.140453136A>T", ["GRCh38"], "BEV017"), + ("NC_000007.13:g.140453136A>T", ["hg19"], None), + ("NC_000007.15:g.1A>T", ["GRCh38"], "BEV016"), + ("NC_000007.14:g.999999999A>T", ["GRCh38"], "BEV016"), + ("NC_000007:g.140753336A>T", ["GRCh38"], "BEV024"), + ("BRAF V600E", ["GRCh38"], "BEV024"), + ("NM_004333.6:c.1799T>A", ["GRCh38"], None), +]) +def test_variant_grounder(text, scope, code): + assert codes(variant(text, scope), VARIANTS) == ([code] if code else []) + + +def test_variant_grounder_mixed_builds_and_locators(): + record = variant("NC_000007.14:g.140753336A>T", scope=["taxon:synthetic"]) + record["evidence_items"][0]["locator"] = "variant=NC_000007.13:g.140453136A>T" + assert codes(record, VARIANTS) == ["BEV017"] + assert VARIANTS.name == "variants:GRCh37+GRCh38" + + +def test_pairs_and_values(): + assert pairs("cluster=3; gene=CD8A") == {"cluster": "3", "gene": "CD8A"} + assert pairs("row 3") == {} and pairs("") == {} + assert same_value("2.4", "2.4137") and not same_value("2.5", "2.4137") and same_value("2", "2.4") + assert same_value("Up", "up") and not same_value("up", "absent") + assert parse_table(b"a,b\n1, 2\n") == (["a", "b"], [{"a": "1", "b": "2"}]) + + +def table_record(locator, text, *, sha=None): + record = base() + record["source_artifacts"][0].update(sha256=sha or hashlib.sha256(TABLE).hexdigest()) + record["source_artifacts"][0]["observed_sha256"] = record["source_artifacts"][0]["sha256"] + record["evidence_items"][0].update(locator=locator, extracted_text=text, evidence_type="marker_gene") + return record + + +TABLES = TableGrounder(SnapshotStore({hashlib.sha256(TABLE).hexdigest(): lambda: TABLE}), ["marker_gene"]) + + +@pytest.mark.parametrize("locator,text,code,hint", [ + ("cluster=3;gene=CD8A", "logfc=2.41; call=up", None, ""), + ("cluster=3;gene=CD8A", "", None, ""), + ("cluster=3;gene=CD8A", "logfc=3.2", "BEV017", "has logfc=2.4137, not logfc=3.2"), + ("cluster=3;gene=CD8B", "call=up", "BEV016", "Closest gene values for cluster=3: CD8A"), + ("cluster=4;gene=CD8A", "", "BEV016", "No row"), + ("gene=CD19", "", "BEV015", "2 rows"), + ("cluster=3;symbol=CD8A", "", "BEV024", "no column 'symbol'"), + ("row 3", "", "BEV024", "key=value"), +]) +def test_table_grounder(locator, text, code, hint): + record = table_record(locator, text) + assert codes(record, TABLES) == ([code] if code else []) and hint in messages(record, TABLES) + assert TABLES.verified_items(record) == ({"bioev:item-1"} if code is None else set()) + + +def test_table_grounder_needs_the_bytes_and_ignores_other_evidence(): + assert codes(table_record("cluster=3;gene=CD8A", "", sha="b" * 64), TABLES) == ["BEV015"] + record = table_record("cluster=3;gene=CD8B", "") + record["evidence_items"][0]["evidence_type"] = "curated_annotation" + assert codes(record, TABLES) == [] + + +MARKERS = [{"subject": "CD19", "predicate": "marker_of", "object": "CL:0000236", "object_label": "B cell"}, + {"subject": "CD8A", "predicate": "marker_of", "object": "CL:0000625"}] + + +def marker_record(gene, target, direction="supports"): + record = cell(target, CL.terms[target].name) + record["evidence_items"][0]["locator"] = f"cluster=3;gene={gene}" + record["statement"]["evidence_lines"][0]["direction"] = direction + return record + + +def test_reference_grounder_evidence_conflicts(): + grounder = ReferenceGrounder(MARKERS, label="ASCT+B", evidence_key="gene", relation="marker_of", ontology=CL) + assert codes(marker_record("CD8A", "CL:0000625"), grounder) == [] + assert codes(marker_record("CD19", "CL:0000788"), grounder) == [] # a B cell marker for a naive B cell + assert codes(marker_record("CD8A", "CL:0000084"), grounder) == [] # related the other way + assert codes(marker_record("GZMB", "CL:0000084"), grounder) == [] # the reference is silent + record = marker_record("CD19", "CL:0000084") + assert codes(record, grounder) == ["BEV025"] + assert "lists CD19 as a marker of CL:0000236 (B cell), not of CL:0000084" in messages(record, grounder) + assert codes(marker_record("CD19", "CL:0000084", direction="contradicts"), grounder) == [] + plain = ReferenceGrounder(MARKERS, label="ASCT+B", evidence_key="gene", relation="marker_of") + assert codes(marker_record("CD19", "CL:0000788"), plain) == ["BEV025"] # without an ontology, only the same id + + +def test_reference_grounder_statement_conflict(): + rows = b"subject\tpredicate\tobject\nSYN:PROTEIN_A\tdoes_not_participate_in\tSYN:PROCESS_A\n" + grounder = ReferenceGrounder.from_table(rows, label="toy", opposites={"participates_in": "does_not_participate_in"}) + assert codes(base(), grounder) == ["BEV025"] and grounder.name == "reference:toy" + assert codes(base(predicate="does_not_participate_in"), grounder) == [] + + +def test_grounders_route_through_the_engine(): + record = copy.deepcopy(cell("CL:0000084", "B cell")) + report = validate_record(record, grounders=[CELLS]) + assert report["overall_status"] == "rejected" and report["grounding"] == [CELLS.name] + report = validate_record(marker_record("CD19", "CL:0000084"), grounders=[ + ReferenceGrounder(MARKERS, label="ASCT+B", evidence_key="gene", relation="marker_of", ontology=CL)]) + assert report["overall_status"] == "review_required" + + +def test_cli_identifier_options(tmp_path): + from bioevidence_validator.cli import main + + paths = {} + for name, data in {"cl.obo": OBO, "hgnc.tsv": HGNC, "grch38.txt": REPORT_38}.items(): + paths[name] = tmp_path / name + paths[name].write_bytes(data) + options = ["--ontology", str(paths["cl.obo"]), "--term-root", "cell_type=CL:0000000", "--genes", str(paths["hgnc.tsv"]), + "--assembly-report", str(paths["grch38.txt"])] + + def run(record): + path, output = tmp_path / "record.json", tmp_path / "report.json" + path.write_text(json.dumps(record), encoding="utf-8") + code = main(["validate", str(path), "--output", str(output), *options]) + return code, json.loads(output.read_text(encoding="utf-8")) if output.exists() else None + + code, report = run(cell("CL:0000625", "CD8-positive, alpha-beta T cell")) + assert code == 0 and report["grounding"] == ["ontology:CL@cl/releases/2026-01-01/cl-basic.obo", + "genes:HGNC@hgnc.tsv", "variants:GRCh38"] + code, report = run(cell("UBERON:0002371", "T cell")) + assert code == 1 and {f["rule_id"] for f in report["findings"]} == {"BEV024"} + options[3] = "cell_type" + assert main(["validate", str(tmp_path / "record.json"), *options]) == 3 From 2eec8257ab568cd25d88280425a0456ea61729e2 Mon Sep 17 00:00:00 2001 From: NingyuSun Date: Wed, 30 Sep 2026 16:42:52 +0900 Subject: [PATCH 2/5] Carry verified evidence only while the claim stays the same Evidence lines are relative to their statement: a revision that states another cell type or the opposite predicate is validated on its own evidence, instead of inheriting stances that meant something else. --- docs/ENGINEERING.md | 2 +- src/bioevidence_validator/feedback.py | 15 +++++++++++++-- tests/test_feedback.py | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/ENGINEERING.md b/docs/ENGINEERING.md index 6ae8932..1b8ebca 100644 --- a/docs/ENGINEERING.md +++ b/docs/ENGINEERING.md @@ -194,7 +194,7 @@ Two kinds of finding are not fed back. The record goes to a person as it is: flags it (BEV022), or it lacks an independent review (BEV021). - **Policy:** the use needs a human decision whatever the proposer does (BEV008–BEV013). -Evidence verified in one attempt is carried into the next, so a revision can fix or replace what failed +Evidence verified in one attempt is carried into the next while the claim stays the same, so a revision can fix or replace what failed but cannot withdraw verified evidence against its answer. The literature benchmark's agent loop (`evaluation/llm_benchmark/agent_loop.py`) showed why: before that rule, an agent whose record held a misquote and a verified quote against its decision dropped the latter and was admitted. diff --git a/src/bioevidence_validator/feedback.py b/src/bioevidence_validator/feedback.py index 5421380..91b8171 100644 --- a/src/bioevidence_validator/feedback.py +++ b/src/bioevidence_validator/feedback.py @@ -9,7 +9,8 @@ evidence against its answer go away. Policy: the use needs a human decision whatever the agent does (LLM-only or string-match-only evidence where the profile forbids it, BEV008/009/013; human acceptance required, deferred or refused, BEV010/012/011). For the same reason, evidence verified in one attempt is carried -into the next: a revision may fix or replace what failed, but not withdraw what was verified. +into the next while the claim stays the same: a revision may fix or replace what failed, but not withdraw what +was verified. `revise` runs the loop around any proposer (a model call, an agent, a person); nothing here calls a model. """ @@ -67,9 +68,19 @@ def _key(record: dict[str, Any], item: dict[str, Any]) -> tuple[str, str, str]: return sha, item["locator"], item.get("extracted_text") or "" +def claim(record: dict[str, Any]) -> tuple[str, str, str]: + statement = record["statement"] + return statement["subject"]["id"], statement["predicate"], statement["object"]["id"] + + def carry(previous: dict[str, Any], verified: set[str], record: dict[str, Any]) -> dict[str, Any]: - """`record` plus the verified evidence of `previous` that it left out, in the evidence line it was in.""" + """`record` plus the verified evidence of `previous` that it left out, in the evidence line it was in. + + Only for the same claim: evidence lines are relative to their statement, so a record that states something + else (another cell type, the opposite predicate) is validated on its own evidence.""" merged = copy.deepcopy(record) + if claim(previous) != claim(record): + return merged present = {_key(merged, item) for item in merged["evidence_items"]} directions = {i: line["direction"] for line in previous["statement"]["evidence_lines"] for i in line["evidence_item_ids"]} ids = {x["id"] for x in merged["evidence_items"]} | {s["id"] for s in merged["source_artifacts"]} diff --git a/tests/test_feedback.py b/tests/test_feedback.py index de3fff5..a343e0e 100644 --- a/tests/test_feedback.py +++ b/tests/test_feedback.py @@ -84,6 +84,8 @@ def test_carry_keeps_verified_evidence_in_its_line(): merged = carry(previous, {"bioev:item-1"}, other) assert [s["id"] for s in merged["source_artifacts"]] == ["bioev:other", "bioev:source-1"] assert merged["evidence_items"][1]["id"] == "bioev:item-1-2" # a fresh id, no collision + changed = annotation("CL:0000625", [("CD8A", "call=up")]) # another claim: nothing is carried + assert carry(previous, {"bioev:item-1", "bioev:item-2"}, changed) == changed def test_revise_fixes_then_admits(): From 46114d26fddd23f20b709b8b92b4dd244f31828f Mon Sep 17 00:00:00 2001 From: NingyuSun Date: Wed, 30 Sep 2026 16:44:58 +0900 Subject: [PATCH 3/5] Let a plain plural match an ontology label 'Hepatocytes' names CL:0000182 as well as 'hepatocyte' does; both sides are normalised the same way, so distinct terms stay distinct. --- docs/ENGINEERING.md | 2 +- src/bioevidence_validator/identifiers.py | 4 +++- tests/test_reference_grounders.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/ENGINEERING.md b/docs/ENGINEERING.md index 1b8ebca..984a10b 100644 --- a/docs/ENGINEERING.md +++ b/docs/ENGINEERING.md @@ -142,7 +142,7 @@ tokens, and evidence locators written as `key=value` pairs separated by `;` (`cl MONDO, UBERON …): - the term exists (BEV016); - it is not obsolete (BEV023, naming its replacement); - - its label is the term's name or an exact synonym, ignoring case and punctuation (BEV017, naming the + - its label is the term's name or an exact synonym, ignoring case, punctuation and a plain plural (BEV017, naming the term that label belongs to); - it is of the right kind: `roots` maps an entity type or locator key to the terms it must descend from, so a `cell_type` must be under CL:0000000 (BEV024). diff --git a/src/bioevidence_validator/identifiers.py b/src/bioevidence_validator/identifiers.py index 1e91c30..db99bb3 100644 --- a/src/bioevidence_validator/identifiers.py +++ b/src/bioevidence_validator/identifiers.py @@ -52,7 +52,9 @@ def _prefix(curie: str) -> str: def _label_key(text: str) -> str: - return " ".join(re.sub(r"[^0-9a-z]+", " ", text.casefold()).split()) + """Case, punctuation and a plain plural ("hepatocytes") do not change a label.""" + words = re.sub(r"[^0-9a-z]+", " ", text.casefold()).split() + return " ".join(w[:-1] if len(w) > 3 and w.endswith("s") and not w.endswith("ss") else w for w in words) def _entities(record: dict[str, Any]) -> Iterator[tuple[str, dict[str, Any]]]: diff --git a/tests/test_reference_grounders.py b/tests/test_reference_grounders.py index 8ecc0bc..e75e295 100644 --- a/tests/test_reference_grounders.py +++ b/tests/test_reference_grounders.py @@ -122,6 +122,7 @@ def test_ontology_grounder_accepts_a_current_matching_term(): assert codes(cell("CL:0000625", "CD8-positive, alpha-beta T cell"), CELLS) == [] assert codes(cell("CL:0000625", "cd8 positive alpha beta t cell"), CELLS) == [] # case and punctuation assert codes(cell("CL:0000084", "T-lymphocyte"), CELLS) == [] # exact synonym + assert codes(cell("CL:0000236", "B cells"), CELLS) == [] # a plain plural def test_ontology_grounder_explains_what_to_fix(): From 8a94da84df692532ebb5debaf6c3f791c87a04c0 Mon Sep 17 00:00:00 2001 From: NingyuSun Date: Wed, 30 Sep 2026 16:45:34 +0900 Subject: [PATCH 4/5] Update the ontology hint test for plural labels --- tests/test_reference_grounders.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_reference_grounders.py b/tests/test_reference_grounders.py index e75e295..d31d26e 100644 --- a/tests/test_reference_grounders.py +++ b/tests/test_reference_grounders.py @@ -133,7 +133,8 @@ def test_ontology_grounder_explains_what_to_fix(): assert codes(record, CELLS) == ["BEV016"] and "CL:0000084 (T cell)" in messages(record, CELLS) record = cell("CL:0000999", "B cell") assert codes(record, CELLS) == ["BEV023"] and "Use CL:0000236 (B cell)" in messages(record, CELLS) - assert "Closest names" in messages(cell("CL:9999999", "naive B-cells"), CELLS) + assert "is the name of CL:0000788" in messages(cell("CL:9999999", "naive B-cells"), CELLS) # plural + assert "Closest names" in messages(cell("CL:9999999", "naive B-cell type"), CELLS) assert "Use CL:0000084 (T cell)" in messages(cell("CL:0000998", "T cell"), CELLS) From ac0218d37f24510761cf89173bd81abafb03a81d Mon Sep 17 00:00:00 2001 From: NingyuSun Date: Wed, 30 Sep 2026 16:49:53 +0900 Subject: [PATCH 5/5] Reference conflicts by ontology disjointness Ontology parses disjoint_from and inherits it through is_a. With conflict="disjoint", ReferenceGrounder reports a supporting marker only when every type the reference lists it for is declared disjoint from the claimed type, so a marker listed for a sibling type is not a conflict. --- docs/ENGINEERING.md | 7 ++++-- src/bioevidence_validator/crosscheck.py | 27 +++++++++++++++++------- src/bioevidence_validator/identifiers.py | 10 ++++++++- tests/test_reference_grounders.py | 17 +++++++++++++++ 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/docs/ENGINEERING.md b/docs/ENGINEERING.md index 984a10b..1cdcc98 100644 --- a/docs/ENGINEERING.md +++ b/docs/ENGINEERING.md @@ -168,8 +168,11 @@ tokens, and evidence locators written as `key=value` pairs separated by `;` (`cl support. It raises BEV025 in two cases: - the reference asserts the opposite predicate for the statement's subject and object; - supporting evidence names an entity (e.g. `gene=CD19`) that the reference relates (e.g. `marker_of`) - only to objects unrelated to the statement's object. With an ontology, an ancestor or descendant - counts as related. + only to objects that conflict with the statement's object. By default a different term conflicts unless + the ontology makes it an ancestor or descendant. With `conflict="disjoint"`, only terms the ontology + declares disjoint conflict (`disjoint_from` on them or their ancestors, e.g. T cell and B cell), so a + marker the reference lists for a sibling type is not reported. That needs an ontology release that keeps + its disjointness axioms: the Cell Ontology's full `cl.obo` does, `cl-basic.obo` does not. ```bash bioevidence validate record.json --ontology cl-basic.obo --term-root cell_type=CL:0000000 --genes hgnc_complete_set.txt --assembly-report GRCh38_assembly_report.txt diff --git a/src/bioevidence_validator/crosscheck.py b/src/bioevidence_validator/crosscheck.py index d2cfd8a..ef74139 100644 --- a/src/bioevidence_validator/crosscheck.py +++ b/src/bioevidence_validator/crosscheck.py @@ -12,9 +12,12 @@ - statement: the reference asserts the opposite predicate for the same subject and object (`opposites` maps each predicate to its opposite); - evidence: supporting evidence names an entity in its locator (`evidence_key`, e.g. `gene=CD19`) that the - reference relates (`relation`, e.g. `marker_of`) only to objects unrelated to the statement's object. With - an ontology, an ancestor or descendant of the object counts as related (a B cell marker supports "naive - B cell"); without one, only the same identifier does. + reference relates (`relation`, e.g. `marker_of`) only to objects that conflict with the statement's object. + With `conflict="unrelated"` (the default), an object conflicts unless it is the same term or, with an + ontology, an ancestor or descendant of it (a B cell marker supports "naive B cell"). With + `conflict="disjoint"`, an object conflicts only when the ontology declares the two disjoint (`disjoint_from` + on them or their ancestors, e.g. T cell and B cell): a marker the reference lists for a sibling type is not a + conflict, and where the ontology declares nothing, nothing is reported. """ from __future__ import annotations @@ -30,7 +33,11 @@ class ReferenceGrounder: def __init__(self, assertions: Iterable[dict[str, str]], *, label: str, opposites: dict[str, str] | None = None, - evidence_key: str | None = None, relation: str | None = None, ontology: Ontology | None = None): + evidence_key: str | None = None, relation: str | None = None, ontology: Ontology | None = None, + conflict: str = "unrelated"): + if conflict not in ("unrelated", "disjoint") or (conflict == "disjoint" and ontology is None): + raise ValueError("conflict must be 'unrelated', or 'disjoint' with an ontology") + self.conflict = conflict self.assertions = list(assertions) self.label, self.ontology = label, ontology self.opposites = dict(opposites or {}) @@ -50,8 +57,10 @@ def from_table(cls, data: bytes, **options: Any) -> ReferenceGrounder: delimiter = "\t" if "\t" in text.partition("\n")[0] else "," return cls(csv.DictReader(io.StringIO(text), delimiter=delimiter), **options) - def _related(self, a: str, b: str) -> bool: - return a == b or bool(self.ontology and self.ontology.related(a, b)) + def _conflicts(self, a: str, b: str) -> bool: + if self.conflict == "disjoint": + return bool(self.ontology and self.ontology.disjoint(a, b)) + return not (a == b or bool(self.ontology and self.ontology.related(a, b))) def _name(self, row: dict[str, str]) -> str: return f"{row['object']} ({row['object_label']})" if row.get("object_label") else row["object"] @@ -71,11 +80,13 @@ def check(self, record: dict[str, Any]) -> list[Finding]: for index, item in enumerate(record["evidence_items"]): entity = pairs(item["locator"]).get(self.key) rows = self._objects.get(entity or "", []) - if item["id"] not in supporting or not rows or any(self._related(r["object"], target) for r in rows): + if item["id"] not in supporting or not rows or not all(self._conflicts(r["object"], target) for r in rows): continue names = sorted({self._name(r) for r in rows}) more = f" and {len(names) - 3} more" if len(names) > 3 else "" out.append(finding(record, "BEV025", f"The {self.label} reference lists {entity} as a {(self.relation or '').replace('_', ' ')} " - f"{'; '.join(names[:3])}{more}, not of {target} or a related term.", + f"{'; '.join(names[:3])}{more}, " + + (f"cell types disjoint from {target}." if self.conflict == "disjoint" + else f"not of {target} or a related term."), f"$.evidence_items[{index}].locator")) return out diff --git a/src/bioevidence_validator/identifiers.py b/src/bioevidence_validator/identifiers.py index db99bb3..50934c6 100644 --- a/src/bioevidence_validator/identifiers.py +++ b/src/bioevidence_validator/identifiers.py @@ -74,10 +74,11 @@ class Term: obsolete: bool = False replaced_by: list[str] = field(default_factory=list) consider: list[str] = field(default_factory=list) + disjoint: list[str] = field(default_factory=list) class Ontology: - """The terms of one pinned OBO release: names, exact synonyms, `is_a` parents and obsoletion.""" + """The terms of one pinned OBO release: names, exact synonyms, `is_a` parents, obsoletion, disjointness.""" def __init__(self, terms: dict[str, Term], prefix: str, version: str): self.terms, self.prefix, self.version = terms, prefix, version @@ -121,6 +122,8 @@ def from_obo(cls, data: bytes) -> Ontology: term.replaced_by.append(value) elif tag == "consider": term.consider.append(value) + elif tag == "disjoint_from": + term.disjoint.append(value) # `ontology: cl` or `ontology: cl/cl-basic`; a release file may also hold a few terms of other ontologies prefix = header.get("ontology", "").split("/")[0].upper() version = header.get("data-version", "") or header.get("date", "") @@ -146,6 +149,11 @@ def related(self, a: str, b: str) -> bool: """The same term, or one is an ancestor of the other.""" return a in self.ancestors(b) or b in self.ancestors(a) + def disjoint(self, a: str, b: str) -> bool: + """Declared disjoint: some ancestor of one is `disjoint_from` some ancestor of the other.""" + mine, theirs = self.ancestors(a), self.ancestors(b) + return any(d in theirs for x in mine if x in self.terms for d in self.terms[x].disjoint) or any(d in mine for x in theirs if x in self.terms for d in self.terms[x].disjoint) + def by_label(self, label: str) -> list[str]: return self._labels.get(_label_key(label), []) diff --git a/tests/test_reference_grounders.py b/tests/test_reference_grounders.py index d31d26e..50107a1 100644 --- a/tests/test_reference_grounders.py +++ b/tests/test_reference_grounders.py @@ -35,6 +35,7 @@ synonym: "T-lymphocyte" EXACT [] synonym: "T lymphocyte" RELATED [] is_a: CL:0000000 ! cell +disjoint_from: CL:0000236 ! B cell [Term] id: CL:0000625 @@ -279,6 +280,22 @@ def test_reference_grounder_evidence_conflicts(): assert codes(marker_record("CD19", "CL:0000788"), plain) == ["BEV025"] # without an ontology, only the same id +def test_ontology_disjointness_is_inherited(): + assert CL.disjoint("CL:0000625", "CL:0000788") and CL.disjoint("CL:0000236", "CL:0000084") + assert not CL.disjoint("CL:0000625", "CL:0000084") and not CL.disjoint("CL:0000625", "CL:0001000") + + +def test_reference_grounder_disjoint_conflicts(): + grounder = ReferenceGrounder(MARKERS, label="ASCT+B", evidence_key="gene", relation="marker_of", ontology=CL, + conflict="disjoint") + record = marker_record("CD19", "CL:0000625") # a B cell marker for a CD8 T cell: disjoint lineages + assert codes(record, grounder) == ["BEV025"] and "cell types disjoint from CL:0000625" in messages(record, grounder) + assert codes(marker_record("CD8A", "CL:0000788"), grounder) == ["BEV025"] + assert codes(marker_record("CD8A", "CL:0001000"), grounder) == [] # nothing declared: nothing reported + with pytest.raises(ValueError): + ReferenceGrounder(MARKERS, label="x", conflict="disjoint") + + def test_reference_grounder_statement_conflict(): rows = b"subject\tpredicate\tobject\nSYN:PROTEIN_A\tdoes_not_participate_in\tSYN:PROCESS_A\n" grounder = ReferenceGrounder.from_table(rows, label="toy", opposites={"participates_in": "does_not_participate_in"})