From 288b97fb64f1931e46afac4c817756c563609b6a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 07:41:05 +0000 Subject: [PATCH 01/33] deepnsm-v2: position decides noun/verb homographs, not the lemma tag (D-LXC-13) A word between subject and object no longer becomes a noun because COCA counts it as a noun more often. - coca::predicate_alternatives: a lemma tag that is Noun or Verb gains the other reading when the lexicon has it; function words keep one tag. bible_wave's tagger applies it to the lemma table's first row. - fsm slot rule: right after a fresh subject (a noun that opened the clause with no determiner or adjective in front), a token's Noun reading is dropped when it also has a Verb reading. Relative, like licensing. A carried object ("gave him charge"), a re-anchored noun ("mothers house") or a determined subject does not trigger it. - ReadingParse::{slot_dropped, unlicensed_dropped}; the KJV gate accepts them as the explanation for a changed verse. A sentence-level "needs a predicate" rule was built and removed: on the KJV its noun-to-verb flips were right about 7 times in 34 (verbless verse fragments). KJV re-pinned: 34,824 ambiguous tokens / 451 words; certain 57,350 + alternative 29,001 (was 69,670 + 1,716); peak 256 configurations, no overflow. Downstream gate verdicts unchanged. Board: D-LXC-13 row, entry 2026-10-04-deepnsm-v2-predicate-by-position.md, D-LXC-2 marked shipped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 5 +- .claude/board/SUPERSESSION-INDEX.md | 6 +- ...-10-04-deepnsm-v2-predicate-by-position.md | 69 +++++++++ .claude/board/entries/README.md | 3 +- ...deepnsm-v2-lexical-evidence-consumer-v1.md | 11 ++ crates/deepnsm-v2/examples/bible_wave.rs | 76 ++++++--- crates/deepnsm-v2/src/coca.rs | 49 ++++++ crates/deepnsm-v2/src/fsm.rs | 144 +++++++++++++++++- 8 files changed, 330 insertions(+), 33 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-predicate-by-position.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 6b2d9c654..db6012782 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -27,8 +27,8 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | D-id | scope | status | gate / falsifier | |---|---|---|---| | **D-LXC-1** | `bible_wave` only: counted pick (fold in the example) replaces the word_forms layer's first-wins; lemma table stays first (variant B, pinned rule F9 from `ec50f07b`); library unchanged. Rewritten 2026-09-29 against the migration documents, every claim re-read | Queued | G1–G7 in the plan; G5 KJV before/after blocking; G6 exactly 25 tag changes within `bible_vocab.txt`; G7 restores the deleted lemma-first test (closes `TD-DEEPNSM-V2-SESSION-RESIDUE` item 3) | -| **D-LXC-2** | FSM (`Tagged`/`parse_to_spo`) takes several readings per token, resolved by role (`E-SURFACE-FORM-COLLAPSE-1`) | In PR | `fsm::parse_readings` + `PosSet` + `coca` fold; T1-T7 disable-verified; KJV accounting pinned in `bible_wave` (3,363 ambiguous tokens / 141 words, 1,908 narrowed, 0 unexplained). Entry `2026-10-03-deepnsm-v2-multi-reading-fsm.md` | -| **D-LXC-3** | lemma-table order: keep first (B, 25 in-vocab changes) or counted-first (A, 141; overrides F9) | Shipped | DECISION 2026-10-03: B — the lemma table order is kept (F9 stands); D-LXC-2 runs on it | +| **D-LXC-2** | FSM (`Tagged`/`parse_to_spo`) takes several readings per token, resolved by role (`E-SURFACE-FORM-COLLAPSE-1`) | Shipped (#1314) | `fsm::parse_readings` + `PosSet` + `coca` fold; T1-T7 disable-verified; KJV accounting pinned in `bible_wave` (3,363 ambiguous tokens / 141 words, 1,908 narrowed, 0 unexplained). Entry `2026-10-03-deepnsm-v2-multi-reading-fsm.md` | +| **D-LXC-3** | lemma-table order: keep first (B, 25 in-vocab changes) or counted-first (A, 141; overrides F9) | Shipped | DECISION 2026-10-03: B — the lemma table order is kept (F9 stands); D-LXC-2 runs on it. ⊘ 2026-10-04: the first row no longer settles a noun/verb homograph — D-LXC-13 widens it and position decides | | **D-LXC-4** | `academic_20k.csv` loader + the duplicate `coca_pos` in `genre_shapes.rs` | Blocked | ruling on 3 duplicate (word, PoS) pairs: wastewater/n, disproportionately/r, instill/v | | **D-LXC-5** | KJV-own counts for the Cam96 vocabulary (`bible_vocab.txt` is surface-only) | Queued | computed from the corpus | | **D-LXC-6** | mask-risc fold result at version v into cycle v+1 as a staged cast | Queued | own plan; touches the reverted CE64 area; not re-verified in the 2026-09-29 rewrite | @@ -36,6 +36,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-8** | scrub the model-naming `Co-Authored-By` trailer from history (120 commits on `main`) | Deferred | operator: no force push | | **D-LXC-9** | `archaic_pos` cannot override COCA-known KJV words (`art` is Noun, `word_forms.csv:1047`) | Queued | pre-existing | | **D-LXC-10** | `Pos::Rel` produced by no in-crate tagger in deepnsm-v2 (only FSM match arms + one unit test) | Queued | pre-existing; external callers can still pass `Pos::Rel` through the public `Tagged`/`parse_to_spo` | +| **D-LXC-13** | noun/verb homograph decided by position: lemma N/V tag widened to its other reading (`coca::predicate_alternatives`), slot rule after a fresh subject | In PR | slot rule + widening disable-verified; KJV re-pinned (34,824 ambiguous / 451 words, certain 57,350 + alternative 29,001, 0 unexplained). Entry `2026-10-04-deepnsm-v2-predicate-by-position.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/SUPERSESSION-INDEX.md b/.claude/board/SUPERSESSION-INDEX.md index 798cd198c..7d79f6c02 100644 --- a/.claude/board/SUPERSESSION-INDEX.md +++ b/.claude/board/SUPERSESSION-INDEX.md @@ -83,7 +83,7 @@ a licence to act on it. | **READ** | `archetype-scaffold-v1` | `GateDecision` | In progress (2026-04-24) | 0/0 | | **READ** | `capstone-cognitive-loop-wiring-nan-census-v1` | `GateDecision` | PROPOSED (2026-06-20). The measurement compa | 0/0 | | **READ** | `deepnsm-v3-convergence-v1` | `StepMask` | PROPOSED (doc-only). Extends `v3-convergence | 0/5 | -| **READ** | `epistemic-quadrant-materialization-v1` | `MergeMode` | PROPOSED.** Operator direction 2026-07-29: * | 1/4 | +| **READ** | `epistemic-quadrant-materialization-v1` | `MergeMode` | PROPOSED.** Operator direction 2026-07-29: * | 2/4 | | **READ** | `graphrag-doc-retrieval-soa-integration-v1` | `GateDecision` | DESIGN + FIRST CODE. **v1.2 (2026-07-17):** | 3/10 | | **READ** | `hexagon-plasticity-v1` | `MergeMode` | ACTIVE — landed 2026-09-14 on operator go. * | 2/10 | | **READ** | `integration-actionhandler-rbac-orchestration-v1` | `GateDecision` | HARDENING (5+3 in progress). | 0/0 | @@ -96,8 +96,8 @@ a licence to act on it. | **READ** | `persistence-artifact-backed-commit-v1` | `persist_cycle` | RATIFIED (operator ruling 2026-08-09). Phase | 0/0 | | **READ** | `post-teardown-buildup-survey-v1` | `StepMask` | SURVEY, read-only, plan-only (no code, no te | 1/5 | | **READ** | `r2il-bpe-typed-genetic-recombination-v1` | `GateDecision` | PROPOSAL, §7's three falsifiers now RUN (see | 1/1 | -| **READ** | `scientific-kg-substrate-v1` | `GateDecision` | PROPOSED — **scoping doc**, no code. Records | 0/9 | -| **READ** | `self-reasoning-substrate-v1` | `GateDecision` | PROPOSED — doc-only. No code, no contract ch | 0/5 | +| **READ** | `scientific-kg-substrate-v1` | `GateDecision` | PROPOSED — **scoping doc**, no code. Records | 3/9 | +| **READ** | `self-reasoning-substrate-v1` | `GateDecision` | PROPOSED — doc-only. No code, no contract ch | 4/5 | | **READ** | `v3-convergence-wiring-v1` | `GateDecision` | ACTIVE (2026-07-01). Operator: "I'm all in f | 0/0 | | **RESCOPE** | `soa-migration-diff-resolution-2026-06-13` | `BindSpace`, `CollapseGateEmission`, `GateDecision`, `MergeMode` … | — | 1/5 | | **RESCOPE** | `cognitive-substrate-convergence-v1` | `BindSpace`, `CollapseGateEmission`, `GateDecision`, `MergeMode` … | PROPOSAL (sprint-10 architectural decisions | 3/13 | diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-predicate-by-position.md b/.claude/board/entries/2026-10-04-deepnsm-v2-predicate-by-position.md new file mode 100644 index 000000000..031436255 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-predicate-by-position.md @@ -0,0 +1,69 @@ +# 2026-10-04 — deepnsm-v2: a noun/verb homograph is decided by position, not by its lemma tag (D-LXC-13) + +**Status:** MEASURED (`cargo run --release --example bible_wave -- pg10.txt`, +`v0.1.0-cam96-data` assets). Reopens D-LXC-3 (operator direction, 2026-10-04: +stop turning a word between subject and object into a noun because the corpus +counts it as a noun more often). + +## What changed + +- `coca::predicate_alternatives(tag, known)`: a lemma tag that is Noun or Verb + gains the other reading when the lexicon has it. Function words keep their + one tag. `bible_wave`'s tagger applies it to the lemma table's first row + (F9 still names that row; it no longer settles a noun/verb homograph). +- `fsm` **slot rule**: right after a fresh subject (a noun that opened the + clause in `Start` with no determiner or adjective in front), a token's Noun + reading is dropped when it also has a Verb reading. Relative, like + licensing: it never empties a token. A carried object ("gave him charge"), + a re-anchored noun ("mothers house") and a determined subject ("the guard + locks") do not trigger it. +- `ReadingParse::{slot_dropped, unlicensed_dropped}` counters; the KJV KILL + gate accepts them as the explanation for a changed verse. + +## Rejected on measurement + +A sentence-level **clause rule** ("drop paths with no predicate when another +path has one") was built and removed. On its own Noun→Verb flips it was right +about 7 times in 34 (sampled): KJV verses are often verbless fragments ("the +goats for sin offering", "without blemish and without spot"), and the rule +forced a verb into them. + +## KJV (`bible_vocab.txt`) + +| | D-LXC-2 (lemma tag forced) | D-LXC-13 | +|---|---|---| +| ambiguous tokens / words | 3,363 / 141 | 34,824 / 451 | +| narrowed / still ambiguous | 1,908 / 1,455 | 14,383 / 20,441 | +| legacy tag eliminated | 179 tokens, 35 words | 1,943 tokens, 162 words | +| triples: certain + alternative | 69,670 + 1,716 | 57,350 + 29,001 | +| legacy→alternative / lost / new certain | 732 / 113 / 122 | 12,984 / 1,231 / 1,190 | +| verses changed | 1,131 | 12,437 | +| peak configurations / overflow | 16 / 0 | 256 / 0 (unchanged at a 1,024 cap) | + +Of the 1,943 eliminated legacy tags, 1,817 are Verb→Noun (licensing after a +determiner or possessive: "his work", "the cry", "the mount"), 121 are +Noun→Verb (slot rule), and 5 are Adj. Hand-scored samples: Verb→Noun about 36/38 correct; +slot-rule Noun→Verb about 19/30. Every slot-rule token was a forced Noun +before, so where the rule fires it is a net gain. These are samples, not a +measured accuracy. + +The certain set shrinks by 12,320: those triples hang on a homograph that +position does not decide, and they are now alternatives instead of being +asserted on the strength of a corpus count. Downstream gates (G1–G4, D-SRS-1, +-2, -4 PASS; D-SRS-3b's three KILLs) keep their verdicts; D-SRS-2's trie +target moves from `let` to `seen` on the smaller graph. + +Ablation: with the lemma tag forced (no widening) and the slot rule on, only +1,112 verses change (certain 69,760) — the rule needs the alternative to be +in the reading set. + +## OPEN + +- Slot-rule errors: archaic possessive `mine`/`thine` read as a subject + ("mine hand"), vocatives ("ye fools", "thou fool"), noun compounds after a + bare noun ("meat offering", "water face"). The first is tagger work + (`mine` before a noun is a determiner), not FSM work. +- `tesseract-paperless::consistency::seam_readings` still forces the lemma + tag ("They record the deeds" → `record:Noun`); it adopts + `coca::predicate_alternatives` after this merges. +- Downstream `--export` still carries certain triples only. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 8c4b5b31f..a16ba6b9f 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -186 entries, 2026-08-06 .. 2026-10-03. +187 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| +| 2026-10-04 | `D-LXC-13` | | [2026-10-04-deepnsm-v2-predicate-by-position.md](2026-10-04-deepnsm-v2-predicate-by-position.md) | | 2026-10-03 | `population-law-crosscheck` | | [2026-10-03-population-law-crosscheck.md](2026-10-03-population-law-crosscheck.md) | | 2026-10-03 | `orkg-reference-harvest` | | [2026-10-03-orkg-reference-harvest.md](2026-10-03-orkg-reference-harvest.md) | | 2026-10-03 | `known-is-not-alpha-premise-gate` | | [2026-10-03-known-is-not-alpha-premise-gate.md](2026-10-03-known-is-not-alpha-premise-gate.md) | diff --git a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md index 37f0921d6..bcd979380 100644 --- a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md +++ b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md @@ -463,3 +463,14 @@ English data first, with the FSM reading-set ABI language-neutral. converge on it. - KJV numbers and the OPEN points: board entry `2026-10-03-deepnsm-v2-multi-reading-fsm.md`. + +## D-LXC-13: position decides noun/verb homographs (2026-10-04) + +Direction (operator, 2026-10-04): a word between subject and object must not +become a noun because COCA counts it as a noun more often. D-LXC-3's B stands +for row order, but the first row no longer settles a noun/verb homograph: +`coca::predicate_alternatives` widens it, and `fsm`'s slot rule (after a fresh +subject) and licensing (after a determiner) pick by position. A sentence-level +"needs a predicate" rule was measured and rejected (KJV fragments). Numbers +and OPEN points: board entry `2026-10-04-deepnsm-v2-predicate-by-position.md`. + diff --git a/crates/deepnsm-v2/examples/bible_wave.rs b/crates/deepnsm-v2/examples/bible_wave.rs index 849046963..c6014baa1 100644 --- a/crates/deepnsm-v2/examples/bible_wave.rs +++ b/crates/deepnsm-v2/examples/bible_wave.rs @@ -33,7 +33,7 @@ //! links farther than ±5 (v1's ring forfeits them) and ±8 (the local reference //! horizon → the Escalate zone). -use deepnsm_v2::coca::{fsm_pos, fsm_pos_tag, reading_set}; +use deepnsm_v2::coca::{fsm_pos, fsm_pos_tag, predicate_alternatives, reading_set}; use deepnsm_v2::{ load_cam96_codes, load_cam96_space, load_word_forms_csv, parse_readings, parse_to_spo, EvidenceError, LexicalEvidence, Nsm, PaletteVocab, Pos, PosSet, Reading, ReadingParse, Spo, @@ -87,6 +87,13 @@ struct LexicalDecodeReport { verses_changed: usize, peak_configs: usize, overflow_flushes: usize, + /// (configuration, reading) pairs the slot rule dropped (D-LXC-13). + slot_dropped: usize, + /// (configuration, reading) pairs the licensing rule dropped. + unlicensed_dropped: usize, + /// Verses whose triples changed although no token's reading set + /// narrowed: a rule dropped a reading on one path only. + combination_only: usize, ambiguous_words: HashMap, eliminated_words: HashMap, } @@ -160,6 +167,8 @@ impl LexicalDecodeReport { self.alternative += parse.alternative.len(); self.peak_configs = self.peak_configs.max(parse.peak_configs); self.overflow_flushes += parse.overflow_flushes; + self.slot_dropped += parse.slot_dropped; + self.unlicensed_dropped += parse.unlicensed_dropped; let lost = legacy .iter() .filter(|t| !parse.certain.contains(t) && !parse.alternative.contains(t)) @@ -181,10 +190,15 @@ impl LexicalDecodeReport { ); } if lost + new_certain > 0 { + let narrowed = parse.ambiguous.iter().any(|sv| sv.survived != sv.entered); assert!( - parse.ambiguous.iter().any(|sv| sv.survived != sv.entered), - "KILL D-LXC-2 unexplained: triples changed but no reading was eliminated" + narrowed || parse.unlicensed_dropped > 0 || parse.slot_dropped > 0, + "KILL D-LXC-2 unexplained: triples changed but no reading or reading \ + combination was eliminated" ); + if !narrowed { + self.combination_only += 1; + } } } @@ -223,8 +237,13 @@ impl LexicalDecodeReport { self.verses_changed ); println!( - "D-LXC-2 configurations: peak {}, overflow flushes {}", - self.peak_configs, self.overflow_flushes + "D-LXC-2 configurations: peak {}, overflow flushes {}; dropped by slot rule {}, \ + by licensing {} (verses changed with no token narrowed: {})", + self.peak_configs, + self.overflow_flushes, + self.slot_dropped, + self.unlicensed_dropped, + self.combination_only ); println!( "D-LXC-2 most ambiguous words: {}", @@ -242,8 +261,12 @@ impl LexicalDecodeReport { assert_eq!(self.overflow_flushes, 0, "KILL D-LXC-2: a verse overflowed"); println!("D-LXC-2 PASS every difference traces to an ambiguous token"); // Pinned for the released `bible_vocab.txt` + Gutenberg `pg10.txt`, - // lemma table order kept (D-LXC-3). A deliberate decoder change - // re-pins these with the difference reported. + // lemma noun/verb tags widened to their predicate alternative and + // decided by position (D-LXC-13, reopens D-LXC-3). Before D-LXC-13: + // reading (771_176, 683_805, 3_363, 141, 84_008, 1_908, 1_455, 179, + // 35), triples (70_393, 69_670, 1_716, 732, 113, 122, 1_131, 16). + // A deliberate decoder change re-pins these with the difference + // reported. assert_eq!( ( self.tokens, @@ -256,7 +279,7 @@ impl LexicalDecodeReport { self.legacy_eliminated, self.eliminated_words.len(), ), - (771_176, 683_805, 3_363, 141, 84_008, 1_908, 1_455, 179, 35), + (771_176, 652_344, 34_824, 451, 84_008, 14_383, 20_441, 1_943, 162), "KILL D-LXC-2: reading accounting moved from the pinned KJV layout" ); assert_eq!( @@ -270,7 +293,7 @@ impl LexicalDecodeReport { self.verses_changed, self.peak_configs, ), - (70_393, 69_670, 1_716, 732, 113, 122, 1_131, 16), + (70_393, 57_350, 29_001, 12_984, 1_231, 1_190, 12_437, 256), "KILL D-LXC-2: triple accounting moved from the pinned KJV layout" ); println!("D-LXC-2 PASS accounting matches the pinned KJV layout"); @@ -1440,6 +1463,9 @@ fn lowercase_word_column(forms_csv: &str) -> String { /// no count, order or band is read to produce a tag. struct Tagger { lemmas: HashMap, + /// Every lemma-table row per lemma, folded: the table's readings, not + /// only its first row. + lemma_readings: HashMap, /// `main`'s tagging, kept verbatim: lemma table, then first /// `word_forms.csv` row. See [`load_pos_legacy_first_wins`]. legacy: HashMap, @@ -1458,6 +1484,7 @@ impl Tagger { vocab: &PaletteVocab, ) -> Result { let mut lemmas = HashMap::new(); + let mut lemma_readings: HashMap = HashMap::new(); for line in lemmas_csv.lines().skip(1) { let f: Vec<&str> = line.split(',').collect(); let (Some(lemma), Some(pos)) = (f.get(1), f.get(2)) else { @@ -1466,11 +1493,14 @@ impl Tagger { lemmas .entry(lemma.to_lowercase()) .or_insert_with(|| coca_pos(pos)); + let set = lemma_readings.entry(lemma.to_lowercase()).or_default(); + *set = set.with(coca_pos(pos)); } let (evidence, report) = load_word_forms_csv(&lowercase_word_column(forms_csv), vocab)?; let (cuts, bands) = calibrate(&lemmas, &evidence, vocab); Ok(Self { lemmas, + lemma_readings, legacy: load_pos_legacy_first_wins(lemmas_csv, forms_csv), evidence, report, @@ -1485,9 +1515,8 @@ impl Tagger { /// Compatibility boundary, not semantic resolution. It reads no count and /// no [`LexicalEvidence`], so frequency cannot change a tag. It does /// depend on source-row order (the first lemma row, the first form row); - /// that is inherited debt, not an authorized resolver, pending a - /// multi-reading parser input (D-LXC-2) and the lemma-table migration - /// (D-LXC-3). + /// that is inherited debt, not an authorized resolver. [`Self::readings`] + /// is the resolver (D-LXC-2, D-LXC-13). fn pos(&self, w: &str) -> Pos { self.legacy .get(w) @@ -1500,18 +1529,27 @@ impl Tagger { /// (D-LXC-2). Same sources and order as [`Self::pos`], but the forms /// layer hands over its whole reading set instead of its first row: /// - /// 1. the lemma table (F9, kept: D-LXC-3 is decided as "keep the lemma - /// table order"), one reading; - /// 2. every [`LexicalEvidence`] reading, folded by [`deepnsm_v2::coca`]; + /// 1. the lemma table's first row (F9). If that tag is a noun or a verb, + /// the word also gets its OTHER noun/verb reading when any lemma row + /// or [`LexicalEvidence`] reading has it (D-LXC-13): the predicate + /// alternative is kept, and the parser's slot rule picks it by + /// position. Function words keep their one tag; + /// 2. otherwise every [`LexicalEvidence`] reading, folded by + /// [`deepnsm_v2::coca`]; /// 3. the archaic list, one reading; /// 4. otherwise [`PosSet::EMPTY`]: unknown, never a guessed reading. /// - /// No count is read. Where the forms layer has one folded reading, the - /// set is exactly the legacy tag; where it has several, the legacy tag is - /// one of them. + /// No count is read. The legacy tag is always one of the readings. fn readings(&self, w: &str, id: WordId) -> PosSet { if let Some(&p) = self.lemmas.get(w) { - return PosSet::single(p); + return predicate_alternatives( + p, + self.lemma_readings + .get(w) + .copied() + .unwrap_or_default() + .union(reading_set(&self.evidence, id).unwrap_or_default()), + ); } reading_set(&self.evidence, id) .or_else(|| archaic_pos(w).map(PosSet::single)) diff --git a/crates/deepnsm-v2/src/coca.rs b/crates/deepnsm-v2/src/coca.rs index 309cd505b..2c871284e 100644 --- a/crates/deepnsm-v2/src/coca.rs +++ b/crates/deepnsm-v2/src/coca.rs @@ -54,8 +54,57 @@ pub fn reading_set(evidence: &LexicalEvidence, id: WordId) -> Option { Some(readings.iter().map(|r| fsm_pos(r.pos)).collect()) } +/// The tag `tag` (a corpus's first-row lemma tag, say), widened by the other +/// of noun/verb when `tag` is one of them and `known` — every reading the +/// lexicon has for the word — contains the other. Anything else is `tag` +/// alone, so function words keep their one tag (D-LXC-13). +/// +/// The widened word enters [`crate::fsm::parse_readings`] with both +/// readings, and position picks: the slot rule makes it the predicate right +/// after a fresh subject ("they record"), licensing makes it a noun right +/// after a determiner ("the record"), and anything else stays ambiguous. A +/// lemma tag is never allowed to settle a noun/verb homograph on its own. +#[must_use] +pub fn predicate_alternatives(tag: Pos, known: PosSet) -> PosSet { + let nv = PosSet::single(Pos::Noun).with(Pos::Verb); + let mut set = PosSet::single(tag); + if nv.contains(tag) { + for p in nv.iter() { + if known.contains(p) { + set = set.with(p); + } + } + } + set +} + #[cfg(test)] mod tests { + /// D-LXC-13: a noun or verb tag gains the other reading only when the + /// lexicon has it; a function word never widens, even when the lexicon + /// knows a noun or verb reading for it. + #[test] + fn only_noun_and_verb_tags_widen() { + let nv = PosSet::single(Pos::Noun).with(Pos::Verb); + assert_eq!(predicate_alternatives(Pos::Noun, nv), nv); + assert_eq!(predicate_alternatives(Pos::Verb, nv), nv); + // No verb reading known: the noun stays a noun. + assert_eq!( + predicate_alternatives(Pos::Noun, PosSet::single(Pos::Adj)), + PosSet::single(Pos::Noun) + ); + // A determiner with a known noun reading stays a determiner. + assert_eq!( + predicate_alternatives(Pos::Det, nv), + PosSet::single(Pos::Det) + ); + // The tag itself is always kept, even if `known` omits it. + assert_eq!( + predicate_alternatives(Pos::Verb, PosSet::EMPTY), + PosSet::single(Pos::Verb) + ); + } + use super::*; use crate::lexical::{LexicalEvidenceBuilder, LexicalReading}; use crate::vocab::PaletteVocab; diff --git a/crates/deepnsm-v2/src/fsm.rs b/crates/deepnsm-v2/src/fsm.rs index a6f4235cb..69ebbb846 100644 --- a/crates/deepnsm-v2/src/fsm.rs +++ b/crates/deepnsm-v2/src/fsm.rs @@ -140,6 +140,12 @@ struct Core { /// open and waiting for its head. Read only by [`parse_readings`]'s /// licensing rule; [`parse_to_spo`] never consults it. nominal_open: bool, + /// The previous token opened the clause as its subject — a noun taken in + /// [`State::Start`] with no determiner or adjective in front ("they", + /// "men", "God"). An object carried into the subject slot ("gave him …") + /// or a re-anchored noun ("mothers house") is not fresh. Read only by + /// the slot rule. + fresh_subject: bool, } impl Core { @@ -151,12 +157,15 @@ impl Core { matrix: 0, antecedent: 0, nominal_open: false, + fresh_subject: false, }; /// Consume one non-`Stop` token; returns the triple it closes, if any. /// This is the whole single-reading transition table, unchanged. fn step(&mut self, t: Tagged) -> Option { debug_assert!(t.pos != Pos::Stop, "Stop is handled by the caller"); + let nominal_was_open = self.nominal_open; + self.fresh_subject = false; self.nominal_open = matches!(t.pos, Pos::Det | Pos::Adj); // While a relative clause is open, its own tiny machine consumes the @@ -231,6 +240,7 @@ impl Core { (State::Start, Pos::Noun) => { self.subject = t.id; self.state = State::HaveSubject; + self.fresh_subject = !nominal_was_open; } (State::HaveSubject, Pos::Verb) => { self.predicate = t.id; @@ -424,12 +434,16 @@ pub struct ReadingParse { /// Times a sentence was force-flushed because it exceeded /// [`MAX_CONFIGS`]. Each flush is a reported clause break, never a pick. pub overflow_flushes: usize, + /// (configuration, reading) pairs the licensing rule dropped. + pub unlicensed_dropped: usize, + /// (configuration, reading) pairs the slot rule dropped. + pub slot_dropped: usize, } /// Bound on live configurations per sentence. Exceeding it flushes the /// sentence at that token (a reported clause break, see /// [`ReadingParse::overflow_flushes`]) instead of choosing a reading. -pub const MAX_CONFIGS: usize = 64; +pub const MAX_CONFIGS: usize = 256; /// One live configuration: the parser registers, the triples this path has /// emitted in the current sentence, and — for each ambiguous token so far — @@ -449,6 +463,22 @@ const fn licensed(core: &Core, pos: Pos) -> bool { !(core.nominal_open && matches!(pos, Pos::Verb)) } +/// The slot rule: directly after a fresh subject noun (one that opened the +/// clause with no determiner or adjective in front of it), a word that can be +/// a verb fills the predicate slot — "they record", "he rose", "men sleep". +/// Its noun reading would make a compound with the subject instead, so it is +/// dropped, however much more often the corpus counts the word as a noun. +/// After a determined noun ("a sin offering"), a carried object ("gave him +/// charge") or a re-anchored noun ("mothers house") both readings stay. +/// Applied relatively, like licensing, and only when the token offers a verb. +fn slot_allows(core: &Core, pos: Pos, set: PosSet) -> bool { + !(core.rel == Rel::None + && core.state == State::HaveSubject + && core.fresh_subject + && pos == Pos::Noun + && set.contains(Pos::Verb)) +} + /// Parse a multi-reading token stream. /// /// Each token enters with every reading in its [`PosSet`]. The parser keeps @@ -464,6 +494,15 @@ const fn licensed(core: &Core, pos: Pos) -> bool { /// single reading is never rejected, and input with one reading per token /// parses exactly as [`parse_to_spo`]. Frequency is never consulted. /// +/// **Slot rule.** Directly after a fresh subject noun, a token that can be a +/// verb loses its noun reading (see [`slot_allows`]): the position between +/// subject and object is the predicate, however often COCA counts the word +/// as a noun. Relative like licensing, so it never empties a token. +/// +/// There is deliberately no "a sentence needs a predicate" rule: KJV verses +/// are often verbless fragments ("the goats for sin offering"), and on the +/// KJV such a rule turned nouns into verbs about three times in four. +/// /// **Output.** At each `Stop` (and at the end of input) the sentence's /// configurations are compared: triples found on all of them are /// [`ReadingParse::certain`], the rest [`ReadingParse::alternative`]. @@ -490,13 +529,17 @@ pub fn parse_readings(tokens: &[Reading]) -> ReadingParse { pending.push((index, set)); } - let any_licensed = configs - .iter() - .any(|c| set.iter().any(|p| licensed(&c.core, p))); + let allowed = |c: &Config, p: Pos| licensed(&c.core, p) && slot_allows(&c.core, p, set); + let any_allowed = configs.iter().any(|c| set.iter().any(|p| allowed(c, p))); let mut next: Vec = Vec::with_capacity(configs.len() * set.len()); for c in &configs { for pos in set.iter() { - if any_licensed && !licensed(&c.core, pos) { + if any_allowed && !allowed(c, pos) { + if licensed(&c.core, pos) { + out.slot_dropped += 1; + } else { + out.unlicensed_dropped += 1; + } continue; } let mut n = c.clone(); @@ -705,12 +748,95 @@ mod tests { /// or none. #[test] fn ambiguity_remains_when_structure_cannot_decide() { - // "men record deeds": verb path → (men, record, deeds); noun path → - // a run of nouns, no triple. - let p = parse_readings(&[one(1, Pos::Noun), noun_or_verb(2), one(3, Pos::Noun)]); + // "the men record deeds rot": verb path → (men, record, deeds) then + // `rot`; noun path → `rot` is the predicate of "the men record + // deeds". The subject is determined, so the slot rule does not apply. + let p = parse_readings(&[ + one(9, Pos::Det), + one(1, Pos::Noun), + noun_or_verb(2), + one(3, Pos::Noun), + one(6, Pos::Verb), + ]); assert_eq!(p.ambiguous[0].survived, NV); assert!(p.certain.is_empty()); assert_eq!(p.alternative, vec![Spo::new(1, 2, 3)]); + assert_eq!(p.slot_dropped, 0); + } + + /// No "a sentence needs a predicate" rule: a verbless fragment keeps + /// its noun reading. "the goats for sin offering" — `offering` follows a + /// re-anchored noun, not a fresh subject, so neither rule touches it. + #[test] + fn a_verbless_fragment_keeps_its_noun_reading() { + let p = parse_readings(&[ + one(9, Pos::Det), + one(1, Pos::Noun), + one(7, Pos::Other), + one(3, Pos::Noun), + noun_or_verb(2), + ]); + assert_eq!(p.ambiguous[0].survived, NV); + assert_eq!(p.slot_dropped, 0); + } + + /// The slot rule: right after a fresh subject noun, a homograph is the + /// predicate, even where a later verb would give the sentence another + /// one ("men record deeds rot"). + #[test] + fn a_homograph_after_a_fresh_subject_is_the_predicate() { + let p = parse_readings(&[ + one(1, Pos::Noun), + noun_or_verb(2), + one(3, Pos::Noun), + one(6, Pos::Verb), + ]); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Verb)); + assert_eq!(p.certain, vec![Spo::new(1, 2, 3)]); + assert!(p.alternative.is_empty()); + assert_eq!(p.slot_dropped, 1); + // Intransitive: "men sleep" — no triple, but the verb reading survives. + let p = parse_readings(&[one(1, Pos::Noun), noun_or_verb(2)]); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Verb)); + assert!(p.certain.is_empty() && p.alternative.is_empty()); + } + + /// Silence twins of the slot rule: it does not fire on a word with no + /// verb reading, after a determined subject, after an object carried + /// into the subject slot, or after a re-anchored noun. + #[test] + fn the_slot_rule_needs_a_fresh_subject_and_a_verb_reading() { + // "men stones rot": `stones` has no verb reading, so it stays. + let p = parse_readings(&[one(1, Pos::Noun), one(3, Pos::Noun), one(6, Pos::Verb)]); + assert_eq!(p.slot_dropped, 0); + // "the men record deeds rot" keeps both readings (no slot drop). + let p = parse_readings(&[ + one(9, Pos::Det), + one(1, Pos::Noun), + noun_or_verb(2), + one(3, Pos::Noun), + one(6, Pos::Verb), + ]); + assert_eq!(p.ambiguous[0].survived, NV); + assert_eq!(p.slot_dropped, 0); + // A homograph after a verb is an object, not a predicate: + // "men saw record" keeps both readings for `record`. + let p = parse_readings(&[one(1, Pos::Noun), one(4, Pos::Verb), noun_or_verb(2)]); + assert_eq!(p.slot_dropped, 0); + // "he gave him charge": `him` closes (he, gave, him) and is carried + // into the subject slot; `charge` keeps both readings. + let p = parse_readings(&[ + one(1, Pos::Noun), + one(4, Pos::Verb), + one(3, Pos::Noun), + noun_or_verb(2), + ]); + assert_eq!(p.ambiguous[0].survived, NV); + assert_eq!(p.slot_dropped, 0); + // "mothers house": `house` follows a re-anchored noun. + let p = parse_readings(&[one(1, Pos::Noun), one(3, Pos::Noun), noun_or_verb(2)]); + assert_eq!(p.ambiguous[0].survived, NV); + assert_eq!(p.slot_dropped, 0); } /// T4: a single-reading word is never rejected, even where the licensing @@ -766,9 +892,11 @@ mod tests { #[test] fn sentences_are_independent() { let toks = [ + one(9, Pos::Det), one(1, Pos::Noun), noun_or_verb(2), one(3, Pos::Noun), + one(6, Pos::Verb), Reading::stop(), one(9, Pos::Det), noun_or_verb(2), From d1297445a864e2cde622074ad0db37f11ee0130d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 07:57:15 +0000 Subject: [PATCH 02/33] deepnsm-v2: adverb tag, measured typology, and the position rules scored on UD gold (D-LXC-14) - Pos::Adv (bit 7): skipped like Other, opens no nominal group; COCA `r` folds to it. - Typology { adjective order, adjective_opens_nominal, attribute_rules } and parse_readings_with; parse_readings = Typology::ENGLISH. Adjective licensing applies only where an adjective always opens a nominal group. - AttributeRule: four adjective/adverb clauses read from the token's neighbours in the borrowed input. Shipped disabled: none beats frequency in every language on gold tags. - examples/ud_pos_eval.rs: scores every rule against Universal Dependencies gold tags (English/German/French, r2.15, fetched not committed), with the typology measured from each train split. Measured: with COCA readings on UD English, position decides 457 noun/verb tokens at 96.7 % against COCA frequency's 85.1 % on the same tokens. Gating adjective licensing on the measured typology fixed German (37.5 % right before) and French. No adjective/adverb clause beats frequency consistently. KJV numbers unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...04-deepnsm-v2-position-rules-on-ud-gold.md | 92 ++++ .claude/board/entries/README.md | 3 +- ...deepnsm-v2-lexical-evidence-consumer-v1.md | 10 + crates/deepnsm-v2/examples/bible_wave.rs | 2 +- crates/deepnsm-v2/examples/ud_pos_eval.rs | 430 ++++++++++++++++++ crates/deepnsm-v2/src/coca.rs | 7 +- crates/deepnsm-v2/src/fsm.rs | 345 +++++++++++++- crates/deepnsm-v2/src/lib.rs | 5 +- 9 files changed, 872 insertions(+), 23 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md create mode 100644 crates/deepnsm-v2/examples/ud_pos_eval.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index db6012782..7d545a6e9 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -37,6 +37,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-9** | `archaic_pos` cannot override COCA-known KJV words (`art` is Noun, `word_forms.csv:1047`) | Queued | pre-existing | | **D-LXC-10** | `Pos::Rel` produced by no in-crate tagger in deepnsm-v2 (only FSM match arms + one unit test) | Queued | pre-existing; external callers can still pass `Pos::Rel` through the public `Tagged`/`parse_to_spo` | | **D-LXC-13** | noun/verb homograph decided by position: lemma N/V tag widened to its other reading (`coca::predicate_alternatives`), slot rule after a fresh subject | In PR | slot rule + widening disable-verified; KJV re-pinned (34,824 ambiguous / 451 words, certain 57,350 + alternative 29,001, 0 unexplained). Entry `2026-10-04-deepnsm-v2-predicate-by-position.md` | +| **D-LXC-14** | `Pos::Adv`; `Typology` (measured, gates adjective licensing); `AttributeRule` adjective/adverb clauses (off: below frequency); `ud_pos_eval` gold harness (en/de/fr) | In PR | noun/verb with COCA: position 96.7 % vs frequency 85.1 % on 457 decided UD-EWT tokens; adjective/adverb: no clause beats frequency in every language. Entry `2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md b/.claude/board/entries/2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md new file mode 100644 index 000000000..9970acf25 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md @@ -0,0 +1,92 @@ +# 2026-10-04 — deepnsm-v2: the position rules scored on gold tags in three languages (D-LXC-14) + +**Status:** MEASURED (`cargo run --release --example ud_pos_eval -- TRAIN TEST [--coca DIR]`, +Universal Dependencies r2.15: UD_English-EWT, UD_German-GSD, UD_French-GSD, +CC BY-SA 4.0, fetched, never committed; sha256 in the example's run notes below). + +Operator direction (2026-10-04): decide adjective vs adverb by position as +D-LXC-13 does noun vs verb, feed the TEKAMOLO tenant, and validate across +languages instead of writing rules per language. + +## What changed + +- `Pos::Adv` — the 8th tag (bit 7, after `Stop`, so no earlier bit moves). + Skipped in the core like `Other`; opens no nominal group. COCA `r` folds to + it. KJV numbers are unchanged (a `{Other}` set became `{Adv}`). +- `Typology { adjective: AdjectiveOrder, adjective_opens_nominal, + attribute_rules }` + `parse_readings_with`. `parse_readings` = + `Typology::ENGLISH`. Determiner licensing applies everywhere; adjective + licensing only where `adjective_opens_nominal`. +- `AttributeRule` (4 clauses) + `attribute_rule` (which clause fires) + + `attribute_readings`. As shipped no clause narrows (`ENGLISH.attribute_rules + = &[]`): each was measured below frequency (table below). +- `examples/ud_pos_eval.rs` — the gold harness. The lexicon is the treebank's + train split (or COCA with `--coca`), folded by one UPOS table; the typology + is measured from train (`amod` head direction; share of adjectives directly + followed by a verb), not written per language. + +## Measured typology + +| | adjective before its noun (`amod`) | adjective followed by a verb | → typology | +|---|---|---|---| +| English | 97.0 % | 1.2 % | Before, opens nominal | +| German | 98.7 % | 7.1 % | Before, does not | +| French | 30.4 % | 7.4 % | Both, does not | + +Gating adjective licensing on this fixed the cross-language failures: with +English licensing everywhere, German "kept Noun" was 37.5 % right ("sehr +lecker **war**", "möglich **ist**") and French 50 %. + +## Noun/verb: position vs frequency on the same tokens + +| lexicon / language | decided | position right | frequency right | +|---|---|---|---| +| **COCA → English EWT** | 457 / 1,632 | **96.7 %** | 85.1 % | +| — licensing (kept Noun) | 349 | 98.0 % | 85.4 % | +| — slot rule (kept Verb) | 108 | 92.6 % | 84.3 % | +| treebank → English | 155 / 2,921 | 89.0 % | 89.7 % | +| treebank → German | 46 / 693 | 87.0 % (slot 100 %, 36) | 91.3 % | +| treebank → French | 30 / 524 | 100 % (slot 100 %, 28) | 90.0 % | + +With COCA — the lexicon `bible_wave` and tesseract read — position beats +COCA's frequency pick by 11.6 points on the tokens it decides +(position-then-frequency 87.4 % vs frequency 84.2 % over all 1,632). This +supports D-LXC-13. With a treebank lexicon (every tag the form ever carried, +in-domain counts) the advantage is gone: almost every word "can be a noun". + +## Adjective/adverb: each clause vs frequency on the same tokens + +| clause | COCA → English | treebank English | German | French | +|---|---|---|---|---| +| after determiner → Adj | 89.3 vs 86.0 (121) | 1 token | 1 token | 1 token | +| before adjective → Adv | 86.7 vs 100 (60) | 63.4 vs 89.6 (164) | 75.0 vs 68.8 (32) | 1 token | +| subject _ verb → Adv | 93.1 vs 86.2 (29) | 29.6 vs 92.6 (108) | 57.9 vs 94.7 (19) | 2 tokens | +| next to noun → Adj | 79.8 vs 92.9 (84) | 85.6 vs 93.6 (327) | 38.3 vs 81.5 (81) | 61.5 vs 82.1 (39) | + +No clause beats frequency in every setting; the two COCA wins are 4 and 2 +tokens. Position does not decide adjective vs adverb better than frequency, +so no clause is enabled. + +## Not done, and why + +- `lance-graph-mask-risc`: the rules read a 3-token window of the borrowed + `&[Reading]` in place; no plane is built and nothing is copied. A mask + program would need ndarray in this zero-dependency crate and either a + materialised reading lane or a byte view of `Reading`. Worth it only for a + bulk pass (archive scale), with a measured clause to run. +- Animal Farm (1945): under US copyright until 2041, so not fetched; the UD + treebanks give gold tags, which a plain text does not. +- TEKAMOLO producer (`insight_spo_tekamolo_read`, planner) still uses its cue + lists; `Pos::Adv` makes a COCA adverbial reading visible to it. Wiring it is + a separate change. + +## OPEN + +- German determiner licensing: 10 decided, 4 right — not inspected. +- The adjective/adverb ambiguity is lexical more than positional; a lexical + class (temporal / manner / place adverb) is what the TEKAMOLO lanes need, + not a position rule. + +UD inputs (r2.15, sha256): en_ewt train `a049fc40…c566`, test `0612e459…036`; +de_gsd train `cb1326ed…c77`, test `9d6776ee…465f`; fr_gsd train `a539328e…30e`, +test `f786c60a…19de`. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index a16ba6b9f..fd74475f9 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -187 entries, 2026-08-06 .. 2026-10-04. +188 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| | 2026-10-04 | `D-LXC-13` | | [2026-10-04-deepnsm-v2-predicate-by-position.md](2026-10-04-deepnsm-v2-predicate-by-position.md) | +| 2026-10-04 | `D-LXC-14` | | [2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md](2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md) | | 2026-10-03 | `population-law-crosscheck` | | [2026-10-03-population-law-crosscheck.md](2026-10-03-population-law-crosscheck.md) | | 2026-10-03 | `orkg-reference-harvest` | | [2026-10-03-orkg-reference-harvest.md](2026-10-03-orkg-reference-harvest.md) | | 2026-10-03 | `known-is-not-alpha-premise-gate` | | [2026-10-03-known-is-not-alpha-premise-gate.md](2026-10-03-known-is-not-alpha-premise-gate.md) | diff --git a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md index bcd979380..d5105f986 100644 --- a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md +++ b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md @@ -474,3 +474,13 @@ subject) and licensing (after a determiner) pick by position. A sentence-level "needs a predicate" rule was measured and rejected (KJV fragments). Numbers and OPEN points: board entry `2026-10-04-deepnsm-v2-predicate-by-position.md`. +## D-LXC-14: the position rules on gold tags (2026-10-04) + +`examples/ud_pos_eval.rs` scores every rule against Universal Dependencies +gold tags (English, German, French), with the typology measured from each +train split rather than written per language. Noun/verb position rules beat +COCA's frequency pick (96.7 % vs 85.1 % on decided tokens); the adjective/adverb +clauses do not beat frequency in any language consistently and ship disabled. +Adjective licensing is gated on the measured typology. Numbers: board entry +`2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md`. + diff --git a/crates/deepnsm-v2/examples/bible_wave.rs b/crates/deepnsm-v2/examples/bible_wave.rs index c6014baa1..fbae1c2be 100644 --- a/crates/deepnsm-v2/examples/bible_wave.rs +++ b/crates/deepnsm-v2/examples/bible_wave.rs @@ -1641,7 +1641,7 @@ mod tests { #[test] fn a_single_reading_keeps_its_tag() { assert_eq!(tag(&forms("1,x,j,9,3,w\n"), "w"), Pos::Adj); - assert_eq!(tag(&forms("1,x,r,9,3,w\n"), "w"), Pos::Other); + assert_eq!(tag(&forms("1,x,r,9,3,w\n"), "w"), Pos::Adv); } // (h) + G7: the lemma table is never overruled by the forms layer diff --git a/crates/deepnsm-v2/examples/ud_pos_eval.rs b/crates/deepnsm-v2/examples/ud_pos_eval.rs new file mode 100644 index 000000000..e4d2e0d32 --- /dev/null +++ b/crates/deepnsm-v2/examples/ud_pos_eval.rs @@ -0,0 +1,430 @@ +//! `ud_pos_eval` — the position rules of [`deepnsm_v2::fsm::parse_readings`] +//! scored against GOLD part-of-speech tags from a Universal Dependencies +//! treebank, in any language UD covers. +//! +//! The lexicon is taken from the treebank's own train split: a form's +//! readings are every tag it carries anywhere in train, folded onto the FSM +//! alphabet by one table ([`fold`]). Nothing in this file is written for a +//! particular language. Test sentences are parsed with those readings, and +//! every token whose readings include both halves of a pair (noun/verb, +//! adjective/adverb) and whose gold tag is one of the two is scored: +//! +//! - **position** — the pair was narrowed to one reading by a parser rule; +//! right when that reading is the gold tag; +//! - **frequency** — the reading the form carries most often in train (the +//! "70 % noun, so noun" pick), on the same tokens. +//! +//! Usage (inputs are fetched, never committed — CC BY-SA 4.0): +//! +//! ```text +//! T=https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/r2.15 +//! curl -O $T/en_ewt-ud-train.conllu -O $T/en_ewt-ud-test.conllu +//! cargo run --release --example ud_pos_eval -- en_ewt-ud-train.conllu en_ewt-ud-test.conllu +//! # COCA readings and COCA's frequency pick instead of the treebank's own: +//! cargo run --release --example ud_pos_eval -- TRAIN TEST --coca ../deepnsm/word_frequency +//! ``` +//! +//! The typology is measured from train; `UD_TYPOLOGY=english` uses +//! [`Typology::ENGLISH`] instead. `UD_DUMP=1` prints every token a rule +//! narrowed away from its gold tag, with context, to stderr. + +use std::collections::HashMap; + +use deepnsm_v2::coca::fsm_pos_tag; +use deepnsm_v2::fsm::{ + attribute_rule, parse_readings_with, AdjectiveOrder, AttributeRule, Pos, PosSet, Reading, + Typology, +}; + +/// One syntactic word of a UD sentence. +struct Word { + form: String, + gold: Pos, + /// The adjective modifies a noun (`amod`) that stands after it + /// (`Some(true)`) or before it (`Some(false)`); `None` otherwise. + amod_head_after: Option, +} + +/// The one UPOS → FSM fold. Language-neutral: it reads only UD's universal +/// tags and the universal `PronType=Rel` feature. +fn fold(upos: &str, feats: &str, form: &str) -> Pos { + let rel = feats.split('|').any(|f| f == "PronType=Rel"); + match upos { + "PRON" | "DET" if rel => Pos::Rel, + "NOUN" | "PROPN" | "PRON" => Pos::Noun, + "VERB" | "AUX" => Pos::Verb, + "ADJ" => Pos::Adj, + "ADV" => Pos::Adv, + "DET" => Pos::Det, + "PUNCT" if matches!(form, "." | "!" | "?") => Pos::Stop, + _ => Pos::Other, + } +} + +/// Sentences of a CoNLL-U file. Multiword-token ranges (`3-4`) and empty +/// nodes (`5.1`) are skipped: the syntactic words carry the tags. +fn read_conllu(path: &str) -> Vec> { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut out = Vec::new(); + let mut cur = Vec::new(); + for line in text.lines() { + if line.is_empty() { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + continue; + } + if line.starts_with('#') { + continue; + } + let cols: Vec<&str> = line.split('\t').collect(); + if cols.len() < 6 || cols[0].contains('-') || cols[0].contains('.') { + continue; + } + let form = cols[1].to_lowercase(); + let gold = fold(cols[3], cols[5], &form); + let amod_head_after = match (cols.get(6), cols.get(7)) { + (Some(head), Some(rel)) if rel.starts_with("amod") => { + match (head.parse::(), cols[0].parse::()) { + (Ok(h), Ok(i)) if h > 0 => Some(h > i), + _ => None, + } + } + _ => None, + }; + cur.push(Word { + form, + gold, + amod_head_after, + }); + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +/// Readings and per-reading counts for every train form. +struct Lexicon { + readings: HashMap, + counts: HashMap<(String, u8), usize>, +} + +impl Lexicon { + fn from_train(sentences: &[Vec]) -> Self { + let mut readings: HashMap = HashMap::new(); + let mut counts = HashMap::new(); + for w in sentences.iter().flatten() { + if w.gold == Pos::Stop { + continue; + } + let set = readings.entry(w.form.clone()).or_default(); + *set = set.with(w.gold); + *counts.entry((w.form.clone(), w.gold as u8)).or_default() += 1; + } + Self { readings, counts } + } + + /// Readings and counts from COCA (`lemmas_5k.csv` + `word_forms.csv` in + /// `dir`), folded by [`deepnsm_v2::coca::fsm_pos_tag`] — the lexicon + /// `bible_wave` reads, so the frequency pick is COCA's, not the + /// treebank's own. Counts are `word_forms.csv`'s per-form `wordFreq`. + fn from_coca(dir: &str) -> Self { + let mut readings: HashMap = HashMap::new(); + let mut counts: HashMap<(String, u8), usize> = HashMap::new(); + let read = |name: &str| { + std::fs::read_to_string(format!("{dir}/{name}")) + .unwrap_or_else(|e| panic!("{dir}/{name}: {e}")) + }; + for line in read("lemmas_5k.csv").lines().skip(1) { + let c: Vec<&str> = line.split(',').collect(); + if c.len() > 2 { + let set = readings.entry(c[1].to_lowercase()).or_default(); + *set = set.with(fsm_pos_tag(c[2])); + } + } + for line in read("word_forms.csv").lines().skip(1) { + let c: Vec<&str> = line.split(',').collect(); + if c.len() > 5 { + let word = c[5].to_lowercase(); + let pos = fsm_pos_tag(c[2]); + let set = readings.entry(word.clone()).or_default(); + *set = set.with(pos); + *counts.entry((word, pos as u8)).or_default() += c[4].parse::().unwrap_or(0); + } + } + Self { readings, counts } + } + + fn get(&self, form: &str) -> PosSet { + self.readings.get(form).copied().unwrap_or(PosSet::EMPTY) + } + + /// The frequency pick between `a` and `b` for `form`: the one train + /// counts more often (first of the pair on a tie). + fn more_frequent(&self, form: &str, a: Pos, b: Pos) -> Pos { + let c = |p: Pos| { + self.counts + .get(&(form.to_string(), p as u8)) + .copied() + .unwrap_or(0) + }; + if c(b) > c(a) { + b + } else { + a + } + } +} + +/// Score for one pair of readings. +#[derive(Default)] +struct PairScore { + /// Tokens whose readings hold both members and whose gold is one of them. + tokens: usize, + /// Of those, narrowed to one member by a parser rule. + decided: usize, + /// Decided and equal to gold. + decided_right: usize, + /// Frequency pick equal to gold, on the decided tokens. + freq_right_on_decided: usize, + /// Frequency pick equal to gold, on every token. + freq_right: usize, + /// Decided ? position pick : frequency pick — equal to gold, on every token. + hybrid_right: usize, + /// Decided tokens, keyed by the member kept: (kept, right, frequency + /// right). + by_kept: HashMap, +} + +impl PairScore { + fn add(&mut self, entered: PosSet, survived: PosSet, gold: Pos, freq: Pos, pair: (Pos, Pos)) { + let (a, b) = pair; + if !(entered.contains(a) && entered.contains(b)) || (gold != a && gold != b) { + return; + } + self.tokens += 1; + let freq_ok = freq == gold; + self.freq_right += usize::from(freq_ok); + let kept = match (survived.contains(a), survived.contains(b)) { + (true, false) => Some(a), + (false, true) => Some(b), + _ => None, + }; + if let Some(k) = kept { + self.decided += 1; + self.decided_right += usize::from(k == gold); + self.freq_right_on_decided += usize::from(freq_ok); + self.hybrid_right += usize::from(k == gold); + let e = self.by_kept.entry(k as u8).or_default(); + e.0 += 1; + e.1 += usize::from(k == gold); + e.2 += usize::from(freq_ok); + } else { + self.hybrid_right += usize::from(freq_ok); + } + } + + fn print(&self, name: &str, pair: (Pos, Pos)) { + let pct = |n: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * n as f64 / d as f64 + } + }; + println!( + "{name}: {} tokens, {} decided by position ({:.1}%)", + self.tokens, + self.decided, + pct(self.decided, self.tokens) + ); + println!( + " on decided tokens: position {:.1}% right, frequency {:.1}% right", + pct(self.decided_right, self.decided), + pct(self.freq_right_on_decided, self.decided) + ); + for p in [pair.0, pair.1] { + let (n, r, f) = self.by_kept.get(&(p as u8)).copied().unwrap_or_default(); + println!( + " kept {p:?}: {n} tokens, {:.1}% right, frequency {:.1}% right", + pct(r, n), + pct(f, n) + ); + } + println!( + " on all tokens: frequency {:.1}% right, position-then-frequency {:.1}% right", + pct(self.freq_right, self.tokens), + pct(self.hybrid_right, self.tokens) + ); + } +} + +/// The word-order facts [`Typology`] names, measured from gold train tags. +/// +/// - adjective order: of adjectives attached to a noun as `amod`, the share +/// whose noun stands after them. Above 0.8 → `Before`, below 0.2 → +/// `After`, else `Both` (policy pins, not measurements). +/// - `adjective_opens_nominal`: the share of adjectives directly followed by +/// a verb. Below 2 % → true (policy pin). +fn measure_typology(train: &[Vec]) -> (Typology, f64, f64) { + let (mut before, mut after, mut adj, mut adj_verb) = (0usize, 0usize, 0usize, 0usize); + for s in train { + for (i, w) in s.iter().enumerate() { + if w.gold != Pos::Adj { + continue; + } + adj += 1; + let next = s.get(i + 1).map(|w| w.gold); + before += usize::from(w.amod_head_after == Some(true)); + after += usize::from(w.amod_head_after == Some(false)); + adj_verb += usize::from(next == Some(Pos::Verb)); + } + } + let share_before = before as f64 / (before + after).max(1) as f64; + let verb_after = adj_verb as f64 / adj.max(1) as f64; + let adjective = if share_before > 0.8 { + AdjectiveOrder::Before + } else if share_before < 0.2 { + AdjectiveOrder::After + } else { + AdjectiveOrder::Both + }; + let typology = Typology { + adjective, + adjective_opens_nominal: verb_after < 0.02, + // As shipped: no clause narrows (D-LXC-14 measured them below + // frequency). The per-clause table scores them regardless. + attribute_rules: Typology::ENGLISH.attribute_rules, + }; + (typology, share_before, verb_after) +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let (train, test, coca) = match args.as_slice() { + [train, test] => (train, test, None), + [train, test, flag, dir] if flag == "--coca" => (train, test, Some(dir.as_str())), + _ => { + eprintln!("usage: ud_pos_eval TRAIN.conllu TEST.conllu [--coca WORD_FREQUENCY_DIR]"); + std::process::exit(2); + } + }; + let train = read_conllu(train); + let (typology, share_before, verb_after) = match std::env::var("UD_TYPOLOGY").as_deref() { + Ok("english") => (Typology::ENGLISH, f64::NAN, f64::NAN), + _ => measure_typology(&train), + }; + println!( + "typology {typology:?} (adjective before its noun {:.1}%, adjective followed by a verb \ + {:.1}%)", + 100.0 * share_before, + 100.0 * verb_after + ); + let lex = match coca { + Some(dir) => Lexicon::from_coca(dir), + None => Lexicon::from_train(&train), + }; + let test = read_conllu(test); + + // One stream: every sentence ends in a stop, whatever its own punctuation. + let mut ids: HashMap<&str, u16> = HashMap::new(); + let mut readings = Vec::new(); + let mut words: Vec> = Vec::new(); + for s in &test { + for w in s { + if w.gold == Pos::Stop { + continue; + } + let next = u16::try_from(ids.len() + 1).unwrap_or(u16::MAX); + let id = *ids.entry(w.form.as_str()).or_insert(next); + readings.push(Reading::new(id, lex.get(&w.form))); + words.push(Some(w)); + } + readings.push(Reading::stop()); + words.push(None); + } + + let parse = parse_readings_with(&readings, typology); + let nv = (Pos::Noun, Pos::Verb); + let aa = (Pos::Adj, Pos::Adv); + let mut nv_score = PairScore::default(); + let mut aa_score = PairScore::default(); + let dump = std::env::var_os("UD_DUMP").is_some(); + for sv in &parse.ambiguous { + let w = words[sv.index].expect("ambiguous tokens are words"); + if dump && sv.survived != sv.entered && !sv.survived.contains(w.gold) { + let lo = sv.index.saturating_sub(3); + let hi = (sv.index + 3).min(words.len()); + let ctx: Vec = (lo..hi) + .map(|j| match words[j] { + Some(x) if j == sv.index => format!("[{}]", x.form), + Some(x) => x.form.clone(), + None => "|".into(), + }) + .collect(); + eprintln!( + "WRONG gold {:?} kept {:?} | {}", + w.gold, + sv.survived, + ctx.join(" ") + ); + } + for (score, pair) in [(&mut nv_score, nv), (&mut aa_score, aa)] { + let freq = lex.more_frequent(&w.form, pair.0, pair.1); + score.add(sv.entered, sv.survived, w.gold, freq, pair); + } + } + let tokens = words.iter().filter(|w| w.is_some()).count(); + println!( + "{tokens} test words, {} ambiguous, {} unknown; slot rule dropped {}, licensing {}, \ + attribute rule {}", + parse.ambiguous.len(), + parse.unknown, + parse.slot_dropped, + parse.unlicensed_dropped, + parse.attribute_narrowed + ); + nv_score.print("noun/verb", nv); + aa_score.print("adjective/adverb", aa); + + // Each attribute clause on its own: every adjective/adverb token whose + // gold is one of the two, the clause that fires, and whether the clause + // or the frequency pick matches gold. Independent of which clauses are + // enabled, and of the parse. + let mut per_rule: HashMap, (usize, usize, usize)> = HashMap::new(); + let edge = |i: Option| match i.and_then(|i| readings.get(i)) { + Some(r) if !r.pos.contains(Pos::Stop) => r.pos, + _ => PosSet::EMPTY, + }; + for (i, w) in words.iter().enumerate() { + let Some(w) = w else { continue }; + let this = readings[i].pos; + if !(this.contains(Pos::Adj) && this.contains(Pos::Adv)) + || !matches!(w.gold, Pos::Adj | Pos::Adv) + { + continue; + } + let rule = attribute_rule(edge(i.checked_sub(1)), this, edge(Some(i + 1)), typology); + let freq = lex.more_frequent(&w.form, Pos::Adj, Pos::Adv); + let e = per_rule.entry(rule).or_default(); + e.0 += 1; + e.1 += usize::from(rule.is_some_and(|r| r.keeps() == w.gold)); + e.2 += usize::from(freq == w.gold); + } + let pct = |n: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * n as f64 / d as f64 + } + }; + for rule in AttributeRule::ALL.map(Some).into_iter().chain([None]) { + let (n, rule_right, freq_right) = per_rule.get(&rule).copied().unwrap_or_default(); + println!( + " clause {rule:?}: {n} tokens, clause {:.1}% right, frequency {:.1}% right", + pct(rule_right, n), + pct(freq_right, n) + ); + } +} diff --git a/crates/deepnsm-v2/src/coca.rs b/crates/deepnsm-v2/src/coca.rs index 2c871284e..7b911ced1 100644 --- a/crates/deepnsm-v2/src/coca.rs +++ b/crates/deepnsm-v2/src/coca.rs @@ -12,8 +12,8 @@ //! //! COCA letters, as used in `lemmas_5k.csv` / `word_forms.csv`: `a` article, //! `d` determiner (`this`, `which`, `all`, `some`; modals are `v`), `n` -//! noun, `p` pronoun, `v` verb, `j` adjective, anything else (`r` adverb, -//! `i` preposition, `c` conjunction, …) is outside the FSM's core slots. +//! noun, `p` pronoun, `v` verb, `j` adjective, `r` adverb; anything else +//! (`i` preposition, `c` conjunction, …) is outside the FSM's core slots. use crate::fsm::{Pos, PosSet}; use crate::lexical::{LexicalEvidence, PosCode}; @@ -27,6 +27,7 @@ pub const fn fsm_pos(code: PosCode) -> Pos { b'v' => Pos::Verb, b'j' => Pos::Adj, b'a' | b'd' => Pos::Det, + b'r' => Pos::Adv, _ => Pos::Other, } } @@ -118,7 +119,7 @@ mod tests { ("j", Pos::Adj), ("a", Pos::Det), ("d", Pos::Det), - ("r", Pos::Other), + ("r", Pos::Adv), ("i", Pos::Other), ("c", Pos::Other), ("", Pos::Other), diff --git a/crates/deepnsm-v2/src/fsm.rs b/crates/deepnsm-v2/src/fsm.rs index 69ebbb846..a9347b74a 100644 --- a/crates/deepnsm-v2/src/fsm.rs +++ b/crates/deepnsm-v2/src/fsm.rs @@ -42,7 +42,7 @@ use crate::spo::Spo; use crate::vocab::WordId; -/// A coarse part-of-speech tag — the six the FSM distinguishes. +/// A coarse part-of-speech tag — the eight the FSM distinguishes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Pos { /// Determiner / article (`the`, `a`) — skipped. @@ -62,10 +62,17 @@ pub enum Pos { /// antecedent pointer in [`crate::wave`] — a cheap positional tag, not the /// full gap→filler resolution. Rel, - /// Adverb / other — skipped for the core triple. + /// Other (preposition, conjunction, numeral, …) — skipped for the core + /// triple. Other, /// End-of-sentence punctuation — flushes any partial clause. Stop, + /// Adverb — skipped for the core triple like [`Pos::Other`], but kept + /// apart so an adjective/adverb homograph can be decided by position + /// (D-LXC-14) and an adverbial reading can feed the TEKAMOLO tenant. + /// Unlike [`Pos::Adj`] it opens no nominal group. Last in bit order so + /// the earlier tags keep their bits. + Adv, } /// One tagged token: its palette [`WordId`] plus its [`Pos`]. @@ -138,8 +145,11 @@ struct Core { antecedent: WordId, /// The previous token was a determiner or adjective: a nominal group is /// open and waiting for its head. Read only by [`parse_readings`]'s - /// licensing rule; [`parse_to_spo`] never consults it. + /// licensing and slot rules; [`parse_to_spo`] never consults it. nominal_open: bool, + /// The open nominal group was opened by an adjective, not a determiner. + /// Licensing reads it only where [`Typology::adjective_opens_nominal`]. + adjective_open: bool, /// The previous token opened the clause as its subject — a noun taken in /// [`State::Start`] with no determiner or adjective in front ("they", /// "men", "God"). An object carried into the subject slot ("gave him …") @@ -157,6 +167,7 @@ impl Core { matrix: 0, antecedent: 0, nominal_open: false, + adjective_open: false, fresh_subject: false, }; @@ -167,13 +178,14 @@ impl Core { let nominal_was_open = self.nominal_open; self.fresh_subject = false; self.nominal_open = matches!(t.pos, Pos::Det | Pos::Adj); + self.adjective_open = t.pos == Pos::Adj; // While a relative clause is open, its own tiny machine consumes the // embedded S-V-O; the matrix subject stays parked in `matrix`. if self.rel != Rel::None { match (self.rel, t.pos) { (_, Pos::Stop) => unreachable!("Stop handled by the caller"), - (_, Pos::Det | Pos::Adj | Pos::Other) => {} + (_, Pos::Det | Pos::Adj | Pos::Adv | Pos::Other) => {} // Object-relative: a noun after the relativizer is the embedded // subject ("rat that [cat] bit …"); the antecedent is its object. (Rel::Open, Pos::Noun) => self.rel = Rel::ObjSubject(t.id), @@ -224,7 +236,7 @@ impl Core { match (self.state, t.pos) { // Skip determiners, modifiers, adverbs — they are not core slots. - (_, Pos::Det | Pos::Adj | Pos::Other) => {} + (_, Pos::Det | Pos::Adj | Pos::Adv | Pos::Other) => {} (_, Pos::Stop) => unreachable!("Stop handled by the caller"), // A relativizer only opens a relative clause when we already have a @@ -302,7 +314,7 @@ pub struct PosSet(u8); impl Pos { /// Every tag, in bit order. - pub const ALL: [Pos; 7] = [ + pub const ALL: [Pos; 8] = [ Pos::Det, Pos::Adj, Pos::Noun, @@ -310,6 +322,7 @@ impl Pos { Pos::Rel, Pos::Other, Pos::Stop, + Pos::Adv, ]; const fn bit(self) -> u8 { @@ -345,6 +358,12 @@ impl PosSet { Self(self.0 | other.0) } + /// This set with `pos` removed. + #[must_use] + pub const fn without(self, pos: Pos) -> Self { + Self(self.0 & !pos.bit()) + } + /// Number of readings. #[must_use] pub const fn len(self) -> usize { @@ -438,6 +457,8 @@ pub struct ReadingParse { pub unlicensed_dropped: usize, /// (configuration, reading) pairs the slot rule dropped. pub slot_dropped: usize, + /// Tokens whose adjective/adverb readings the attribute rule narrowed. + pub attribute_narrowed: usize, } /// Bound on live configurations per sentence. Exceeding it flushes the @@ -455,12 +476,149 @@ struct Config { support: Vec, } -/// The licensing rule: a verb cannot follow a determiner or adjective, whose -/// nominal group is still waiting for its head. This is the only -/// elimination this layer makes, and [`parse_readings`] applies it -/// relatively: it never removes a token's last admissible reading. -const fn licensed(core: &Core, pos: Pos) -> bool { - !(core.nominal_open && matches!(pos, Pos::Verb)) +/// Word order of an attributive adjective relative to its noun (WALS 87A). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdjectiveOrder { + /// "the old man", "der alte Mann". + Before, + /// "l'homme âgé". + After, + /// Both orders are common ("un bon homme", "un homme bon"). + Both, +} + +/// The word-order facts the position rules read. A language is described by +/// these, never by rules of its own; [`Typology::ENGLISH`] is the default, +/// and a treebank's train split can measure them (see the `ud_pos_eval` +/// example). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Typology { + /// Where an attributive adjective stands. + pub adjective: AdjectiveOrder, + /// An adjective is always followed by the rest of its nominal group, so a + /// verb never comes straight after it. True for English; false where a + /// predicative adjective precedes its verb ("weil es möglich ist") or an + /// adjective follows its noun ("un projet important est"). + pub adjective_opens_nominal: bool, + /// The [`AttributeRule`] clauses allowed to narrow a token. Empty means + /// the attribute rule only reports ([`attribute_rule`]) and never drops + /// a reading. + pub attribute_rules: &'static [AttributeRule], +} + +impl Typology { + /// English: adjectives before the noun, predicatives after the verb. + pub const ENGLISH: Self = Self { + adjective: AdjectiveOrder::Before, + adjective_opens_nominal: true, + attribute_rules: &[], + }; +} + +impl Default for Typology { + fn default() -> Self { + Self::ENGLISH + } +} + +/// The licensing rule: a verb cannot follow a determiner, whose nominal +/// group is still waiting for its head — nor an adjective, where the +/// [`Typology`] says an adjective always opens one. [`parse_readings`] +/// applies it relatively: it never removes a token's last admissible +/// reading. +const fn licensed(core: &Core, pos: Pos, typology: Typology) -> bool { + let open = core.nominal_open && (!core.adjective_open || typology.adjective_opens_nominal); + !(open && matches!(pos, Pos::Verb)) +} + +/// One clause of the attribute rule (D-LXC-14). Each names the position it +/// reads and the reading it keeps. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AttributeRule { + /// Right after a determiner-only word → adjective ("the **only** son"). + AfterDeterminer, + /// Right before a word that can be an adjective → adverb, a degree + /// modifier ("**very** good", "**pretty** high", "**sehr** gut"). + BeforeAdjective, + /// Between a word that can be a subject and a word that can be a verb → + /// adverb ("I **just** need", "airfare **alone** will"). + BetweenSubjectAndVerb, + /// Next to a noun on the side [`Typology::adjective`] allows → adjective + /// ("a **high** rate", "un homme **bon**"). + NextToNoun, +} + +impl AttributeRule { + /// Every clause, in the order [`attribute_rule`] tries them. + pub const ALL: [AttributeRule; 4] = [ + AttributeRule::AfterDeterminer, + AttributeRule::BeforeAdjective, + AttributeRule::BetweenSubjectAndVerb, + AttributeRule::NextToNoun, + ]; + + /// The reading this clause keeps (the other of Adj/Adv is dropped). + #[must_use] + pub const fn keeps(self) -> Pos { + match self { + AttributeRule::AfterDeterminer | AttributeRule::NextToNoun => Pos::Adj, + AttributeRule::BeforeAdjective | AttributeRule::BetweenSubjectAndVerb => Pos::Adv, + } + } +} + +/// The clause of the attribute rule that decides an adjective/adverb +/// homograph from its neighbours, never from which reading a corpus counts +/// more often — or `None` (the token lacks one of Adj/Adv, or no clause +/// matches: after a verb, "was **good**" and "ran **fast**" look the same). +/// Clauses are tried in [`AttributeRule::ALL`] order; `prev` / `next` are the +/// neighbours' readings within the sentence ([`PosSet::EMPTY`] at an edge). +#[must_use] +pub fn attribute_rule( + prev: PosSet, + this: PosSet, + next: PosSet, + typology: Typology, +) -> Option { + if !(this.contains(Pos::Adj) && this.contains(Pos::Adv)) { + return None; + } + let noun_after = next.contains(Pos::Noun) + && matches!( + typology.adjective, + AdjectiveOrder::Before | AdjectiveOrder::Both + ); + let noun_before = prev.contains(Pos::Noun) + && matches!( + typology.adjective, + AdjectiveOrder::After | AdjectiveOrder::Both + ); + AttributeRule::ALL.into_iter().find(|rule| match rule { + AttributeRule::AfterDeterminer => prev == PosSet::single(Pos::Det), + AttributeRule::BeforeAdjective => next.contains(Pos::Adj), + AttributeRule::BetweenSubjectAndVerb => { + prev.contains(Pos::Noun) && next.contains(Pos::Verb) + } + AttributeRule::NextToNoun => noun_after || noun_before, + }) +} + +/// `this` narrowed by [`attribute_rule`]'s clause, if one is enabled in +/// `typology` and matches; otherwise `this` unchanged. Only one of Adj/Adv is +/// ever removed; every other reading stays. +#[must_use] +pub fn attribute_readings(prev: PosSet, this: PosSet, next: PosSet, typology: Typology) -> PosSet { + match attribute_rule(prev, this, next, typology) { + Some(rule) if typology.attribute_rules.contains(&rule) => { + let drop = if rule.keeps() == Pos::Adj { + Pos::Adv + } else { + Pos::Adj + }; + this.without(drop) + } + _ => this, + } } /// The slot rule: directly after a fresh subject noun (one that opened the @@ -508,6 +666,12 @@ fn slot_allows(core: &Core, pos: Pos, set: PosSet) -> bool { /// [`ReadingParse::certain`], the rest [`ReadingParse::alternative`]. #[must_use] pub fn parse_readings(tokens: &[Reading]) -> ReadingParse { + parse_readings_with(tokens, Typology::ENGLISH) +} + +/// [`parse_readings`] for a language described by `typology`. +#[must_use] +pub fn parse_readings_with(tokens: &[Reading], typology: Typology) -> ReadingParse { let mut out = ReadingParse::default(); let mut configs = vec![fresh()]; // (input index, entered set) for each ambiguous token of this sentence. @@ -522,20 +686,33 @@ pub fn parse_readings(tokens: &[Reading]) -> ReadingParse { out.unknown += 1; PosSet::single(Pos::Other) } else { - tok.pos + // Neighbours are read in place from the borrowed input; a stop + // is a sentence edge. + let edge = |t: Option<&Reading>| match t { + Some(t) if !t.pos.contains(Pos::Stop) => t.pos, + _ => PosSet::EMPTY, + }; + let prev = edge(index.checked_sub(1).and_then(|i| tokens.get(i))); + let next = edge(tokens.get(index + 1)); + let narrowed = attribute_readings(prev, tok.pos, next, typology); + if narrowed != tok.pos { + out.attribute_narrowed += 1; + } + narrowed }; - let ambiguous = set.len() > 1; + let ambiguous = tok.pos.len() > 1; if ambiguous { - pending.push((index, set)); + pending.push((index, tok.pos)); } - let allowed = |c: &Config, p: Pos| licensed(&c.core, p) && slot_allows(&c.core, p, set); + let allowed = + |c: &Config, p: Pos| licensed(&c.core, p, typology) && slot_allows(&c.core, p, set); let any_allowed = configs.iter().any(|c| set.iter().any(|p| allowed(c, p))); let mut next: Vec = Vec::with_capacity(configs.len() * set.len()); for c in &configs { for pos in set.iter() { if any_allowed && !allowed(c, pos) { - if licensed(&c.core, pos) { + if licensed(&c.core, pos, typology) { out.slot_dropped += 1; } else { out.unlicensed_dropped += 1; @@ -544,6 +721,11 @@ pub fn parse_readings(tokens: &[Reading]) -> ReadingParse { } let mut n = c.clone(); n.emitted.extend(n.core.step(Tagged::new(tok.id, pos))); + if typology.adjective_opens_nominal { + // Determiner and adjective open the same group here, so + // keep their paths mergeable. + n.core.adjective_open = false; + } if ambiguous { n.support.push(PosSet::single(pos)); } @@ -839,6 +1021,135 @@ mod tests { assert_eq!(p.slot_dropped, 0); } + /// D-LXC-14: an adjective that may precede its verb ("weil es möglich + /// ist", "un projet important est") does not license away a verb + /// reading; in English word order it does. + #[test] + fn adjective_licensing_follows_the_typology() { + // "the old ist N|V": verb reading after an adjective. + let toks = [ + one(9, Pos::Det), + one(1, Pos::Noun), + one(5, Pos::Adj), + noun_or_verb(2), + ]; + let english = parse_readings_with(&toks, Typology::ENGLISH); + assert_eq!(english.ambiguous[0].survived, PosSet::single(Pos::Noun)); + let verb_after_adjective = Typology { + adjective_opens_nominal: false, + ..Typology::ENGLISH + }; + let german = parse_readings_with(&toks, verb_after_adjective); + assert_eq!(german.ambiguous[0].survived, NV); + // A determiner still licenses away the verb in every typology. + let toks = [one(9, Pos::Det), noun_or_verb(2)]; + let p = parse_readings_with(&toks, verb_after_adjective); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Noun)); + } + + const ADJ_ADV: PosSet = PosSet::single(Pos::Adj).with(Pos::Adv); + + /// D-LXC-14: each attribute clause matches its own position, and none + /// matches after a verb. + #[test] + fn each_attribute_clause_reads_its_position() { + let t = Typology::ENGLISH; + let det = PosSet::single(Pos::Det); + let noun = PosSet::single(Pos::Noun); + let verb = PosSet::single(Pos::Verb); + let adj = PosSet::single(Pos::Adj); + let e = PosSet::EMPTY; + // "the [only] son" + assert_eq!( + attribute_rule(det, ADJ_ADV, noun, t), + Some(AttributeRule::AfterDeterminer) + ); + // "was [very] good" + assert_eq!( + attribute_rule(verb, ADJ_ADV, adj, t), + Some(AttributeRule::BeforeAdjective) + ); + // "I [just] need" + assert_eq!( + attribute_rule(noun, ADJ_ADV, verb, t), + Some(AttributeRule::BetweenSubjectAndVerb) + ); + // "of [high] rate" (`of` is Other) + let other = PosSet::single(Pos::Other); + assert_eq!( + attribute_rule(other, ADJ_ADV, noun, t), + Some(AttributeRule::NextToNoun) + ); + // Silence: "was [good] ." and "ran [fast] ." — after a verb, nothing. + assert_eq!(attribute_rule(verb, ADJ_ADV, e, t), None); + // A token without both readings is never touched. + assert_eq!(attribute_rule(det, adj, noun, t), None); + // Post-nominal order: "homme [bon] ." is attributive only where the + // typology allows an adjective after its noun. + let both = Typology { + adjective: AdjectiveOrder::Both, + ..t + }; + assert_eq!( + attribute_rule(noun, ADJ_ADV, e, both), + Some(AttributeRule::NextToNoun) + ); + assert_eq!(attribute_rule(noun, ADJ_ADV, e, t), None); + } + + /// D-LXC-14: as shipped no clause narrows (each was measured below + /// frequency on gold tags); an enabled clause narrows exactly its token. + #[test] + fn attribute_clauses_narrow_only_when_enabled() { + let toks = [ + one(9, Pos::Det), + Reading::new(5, ADJ_ADV), + one(1, Pos::Noun), + ]; + let p = parse_readings(&toks); + assert_eq!(p.ambiguous[0].survived, ADJ_ADV); + assert_eq!(p.attribute_narrowed, 0); + let enabled = Typology { + attribute_rules: &[AttributeRule::AfterDeterminer], + ..Typology::ENGLISH + }; + let p = parse_readings_with(&toks, enabled); + assert_eq!(p.ambiguous[0].entered, ADJ_ADV); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Adj)); + assert_eq!(p.attribute_narrowed, 1); + // A clause that is not enabled does not fire: "was [very] good". + let toks = [ + one(4, Pos::Verb), + Reading::new(5, ADJ_ADV), + one(6, Pos::Adj), + ]; + let p = parse_readings_with(&toks, enabled); + assert_eq!(p.ambiguous[0].survived, ADJ_ADV); + // A stop is a sentence edge: "homme . [bon] ," does not read + // `homme`, so the post-nominal clause cannot fire across it — while + // without the stop it does. + let both = Typology { + adjective: AdjectiveOrder::Both, + attribute_rules: &AttributeRule::ALL, + ..Typology::ENGLISH + }; + let across = [ + one(1, Pos::Noun), + Reading::stop(), + Reading::new(5, ADJ_ADV), + one(7, Pos::Other), + ]; + let p = parse_readings_with(&across, both); + assert_eq!(p.ambiguous[0].survived, ADJ_ADV); + let within = [ + one(1, Pos::Noun), + Reading::new(5, ADJ_ADV), + one(7, Pos::Other), + ]; + let p = parse_readings_with(&within, both); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Adj)); + } + /// T4: a single-reading word is never rejected, even where the licensing /// rule would forbid it — a word is not dropped for having one reading. #[test] diff --git a/crates/deepnsm-v2/src/lib.rs b/crates/deepnsm-v2/src/lib.rs index 5954a75ac..1a2a8d0f9 100644 --- a/crates/deepnsm-v2/src/lib.rs +++ b/crates/deepnsm-v2/src/lib.rs @@ -64,7 +64,10 @@ pub use evidence::{ evidence_basin, forward_gate, novelty_rate, open_question_yield, partial_spearman, shuffle_beliefs_null, shuffle_rungs_null, EvidenceBasin, ForwardGateReport, }; -pub use fsm::{parse_readings, parse_to_spo, Pos, PosSet, Reading, ReadingParse, Survivor, Tagged}; +pub use fsm::{ + attribute_readings, attribute_rule, parse_readings, parse_readings_with, parse_to_spo, + AdjectiveOrder, AttributeRule, Pos, PosSet, Reading, ReadingParse, Survivor, Tagged, Typology, +}; pub use introspect::{ confidence_delta_recount, confidence_delta_self, most_frequent_belief, provenance_check, ConfidenceAnswer, ProvenanceReport, From 31c673660a1afcda6d3b82585f53aa3b49b8b266 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:06:47 +0000 Subject: [PATCH 03/33] deepnsm-v2: KJV silver labels from Luther 1545, clause rule as a switch, WordNet measured (D-LXC-15) - Typology::predicate_required: the "no verb yet, which word could it be" rule as an off-by-default switch. - bible_wave optional Rosetta arm (ROSETTA_DIR): German casing in the aligned Luther 1545 verse labels KJV noun/verb homographs (contrastive associates only, Psalms excluded). serde_json is a dev-dependency only. - ud_pos_eval: UD_CLAUSE, UD_WORDNET, UD_WORDNET_FILTER. Measured: on 756 KJV tokens decided by position, slot + licensing are 85.8 % right against the German label, the COCA lemma tag 74.9 % on the same tokens. The clause rule's additions are below the slot rule in English, German, French and on the KJV, and below frequency, so it stays off. WordNet: 59.3 % as a chooser; as a reading filter it changes 1 form. KJV pins unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...eepnsm-v2-kjv-silver-labels-from-luther.md | 80 +++++ .claude/board/entries/README.md | 3 +- ...deepnsm-v2-lexical-evidence-consumer-v1.md | 10 + crates/deepnsm-v2/Cargo.toml | 5 + crates/deepnsm-v2/examples/bible_wave.rs | 284 +++++++++++++++++- crates/deepnsm-v2/examples/ud_pos_eval.rs | 151 +++++++++- crates/deepnsm-v2/src/fsm.rs | 59 +++- 8 files changed, 580 insertions(+), 13 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 7d545a6e9..240aaeb81 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -38,6 +38,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-10** | `Pos::Rel` produced by no in-crate tagger in deepnsm-v2 (only FSM match arms + one unit test) | Queued | pre-existing; external callers can still pass `Pos::Rel` through the public `Tagged`/`parse_to_spo` | | **D-LXC-13** | noun/verb homograph decided by position: lemma N/V tag widened to its other reading (`coca::predicate_alternatives`), slot rule after a fresh subject | In PR | slot rule + widening disable-verified; KJV re-pinned (34,824 ambiguous / 451 words, certain 57,350 + alternative 29,001, 0 unexplained). Entry `2026-10-04-deepnsm-v2-predicate-by-position.md` | | **D-LXC-14** | `Pos::Adv`; `Typology` (measured, gates adjective licensing); `AttributeRule` adjective/adverb clauses (off: below frequency); `ud_pos_eval` gold harness (en/de/fr) | In PR | noun/verb with COCA: position 96.7 % vs frequency 85.1 % on 457 decided UD-EWT tokens; adjective/adverb: no clause beats frequency in every language. Entry `2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md` | +| **D-LXC-15** | KJV silver noun/verb labels from Luther 1545 casing (release `v0.1.0-codebooks-2026-07-26`); clause rule as an off-by-default switch; WordNet as a parallel coordinate | In PR | KJV: slot + licensing 85.8 % vs COCA lemma tag 74.9 % on 756 decided; clause rule below slot rule and frequency in en/de/fr/KJV; WordNet chooser 59.3 %, existence filter changes 1 form. Entry `2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md b/.claude/board/entries/2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md new file mode 100644 index 000000000..19f34d5d5 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md @@ -0,0 +1,80 @@ +# 2026-10-04 — deepnsm-v2: KJV noun/verb decisions checked against German casing; slot rule vs clause rule; WordNet as a parallel coordinate (D-LXC-15) + +**Status:** MEASURED. KJV arm: `ROSETTA_DIR= cargo run --release --example +bible_wave -- pg10.txt`, with `` holding `pd-texts/`, `rosetta-gpl/`, `de/` +from release `v0.1.0-codebooks-2026-07-26` (sha256 de a5575ad1…, pd-texts +d141d76c…, rosetta-gpl 66acc1ed…, wordnet f670f1f3…). UD arm: `ud_pos_eval` +with `UD_CLAUSE=1` / `UD_WORDNET=…` / `UD_WORDNET_FILTER=1`. + +Operator questions (2026-10-04): the slot rule vs "we have no verb in this +sentence, which one could it be"; use the German/Czech/Greek Bible releases; +WordNet as a coordinate in parallel. + +## What changed + +- `Typology::predicate_required` — the clause rule as a switch, off by + default. At a sentence end it drops reading combinations on which no verb + took a predicate slot, if some combination's verb did. +- `bible_wave` optional Rosetta arm (`ROSETTA_DIR`): silver noun/verb labels + for KJV homographs from Luther 1545 casing via the release's en→de + alignment. Contrastive: a word is used only if its associates include a + German lexicon noun AND a verb; capitalised non-initial noun → Noun, + lowercase verb → Verb; all associates found must agree. Psalms excluded + (documented Luther 1545 versification offset). Verse key (book, chapter, + verse) from `corpus` markers, 66 books asserted. `serde_json` is a + dev-dependency only. +- `ud_pos_eval`: `UD_CLAUSE`, `UD_WORDNET` (sense-count pick), `UD_WORDNET_FILTER` + (WordNet as an existence filter on COCA readings). + +## Slot rule vs clause rule (kept Verb) + +| | slot rule only | + clause rule | clause rule's own additions | +|---|---|---|---| +| UD English, COCA readings | 108 at 92.6 % | 177 at 68.4 % | ~69, ~30 % right | +| UD German | 36 at 100 % | 133 at 94.7 % | ~97, ~93 % (frequency on them ~97 %) | +| UD French | 28 at 100 % | 56 at 89.3 % | ~28, ~79 % | +| KJV, German-casing silver | 20 at 60.0 % | 34 at 58.8 % | 14, ~57 % | + +The clause rule stays off: its additions are below the slot rule everywhere +and below frequency in every language. + +## KJV against German casing (1,751 labelled of 34,608 homograph tokens) + +| policy | decided | right | COCA lemma tag on the same tokens | +|---|---|---|---| +| COCA lemma tag (forced) | 1,751 | 67.2 % | — | +| slot + licensing | 756 | **85.8 %** | 74.9 % | +| — licensing (kept Noun) | 736 | 86.5 % | | +| — slot rule (kept Verb) | 20 | 60.0 % | | + +Labels are silver. Of the slot rule's 8 misses, 3 are label noise (collocate +associates: regard→Gott, light→Licht, walking→Meer) and 5 are vocatives / +a list ("ye fools", "thou judge", "faith hope charity"). The non-contrastive +first version (7,531 labels) gave the same ordering — licensing 95.9 %, +slot 56.0 %, clause 45.3 %, lemma tag 80.3 % — but with collocate noise +(`rose` → `des Morgens`). + +## WordNet as a parallel coordinate (UD English, COCA readings) + +- As a chooser (reading with more WordNet senses): **59.3 %** right vs COCA + frequency 84.2 % — sense counts do not track usage. +- As an existence filter (drop a COCA reading WordNet lacks): changes **1** + form; WordNet confirms nearly every COCA noun/verb homograph. Neutral. + +## Not used yet + +- Czech (BKR) and Greek (Tischendorf): no noun capitalisation; labels would + need a morphological lexicon. The en-cs / en-el alignments are in the + release. +- `de/tekamolo.tsv` (adverbial lemma → TEKAMOLO lane, from German UD + `advmod`/`mark`) is the lexical adverb class the TEKAMOLO tenant needs; + through the en→de alignment it can reach English adverbs. Not wired. + +## OPEN + +- Slot-rule errors: vocatives/address ("ye fools", "thou judge"), archaic + `mine`/`thine` as subject, nouns after a carried object. +- Licensing errors on the KJV: a verb after a predicative adjective ("make + alive wound"), `both`/`that` read as determiners. +- Silver-label noise: collocates and translation shifts; a cleaner label + would align word to word, not verse to verse. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index fd74475f9..4f4f76c74 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,12 +25,13 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -188 entries, 2026-08-06 .. 2026-10-04. +189 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| | 2026-10-04 | `D-LXC-13` | | [2026-10-04-deepnsm-v2-predicate-by-position.md](2026-10-04-deepnsm-v2-predicate-by-position.md) | | 2026-10-04 | `D-LXC-14` | | [2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md](2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md) | +| 2026-10-04 | `D-LXC-15` | | [2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md](2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md) | | 2026-10-03 | `population-law-crosscheck` | | [2026-10-03-population-law-crosscheck.md](2026-10-03-population-law-crosscheck.md) | | 2026-10-03 | `orkg-reference-harvest` | | [2026-10-03-orkg-reference-harvest.md](2026-10-03-orkg-reference-harvest.md) | | 2026-10-03 | `known-is-not-alpha-premise-gate` | | [2026-10-03-known-is-not-alpha-premise-gate.md](2026-10-03-known-is-not-alpha-premise-gate.md) | diff --git a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md index d5105f986..3ce84d2ff 100644 --- a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md +++ b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md @@ -484,3 +484,13 @@ clauses do not beat frequency in any language consistently and ship disabled. Adjective licensing is gated on the measured typology. Numbers: board entry `2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md`. +## D-LXC-15: the KJV checked against German casing (2026-10-04) + +The codebook release's Luther 1545 lane and en→de alignment give silver +noun/verb labels for KJV homographs (German capitalises nouns). On those, +slot + licensing beat the COCA lemma tag (85.8 % vs 74.9 % on the same +tokens); the clause rule ("no verb yet, which could it be") stays an off +switch, below the slot rule in every language. WordNet helps neither as a +chooser nor as a reading filter for this question. Board entry +`2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md`. + diff --git a/crates/deepnsm-v2/Cargo.toml b/crates/deepnsm-v2/Cargo.toml index aaf22ba67..af2afd177 100644 --- a/crates/deepnsm-v2/Cargo.toml +++ b/crates/deepnsm-v2/Cargo.toml @@ -34,3 +34,8 @@ lance-graph-contract = { path = "../lance-graph-contract", features = ["guid-v3- [[example]] name = "bible_wave" test = true + +# Examples only: `bible_wave`'s optional Rosetta arm reads the release's +# verse-keyed Bible JSON. The library stays dependency-free. +[dev-dependencies] +serde_json = "1" diff --git a/crates/deepnsm-v2/examples/bible_wave.rs b/crates/deepnsm-v2/examples/bible_wave.rs index fbae1c2be..801d5690b 100644 --- a/crates/deepnsm-v2/examples/bible_wave.rs +++ b/crates/deepnsm-v2/examples/bible_wave.rs @@ -35,9 +35,9 @@ use deepnsm_v2::coca::{fsm_pos, fsm_pos_tag, predicate_alternatives, reading_set}; use deepnsm_v2::{ - load_cam96_codes, load_cam96_space, load_word_forms_csv, parse_readings, parse_to_spo, - EvidenceError, LexicalEvidence, Nsm, PaletteVocab, Pos, PosSet, Reading, ReadingParse, Spo, - Tagged, TemporalStream, WordFormsReport, WordId, + load_cam96_codes, load_cam96_space, load_word_forms_csv, parse_readings, parse_readings_with, + parse_to_spo, EvidenceError, LexicalEvidence, Nsm, PaletteVocab, Pos, PosSet, Reading, + ReadingParse, Spo, Tagged, TemporalStream, Typology, WordFormsReport, WordId, }; use std::collections::HashMap; use std::path::PathBuf; @@ -318,6 +318,247 @@ fn lexicon_file(name: &str) -> String { .unwrap_or_else(|e| panic!("missing lexicon table {} ({e})", path.display())) } +/// D-LXC-15: German casing as a silver noun/verb label for KJV tokens. +/// +/// For a KJV noun/verb homograph, its German associates come from the +/// release's corpus-derived `alignment_en-de.tsv` (KJV → Luther 1545, Dice / +/// PMI, cooc ≥ 5). Luther 1545 capitalises nouns, so an associate found in +/// the same verse labels the token: capitalised, not sentence-initial and a +/// noun in the German lexicon → noun; lowercase and a verb there → verb. +/// The label is contrastive: a word is used only when its associates attest +/// BOTH readings (one-sided associates — collocates like `rose` → `Morgens` +/// — could only ever say one thing). A token is labelled only when every +/// associate found agrees. Psalms are excluded: Luther 1545 numbers them +/// differently (the release's documented versification offset). Inputs: the +/// `v0.1.0-codebooks-2026-07-26` release (`pd-texts`, `rosetta-gpl`, `de`), +/// extracted under `ROSETTA_DIR`. +struct RosettaSilver { + verses: HashMap<(u16, u16, u16), String>, + align: HashMap>, + /// German word form → its lexicon readings (noun, verb). + de_pos: HashMap, + /// Per policy: (labelled tokens decided, decided right). + legacy: (usize, usize), + shipped: (usize, usize), + clause: (usize, usize), + /// The shipped rules' decisions by kept reading: (Noun n, right, Verb n, right). + shipped_by_kept: [(usize, usize); 2], + clause_by_kept: [(usize, usize); 2], + /// Legacy right on the tokens each policy decided: (shipped, clause). + legacy_on_shipped: usize, + legacy_on_clause: usize, + labelled: usize, + candidates: usize, +} + +impl RosettaSilver { + fn load(dir: &str) -> Self { + let read = |p: &str| { + std::fs::read_to_string(format!("{dir}/{p}")) + .unwrap_or_else(|e| panic!("{dir}/{p}: {e}")) + }; + let json: serde_json::Value = + serde_json::from_str(&read("pd-texts/bible_luther1545.json")).expect("luther1545 json"); + let mut verses = HashMap::new(); + for book in json["books"].as_array().expect("books") { + let nr = book["nr"].as_u64().expect("book nr") as u16; + for ch in book["chapters"].as_array().expect("chapters") { + for v in ch["verses"].as_array().expect("verses") { + let c = v["chapter"].as_u64().expect("chapter") as u16; + let n = v["verse"].as_u64().expect("verse") as u16; + let t = v["text"].as_str().expect("text").to_string(); + verses.insert((nr, c, n), t); + } + } + } + let mut align: HashMap> = HashMap::new(); + for line in read("rosetta-gpl/alignment_en-de.tsv").lines().skip(1) { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() >= 2 { + align + .entry(c[0].to_string()) + .or_default() + .push(c[1].to_string()); + } + } + let mut de_pos: HashMap = HashMap::new(); + for l in read("de/lexicon.tsv") + .lines() + .filter(|l| !l.starts_with('#')) + { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() >= 3 { + let e = de_pos.entry(c[0].to_lowercase()).or_default(); + e.0 |= c[2] == "n"; + e.1 |= c[2] == "v"; + } + } + Self { + verses, + align, + de_pos, + legacy: (0, 0), + shipped: (0, 0), + clause: (0, 0), + shipped_by_kept: [(0, 0); 2], + clause_by_kept: [(0, 0); 2], + legacy_on_shipped: 0, + legacy_on_clause: 0, + labelled: 0, + candidates: 0, + } + } + + /// The silver label for English `word` in verse `key`, or `None`. + fn label(&self, key: (u16, u16, u16), word: &str) -> Option { + const PSALMS: u16 = 19; + if key.0 == PSALMS { + return None; + } + let text = self.verses.get(&key)?; + let assoc = self.align.get(word)?; + let pos_of = |a: &String| self.de_pos.get(a).copied().unwrap_or_default(); + if !(assoc.iter().any(|a| pos_of(a).0) && assoc.iter().any(|a| pos_of(a).1)) { + return None; + } + let mut found: Option = None; + let mut initial = true; + for raw in text.split_whitespace() { + let tok: String = raw.chars().filter(|c| c.is_alphabetic()).collect(); + let this_initial = initial; + initial = raw.ends_with(['.', '!', '?', ':']); + if tok.is_empty() || this_initial { + continue; + } + let lower = tok.to_lowercase(); + if !assoc.contains(&lower) { + continue; + } + let (noun, verb) = pos_of(&lower); + let pos = if tok.starts_with(char::is_uppercase) && noun { + Pos::Noun + } else if !tok.starts_with(char::is_uppercase) && verb { + Pos::Verb + } else { + continue; + }; + match found { + None => found = Some(pos), + Some(p) if p == pos => {} + Some(_) => return None, + } + } + found + } + + fn kept(set: PosSet) -> Option { + match (set.contains(Pos::Noun), set.contains(Pos::Verb)) { + (true, false) => Some(Pos::Noun), + (false, true) => Some(Pos::Verb), + _ => None, + } + } + + #[allow(clippy::too_many_arguments)] + fn verse( + &mut self, + key: (u16, u16, u16), + readings: &[Reading], + legacy_tags: &[Tagged], + words: &[String], + shipped: &ReadingParse, + clause: &ReadingParse, + ) { + let clause_by_index: HashMap = clause + .ambiguous + .iter() + .map(|sv| (sv.index, sv.survived)) + .collect(); + for sv in &shipped.ambiguous { + let entered = readings[sv.index].pos; + if !(entered.contains(Pos::Noun) && entered.contains(Pos::Verb)) { + continue; + } + self.candidates += 1; + let Some(gold) = self.label(key, &words[sv.index]) else { + continue; + }; + self.labelled += 1; + let legacy_ok = legacy_tags[sv.index].pos == gold; + self.legacy.0 += 1; + self.legacy.1 += usize::from(legacy_ok); + let slot = |p: Pos| usize::from(p == Pos::Verb); + if let Some(k) = Self::kept(sv.survived) { + if k != gold && std::env::var_os("ROSETTA_DUMP").is_some() { + eprintln!( + "SILVER kept {k:?} german {gold:?} [{}] | {} | {}", + words[sv.index], + words.join(" "), + self.verses.get(&key).map_or("", String::as_str) + ); + } + self.shipped.0 += 1; + self.shipped.1 += usize::from(k == gold); + self.legacy_on_shipped += usize::from(legacy_ok); + let e = &mut self.shipped_by_kept[slot(k)]; + e.0 += 1; + e.1 += usize::from(k == gold); + } + if let Some(k) = clause_by_index.get(&sv.index).copied().and_then(Self::kept) { + self.clause.0 += 1; + self.clause.1 += usize::from(k == gold); + self.legacy_on_clause += usize::from(legacy_ok); + let e = &mut self.clause_by_kept[slot(k)]; + e.0 += 1; + e.1 += usize::from(k == gold); + } + } + } + + fn print(&self) { + let pct = |n: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * n as f64 / d as f64 + } + }; + println!( + "D-LXC-15 KJV noun/verb homographs: {} ({} with a German-casing label)", + self.candidates, self.labelled + ); + println!( + "D-LXC-15 legacy lemma tag: {:.1}% right on all labelled", + pct(self.legacy.1, self.legacy.0) + ); + for (name, (n, r), legacy_r, by_kept) in [ + ( + "slot + licensing", + self.shipped, + self.legacy_on_shipped, + self.shipped_by_kept, + ), + ( + "+ clause rule", + self.clause, + self.legacy_on_clause, + self.clause_by_kept, + ), + ] { + println!( + "D-LXC-15 {name}: decided {n}, {:.1}% right (legacy tag on the same tokens \ + {:.1}%); kept Noun {} at {:.1}%, kept Verb {} at {:.1}%", + pct(r, n), + pct(legacy_r, n), + by_kept[0].0, + pct(by_kept[0].1, by_kept[0].0), + by_kept[1].0, + pct(by_kept[1].1, by_kept[1].0) + ); + } + } +} + fn main() { let path = std::env::args() .nth(1) @@ -456,6 +697,28 @@ fn main() { let mut readings_buf: Vec = Vec::new(); let mut legacy_buf: Vec = Vec::new(); let mut words_buf: Vec = Vec::new(); + // D-LXC-15: optional Rosetta silver labels. The verse key is + // (book, chapter, verse); a book starts where the markers restart at 1:1. + let mut rosetta = std::env::var("ROSETTA_DIR") + .ok() + .filter(|d| !d.is_empty()) + .map(|d| RosettaSilver::load(&d)); + let mut book = 0u16; + let keys: Vec<(u16, u16, u16)> = split + .markers + .iter() + .map(|&(c, v)| { + if (c, v) == (1, 1) { + book += 1; + } + (book, c, v) + }) + .collect(); + assert_eq!(keys.len(), verses.len(), "KILL: one marker per verse"); + let with_clause = Typology { + predicate_required: true, + ..Typology::ENGLISH + }; for (vi, verse) in verses.iter().enumerate() { readings_buf.clear(); legacy_buf.clear(); @@ -472,12 +735,27 @@ fn main() { let parse = parse_readings(&readings_buf); let legacy = parse_to_spo(&legacy_buf); lx.verse(&readings_buf, &legacy_buf, &words_buf, &parse, &legacy); + if let Some(r) = rosetta.as_mut() { + let clause = parse_readings_with(&readings_buf, with_clause); + r.verse( + keys[vi], + &readings_buf, + &legacy_buf, + &words_buf, + &parse, + &clause, + ); + } for &t in &parse.certain { stream.push(vi as u64, t); all.push((vi as u64, t)); } } lx.print_and_gate(); + if let Some(r) = &rosetta { + assert_eq!(book, 66, "KILL D-LXC-15: the KJV must split into 66 books"); + r.print(); + } // ── SoC seam (text): emit labelled verse text for the reasoning layer ── if let Some(out) = &export_verses { diff --git a/crates/deepnsm-v2/examples/ud_pos_eval.rs b/crates/deepnsm-v2/examples/ud_pos_eval.rs index e4d2e0d32..64a2e55cc 100644 --- a/crates/deepnsm-v2/examples/ud_pos_eval.rs +++ b/crates/deepnsm-v2/examples/ud_pos_eval.rs @@ -24,7 +24,9 @@ //! cargo run --release --example ud_pos_eval -- TRAIN TEST --coca ../deepnsm/word_frequency //! ``` //! -//! The typology is measured from train; `UD_TYPOLOGY=english` uses +//! `UD_CLAUSE=1` switches on the clause rule +//! ([`Typology::predicate_required`]). The typology is measured from train; +//! `UD_TYPOLOGY=english` uses //! [`Typology::ENGLISH`] instead. `UD_DUMP=1` prints every token a rule //! narrowed away from its gold tag, with context, to stderr. @@ -108,6 +110,8 @@ fn read_conllu(path: &str) -> Vec> { struct Lexicon { readings: HashMap, counts: HashMap<(String, u8), usize>, + /// Form → its COCA lemmas (COCA mode only), for the WordNet lookup. + lemmas: HashMap>, } impl Lexicon { @@ -122,7 +126,11 @@ impl Lexicon { *set = set.with(w.gold); *counts.entry((w.form.clone(), w.gold as u8)).or_default() += 1; } - Self { readings, counts } + Self { + readings, + counts, + lemmas: HashMap::new(), + } } /// Readings and counts from COCA (`lemmas_5k.csv` + `word_forms.csv` in @@ -132,6 +140,7 @@ impl Lexicon { fn from_coca(dir: &str) -> Self { let mut readings: HashMap = HashMap::new(); let mut counts: HashMap<(String, u8), usize> = HashMap::new(); + let mut lemmas: HashMap> = HashMap::new(); let read = |name: &str| { std::fs::read_to_string(format!("{dir}/{name}")) .unwrap_or_else(|e| panic!("{dir}/{name}: {e}")) @@ -148,12 +157,21 @@ impl Lexicon { if c.len() > 5 { let word = c[5].to_lowercase(); let pos = fsm_pos_tag(c[2]); + let lemma = c[1].to_lowercase(); + let ls = lemmas.entry(word.clone()).or_default(); + if !ls.contains(&lemma) { + ls.push(lemma); + } let set = readings.entry(word.clone()).or_default(); *set = set.with(pos); *counts.entry((word, pos as u8)).or_default() += c[4].parse::().unwrap_or(0); } } - Self { readings, counts } + Self { + readings, + counts, + lemmas, + } } fn get(&self, form: &str) -> PosSet { @@ -177,6 +195,51 @@ impl Lexicon { } } +/// WordNet noun and verb sense counts per lemma, from the release's +/// `wordnet31_isa_v2.tsv` (7 columns: word, pos, sense_num, …). +fn load_wordnet(path: &str) -> HashMap { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut senses: HashMap<(String, bool), std::collections::HashSet> = HashMap::new(); + for line in text.lines().filter(|l| !l.starts_with('#')) { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() != 7 || !matches!(c[1], "n" | "v") { + continue; + } + let word = c[0].to_lowercase().replace('_', " "); + senses + .entry((word, c[1] == "v")) + .or_default() + .insert(c[2].to_string()); + } + let mut out: HashMap = HashMap::new(); + for ((word, verb), s) in senses { + let e = out.entry(word).or_default(); + if verb { + e.1 = s.len(); + } else { + e.0 = s.len(); + } + } + out +} + +/// The WordNet pick for `form`: the reading with more senses over its COCA +/// lemmas, or `None` when WordNet has neither. +fn wordnet_pick(lex: &Lexicon, wn: &HashMap, form: &str) -> Option { + let (mut n, mut v) = (0, 0); + for lemma in lex.lemmas.get(form)? { + if let Some(&(a, b)) = wn.get(lemma) { + n += a; + v += b; + } + } + match (n, v) { + (0, 0) => None, + _ if n >= v => Some(Pos::Noun), + _ => Some(Pos::Verb), + } +} + /// Score for one pair of readings. #[derive(Default)] struct PairScore { @@ -296,6 +359,7 @@ fn measure_typology(train: &[Vec]) -> (Typology, f64, f64) { // As shipped: no clause narrows (D-LXC-14 measured them below // frequency). The per-clause table scores them regardless. attribute_rules: Typology::ENGLISH.attribute_rules, + predicate_required: false, }; (typology, share_before, verb_after) } @@ -321,10 +385,46 @@ fn main() { 100.0 * share_before, 100.0 * verb_after ); - let lex = match coca { + let typology = Typology { + predicate_required: std::env::var("UD_CLAUSE").is_ok_and(|v| !v.is_empty()), + ..typology + }; + let mut lex = match coca { Some(dir) => Lexicon::from_coca(dir), None => Lexicon::from_train(&train), }; + // WordNet as a second lexicon on which readings EXIST: with + // `UD_WORDNET_FILTER=1`, a COCA noun/verb homograph keeps a reading only + // if one of its lemmas has that sense in WordNet. Reports what it costs. + if std::env::var("UD_WORDNET_FILTER").is_ok_and(|v| !v.is_empty()) { + let path = std::env::var("UD_WORDNET").expect("UD_WORDNET_FILTER needs UD_WORDNET"); + let wn = load_wordnet(&path); + let mut narrowed = 0; + let forms: Vec = lex.readings.keys().cloned().collect(); + for form in forms { + let set = lex.readings[&form]; + if !(set.contains(Pos::Noun) && set.contains(Pos::Verb)) { + continue; + } + let (mut n, mut v) = (0, 0); + for lemma in lex.lemmas.get(&form).into_iter().flatten() { + if let Some(&(a, b)) = wn.get(lemma) { + n += a; + v += b; + } + } + let kept = match (n > 0, v > 0) { + (true, false) => set.without(Pos::Verb), + (false, true) => set.without(Pos::Noun), + _ => set, + }; + if kept != set { + narrowed += 1; + lex.readings.insert(form, kept); + } + } + println!("WordNet filter: {narrowed} COCA noun/verb forms lost a reading WordNet lacks"); + } let test = read_conllu(test); // One stream: every sentence ends in a stop, whatever its own punctuation. @@ -378,14 +478,53 @@ fn main() { let tokens = words.iter().filter(|w| w.is_some()).count(); println!( "{tokens} test words, {} ambiguous, {} unknown; slot rule dropped {}, licensing {}, \ - attribute rule {}", + attribute rule {}, clause rule {}", parse.ambiguous.len(), parse.unknown, parse.slot_dropped, parse.unlicensed_dropped, - parse.attribute_narrowed + parse.attribute_narrowed, + parse.unpredicated_dropped ); nv_score.print("noun/verb", nv); + // A parallel coordinate: WordNet's own noun/verb sense counts as the + // prior, scored on the same tokens (COCA mode supplies the lemmas). + if let Some(path) = std::env::var("UD_WORDNET").ok().filter(|p| !p.is_empty()) { + let wn = load_wordnet(&path); + let (mut all, mut all_right, mut on_dec, mut on_dec_right, mut covered) = + (0usize, 0usize, 0usize, 0usize, 0usize); + for sv in &parse.ambiguous { + let w = words[sv.index].expect("ambiguous tokens are words"); + if !(sv.entered.contains(Pos::Noun) && sv.entered.contains(Pos::Verb)) + || !matches!(w.gold, Pos::Noun | Pos::Verb) + { + continue; + } + all += 1; + let Some(pick) = wordnet_pick(&lex, &wn, &w.form) else { + continue; + }; + covered += 1; + all_right += usize::from(pick == w.gold); + if sv.survived.contains(Pos::Noun) != sv.survived.contains(Pos::Verb) { + on_dec += 1; + on_dec_right += usize::from(pick == w.gold); + } + } + let pct = |n: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * n as f64 / d as f64 + } + }; + println!( + " WordNet sense-count pick: covers {covered} of {all}, {:.1}% right on those; \ + on position-decided tokens {:.1}% right ({on_dec})", + pct(all_right, covered), + pct(on_dec_right, on_dec) + ); + } aa_score.print("adjective/adverb", aa); // Each attribute clause on its own: every adjective/adverb token whose diff --git a/crates/deepnsm-v2/src/fsm.rs b/crates/deepnsm-v2/src/fsm.rs index a9347b74a..5d43adcef 100644 --- a/crates/deepnsm-v2/src/fsm.rs +++ b/crates/deepnsm-v2/src/fsm.rs @@ -156,6 +156,10 @@ struct Core { /// or a re-anchored noun ("mothers house") is not fresh. Read only by /// the slot rule. fresh_subject: bool, + /// A verb has taken a predicate slot in this sentence (matrix or + /// relative clause). Read only by the clause rule + /// ([`Typology::predicate_required`]). + predicated: bool, } impl Core { @@ -169,6 +173,7 @@ impl Core { nominal_open: false, adjective_open: false, fresh_subject: false, + predicated: false, }; /// Consume one non-`Stop` token; returns the triple it closes, if any. @@ -192,6 +197,7 @@ impl Core { // Subject-relative: a verb right after the relativizer means the // antecedent is the embedded subject ("dog that [chased] …"). (Rel::Open, Pos::Verb) => { + self.predicated = true; self.rel = Rel::HaveVerb { subj: self.antecedent, verb: t.id, @@ -200,6 +206,7 @@ impl Core { // Object-relative embedded verb: emit (embedded_subj, verb, // antecedent) and close — the antecedent IS the object. (Rel::ObjSubject(es), Pos::Verb) => { + self.predicated = true; let out = Spo::new(es, t.id, self.antecedent); self.subject = self.matrix; self.state = State::HaveSubject; @@ -223,6 +230,7 @@ impl Core { // close the (intransitive → no triple) embedded clause and let // the matrix subject take this verb as its predicate. (Rel::HaveVerb { .. }, Pos::Verb) => { + self.predicated = true; self.subject = self.matrix; self.predicate = t.id; self.state = State::HaveVerb; @@ -255,6 +263,7 @@ impl Core { self.fresh_subject = !nominal_was_open; } (State::HaveSubject, Pos::Verb) => { + self.predicated = true; self.predicate = t.id; self.state = State::HaveVerb; } @@ -459,6 +468,8 @@ pub struct ReadingParse { pub slot_dropped: usize, /// Tokens whose adjective/adverb readings the attribute rule narrowed. pub attribute_narrowed: usize, + /// Configurations the clause rule dropped at a sentence end. + pub unpredicated_dropped: usize, } /// Bound on live configurations per sentence. Exceeding it flushes the @@ -504,6 +515,12 @@ pub struct Typology { /// the attribute rule only reports ([`attribute_rule`]) and never drops /// a reading. pub attribute_rules: &'static [AttributeRule], + /// The clause rule — "this sentence has no verb yet; which word could it + /// be": at a sentence end, drop the reading combinations on which no verb + /// took a predicate slot, if some combination's verb did. A rule switch, + /// not a word-order fact; off in [`Typology::ENGLISH`] (KJV verses are + /// often verbless fragments). + pub predicate_required: bool, } impl Typology { @@ -512,6 +529,7 @@ impl Typology { adjective: AdjectiveOrder::Before, adjective_opens_nominal: true, attribute_rules: &[], + predicate_required: false, }; } @@ -679,7 +697,7 @@ pub fn parse_readings_with(tokens: &[Reading], typology: Typology) -> ReadingPar for (index, tok) in tokens.iter().enumerate() { if tok.pos.contains(Pos::Stop) { - finish_sentence(&mut out, &mut configs, &mut pending); + finish_sentence(&mut out, &mut configs, &mut pending, typology); continue; } let set = if tok.pos.is_empty() { @@ -736,10 +754,10 @@ pub fn parse_readings_with(tokens: &[Reading], typology: Typology) -> ReadingPar out.peak_configs = out.peak_configs.max(configs.len()); if configs.len() > MAX_CONFIGS { out.overflow_flushes += 1; - finish_sentence(&mut out, &mut configs, &mut pending); + finish_sentence(&mut out, &mut configs, &mut pending, typology); } } - finish_sentence(&mut out, &mut configs, &mut pending); + finish_sentence(&mut out, &mut configs, &mut pending, typology); out } @@ -772,7 +790,13 @@ fn finish_sentence( out: &mut ReadingParse, configs: &mut Vec, pending: &mut Vec<(usize, PosSet)>, + typology: Typology, ) { + if typology.predicate_required && configs.iter().any(|c| c.core.predicated) { + let before = configs.len(); + configs.retain(|c| c.core.predicated); + out.unpredicated_dropped += before - configs.len(); + } if let Some((first, rest)) = configs.split_first() { for t in &first.emitted { if rest.iter().all(|c| c.emitted.contains(t)) { @@ -1150,6 +1174,35 @@ mod tests { assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Adj)); } + /// The clause rule, when switched on: "the men record deeds" has no + /// verb unless `record` is one, so its noun reading goes. Off (the + /// default) both stay. With no path predicated it drops nothing. + #[test] + fn the_clause_rule_finds_the_only_possible_verb_when_enabled() { + let toks = [ + one(9, Pos::Det), + one(1, Pos::Noun), + noun_or_verb(2), + one(3, Pos::Noun), + ]; + let off = parse_readings(&toks); + assert_eq!(off.ambiguous[0].survived, NV); + assert_eq!(off.unpredicated_dropped, 0); + let on = Typology { + predicate_required: true, + ..Typology::ENGLISH + }; + let p = parse_readings_with(&toks, on); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Verb)); + assert_eq!(p.certain, vec![Spo::new(1, 2, 3)]); + assert_eq!(p.unpredicated_dropped, 1); + // No path has a predicate: a sentence-initial homograph takes no + // slot either way, so nothing is dropped. + let p = parse_readings_with(&[noun_or_verb(2), one(3, Pos::Det)], on); + assert_eq!(p.ambiguous[0].survived, NV); + assert_eq!(p.unpredicated_dropped, 0); + } + /// T4: a single-reading word is never rejected, even where the licensing /// rule would forbid it — a word is not dropped for having one reading. #[test] From ea7c07731b4b8f4938af56a286421fc6b6861e19 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:14:15 +0000 Subject: [PATCH 04/33] deepnsm-v2: learned position table, German case kept, Animal Farm reference (D-LXC-16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ud_pos_eval learns, per ambiguous pair, P(reading | previous readings, previous word is a copula, next readings) from a treebank's train split — the neighbours only, never the word — and scores it alone and combined with the frequency pick. Copulas are train's `cop` forms; whether a language capitalises nouns is measured from train, and German keys keep their case. Position x frequency beats frequency on adjective/adverb in all five settings (Animal Farm 78.1 -> 88.4 %, UD English 89.7 -> 92.0 %, German 80.8 -> 85.5 %, French 82.0 -> 92.0 %) and on noun/verb in four of five. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...10-04-deepnsm-v2-learned-position-table.md | 55 ++++ .claude/board/entries/README.md | 3 +- crates/deepnsm-v2/examples/ud_pos_eval.rs | 282 +++++++++++++++++- 4 files changed, 336 insertions(+), 5 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-learned-position-table.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 240aaeb81..60e255e62 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -39,6 +39,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-13** | noun/verb homograph decided by position: lemma N/V tag widened to its other reading (`coca::predicate_alternatives`), slot rule after a fresh subject | In PR | slot rule + widening disable-verified; KJV re-pinned (34,824 ambiguous / 451 words, certain 57,350 + alternative 29,001, 0 unexplained). Entry `2026-10-04-deepnsm-v2-predicate-by-position.md` | | **D-LXC-14** | `Pos::Adv`; `Typology` (measured, gates adjective licensing); `AttributeRule` adjective/adverb clauses (off: below frequency); `ud_pos_eval` gold harness (en/de/fr) | In PR | noun/verb with COCA: position 96.7 % vs frequency 85.1 % on 457 decided UD-EWT tokens; adjective/adverb: no clause beats frequency in every language. Entry `2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md` | | **D-LXC-15** | KJV silver noun/verb labels from Luther 1545 casing (release `v0.1.0-codebooks-2026-07-26`); clause rule as an off-by-default switch; WordNet as a parallel coordinate | In PR | KJV: slot + licensing 85.8 % vs COCA lemma tag 74.9 % on 756 decided; clause rule below slot rule and frequency in en/de/fr/KJV; WordNet chooser 59.3 %, existence filter changes 1 form. Entry `2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md` | +| **D-LXC-16** | learned position table (neighbours' readings + copula → P(reading)), German case kept, Animal Farm silver reference | In PR | position × frequency beats frequency on adjective/adverb in 5/5 settings (Animal Farm 78.1 → 88.4 %), noun/verb in 4/5. Harness only. Entry `2026-10-04-deepnsm-v2-learned-position-table.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-learned-position-table.md b/.claude/board/entries/2026-10-04-deepnsm-v2-learned-position-table.md new file mode 100644 index 000000000..e8226f8e4 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-learned-position-table.md @@ -0,0 +1,55 @@ +# 2026-10-04 — deepnsm-v2: a learned position table beats frequency on adjective/adverb in every language (D-LXC-16) + +**Status:** MEASURED (`ud_pos_eval`, UD r2.15 en/de/fr; Animal Farm silver-tagged +by spaCy `en_core_web_sm` 3.8.0 from Project Gutenberg Australia `0100011.txt`, +sha256 d67d621f…, public domain in AU/EU, fetched not committed). + +Operator direction (2026-10-04): research mode — if a position method does not +work yet, work harder; Animal Farm as the SPO / frequency-vs-position +reference; keep German case for morphology. + +## Method + +Hand-written clauses (D-LXC-14) captured a sliver. The principled version is +learned: for each ambiguous pair, count gold tags per **positional context** +— (previous readings, previous word is a copula, next readings) — on a +treebank's train split, with the evaluation lexicon's readings. The word +itself is never part of the key. Contexts with < 10 train tokens back off to +the more decisive half (next-only / previous-only). Scored alone (decide at +p ≥ 0.75 or ≤ 0.25) and combined with the word's frequency as log-odds +(`logit p_word + logit p_ctx − logit prior`). + +Copulas: the forms train marks `cop` (measured). Case: a language +capitalises nouns when non-initial NOUN is > 90 % capitalised and other +words < 10 % (German 98.1 % / 0.8 % → kept; English, French folded). + +## Results (all tokens of the pair; frequency vs position × frequency) + +| setting | adj/adv freq | adj/adv pos×freq | noun/verb freq | noun/verb pos×freq | +|---|---|---|---|---| +| Animal Farm (COCA; table from UD English train) | 78.1 % | **88.4 %** | 82.6 % | **92.6 %** | +| UD English (COCA) | 89.7 % | **92.0 %** | 84.2 % | **90.8 %** | +| UD English (treebank lexicon) | 90.0 % | **90.6 %** | 90.7 % | **92.8 %** | +| UD German (case kept) | 80.8 % | **85.5 %** | 97.9 % | 97.6 % | +| UD French | 82.0 % | **92.0 %** | 88.9 % | **94.7 %** | + +Adjective/adverb: 5 of 5 above frequency. Noun/verb: 4 of 5 (German is +−0.3 once case is kept, because capitalisation already halves its noun/verb +ambiguity, 693 → 333 tokens, and frequency is at 97.9 %). + +On Animal Farm the hand clauses already beat or tied frequency (after +determiner 84.5 vs 78.3, before adjective 92.1 vs 81.6); they lost only on UD +English web text. + +## Caveats + +- Animal Farm labels are silver (spaCy, itself contextual) — they may favour + contextual methods. Its table is learned on a different text (UD English). +- The table is a CC BY-SA derivative of UD (counts per context). + +## OPEN + +- Library form: a `PositionTable` type and whether the FSM should narrow on + a confident combined posterior — not built; harness only. +- The KJV (German-casing silver) has not been scored with the table: its + tagger's reading sets differ from the table's. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 4f4f76c74..46a62bd6c 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,12 +25,13 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -189 entries, 2026-08-06 .. 2026-10-04. +190 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| | 2026-10-04 | `D-LXC-13` | | [2026-10-04-deepnsm-v2-predicate-by-position.md](2026-10-04-deepnsm-v2-predicate-by-position.md) | | 2026-10-04 | `D-LXC-14` | | [2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md](2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md) | +| 2026-10-04 | `D-LXC-16` | | [2026-10-04-deepnsm-v2-learned-position-table.md](2026-10-04-deepnsm-v2-learned-position-table.md) | | 2026-10-04 | `D-LXC-15` | | [2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md](2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md) | | 2026-10-03 | `population-law-crosscheck` | | [2026-10-03-population-law-crosscheck.md](2026-10-03-population-law-crosscheck.md) | | 2026-10-03 | `orkg-reference-harvest` | | [2026-10-03-orkg-reference-harvest.md](2026-10-03-orkg-reference-harvest.md) | diff --git a/crates/deepnsm-v2/examples/ud_pos_eval.rs b/crates/deepnsm-v2/examples/ud_pos_eval.rs index 64a2e55cc..c2353b4b3 100644 --- a/crates/deepnsm-v2/examples/ud_pos_eval.rs +++ b/crates/deepnsm-v2/examples/ud_pos_eval.rs @@ -24,6 +24,16 @@ //! cargo run --release --example ud_pos_eval -- TRAIN TEST --coca ../deepnsm/word_frequency //! ``` //! +//! Beside the parser's rules it learns a **position table** from train: for +//! each ambiguous pair, P(first reading | previous readings, previous word is +//! a copula, next readings) — the neighbours only, never the word — and scores +//! it alone and combined with the frequency pick (log-odds). Whether the +//! language capitalises nouns is measured from train; where it does (German) +//! lexicon keys keep their case, and a sentence-initial word adds its +//! lowercase readings. The copula set is train's `cop` forms. A plain text +//! can be scored the same way after silver tagging, e.g. Animal Farm through +//! spaCy `en_core_web_sm` (lab step, not committed). +//! //! `UD_CLAUSE=1` switches on the clause rule //! ([`Typology::predicate_required`]). The typology is measured from train; //! `UD_TYPOLOGY=english` uses @@ -40,7 +50,15 @@ use deepnsm_v2::fsm::{ /// One syntactic word of a UD sentence. struct Word { + /// Lexicon key: lowercased, or the surface where the language + /// capitalises nouns (see [`capitalises_nouns`]). form: String, + /// The form as written. + surface: String, + /// UPOS is NOUN (not PROPN/PRON) — for the capitalisation measurement. + upos_noun: bool, + /// The word is a copula (UD relation `cop`). + copula: bool, gold: Pos, /// The adjective modifies a noun (`amod`) that stands after it /// (`Some(true)`) or before it (`Some(false)`); `None` otherwise. @@ -96,6 +114,9 @@ fn read_conllu(path: &str) -> Vec> { }; cur.push(Word { form, + surface: cols[1].to_string(), + upos_noun: cols[3] == "NOUN", + copula: cols.get(7).is_some_and(|r| *r == "cop"), gold, amod_head_after, }); @@ -240,6 +261,220 @@ fn wordnet_pick(lex: &Lexicon, wn: &HashMap, form: &str) } } +/// Whether the language capitalises nouns, measured from gold: of +/// non-initial words, the share of NOUN capitalised and the share of every +/// other non-PROPN word capitalised. Nouns above 90 % and the rest below 10 % +/// → true (policy pins). Returns (decision, noun share, other share). +fn capitalises_nouns(train: &[Vec]) -> (bool, f64, f64) { + let (mut n, mut nc, mut o, mut oc) = (0usize, 0usize, 0usize, 0usize); + for s in train { + for w in s.iter().skip(1) { + let cap = w.surface.starts_with(char::is_uppercase); + if w.upos_noun { + n += 1; + nc += usize::from(cap); + } else if w.gold != Pos::Noun && w.surface.chars().any(char::is_alphabetic) { + o += 1; + oc += usize::from(cap); + } + } + } + let ns = nc as f64 / n.max(1) as f64; + let os = oc as f64 / o.max(1) as f64; + (ns > 0.9 && os < 0.1, ns, os) +} + +/// Re-key every word on its surface (case kept). +fn keep_case(sentences: &mut [Vec]) { + for w in sentences.iter_mut().flatten() { + w.form = w.surface.clone(); + } +} + +/// The readings of word `k` of a sentence: its own key, and — for the first +/// word when case is kept — also its lowercase form (sentence-initial +/// capitals say nothing about the word class). +fn readings_of(lex: &Lexicon, s: &[Word], k: usize, cased: bool) -> PosSet { + let own = lex.get(&s[k].form); + if cased && k == 0 { + own.union(lex.get(&s[k].form.to_lowercase())) + } else { + own + } +} + +/// Positional context of a token: the neighbours' reading sets and whether +/// the previous word is a copula. Never the word itself. +type Context = (u8, bool, u8); + +fn bits(p: PosSet) -> u8 { + Pos::ALL + .iter() + .enumerate() + .filter(|(_, x)| p.contains(**x)) + .fold(0u8, |a, (i, _)| a | (1 << i)) +} + +/// P(first | context) for one reading pair, learned from gold train tags +/// with the evaluation lexicon's readings. Backs off from the full context +/// to its next-only and previous-only halves when support is thin. +struct PositionTable { + full: HashMap, + next: HashMap, + prev: HashMap<(u8, bool), [usize; 2]>, +} + +/// Minimum train tokens behind a context before it may decide (policy pin). +const MIN_SUPPORT: usize = 10; + +impl PositionTable { + fn learn( + train: &[Vec], + lex: &Lexicon, + copulas: &std::collections::HashSet, + cased: bool, + pair: (Pos, Pos), + ) -> Self { + let mut t = Self { + full: HashMap::new(), + next: HashMap::new(), + prev: HashMap::new(), + }; + for s in train { + let set = |k: usize| readings_of(lex, s, k, cased); + for k in 0..s.len() { + let this = set(k); + let w = &s[k]; + if !(this.contains(pair.0) && this.contains(pair.1)) + || (w.gold != pair.0 && w.gold != pair.1) + { + continue; + } + let ctx = Self::context(s, k, &set, copulas); + let slot = usize::from(w.gold == pair.1); + t.full.entry(ctx).or_default()[slot] += 1; + t.next.entry(ctx.2).or_default()[slot] += 1; + t.prev.entry((ctx.0, ctx.1)).or_default()[slot] += 1; + } + } + t + } + + fn context( + s: &[Word], + k: usize, + set: &dyn Fn(usize) -> PosSet, + copulas: &std::collections::HashSet, + ) -> Context { + let prev = k.checked_sub(1); + ( + prev.map_or(0, |j| bits(set(j))), + prev.is_some_and(|j| copulas.contains(&s[j].form.to_lowercase())), + if k + 1 < s.len() { bits(set(k + 1)) } else { 0 }, + ) + } + + /// P(first member | context), with the support it rests on. + fn p_first(&self, ctx: Context) -> Option<(f64, usize)> { + let pick = |c: [usize; 2]| { + let n = c[0] + c[1]; + (n >= MIN_SUPPORT).then(|| ((c[0] as f64 + 0.5) / (n as f64 + 1.0), n)) + }; + if let Some(r) = self.full.get(&ctx).copied().and_then(pick) { + return Some(r); + } + // Back off: whichever half is more decisive. + let a = self.next.get(&ctx.2).copied().and_then(pick); + let b = self.prev.get(&(ctx.0, ctx.1)).copied().and_then(pick); + match (a, b) { + (Some(x), Some(y)) => Some(if (x.0 - 0.5).abs() >= (y.0 - 0.5).abs() { + x + } else { + y + }), + (x, y) => x.or(y), + } + } +} + +/// Learn a [`PositionTable`] on train and score it on test, against the +/// frequency pick and their log-odds combination, on the same tokens. +fn position_table_report( + train: &[Vec], + test: &[Vec], + lex: &Lexicon, + copulas: &std::collections::HashSet, + cased: bool, + pair: (Pos, Pos), + name: &str, +) { + let table = PositionTable::learn(train, lex, copulas, cased, pair); + let (a, b) = pair; + let prior = { + let (x, y) = table + .full + .values() + .fold((0, 0), |s, c| (s.0 + c[0], s.1 + c[1])); + (x as f64 + 0.5) / ((x + y) as f64 + 1.0) + }; + let logit = |p: f64| (p / (1.0 - p)).ln(); + let (mut n, mut dec, mut dec_right, mut freq_on_dec, mut freq_all, mut comb_all) = + (0usize, 0usize, 0usize, 0usize, 0usize, 0usize); + for s in test { + let set = |k: usize| readings_of(lex, s, k, cased); + for k in 0..s.len() { + let w = &s[k]; + let this = set(k); + if !(this.contains(a) && this.contains(b)) || (w.gold != a && w.gold != b) { + continue; + } + n += 1; + let ca = lex + .counts + .get(&(w.form.clone(), a as u8)) + .copied() + .unwrap_or(0); + let cb = lex + .counts + .get(&(w.form.clone(), b as u8)) + .copied() + .unwrap_or(0); + let p_word = (ca as f64 + 0.5) / ((ca + cb) as f64 + 1.0); + let freq = if cb > ca { b } else { a }; + freq_all += usize::from(freq == w.gold); + let ctx = PositionTable::context(s, k, &set, copulas); + let p_ctx = table.p_first(ctx); + if let Some((p, _)) = p_ctx { + if !(0.25..0.75).contains(&p) { + let pick = if p >= 0.75 { a } else { b }; + dec += 1; + dec_right += usize::from(pick == w.gold); + freq_on_dec += usize::from(freq == w.gold); + } + } + let score = logit(p_word) + p_ctx.map_or(0.0, |(p, _)| logit(p) - logit(prior)); + let comb = if score >= 0.0 { a } else { b }; + comb_all += usize::from(comb == w.gold); + } + } + let pct = |x: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * x as f64 / d as f64 + } + }; + println!( + " {name} position table ({} contexts): decides {dec} of {n} at {:.1}% (frequency \ + {:.1}% on them); all tokens: frequency {:.1}%, position × frequency {:.1}%", + table.full.len(), + pct(dec_right, dec), + pct(freq_on_dec, dec), + pct(freq_all, n), + pct(comb_all, n) + ); +} + /// Score for one pair of readings. #[derive(Default)] struct PairScore { @@ -374,7 +609,26 @@ fn main() { std::process::exit(2); } }; - let train = read_conllu(train); + let mut train = read_conllu(train); + // German capitalises nouns: case is morphology there, so keys keep it. + // Measured per language; COCA (English) stays lowercase. + let (caps, noun_share, other_share) = capitalises_nouns(&train); + let cased = caps && coca.is_none(); + println!( + "capitalisation: non-initial nouns {:.1}% capitalised, other words {:.1}% → case {}", + 100.0 * noun_share, + 100.0 * other_share, + if cased { "kept" } else { "folded" } + ); + if cased { + keep_case(&mut train); + } + let copulas: std::collections::HashSet = train + .iter() + .flatten() + .filter(|w| w.copula) + .map(|w| w.form.to_lowercase()) + .collect(); let (typology, share_before, verb_after) = match std::env::var("UD_TYPOLOGY").as_deref() { Ok("english") => (Typology::ENGLISH, f64::NAN, f64::NAN), _ => measure_typology(&train), @@ -425,20 +679,23 @@ fn main() { } println!("WordNet filter: {narrowed} COCA noun/verb forms lost a reading WordNet lacks"); } - let test = read_conllu(test); + let mut test = read_conllu(test); + if cased { + keep_case(&mut test); + } // One stream: every sentence ends in a stop, whatever its own punctuation. let mut ids: HashMap<&str, u16> = HashMap::new(); let mut readings = Vec::new(); let mut words: Vec> = Vec::new(); for s in &test { - for w in s { + for (k, w) in s.iter().enumerate() { if w.gold == Pos::Stop { continue; } let next = u16::try_from(ids.len() + 1).unwrap_or(u16::MAX); let id = *ids.entry(w.form.as_str()).or_insert(next); - readings.push(Reading::new(id, lex.get(&w.form))); + readings.push(Reading::new(id, readings_of(&lex, s, k, cased))); words.push(Some(w)); } readings.push(Reading::stop()); @@ -487,6 +744,7 @@ fn main() { parse.unpredicated_dropped ); nv_score.print("noun/verb", nv); + position_table_report(&train, &test, &lex, &copulas, cased, nv, "noun/verb"); // A parallel coordinate: WordNet's own noun/verb sense counts as the // prior, scored on the same tokens (COCA mode supplies the lemmas). if let Some(path) = std::env::var("UD_WORDNET").ok().filter(|p| !p.is_empty()) { @@ -526,6 +784,7 @@ fn main() { ); } aa_score.print("adjective/adverb", aa); + position_table_report(&train, &test, &lex, &copulas, cased, aa, "adjective/adverb"); // Each attribute clause on its own: every adjective/adverb token whose // gold is one of the two, the clause that fires, and whether the clause @@ -546,6 +805,21 @@ fn main() { } let rule = attribute_rule(edge(i.checked_sub(1)), this, edge(Some(i + 1)), typology); let freq = lex.more_frequent(&w.form, Pos::Adj, Pos::Adv); + if std::env::var_os("UD_CLAUSE_DUMP").is_some() { + let word = |j: Option| { + j.and_then(|j| words.get(j).copied().flatten()) + .map_or("|".to_string(), |x| format!("{}/{:?}", x.form, x.gold)) + }; + eprintln!( + "CLAUSE {:?} gold {:?} freq {:?} | {} [{}] {}", + rule, + w.gold, + freq, + word(i.checked_sub(1)), + w.form, + word(Some(i + 1)) + ); + } let e = per_rule.entry(rule).or_default(); e.0 += 1; e.1 += usize::from(rule.is_some_and(|r| r.keeps() == w.gold)); From 835c53e0583590e79fd452842fcd69b2bed1114b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:20:43 +0000 Subject: [PATCH 05/33] deepnsm-v2: German TEKAMOLO cues by verb position; adverbial adjectives fold to Adv (D-LXC-17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - examples/tekamolo_de.rs: German cue words (da, als, während, bis, …) classed as subordinator / preposition / adverb by where the finite verb stands (V2, verb-final, noun phrase), learned word-independently from UD German train and combined with frequency. Finiteness is measured from train; nothing at test reads gold. 88.5 % vs frequency 84.1 % on 208 tokens; `da` 87.5 % vs 37.5 %. - ud_pos_eval: an adjective with relation advmod folds to Adv (it fills an adverbial slot); optional "copula anywhere in the clause" context (UD_NO_CLAUSE_CTX ablates it) — helps German, costs English/French. - Board: the Te-Ka-Mo-Lo order law measured on gold dependencies (46-65 %, not usable on this text); where TEKAMOLO lived in DeepNSM v1/v2. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...eepnsm-v2-german-tekamolo-verb-position.md | 65 ++++++ .claude/board/entries/README.md | 3 +- crates/deepnsm-v2/examples/tekamolo_de.rs | 215 ++++++++++++++++++ crates/deepnsm-v2/examples/ud_pos_eval.rs | 47 ++-- 5 files changed, 315 insertions(+), 16 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md create mode 100644 crates/deepnsm-v2/examples/tekamolo_de.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 60e255e62..c6d2dbdbb 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -40,6 +40,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-14** | `Pos::Adv`; `Typology` (measured, gates adjective licensing); `AttributeRule` adjective/adverb clauses (off: below frequency); `ud_pos_eval` gold harness (en/de/fr) | In PR | noun/verb with COCA: position 96.7 % vs frequency 85.1 % on 457 decided UD-EWT tokens; adjective/adverb: no clause beats frequency in every language. Entry `2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md` | | **D-LXC-15** | KJV silver noun/verb labels from Luther 1545 casing (release `v0.1.0-codebooks-2026-07-26`); clause rule as an off-by-default switch; WordNet as a parallel coordinate | In PR | KJV: slot + licensing 85.8 % vs COCA lemma tag 74.9 % on 756 decided; clause rule below slot rule and frequency in en/de/fr/KJV; WordNet chooser 59.3 %, existence filter changes 1 form. Entry `2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md` | | **D-LXC-16** | learned position table (neighbours' readings + copula → P(reading)), German case kept, Animal Farm silver reference | In PR | position × frequency beats frequency on adjective/adverb in 5/5 settings (Animal Farm 78.1 → 88.4 %), noun/verb in 4/5. Harness only. Entry `2026-10-04-deepnsm-v2-learned-position-table.md` | +| **D-LXC-17** | German TEKAMOLO cues by verb position (`tekamolo_de`); adverbial adjectives → Adv fold; Te-Ka-Mo-Lo order law measured | In PR | cue words: position × frequency 88.5 % vs frequency 84.1 % (208 tokens, `da` 87.5 vs 37.5); order law 46–65 % on gold, not usable. Entry `2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md b/.claude/board/entries/2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md new file mode 100644 index 000000000..299870e7d --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md @@ -0,0 +1,65 @@ +# 2026-10-04 — deepnsm-v2: German TEKAMOLO cues decided by verb position; the Te-Ka-Mo-Lo order law fails on real text (D-LXC-17) + +**Status:** MEASURED (`cargo run --release --example tekamolo_de -- de_gsd-ud-train.conllu +de_gsd-ud-test.conllu`; `ud_pos_eval` for the adverbial-adjective fold; UD r2.15). + +Operator (2026-10-04): "In German we have TEKAMOLO" — and DeepNSM had it. + +## Where TEKAMOLO lived + +- deepnsm-v2 had `src/tekamolo.rs` (718 lines, `7bae6c84`): German left-corner + lane hypotheses over Luther 1545 / Elberfelder 1905 from `de/tekamolo.tsv`, + committed at the clause's right corner. Deleted in `68955ecb` as redundant to + `lance-graph-planner/examples/insight_coca_read.rs` (English, COCA tables). +- deepnsm v1: TEKAMOLO role slices in `markov_bundle`; `parser.rs` leaves + `TekamoloSlots` empty ("until D3"). +- `de/tekamolo.tsv` (release `v0.1.0-codebooks-2026-07-26`): lanes from a + hand cue list (`build_de_codebook.py`), counted by UD relation — no gold lanes. + +## 1. The Te < Ka < Mo < Lo order law (lab, UD German GSD gold dependencies) + +| test | pairs | agree | +|---|---|---| +| cue-lexicon adverbials, same head, any position | 297 train / 46 test | 60.3 % / 65.2 % | +| Mittelfeld only (after the finite verb), advmod/obl | 92 / 17 | 45.7 % / 58.8 % | +| `obl` laned by a single-lane preposition | 37 / 0 | 54.1 % / — | + +Ka-before-Te violations are subordinators opening their clause; the cue +lexicon mostly hits particles (noch, schon, so, sehr) whose position follows +scope. Laned PPs rarely co-occur. On this text the order law cannot decide a +lane. + +## 2. Cue words by verb position (Rust, `tekamolo_de`, no gold at test) + +Signatures learned over every cue token of train (word-independent): +V2 (finite verb next) → adv 180 / mark 7 / case 1; verb-final segment → +mark 507 / case 293 / adv 84; noun phrase → case 2160 / adv 336 / mark 179. +Finiteness measured from train (2,394 forms), case kept. + +| | n | position | frequency | position × frequency | +|---|---|---|---|---| +| all 11 cues | 208 | 57.2 % | 84.1 % | **88.5 %** | +| `da` | 16 | 68.8 % | 37.5 % | **87.5 %** | +| `während` | 12 | 50.0 % | 33.3 % | 50.0 % | +| `als` | 44 | 81.8 % | 77.3 % | 77.3 % | + +Lane mapping the classes feed: `da`+mark → Kausal, `da`+adv → Lokal/Temporal; +`als`/`während`/`bis`/`seit`/`nachdem` → Temporal; `damit`+mark → Kausal +(final); `so`+adv → Modal. + +## 3. Adverbially used adjectives fill the Modal lane + +`ud_pos_eval`'s fold now maps UPOS ADJ with relation `advmod` to `Adv` +(language-neutral). German Adj/Adv tokens 234 → 315; the position table alone +decides 182 at 87.4 % vs frequency 82.4 % on them; position × frequency +82.9 % vs 80.0 % overall. A "copula anywhere in the clause" context feature +(`UD_NO_CLAUSE_CTX` ablates it) helps German (+0.4 adj/adv, +0.6 noun/verb) +and costs English/French (up to −2.0); per-language selection on dev is OPEN. + +## OPEN + +- Lane-level gold does not exist; lanes stay a mapping from the decided class. +- The deleted left-corner module could be re-measured against this signature + method on the Luther/Elberfelder lanes. +- `so`/`damit`: position alone is wrong (mid-clause adverbs in verb-final + segments); frequency carries them. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 46a62bd6c..e4d0ace7a 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -190 entries, 2026-08-06 .. 2026-10-04. +191 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| @@ -33,6 +33,7 @@ exactly one of them, never both. | 2026-10-04 | `D-LXC-14` | | [2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md](2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md) | | 2026-10-04 | `D-LXC-16` | | [2026-10-04-deepnsm-v2-learned-position-table.md](2026-10-04-deepnsm-v2-learned-position-table.md) | | 2026-10-04 | `D-LXC-15` | | [2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md](2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md) | +| 2026-10-04 | `D-LXC-17` | | [2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md](2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md) | | 2026-10-03 | `population-law-crosscheck` | | [2026-10-03-population-law-crosscheck.md](2026-10-03-population-law-crosscheck.md) | | 2026-10-03 | `orkg-reference-harvest` | | [2026-10-03-orkg-reference-harvest.md](2026-10-03-orkg-reference-harvest.md) | | 2026-10-03 | `known-is-not-alpha-premise-gate` | | [2026-10-03-known-is-not-alpha-premise-gate.md](2026-10-03-known-is-not-alpha-premise-gate.md) | diff --git a/crates/deepnsm-v2/examples/tekamolo_de.rs b/crates/deepnsm-v2/examples/tekamolo_de.rs new file mode 100644 index 000000000..f79676e14 --- /dev/null +++ b/crates/deepnsm-v2/examples/tekamolo_de.rs @@ -0,0 +1,215 @@ +//! `tekamolo_de` — German TEKAMOLO cue words decided by **verb position**, +//! calibrated against frequency, on Universal Dependencies German. +//! +//! The ambiguous German cues (`da`, `als`, `während`, `bis`, `wenn`, …) are a +//! subordinator (`weil`-like, Kausal/Temporal clause), a preposition (`seit +//! 1990`, `als Kind`) or an adverb (`da` = there/then, Lokal/Temporal). German +//! word order separates them by where the finite verb stands: +//! +//! - **V2** — a finite verb right after the cue: the cue fills the Vorfeld, an +//! adverb (`Da kommt er`); +//! - **verb-final** — the segment the cue opens (to the next punctuation) ends +//! in a finite verb: a subordinate clause (`da er krank war`); +//! - **noun phrase** — anything else: a preposition (`seit dem Krieg`). +//! +//! P(class | signature) is learned over every cue token of train (the cue word +//! is not part of it), P(class | word) is the word's train frequency, and the +//! two are combined as log-odds against the class prior. Gold is the UD +//! relation (`mark` / `case` / other). Nothing at test reads gold: finiteness +//! is the share of train tokens of the form marked `VerbForm=Fin` (≥ ½), and +//! case is kept (German capitalises nouns; see `ud_pos_eval`). +//! +//! ```text +//! T=https://raw.githubusercontent.com/UniversalDependencies/UD_German-GSD/r2.15 +//! curl -O $T/de_gsd-ud-train.conllu -O $T/de_gsd-ud-test.conllu +//! cargo run --release --example tekamolo_de -- de_gsd-ud-train.conllu de_gsd-ud-test.conllu +//! ``` + +use std::collections::{HashMap, HashSet}; + +/// The ambiguous cue words scored. +const CUES: [&str; 11] = [ + "da", "so", "wenn", "als", "während", "seit", "bis", "nachdem", "ob", "weil", "damit", +]; + +/// Subordinator, preposition, adverb. +const CLASSES: [&str; 3] = ["mark", "case", "adv"]; + +struct Word { + form: String, + finite: bool, + punct: bool, + rel: String, +} + +fn read(path: &str) -> Vec> { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut out = Vec::new(); + let mut cur = Vec::new(); + for line in text.lines() { + if line.is_empty() { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if line.starts_with('#') || c.len() < 8 || c[0].contains(['-', '.']) { + continue; + } + cur.push(Word { + form: c[1].to_string(), + finite: c[5].split('|').any(|f| f == "VerbForm=Fin"), + punct: c[3] == "PUNCT", + rel: c[7].to_string(), + }); + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +/// The gold class of a cue token, or `None` if its relation is not one a cue +/// takes. +fn class(w: &Word) -> Option { + match w.rel.as_str() { + "mark" => Some(0), + "case" => Some(1), + "advmod" | "cc" => Some(2), + _ => None, + } +} + +/// What train knows about forms, measured — never gold at test. +struct Lexicon { + finite: HashSet, + punct: HashSet, +} + +impl Lexicon { + fn from_train(train: &[Vec]) -> Self { + let mut seen: HashMap<&str, (usize, usize)> = HashMap::new(); + let mut punct = HashSet::new(); + for w in train.iter().flatten() { + let e = seen.entry(&w.form).or_default(); + e.0 += 1; + e.1 += usize::from(w.finite); + if w.punct { + punct.insert(w.form.clone()); + } + } + let finite = seen + .into_iter() + .filter(|(_, (n, f))| 2 * f >= *n) + .map(|(w, _)| w.to_string()) + .collect(); + Self { finite, punct } + } + + /// The verb-position signature of the cue at `i`: V2, verb-final, noun + /// phrase, or sentence edge. + fn signature(&self, s: &[Word], i: usize) -> usize { + let fin = |w: &Word| self.finite.contains(&w.form); + if s.get(i + 1).is_some_and(fin) { + return 0; + } + let end = (i + 1..s.len()) + .find(|&j| self.punct.contains(&s[j].form)) + .unwrap_or(s.len()); + match s[i + 1..end].last() { + Some(last) if fin(last) => 1, + Some(_) => 2, + None => 3, + } + } +} + +const SIGNATURES: [&str; 4] = ["V2", "verb-final", "noun phrase", "edge"]; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let [train, test] = args.as_slice() else { + eprintln!("usage: tekamolo_de TRAIN.conllu TEST.conllu"); + std::process::exit(2); + }; + let train = read(train); + let test = read(test); + let lex = Lexicon::from_train(&train); + + let is_cue = |w: &Word| CUES.contains(&w.form.to_lowercase().as_str()); + let mut by_word: HashMap = HashMap::new(); + let mut by_sig = [[0usize; 3]; 4]; + let mut prior = [0usize; 3]; + for s in &train { + for (i, w) in s.iter().enumerate() { + let Some(c) = class(w).filter(|_| is_cue(w)) else { + continue; + }; + by_word.entry(w.form.to_lowercase()).or_default()[c] += 1; + by_sig[lex.signature(s, i)][c] += 1; + prior[c] += 1; + } + } + let p = |cnt: &[usize; 3], c: usize| { + (cnt[c] as f64 + 0.5) / (cnt.iter().sum::() as f64 + 1.5) + }; + let argmax = |f: &dyn Fn(usize) -> f64| (0..3).max_by(|&a, &b| f(a).total_cmp(&f(b))).unwrap(); + + let (mut n, mut pos, mut freq, mut comb) = (0, 0, 0, 0); + let mut per: HashMap = HashMap::new(); + for s in &test { + for (i, w) in s.iter().enumerate() { + let Some(gold) = class(w).filter(|_| is_cue(w)) else { + continue; + }; + let word = w.form.to_lowercase(); + let wc = by_word.get(&word).copied().unwrap_or_default(); + let sc = by_sig[lex.signature(s, i)]; + let fp = argmax(&|c| wc[c] as f64); + let pp = argmax(&|c| sc[c] as f64); + let cp = argmax(&|c| p(&wc, c).ln() + p(&sc, c).ln() - p(&prior, c).ln()); + n += 1; + pos += usize::from(pp == gold); + freq += usize::from(fp == gold); + comb += usize::from(cp == gold); + let r = per.entry(word).or_default(); + r[0] += 1; + r[1] += usize::from(pp == gold); + r[2] += usize::from(fp == gold); + r[3] += usize::from(cp == gold); + } + } + let pct = |x: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * x as f64 / d as f64 + } + }; + println!("{} finite forms measured from train", lex.finite.len()); + for (k, name) in SIGNATURES.iter().enumerate() { + let c = by_sig[k]; + println!( + " train signature {name:11}: {} {}, {} {}, {} {}", + CLASSES[0], c[0], CLASSES[1], c[1], CLASSES[2], c[2] + ); + } + let mut words: Vec<_> = per.into_iter().collect(); + words.sort(); + for (w, r) in words { + println!( + " {w:9} {:3}: position {:.1}%, frequency {:.1}%, combined {:.1}%", + r[0], + pct(r[1], r[0]), + pct(r[2], r[0]), + pct(r[3], r[0]) + ); + } + println!( + "all {n}: position {:.1}%, frequency {:.1}%, position × frequency {:.1}%", + pct(pos, n), + pct(freq, n), + pct(comb, n) + ); +} diff --git a/crates/deepnsm-v2/examples/ud_pos_eval.rs b/crates/deepnsm-v2/examples/ud_pos_eval.rs index c2353b4b3..f6e1b24d5 100644 --- a/crates/deepnsm-v2/examples/ud_pos_eval.rs +++ b/crates/deepnsm-v2/examples/ud_pos_eval.rs @@ -66,10 +66,14 @@ struct Word { } /// The one UPOS → FSM fold. Language-neutral: it reads only UD's universal -/// tags and the universal `PronType=Rel` feature. -fn fold(upos: &str, feats: &str, form: &str) -> Pos { +/// tags, the universal `PronType=Rel` feature and the universal `advmod` +/// relation — an adjective used as an adverbial modifier (German "er läuft +/// **schnell**") is folded to [`Pos::Adv`]: it fills an adverbial (TEKAMOLO) +/// slot, which is the distinction the Adj/Adv pair stands for. +fn fold(upos: &str, feats: &str, form: &str, deprel: &str) -> Pos { let rel = feats.split('|').any(|f| f == "PronType=Rel"); match upos { + "ADJ" if deprel.starts_with("advmod") => Pos::Adv, "PRON" | "DET" if rel => Pos::Rel, "NOUN" | "PROPN" | "PRON" => Pos::Noun, "VERB" | "AUX" => Pos::Verb, @@ -102,7 +106,7 @@ fn read_conllu(path: &str) -> Vec> { continue; } let form = cols[1].to_lowercase(); - let gold = fold(cols[3], cols[5], &form); + let gold = fold(cols[3], cols[5], &form, cols.get(7).copied().unwrap_or("_")); let amod_head_after = match (cols.get(6), cols.get(7)) { (Some(head), Some(rel)) if rel.starts_with("amod") => { match (head.parse::(), cols[0].parse::()) { @@ -305,7 +309,7 @@ fn readings_of(lex: &Lexicon, s: &[Word], k: usize, cased: bool) -> PosSet { /// Positional context of a token: the neighbours' reading sets and whether /// the previous word is a copula. Never the word itself. -type Context = (u8, bool, u8); +type Context = (u8, bool, u8, bool); fn bits(p: PosSet) -> u8 { Pos::ALL @@ -322,6 +326,8 @@ struct PositionTable { full: HashMap, next: HashMap, prev: HashMap<(u8, bool), [usize; 2]>, + /// Next readings + copula anywhere in the clause. + clause: HashMap<(u8, bool), [usize; 2]>, } /// Minimum train tokens behind a context before it may decide (policy pin). @@ -339,6 +345,7 @@ impl PositionTable { full: HashMap::new(), next: HashMap::new(), prev: HashMap::new(), + clause: HashMap::new(), }; for s in train { let set = |k: usize| readings_of(lex, s, k, cased); @@ -355,6 +362,7 @@ impl PositionTable { t.full.entry(ctx).or_default()[slot] += 1; t.next.entry(ctx.2).or_default()[slot] += 1; t.prev.entry((ctx.0, ctx.1)).or_default()[slot] += 1; + t.clause.entry((ctx.2, ctx.3)).or_default()[slot] += 1; } } t @@ -367,10 +375,21 @@ impl PositionTable { copulas: &std::collections::HashSet, ) -> Context { let prev = k.checked_sub(1); + // The clause: the words between punctuation marks around `k`. + let is_break = |w: &Word| !w.surface.chars().any(char::is_alphanumeric); + let lo = (0..k).rev().find(|&j| is_break(&s[j])).map_or(0, |j| j + 1); + let hi = (k + 1..s.len()) + .find(|&j| is_break(&s[j])) + .unwrap_or(s.len()); + let clause_copula = std::env::var_os("UD_NO_CLAUSE_CTX").is_none() + && (lo..hi) + .filter(|&j| j != k) + .any(|j| copulas.contains(&s[j].form.to_lowercase())); ( prev.map_or(0, |j| bits(set(j))), prev.is_some_and(|j| copulas.contains(&s[j].form.to_lowercase())), if k + 1 < s.len() { bits(set(k + 1)) } else { 0 }, + clause_copula, ) } @@ -383,17 +402,15 @@ impl PositionTable { if let Some(r) = self.full.get(&ctx).copied().and_then(pick) { return Some(r); } - // Back off: whichever half is more decisive. - let a = self.next.get(&ctx.2).copied().and_then(pick); - let b = self.prev.get(&(ctx.0, ctx.1)).copied().and_then(pick); - match (a, b) { - (Some(x), Some(y)) => Some(if (x.0 - 0.5).abs() >= (y.0 - 0.5).abs() { - x - } else { - y - }), - (x, y) => x.or(y), - } + // Back off: whichever partial context is most decisive. + [ + self.clause.get(&(ctx.2, ctx.3)).copied().and_then(pick), + self.next.get(&ctx.2).copied().and_then(pick), + self.prev.get(&(ctx.0, ctx.1)).copied().and_then(pick), + ] + .into_iter() + .flatten() + .max_by(|x, y| (x.0 - 0.5).abs().total_cmp(&(y.0 - 0.5).abs())) } } From 9e8a552cf2e931d5cc1cda7a6378bedefc2519ec Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:28:18 +0000 Subject: [PATCH 06/33] =?UTF-8?q?deepnsm-v2:=20the=20question=20test=20as?= =?UTF-8?q?=20the=20Pearl=202=C2=B3=20mask;=20blast=20radius=20of=20delete?= =?UTF-8?q?d=20grammar=20heuristics=20(D-LXC-18)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fsm::answered_questions: per token, the 2³ mask of SPO questions its clause has answered before it (who?/does what?/whom?), stepped with the FSM's own table — the school question test as a position. - ud_pos_eval: UD_CTX=neigh|q|both keys the position table on neighbours, on the answered-question mask, or both; the left context is tagged by frequency only. Measured: the 2³ context gives the most precise adjective/adverb decisions on Animal Farm (91.8 % on 255, frequency 78.4 %) and the best German adjective/adverb result (85.1 %); noun/verb stays better with neighbours. Board: blast radius from the history sweep — the German TEKAMOLO reader was deleted without a German replacement, and the TEKAMOLO tenant writer still reads the v1 WordNet rail the release audit condemns. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...v2-question-test-2cube-and-blast-radius.md | 61 +++++++++++++++++ .claude/board/entries/README.md | 3 +- crates/deepnsm-v2/examples/ud_pos_eval.rs | 67 ++++++++++++++++--- crates/deepnsm-v2/src/fsm.rs | 53 +++++++++++++++ crates/deepnsm-v2/src/lib.rs | 5 +- 6 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index c6d2dbdbb..221e983bd 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -41,6 +41,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-15** | KJV silver noun/verb labels from Luther 1545 casing (release `v0.1.0-codebooks-2026-07-26`); clause rule as an off-by-default switch; WordNet as a parallel coordinate | In PR | KJV: slot + licensing 85.8 % vs COCA lemma tag 74.9 % on 756 decided; clause rule below slot rule and frequency in en/de/fr/KJV; WordNet chooser 59.3 %, existence filter changes 1 form. Entry `2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md` | | **D-LXC-16** | learned position table (neighbours' readings + copula → P(reading)), German case kept, Animal Farm silver reference | In PR | position × frequency beats frequency on adjective/adverb in 5/5 settings (Animal Farm 78.1 → 88.4 %), noun/verb in 4/5. Harness only. Entry `2026-10-04-deepnsm-v2-learned-position-table.md` | | **D-LXC-17** | German TEKAMOLO cues by verb position (`tekamolo_de`); adverbial adjectives → Adv fold; Te-Ka-Mo-Lo order law measured | In PR | cue words: position × frequency 88.5 % vs frequency 84.1 % (208 tokens, `da` 87.5 vs 37.5); order law 46–65 % on gold, not usable. Entry `2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md` | +| **D-LXC-18** | question test as the Pearl 2³ mask (`fsm::answered_questions`, `UD_CTX`); blast radius of deleted grammar heuristics | In PR | 2³ context: Animal Farm adj/adv 91.8 % on decided (frequency 78.4 %), German adj/adv 85.1 % (best); noun/verb better with neighbours. German TEKAMOLO capability was lost in `68955ecb`; tenant writer reads the condemned v1 WordNet rail. Entry `2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md b/.claude/board/entries/2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md new file mode 100644 index 000000000..49544ce44 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md @@ -0,0 +1,61 @@ +# 2026-10-04 — deepnsm-v2: the school question test as the Pearl 2³ mask; blast radius of the deleted grammar heuristics (D-LXC-18) + +**Status:** MEASURED (`UD_CTX=neigh|q|both cargo run --release --example ud_pos_eval …`, +UD r2.15 + Animal Farm silver) and VERIFIED-IN-HISTORY (`git log --diff-filter=D`, +pickaxe `-S` over the unshallowed `origin/main`, 6,042 commits). + +Operator (2026-10-04): use the SPO 2³ Pearl ladder for grammar ambiguity — +the school way of resolving word classes by test questions; WordNet's value is +its HHTL DN parent/child address; TEKAMOLO lanes are 4× SPO; find the blast +radius of destroyed grammar heuristics. + +## The question test as a position + +`fsm::answered_questions` — for every token, the 2³ mask of SPO questions its +clause has answered before it arrives (*who/what?* S, *does what?* P, +*whom/what?* O; stepped with the FSM's own transition table; a stop starts a +new clause). The same eight masks as `nars_engine::ALL_MASKS` / +`SpoDistances::all_projections` (P3b, P6 in `lance-graph-osint`). In +`ud_pos_eval` the left context is tagged by frequency only (never gold) and the +position table can key on the mask (`UD_CTX=q`). + +| context | Animal Farm adj/adv decided | German adj/adv all | Animal Farm noun/verb all | +|---|---|---|---| +| neighbours | 82.9 % on 374 | 82.9 % | **91.6 %** | +| **2³ answered questions + next** | **91.8 % on 255** (frequency 78.4 %) | **85.1 %** | 84.5 % | +| both | 82.1 % on 397 | 83.8 % | 90.5 % | + +Adjective/adverb follows the question test (an adverb answers *how/when* once +*does what?* is answered; an adjective sits inside an unanswered noun phrase); +noun/verb follows adjacency (determiner→noun, subject→verb). UD English is the +exception for adj/adv (q 87.9 % vs neighbours 91.0 % with COCA). Per-pair / +per-language selection on dev is OPEN. + +## Blast radius — deleted grammar heuristics and where they went + +| deleted (commit) | named replacement | status of the replacement | +|---|---|---| +| deepnsm-v2 `tekamolo.rs` — German left-corner lanes (`68955ecb`) | planner `insight_coca_read` | LIVE, but **English COCA only**: the German capability was lost, not replaced | +| deepnsm-v2 `lexicon.rs` (`68955ecb`) | taggers inlined in `bible_wave`, `genre_shapes` | LIVE | +| deepnsm-v2 `loci.rs` (`68955ecb`) | deepnsm `spo_anaphora_nibble`, jc `l9_loci_real_text`, planner `probe_antecedent_binder` | LIVE (all three) | +| deepnsm-v2 `toc/hydrate/promote` (`68955ecb`) | planner path writes the TEKAMOLO tenant (`insight_reason_wired`) | LIVE, but reads the **v1** WordNet rail the release's own audit condemns (12.76 % wrong sense, 33.84 % on verbs) | +| deepnsm `ontology_vocab.rs` (`48405aa2`) | none — reverted on purpose: concept ids are identities, CAM-PQ is prohibited for ontologies | constraint stands; WordNet enters only as an exact address (`probe_wordnet_44_activation`, 4⁴ ancestry) | +| planner `data/{coca,wordnet}` generators (`b8de41a2`) | moved to branch `claude/rosetta-codebook-bakes-z30uij` | outputs in releases | +| deepnsm v1 `markov_bundle` TEKAMOLO slices (`0ae9f906`) | restored; `parser.rs` leaves `TekamoloSlots` empty "until D3" | slots exist, nothing fills them | + +Pickaxe: `Frageprobe` / `wer oder was` / `QuestionProbe` / `wh_question` — 0 +code commits; the school question test was never implemented before this. + +## TEKAMOLO = 4 × SPO + +`TekamoloFacet` is the G4D3 carving — the same bytes as the L5 SPO triplets — +so each lane can carry its own adverbial triple (Lokal = (carried, into, +hall)) with its own 2³ mask. Not built. + +## OPEN + +- WordNet ancestry as the lane of a PP's noun (*Wann?* → is_a* time period, + *Wo?* → is_a* location), via the exact 4⁴ address — testable against UD + English `obl:tmod`; not run. +- Re-measure the deleted German left-corner reader against `tekamolo_de`. +- Move `insight_reason_wired` to the v2 WordNet rail. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index e4d0ace7a..b645ebbc2 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -191 entries, 2026-08-06 .. 2026-10-04. +192 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| +| 2026-10-04 | `D-LXC-18` | | [2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md](2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md) | | 2026-10-04 | `D-LXC-13` | | [2026-10-04-deepnsm-v2-predicate-by-position.md](2026-10-04-deepnsm-v2-predicate-by-position.md) | | 2026-10-04 | `D-LXC-14` | | [2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md](2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md) | | 2026-10-04 | `D-LXC-16` | | [2026-10-04-deepnsm-v2-learned-position-table.md](2026-10-04-deepnsm-v2-learned-position-table.md) | diff --git a/crates/deepnsm-v2/examples/ud_pos_eval.rs b/crates/deepnsm-v2/examples/ud_pos_eval.rs index f6e1b24d5..7d284abc9 100644 --- a/crates/deepnsm-v2/examples/ud_pos_eval.rs +++ b/crates/deepnsm-v2/examples/ud_pos_eval.rs @@ -44,8 +44,8 @@ use std::collections::HashMap; use deepnsm_v2::coca::fsm_pos_tag; use deepnsm_v2::fsm::{ - attribute_rule, parse_readings_with, AdjectiveOrder, AttributeRule, Pos, PosSet, Reading, - Typology, + answered_questions, attribute_rule, parse_readings_with, AdjectiveOrder, AttributeRule, Pos, + PosSet, Reading, Tagged, Typology, }; /// One syntactic word of a UD sentence. @@ -309,7 +309,41 @@ fn readings_of(lex: &Lexicon, s: &[Word], k: usize, cased: bool) -> PosSet { /// Positional context of a token: the neighbours' reading sets and whether /// the previous word is a copula. Never the word itself. -type Context = (u8, bool, u8, bool); +type Context = (u8, bool, u8, bool, u8); + +/// Which positional evidence the table keys on (`UD_CTX`): the neighbours +/// (`neigh`, default), the question test — the 2³ mask of SPO questions the +/// clause has answered before the word, plus the next readings (`q`) — or +/// both (`both`). +fn ctx_mode() -> &'static str { + match std::env::var("UD_CTX").as_deref() { + Ok("q") => "q", + Ok("both") => "both", + _ => "neigh", + } +} + +/// The 2³ answered-question mask before every word of a sentence. The left +/// context is tagged by the frequency pick of each word's readings — never +/// gold. +fn answered_masks(lex: &Lexicon, s: &[Word], cased: bool) -> Vec { + let tags: Vec = (0..s.len()) + .map(|k| { + let set = readings_of(lex, s, k, cased); + let pos = set + .iter() + .max_by_key(|p| { + lex.counts + .get(&(s[k].form.clone(), *p as u8)) + .copied() + .unwrap_or(0) + }) + .unwrap_or(Pos::Other); + Tagged::new(0, pos) + }) + .collect(); + answered_questions(&tags) +} fn bits(p: PosSet) -> u8 { Pos::ALL @@ -328,6 +362,8 @@ struct PositionTable { prev: HashMap<(u8, bool), [usize; 2]>, /// Next readings + copula anywhere in the clause. clause: HashMap<(u8, bool), [usize; 2]>, + /// Answered-question mask + next readings. + question: HashMap<(u8, u8), [usize; 2]>, } /// Minimum train tokens behind a context before it may decide (policy pin). @@ -346,9 +382,11 @@ impl PositionTable { next: HashMap::new(), prev: HashMap::new(), clause: HashMap::new(), + question: HashMap::new(), }; for s in train { let set = |k: usize| readings_of(lex, s, k, cased); + let masks = answered_masks(lex, s, cased); for k in 0..s.len() { let this = set(k); let w = &s[k]; @@ -357,12 +395,13 @@ impl PositionTable { { continue; } - let ctx = Self::context(s, k, &set, copulas); + let ctx = Self::context(s, k, &set, copulas, &masks); let slot = usize::from(w.gold == pair.1); t.full.entry(ctx).or_default()[slot] += 1; t.next.entry(ctx.2).or_default()[slot] += 1; t.prev.entry((ctx.0, ctx.1)).or_default()[slot] += 1; t.clause.entry((ctx.2, ctx.3)).or_default()[slot] += 1; + t.question.entry((ctx.4, ctx.2)).or_default()[slot] += 1; } } t @@ -373,6 +412,7 @@ impl PositionTable { k: usize, set: &dyn Fn(usize) -> PosSet, copulas: &std::collections::HashSet, + masks: &[u8], ) -> Context { let prev = k.checked_sub(1); // The clause: the words between punctuation marks around `k`. @@ -385,11 +425,18 @@ impl PositionTable { && (lo..hi) .filter(|&j| j != k) .any(|j| copulas.contains(&s[j].form.to_lowercase())); + let next = if k + 1 < s.len() { bits(set(k + 1)) } else { 0 }; + let neigh = ctx_mode() != "q"; ( - prev.map_or(0, |j| bits(set(j))), - prev.is_some_and(|j| copulas.contains(&s[j].form.to_lowercase())), - if k + 1 < s.len() { bits(set(k + 1)) } else { 0 }, - clause_copula, + if neigh { + prev.map_or(0, |j| bits(set(j))) + } else { + 0 + }, + neigh && prev.is_some_and(|j| copulas.contains(&s[j].form.to_lowercase())), + next, + neigh && clause_copula, + if ctx_mode() == "neigh" { 0 } else { masks[k] }, ) } @@ -405,6 +452,7 @@ impl PositionTable { // Back off: whichever partial context is most decisive. [ self.clause.get(&(ctx.2, ctx.3)).copied().and_then(pick), + self.question.get(&(ctx.4, ctx.2)).copied().and_then(pick), self.next.get(&ctx.2).copied().and_then(pick), self.prev.get(&(ctx.0, ctx.1)).copied().and_then(pick), ] @@ -439,6 +487,7 @@ fn position_table_report( (0usize, 0usize, 0usize, 0usize, 0usize, 0usize); for s in test { let set = |k: usize| readings_of(lex, s, k, cased); + let masks = answered_masks(lex, s, cased); for k in 0..s.len() { let w = &s[k]; let this = set(k); @@ -459,7 +508,7 @@ fn position_table_report( let p_word = (ca as f64 + 0.5) / ((ca + cb) as f64 + 1.0); let freq = if cb > ca { b } else { a }; freq_all += usize::from(freq == w.gold); - let ctx = PositionTable::context(s, k, &set, copulas); + let ctx = PositionTable::context(s, k, &set, copulas, &masks); let p_ctx = table.p_first(ctx); if let Some((p, _)) = p_ctx { if !(0.25..0.75).contains(&p) { diff --git a/crates/deepnsm-v2/src/fsm.rs b/crates/deepnsm-v2/src/fsm.rs index 5d43adcef..97206b740 100644 --- a/crates/deepnsm-v2/src/fsm.rs +++ b/crates/deepnsm-v2/src/fsm.rs @@ -307,6 +307,48 @@ pub fn parse_to_spo(tokens: &[Tagged]) -> Vec { out } +/// Question-test bits of [`answered_questions`]: which of the SPO questions +/// the clause has already answered — *who/what?* (subject), *does what?* +/// (predicate), *whom/what?* (object). The 8 values are the Pearl 2³ masks. +pub const ANSWERED_S: u8 = 1; +/// See [`ANSWERED_S`]. +pub const ANSWERED_P: u8 = 2; +/// See [`ANSWERED_S`]. +pub const ANSWERED_O: u8 = 4; + +/// For each token, the 2³ mask of SPO questions its clause has answered +/// BEFORE it arrives — the school question test (*Frageprobe*) as a position: +/// a word arriving after "who?" is answered and before "does what?" is, is +/// asked "does what?". Steps the same transition table as [`parse_to_spo`]; +/// a `Stop` starts a new clause (mask 0). +#[must_use] +pub fn answered_questions(tokens: &[Tagged]) -> Vec { + let mut out = Vec::with_capacity(tokens.len()); + let mut core = Core::START; + let mut emitted = false; + for &t in tokens { + let mut mask = 0; + if matches!(core.state, State::HaveSubject | State::HaveVerb) { + mask |= ANSWERED_S; + } + if core.state == State::HaveVerb || emitted { + mask |= ANSWERED_P; + } + if emitted { + mask |= ANSWERED_O; + } + out.push(mask); + if t.pos == Pos::Stop { + core.state = State::Start; + core.rel = Rel::None; + emitted = false; + continue; + } + emitted |= core.step(t).is_some(); + } + out +} + // ───────────────────────────────────────────────────────────────────────────── // Multi-reading input (D-LXC-2) // ───────────────────────────────────────────────────────────────────────────── @@ -1203,6 +1245,17 @@ mod tests { assert_eq!(p.unpredicated_dropped, 0); } + /// The question test as a position: "dog bit man . cat" — before `dog` + /// nothing is answered, before `bit` who?, before `man` who? + does what?, + /// after the triple all three, and a stop starts over. + #[test] + fn answered_questions_follow_the_clause() { + let toks = [n(1), v(2), n(3), Tagged::new(0, Pos::Stop), n(4)]; + let m = answered_questions(&toks); + let spo = ANSWERED_S | ANSWERED_P | ANSWERED_O; + assert_eq!(m, vec![0, ANSWERED_S, ANSWERED_S | ANSWERED_P, spo, 0]); + } + /// T4: a single-reading word is never rejected, even where the licensing /// rule would forbid it — a word is not dropped for having one reading. #[test] diff --git a/crates/deepnsm-v2/src/lib.rs b/crates/deepnsm-v2/src/lib.rs index 1a2a8d0f9..b625f07fb 100644 --- a/crates/deepnsm-v2/src/lib.rs +++ b/crates/deepnsm-v2/src/lib.rs @@ -65,8 +65,9 @@ pub use evidence::{ shuffle_beliefs_null, shuffle_rungs_null, EvidenceBasin, ForwardGateReport, }; pub use fsm::{ - attribute_readings, attribute_rule, parse_readings, parse_readings_with, parse_to_spo, - AdjectiveOrder, AttributeRule, Pos, PosSet, Reading, ReadingParse, Survivor, Tagged, Typology, + answered_questions, attribute_readings, attribute_rule, parse_readings, parse_readings_with, + parse_to_spo, AdjectiveOrder, AttributeRule, Pos, PosSet, Reading, ReadingParse, Survivor, + Tagged, Typology, }; pub use introspect::{ confidence_delta_recount, confidence_delta_self, most_frequent_belief, provenance_check, From ec65f274f324b176f9e2aefd241e7e65dd3fb1e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:37:41 +0000 Subject: [PATCH 07/33] deepnsm-v2: literal rules, priorities and a quorum for word-class homographs (D-LXC-19) ud_pos_eval now decides noun/verb and adjective/adverb with 14-20 literal rules: three learned position tables, frequency, determiner/slot/infinitive/ copula/attribute rules, English -ly, German capitalisation, and (with UD_DE_INVENTORY) German inflection against the lemma plus TEKAMOLO cues. Each rule is weighted by its precision on a held-out 10 % of train; test is scored by priority, by summed logits, and by a logistic regression fitted jointly on held-out. The joint quorum is best or tied in 7 of 10 language x pair rows. Summing the weights fails on correlated voters (German noun/verb 84.9 % vs 98.2 % frequency); a disable run shows that collapse is not lexicon leakage. Animal Farm and German adjective/adverb stay best with position x frequency. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + .../2026-10-04-deepnsm-v2-rule-quorum.md | 74 +++ .claude/board/entries/README.md | 3 +- ...deepnsm-v2-lexical-evidence-consumer-v1.md | 10 + crates/deepnsm-v2/examples/ud_pos_eval.rs | 625 +++++++++++++++++- 5 files changed, 694 insertions(+), 19 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-rule-quorum.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 221e983bd..78352ce23 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -42,6 +42,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-16** | learned position table (neighbours' readings + copula → P(reading)), German case kept, Animal Farm silver reference | In PR | position × frequency beats frequency on adjective/adverb in 5/5 settings (Animal Farm 78.1 → 88.4 %), noun/verb in 4/5. Harness only. Entry `2026-10-04-deepnsm-v2-learned-position-table.md` | | **D-LXC-17** | German TEKAMOLO cues by verb position (`tekamolo_de`); adverbial adjectives → Adv fold; Te-Ka-Mo-Lo order law measured | In PR | cue words: position × frequency 88.5 % vs frequency 84.1 % (208 tokens, `da` 87.5 vs 37.5); order law 46–65 % on gold, not usable. Entry `2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md` | | **D-LXC-18** | question test as the Pearl 2³ mask (`fsm::answered_questions`, `UD_CTX`); blast radius of deleted grammar heuristics | In PR | 2³ context: Animal Farm adj/adv 91.8 % on decided (frequency 78.4 %), German adj/adv 85.1 % (best); noun/verb better with neighbours. German TEKAMOLO capability was lost in `68955ecb`; tenant writer reads the condemned v1 WordNet rail. Entry `2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md` | +| **D-LXC-19** | literal rules, priorities and a quorum for word-class homographs (`ud_pos_eval` `UD_RULES`, `UD_DE_INVENTORY`) | In PR | 14–20 rules per pair; held-out-fitted joint quorum best or tied in 7/10 rows (UD English n/v 93.7 %, de n/v 98.8 %); summed quorum fails on correlated voters (de n/v 84.9 %); Animal Farm adj/adv and German adj/adv stay best with position × frequency. Entry `2026-10-04-deepnsm-v2-rule-quorum.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-rule-quorum.md b/.claude/board/entries/2026-10-04-deepnsm-v2-rule-quorum.md new file mode 100644 index 000000000..2570ecab2 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-rule-quorum.md @@ -0,0 +1,74 @@ +# 2026-10-04 — deepnsm-v2: literal rules, priorities and a quorum for word-class homographs (D-LXC-19) + +**Status:** MEASURED (`UD_RULES=1 [UD_DE_INVENTORY=DIR] cargo run --release +--example ud_pos_eval -- TRAIN TEST [--coca ../deepnsm/word_frequency]`, UD r2.15 +en_ewt / de_gsd / fr_gsd, Animal Farm spaCy silver; debug-0 profile). + +Operator (2026-10-04): instead of picking one rule, build ~20 rules with +priorities and a quorum; collect the literal grammar rules (Wechsel, case, +TEKAMOLO). + +## What was built (`examples/ud_pos_eval.rs`, eval harness only) + +- **Rules:** 14 for noun/verb and 16–20 for adj/adv. + - three learned position tables (neighbours, the 2³ answered-question mask, both); + - frequency, split at a 0.9 share; + - determiner, adjective, slot, infinitive, modal and copula rules; + - the four attribute clauses; + - English `-ly`; + - German capitalisation; + - with `UD_DE_INVENTORY`: German inflection against the lemma (uninflected → adv; lemma + ending → adj; uninflected right before a noun → adv) and TEKAMOLO adverbial cues. +- **Training:** tables learn on 90 % of train. Each rule's precision is measured on the other 10 %, and that held-out precision is its weight. +- **Three decisions on test:** + - priority — the highest-precision rule that fires; + - summed quorum — Σ ±logit(precision); + - joint quorum — a logistic regression over all votes plus the frequency log-odds, fitted on held-out. +- **German inventories:** built with `build_de_codebook.py` from `de_gsd-ud-train.conllu` only. The release `de-codebook` was built from GSD + HDT, and its manifest does not name the splits, so it may contain the test sentences. It is not used against GSD test. + +## Results (all tokens of the pair; the same run's position × frequency for reference) + +| run | pair | frequency | position × frequency | priority | summed quorum | joint quorum | +|---|---|---|---|---|---|---| +| UD English (train lexicon) | noun/verb | 89.5 | 92.7 | 93.3 | 91.4 | **93.7** | +| | adj/adv | 89.0 | 90.1 | 88.8 | 85.4 | **90.4** | +| UD English (COCA) | noun/verb | 84.2 | **91.1** | 89.0 | 88.1 | 90.5 | +| | adj/adv | 89.3 | 91.0 | 88.5 | 87.1 | **92.0** | +| Animal Farm (COCA) | noun/verb | 82.6 | 91.6 | 90.8 | 89.4 | **91.9** | +| | adj/adv | 77.1 | **88.2** | 86.5 | 85.7 | 84.8 | +| German (train lexicon) | noun/verb | 98.2 | 97.6 | 97.3 | 84.9 | **98.8** | +| | adj/adv, + inventories | 79.3 | **82.9** | 82.5 | 81.2 | 81.6 | +| French (train lexicon) | noun/verb | 89.9 | 94.8 | 94.0 | 89.7 | **94.8** | +| | adj/adv (50 tokens) | 82.0 | 90.0 | 88.0 | 88.0 | **90.0** | + +Train-lexicon runs build the lexicon from the 90 % fit split. Their quorum +token sets are therefore a few percent smaller than the position-table line's, +and the frequency column is the quorum line's own. + +## Findings + +1. **A summed quorum fails.** The three position tables agree almost always, + so summing logits counts one opinion three times. German noun/verb drops to + 84.9 % against 98.2 % frequency. The joint fit spreads one weight across + correlated voters and is best or tied in 7 of 10 rows. +2. **The collapse is correlation, not lexicon leakage.** Disable run: with the + all-train lexicon (held-out sentences included), German noun/verb's summed + quorum is still 85.0 %. The fit-only lexicon makes the held-out weights + honest. It also has a cost: it covers 90 % of train, and with the full + lexicon German adj/adv's joint quorum reaches 82.9 % (315 tokens instead + of 309). +3. **Animal Farm adj/adv loses with the joint fit (84.8 % vs 88.2 %).** The + weights are fitted on EWT held-out (web text) and do not transfer to + Orwell's prose, where the question table alone is 91.9 %. Weights are + genre-dependent; a quorum fitted on one register is not the default for + another. +4. **German rule precision (test):** + - "Next to a noun → adj" is right only 27.8 % of the time; "uninflected right before a noun → adv" 80.0 %. A German attributive adjective must inflect, so the English noun-adjacency rule is wrong for German, and inflection is the German position signal. + - "Before an adjective → adv" 96.7 %. + - TEKAMOLO adverbial cues 71.0 %. + +## OPEN + +- German adj/adv stays at 81–83 %; no combination beats position × frequency. +- Fit the weights per register (or on dev of the target genre) instead of one held-out set. +- The case and PP-lane questions (Dativ/Akkusativ, Wechsel static vs. directional → Lokal, relative-pronoun case) are separate ambiguities. They are not scored here, and their inventories (`wechsel.tsv`, `article_case.tsv`, `valency.tsv`, `relative_pronoun.tsv`) need their own gold harness on UD `Case`. +- Re-measure right-corner TEKAMOLO on edited text (GSD) against Luther. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index b645ebbc2..5bf155ac0 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -192 entries, 2026-08-06 .. 2026-10-04. +193 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| +| 2026-10-04 | `D-LXC-19` | | [2026-10-04-deepnsm-v2-rule-quorum.md](2026-10-04-deepnsm-v2-rule-quorum.md) | | 2026-10-04 | `D-LXC-18` | | [2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md](2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md) | | 2026-10-04 | `D-LXC-13` | | [2026-10-04-deepnsm-v2-predicate-by-position.md](2026-10-04-deepnsm-v2-predicate-by-position.md) | | 2026-10-04 | `D-LXC-14` | | [2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md](2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md) | diff --git a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md index 3ce84d2ff..e3b7eaca6 100644 --- a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md +++ b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md @@ -494,3 +494,13 @@ switch, below the slot rule in every language. WordNet helps neither as a chooser nor as a reading filter for this question. Board entry `2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md`. + +## D-LXC-19: rules, priorities and a quorum (2026-10-04) + +`ud_pos_eval` decides each pair with 14–20 literal rules, each weighted by its +precision on a held-out 10 % of train. A logistic regression over all votes +(joint quorum) is best or tied in 7 of 10 language × pair rows. Summing the +weights instead fails, because the position tables are correlated voters. +German inflection against the lemma replaces the English noun-adjacency rule +as the German attribute signal. Board entry +`2026-10-04-deepnsm-v2-rule-quorum.md`. diff --git a/crates/deepnsm-v2/examples/ud_pos_eval.rs b/crates/deepnsm-v2/examples/ud_pos_eval.rs index 7d284abc9..c59e78175 100644 --- a/crates/deepnsm-v2/examples/ud_pos_eval.rs +++ b/crates/deepnsm-v2/examples/ud_pos_eval.rs @@ -34,6 +34,18 @@ //! can be scored the same way after silver tagging, e.g. Animal Farm through //! spaCy `en_core_web_sm` (lab step, not committed). //! +//! **Rules, priorities and a quorum.** Every pair is also decided by a set of +//! literal rules (three learned position tables, frequency, determiner / +//! slot / infinitive / copula / attribute-clause rules, English `-ly`, German +//! capitalisation). Tables learn on 90 % of train; each rule is weighted on the +//! other 10 %, and test is scored three ways: the highest-precision rule that +//! fires (priority), Σ ±logit(precision) (summed quorum) and a logistic +//! regression over all votes fitted on the held-out 10 % (joint quorum). +//! `UD_RULES=1` prints every rule's held-out and test precision and joint +//! weight. `UD_DE_INVENTORY=DIR` adds the German literal inventories +//! (`build_de_codebook.py` output: inflection against the lemma, TEKAMOLO +//! adverbial cues); build them from TRAIN only, never from the test file. +//! //! `UD_CLAUSE=1` switches on the clause rule //! ([`Typology::predicate_required`]). The typology is measured from train; //! `UD_TYPOLOGY=english` uses @@ -45,7 +57,7 @@ use std::collections::HashMap; use deepnsm_v2::coca::fsm_pos_tag; use deepnsm_v2::fsm::{ answered_questions, attribute_rule, parse_readings_with, AdjectiveOrder, AttributeRule, Pos, - PosSet, Reading, Tagged, Typology, + PosSet, Reading, Tagged, Typology, ANSWERED_S, }; /// One syntactic word of a UD sentence. @@ -353,10 +365,25 @@ fn bits(p: PosSet) -> u8 { .fold(0u8, |a, (i, _)| a | (1 << i)) } +/// `ctx` with the fields `mode` does not read zeroed. +fn project(ctx: Context, mode: &str) -> Context { + let neigh = mode != "q"; + let q = mode != "neigh"; + ( + if neigh { ctx.0 } else { 0 }, + neigh && ctx.1, + ctx.2, + neigh && ctx.3, + if q { ctx.4 } else { 0 }, + ) +} + /// P(first | context) for one reading pair, learned from gold train tags /// with the evaluation lexicon's readings. Backs off from the full context /// to its next-only and previous-only halves when support is thin. struct PositionTable { + /// Which context fields this table keys on (see [`ctx_mode`]). + mode: &'static str, full: HashMap, next: HashMap, prev: HashMap<(u8, bool), [usize; 2]>, @@ -376,8 +403,10 @@ impl PositionTable { copulas: &std::collections::HashSet, cased: bool, pair: (Pos, Pos), + mode: &'static str, ) -> Self { let mut t = Self { + mode, full: HashMap::new(), next: HashMap::new(), prev: HashMap::new(), @@ -395,7 +424,7 @@ impl PositionTable { { continue; } - let ctx = Self::context(s, k, &set, copulas, &masks); + let ctx = project(Self::context(s, k, &set, copulas, &masks), mode); let slot = usize::from(w.gold == pair.1); t.full.entry(ctx).or_default()[slot] += 1; t.next.entry(ctx.2).or_default()[slot] += 1; @@ -425,23 +454,18 @@ impl PositionTable { && (lo..hi) .filter(|&j| j != k) .any(|j| copulas.contains(&s[j].form.to_lowercase())); - let next = if k + 1 < s.len() { bits(set(k + 1)) } else { 0 }; - let neigh = ctx_mode() != "q"; ( - if neigh { - prev.map_or(0, |j| bits(set(j))) - } else { - 0 - }, - neigh && prev.is_some_and(|j| copulas.contains(&s[j].form.to_lowercase())), - next, - neigh && clause_copula, - if ctx_mode() == "neigh" { 0 } else { masks[k] }, + prev.map_or(0, |j| bits(set(j))), + prev.is_some_and(|j| copulas.contains(&s[j].form.to_lowercase())), + if k + 1 < s.len() { bits(set(k + 1)) } else { 0 }, + clause_copula, + masks[k], ) } /// P(first member | context), with the support it rests on. fn p_first(&self, ctx: Context) -> Option<(f64, usize)> { + let ctx = project(ctx, self.mode); let pick = |c: [usize; 2]| { let n = c[0] + c[1]; (n >= MIN_SUPPORT).then(|| ((c[0] as f64 + 0.5) / (n as f64 + 1.0), n)) @@ -450,14 +474,25 @@ impl PositionTable { return Some(r); } // Back off: whichever partial context is most decisive. + // Each mode backs off only to its own partial contexts, so a field it + // does not read can never collapse into a constant key (the class + // prior posing as a position). + let neigh = self.mode != "q"; + let q = self.mode != "neigh"; [ - self.clause.get(&(ctx.2, ctx.3)).copied().and_then(pick), - self.question.get(&(ctx.4, ctx.2)).copied().and_then(pick), - self.next.get(&ctx.2).copied().and_then(pick), - self.prev.get(&(ctx.0, ctx.1)).copied().and_then(pick), + neigh + .then(|| self.clause.get(&(ctx.2, ctx.3)).copied()) + .flatten(), + q.then(|| self.question.get(&(ctx.4, ctx.2)).copied()) + .flatten(), + self.next.get(&ctx.2).copied(), + neigh + .then(|| self.prev.get(&(ctx.0, ctx.1)).copied()) + .flatten(), ] .into_iter() .flatten() + .filter_map(pick) .max_by(|x, y| (x.0 - 0.5).abs().total_cmp(&(y.0 - 0.5).abs())) } } @@ -473,7 +508,7 @@ fn position_table_report( pair: (Pos, Pos), name: &str, ) { - let table = PositionTable::learn(train, lex, copulas, cased, pair); + let table = PositionTable::learn(train, lex, copulas, cased, pair, ctx_mode()); let (a, b) = pair; let prior = { let (x, y) = table @@ -541,6 +576,538 @@ fn position_table_report( ); } +/// What one rule sees of a token: its readings, its neighbours' readings and +/// forms, the positional context, and the word's counts for the pair. +struct Tok<'a> { + this: PosSet, + prev: PosSet, + next: PosSet, + prev_form: Option<&'a str>, + surface: &'a str, + form: &'a str, + initial: bool, + ctx: Context, + /// Lexicon counts of the pair's first and second reading for the word. + counts: (usize, usize), +} + +type Vote = Box Option>; + +/// A literal grammar rule: votes for one reading of the pair, or abstains. +struct Rule { + name: &'static str, + vote: Vote, +} + +fn only(p: PosSet, x: Pos) -> bool { + p == PosSet::single(x) +} + +/// Every rule for `pair`: three learned position tables (vote when +/// decisive), frequency split by how lopsided the word is, and literal +/// grammar rules. Each is weighted later by its held-out precision, so a +/// weak or language-specific rule costs nothing. +/// German literal inventories (`build_de_codebook.py` output, built from +/// train only), read when `UD_DE_INVENTORY` names their directory. +#[derive(Default)] +struct Inventory { + /// Adjective/adverb form → its lemma (`lexicon.tsv`, rows tagged j or r). + lemma: HashMap, + /// TEKAMOLO cue lemmas attested as `advmod` (`tekamolo.tsv`). + adverbial_cue: std::collections::HashSet, +} + +impl Inventory { + fn load() -> Option { + let dir = std::env::var("UD_DE_INVENTORY") + .ok() + .filter(|d| !d.is_empty())?; + let read = |name: &str| { + std::fs::read_to_string(format!("{dir}/{name}")) + .unwrap_or_else(|e| panic!("{dir}/{name}: {e}")) + }; + let mut inv = Self::default(); + for line in read("lexicon.tsv").lines().filter(|l| !l.starts_with('#')) { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() > 2 && matches!(c[2], "j" | "r") { + inv.lemma + .entry(c[0].to_string()) + .or_insert_with(|| c[1].to_string()); + } + } + for line in read("tekamolo.tsv").lines().filter(|l| !l.starts_with('#')) { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() > 2 && c[2] == "advmod" { + inv.adverbial_cue.insert(c[1].to_string()); + } + } + Some(inv) + } +} + +fn rules( + inventory: Option>, + pair: (Pos, Pos), + tables: [std::rc::Rc; 3], + copulas: std::rc::Rc>, + typology: Typology, + cased: bool, +) -> Vec { + let (a, b) = pair; + let mut out: Vec = Vec::new(); + for (name, t) in [ + "table: neighbours", + "table: answered questions", + "table: both", + ] + .into_iter() + .zip(tables) + { + out.push(Rule { + name, + vote: Box::new(move |x| { + t.p_first(x.ctx).and_then(|(p, _)| { + if p >= 0.75 { + Some(a) + } else if p <= 0.25 { + Some(b) + } else { + None + } + }) + }), + }); + } + let lopsided = |(ca, cb): (usize, usize)| { + let n = ca + cb; + n > 0 && (ca * 10 >= n * 9 || cb * 10 >= n * 9) + }; + out.push(Rule { + name: "frequency, share >= 0.9", + vote: Box::new(move |x| { + lopsided(x.counts).then_some(if x.counts.0 >= x.counts.1 { a } else { b }) + }), + }); + out.push(Rule { + name: "frequency, share < 0.9", + vote: Box::new(move |x| { + (x.counts.0 + x.counts.1 > 0 && !lopsided(x.counts)) + .then_some(if x.counts.0 >= x.counts.1 { a } else { b }) + }), + }); + let cop = move |f: Option<&str>| f.is_some_and(|f| copulas.contains(&f.to_lowercase())); + let mut fixed: Vec<(&'static str, Vote)> = Vec::new(); + if pair == (Pos::Noun, Pos::Verb) { + fixed.push(( + "after a determiner -> noun", + Box::new(|x: &Tok| only(x.prev, Pos::Det).then_some(Pos::Noun)), + )); + fixed.push(( + "after an adjective -> noun", + Box::new(move |x: &Tok| { + (only(x.prev, Pos::Adj) && typology.adjective_opens_nominal).then_some(Pos::Noun) + }), + )); + fixed.push(( + "slot: subject answered, prev a noun -> verb", + Box::new(|x: &Tok| { + (x.ctx.4 == ANSWERED_S && only(x.prev, Pos::Noun)).then_some(Pos::Verb) + }), + )); + fixed.push(( + "before a determiner -> verb", + Box::new(|x: &Tok| only(x.next, Pos::Det).then_some(Pos::Verb)), + )); + fixed.push(( + "before a verb-only word -> noun", + Box::new(|x: &Tok| only(x.next, Pos::Verb).then_some(Pos::Noun)), + )); + fixed.push(( + "after an infinitive marker -> verb", + Box::new(|x: &Tok| { + x.prev_form + .is_some_and(|f| f.eq_ignore_ascii_case("to") || f.eq_ignore_ascii_case("zu")) + .then_some(Pos::Verb) + }), + )); + let c = cop.clone(); + fixed.push(( + "after a non-copula verb-only word -> verb", + Box::new(move |x: &Tok| { + (only(x.prev, Pos::Verb) && !c(x.prev_form)).then_some(Pos::Verb) + }), + )); + fixed.push(( + "capitalised, not initial -> noun", + Box::new(move |x: &Tok| { + (cased && !x.initial && x.surface.starts_with(char::is_uppercase)) + .then_some(Pos::Noun) + }), + )); + fixed.push(( + "lowercase where nouns are capitalised -> verb", + Box::new(move |x: &Tok| { + (cased && x.surface.starts_with(char::is_lowercase)).then_some(Pos::Verb) + }), + )); + } else { + for (name, rule) in [ + ( + "clause: after a determiner -> adj", + AttributeRule::AfterDeterminer, + ), + ( + "clause: before an adjective -> adv", + AttributeRule::BeforeAdjective, + ), + ( + "clause: between subject and verb -> adv", + AttributeRule::BetweenSubjectAndVerb, + ), + ("clause: next to a noun -> adj", AttributeRule::NextToNoun), + ] { + fixed.push(( + name, + Box::new(move |x: &Tok| { + (attribute_rule(x.prev, x.this, x.next, typology) == Some(rule)) + .then(|| rule.keeps()) + }), + )); + } + let c = cop.clone(); + fixed.push(( + "after a copula -> adj", + Box::new(move |x: &Tok| c(x.prev_form).then_some(Pos::Adj)), + )); + let c = cop.clone(); + fixed.push(( + "after a non-copula verb, no noun next -> adv", + Box::new(move |x: &Tok| { + (x.prev.contains(Pos::Verb) && !c(x.prev_form) && !x.next.contains(Pos::Noun)) + .then_some(Pos::Adv) + }), + )); + fixed.push(( + "before a determiner -> adv", + Box::new(|x: &Tok| only(x.next, Pos::Det).then_some(Pos::Adv)), + )); + fixed.push(( + "before a verb-only word -> adv", + Box::new(|x: &Tok| only(x.next, Pos::Verb).then_some(Pos::Adv)), + )); + fixed.push(( + "-ly ending -> adv", + Box::new(|x: &Tok| x.form.ends_with("ly").then_some(Pos::Adv)), + )); + fixed.push(( + "inflected ending before a noun -> adj", + Box::new(move |x: &Tok| { + (cased + && ["e", "en", "em", "er", "es"] + .iter() + .any(|e| x.form.ends_with(e)) + && x.next.contains(Pos::Noun)) + .then_some(Pos::Adj) + }), + )); + if let Some(inv) = inventory { + let i = inv.clone(); + fixed.push(( + "uninflected (form is its lemma) -> adv", + Box::new(move |x: &Tok| { + (i.lemma.get(x.form).map(String::as_str) == Some(x.form)).then_some(Pos::Adv) + }), + )); + let i = inv.clone(); + fixed.push(( + "inflected (lemma + ending) -> adj", + Box::new(move |x: &Tok| { + i.lemma + .get(x.form) + .and_then(|l| x.form.strip_prefix(l.as_str())) + .and_then(|e| { + ["e", "en", "em", "er", "es"] + .contains(&e) + .then_some(Pos::Adj) + }) + }), + )); + let i = inv.clone(); + fixed.push(( + "uninflected right before a noun -> adv", + Box::new(move |x: &Tok| { + (i.lemma.get(x.form).map(String::as_str) == Some(x.form) + && x.next.contains(Pos::Noun)) + .then_some(Pos::Adv) + }), + )); + let i = inv; + fixed.push(( + "TEKAMOLO adverbial cue lemma -> adv", + Box::new(move |x: &Tok| { + let l = i.lemma.get(x.form).map_or(x.form, String::as_str); + i.adverbial_cue.contains(l).then_some(Pos::Adv) + }), + )); + } + let c = cop.clone(); + fixed.push(( + "clause-final, no copula before -> adv", + Box::new(move |x: &Tok| { + (x.next.is_empty() && !x.prev.is_empty() && !c(x.prev_form)).then_some(Pos::Adv) + }), + )); + } + out.extend(fixed.into_iter().map(|(name, vote)| Rule { name, vote })); + out +} + +/// Every token of the pair in `sentences`, as rules see it, with its gold. +fn pair_tokens<'a>( + sentences: &'a [Vec], + lex: &Lexicon, + copulas: &std::collections::HashSet, + cased: bool, + pair: (Pos, Pos), +) -> Vec<(Tok<'a>, Pos)> { + let mut out = Vec::new(); + for s in sentences { + let set = |k: usize| readings_of(lex, s, k, cased); + let masks = answered_masks(lex, s, cased); + for k in 0..s.len() { + let w = &s[k]; + let this = set(k); + if !(this.contains(pair.0) && this.contains(pair.1)) + || (w.gold != pair.0 && w.gold != pair.1) + { + continue; + } + let c = |p: Pos| { + lex.counts + .get(&(w.form.clone(), p as u8)) + .copied() + .unwrap_or(0) + }; + out.push(( + Tok { + this, + prev: k.checked_sub(1).map_or(PosSet::EMPTY, &set), + next: if k + 1 < s.len() { + set(k + 1) + } else { + PosSet::EMPTY + }, + prev_form: k.checked_sub(1).map(|j| s[j].form.as_str()), + surface: &w.surface, + form: &w.form, + initial: k == 0, + ctx: PositionTable::context(s, k, &set, copulas, &masks), + counts: (c(pair.0), c(pair.1)), + }, + w.gold, + )); + } + } + out +} + +/// Rules, priorities and a quorum. The position tables learn on 90 % of +/// train; each rule's weight is its precision when it fires on the other +/// 10 %. Test is scored by (a) the highest-weight rule that fires +/// (priority) and (b) the weighted vote Σ ±logit(precision) of every rule +/// that fires (quorum); both fall back to frequency when nothing fires. +/// The lexicon is the run's own (from all of train, or COCA). +#[allow(clippy::too_many_arguments)] +fn quorum_report( + train: &[Vec], + test: &[Vec], + lex: &Lexicon, + coca: bool, + copulas: &std::collections::HashSet, + cased: bool, + typology: Typology, + pair: (Pos, Pos), + name: &str, +) { + let fit: Vec<&Vec> = train + .iter() + .enumerate() + .filter(|(i, _)| i % 10 != 0) + .map(|(_, s)| s) + .collect(); + let held: Vec<&Vec> = train + .iter() + .enumerate() + .filter(|(i, _)| i % 10 == 0) + .map(|(_, s)| s) + .collect(); + let fit: Vec> = fit.into_iter().map(|s| clone_sentence(s)).collect(); + let held: Vec> = held.into_iter().map(|s| clone_sentence(s)).collect(); + // A train-built lexicon must not have seen the held-out sentences, or + // held-out words carry only the readings they were seen with and every + // weight is fitted on an easier set than test. + let own; + let lex = if coca { + lex + } else { + own = Lexicon::from_train(&fit); + &own + }; + let tables = ["neigh", "q", "both"] + .map(|m| std::rc::Rc::new(PositionTable::learn(&fit, lex, copulas, cased, pair, m))); + let rules = rules( + Inventory::load().map(std::rc::Rc::new), + pair, + tables, + std::rc::Rc::new(copulas.clone()), + typology, + cased, + ); + + let held = pair_tokens(&held, lex, copulas, cased, pair); + let weights: Vec<(usize, f64)> = rules + .iter() + .map(|r| { + let (mut n, mut right) = (0usize, 0usize); + for (t, g) in &held { + if let Some(v) = (r.vote)(t) { + n += 1; + right += usize::from(v == *g); + } + } + (n, (right as f64 + 0.5) / (n as f64 + 1.0)) + }) + .collect(); + // A rule votes only with held-out support and better than chance. + let mut order: Vec = (0..rules.len()) + .filter(|&i| weights[i].0 >= 5 && weights[i].1 > 0.5) + .collect(); + order.sort_by(|&x, &y| weights[y].1.total_cmp(&weights[x].1)); + let logit = |p: f64| { + let p = p.min(0.99); + (p / (1.0 - p)).ln() + }; + + // The joint quorum: a logistic regression over every rule's vote (+1 + // first reading, -1 second, 0 abstain) and the word's frequency log-odds, + // fitted on held-out. Correlated rules (the three tables agree almost + // always) share one weight between them instead of each counting in full. + let (a, b) = pair; + let features = |t: &Tok| -> Vec { + let mut f: Vec = rules + .iter() + .map(|r| match (r.vote)(t) { + Some(v) if v == a => 1.0, + Some(_) => -1.0, + None => 0.0, + }) + .collect(); + let lo = ((t.counts.0 as f64 + 0.5) / (t.counts.1 as f64 + 0.5)).ln(); + f.push(lo.clamp(-5.0, 5.0) / 5.0); + f.push(1.0); + f + }; + let xs: Vec<(Vec, f64)> = held + .iter() + .map(|(t, g)| (features(t), f64::from(u8::from(*g == a)))) + .collect(); + let dim = rules.len() + 2; + let mut w = vec![0.0f64; dim]; + let lambda = 1e-3; + for _ in 0..3000 { + let mut grad = vec![0.0f64; dim]; + for (x, y) in &xs { + let z: f64 = x.iter().zip(&w).map(|(xi, wi)| xi * wi).sum(); + let e = 1.0 / (1.0 + (-z).exp()) - y; + for (gi, xi) in grad.iter_mut().zip(x) { + *gi += e * xi; + } + } + let n = xs.len().max(1) as f64; + for (wi, gi) in w.iter_mut().zip(&grad) { + *wi -= 0.5 * (gi / n + lambda * *wi); + } + } + + let toks = pair_tokens(test, lex, copulas, cased, pair); + let mut joint_ok = 0usize; + let (mut freq_ok, mut prio_ok, mut quorum_ok, mut silent) = (0usize, 0usize, 0usize, 0usize); + let mut fired = vec![(0usize, 0usize); rules.len()]; + for (t, g) in &toks { + let freq = if t.counts.1 > t.counts.0 { b } else { a }; + freq_ok += usize::from(freq == *g); + let votes: Vec> = rules.iter().map(|r| (r.vote)(t)).collect(); + for (i, v) in votes.iter().enumerate() { + if let Some(v) = v { + fired[i].0 += 1; + fired[i].1 += usize::from(v == g); + } + } + let prio = order.iter().find_map(|&i| votes[i]); + silent += usize::from(prio.is_none()); + prio_ok += usize::from(prio.unwrap_or(freq) == *g); + let score: f64 = order + .iter() + .filter_map(|&i| { + votes[i].map(|v| if v == a { 1.0 } else { -1.0 } * logit(weights[i].1)) + }) + .sum(); + let quorum = match score.partial_cmp(&0.0) { + Some(std::cmp::Ordering::Greater) => a, + Some(std::cmp::Ordering::Less) => b, + _ => freq, + }; + quorum_ok += usize::from(quorum == *g); + let z: f64 = features(t).iter().zip(&w).map(|(xi, wi)| xi * wi).sum(); + joint_ok += usize::from(if z > 0.0 { a } else { b } == *g); + } + let pct = |x: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * x as f64 / d as f64 + } + }; + let n = toks.len(); + println!( + " {name} quorum: {} rules, {} voting; {n} test tokens ({silent} with no vote) — \ + frequency {:.1}%, priority {:.1}%, summed quorum {:.1}%, joint quorum {:.1}%", + rules.len(), + order.len(), + pct(freq_ok, n), + pct(prio_ok, n), + pct(quorum_ok, n), + pct(joint_ok, n) + ); + if std::env::var_os("UD_RULES").is_some() { + for i in 0..rules.len() { + let voting = if order.contains(&i) { ' ' } else { 'x' }; + println!( + " {voting} {:46} held-out {:5} at {:5.1}% | test {:5} at {:5.1}% | joint w {:+.2}", + rules[i].name, + weights[i].0, + 100.0 * weights[i].1, + fired[i].0, + pct(fired[i].1, fired[i].0), + w[i] + ); + } + } +} + +fn clone_sentence(s: &[Word]) -> Vec { + s.iter() + .map(|w| Word { + form: w.form.clone(), + surface: w.surface.clone(), + upos_noun: w.upos_noun, + copula: w.copula, + gold: w.gold, + amod_head_after: w.amod_head_after, + }) + .collect() +} + /// Score for one pair of readings. #[derive(Default)] struct PairScore { @@ -811,6 +1378,17 @@ fn main() { ); nv_score.print("noun/verb", nv); position_table_report(&train, &test, &lex, &copulas, cased, nv, "noun/verb"); + quorum_report( + &train, + &test, + &lex, + coca.is_some(), + &copulas, + cased, + typology, + nv, + "noun/verb", + ); // A parallel coordinate: WordNet's own noun/verb sense counts as the // prior, scored on the same tokens (COCA mode supplies the lemmas). if let Some(path) = std::env::var("UD_WORDNET").ok().filter(|p| !p.is_empty()) { @@ -851,6 +1429,17 @@ fn main() { } aa_score.print("adjective/adverb", aa); position_table_report(&train, &test, &lex, &copulas, cased, aa, "adjective/adverb"); + quorum_report( + &train, + &test, + &lex, + coca.is_some(), + &copulas, + cased, + typology, + aa, + "adjective/adverb", + ); // Each attribute clause on its own: every adjective/adverb token whose // gold is one of the two, the clause that fires, and whether the clause From 90c3750f62ca9fc83f80014efc1ee1fe95564e87 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:41:57 +0000 Subject: [PATCH 08/33] deepnsm-v2: the Frageprobe through the Satzklammer; question mask is CausalMask (D-LXC-20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fsm::answered_questions now returns causal_edge::pearl::CausalMask, the Pearl 2³ ladder CausalEdge64 packs, instead of private constants in the reverse bit order. deepnsm-v2 takes a path dependency on the zero-dependency causal-edge crate. ud_pos_eval adds the question test as German places its answer: a predicative at the right edge of a copula clause (88.5 % right on German test, against 35.3 % for "after a copula"), a manner adverb before the right bracket of a full-verb clause (90.0 %). German adjective/adverb joint quorum 81.6 -> 85.1 %; with the two rules silenced it is 81.9 %. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...10-04-deepnsm-v2-satzklammer-frageprobe.md | 52 +++++++++++++++++++ .claude/board/entries/README.md | 3 +- ...deepnsm-v2-lexical-evidence-consumer-v1.md | 8 +++ crates/deepnsm-v2/Cargo.toml | 3 ++ crates/deepnsm-v2/examples/ud_pos_eval.rs | 46 +++++++++++++++- crates/deepnsm-v2/src/fsm.rs | 35 ++++++------- crates/deepnsm-v2/src/lib.rs | 1 + 8 files changed, 126 insertions(+), 23 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-satzklammer-frageprobe.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 78352ce23..9abd797d6 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -43,6 +43,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-17** | German TEKAMOLO cues by verb position (`tekamolo_de`); adverbial adjectives → Adv fold; Te-Ka-Mo-Lo order law measured | In PR | cue words: position × frequency 88.5 % vs frequency 84.1 % (208 tokens, `da` 87.5 vs 37.5); order law 46–65 % on gold, not usable. Entry `2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md` | | **D-LXC-18** | question test as the Pearl 2³ mask (`fsm::answered_questions`, `UD_CTX`); blast radius of deleted grammar heuristics | In PR | 2³ context: Animal Farm adj/adv 91.8 % on decided (frequency 78.4 %), German adj/adv 85.1 % (best); noun/verb better with neighbours. German TEKAMOLO capability was lost in `68955ecb`; tenant writer reads the condemned v1 WordNet rail. Entry `2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md` | | **D-LXC-19** | literal rules, priorities and a quorum for word-class homographs (`ud_pos_eval` `UD_RULES`, `UD_DE_INVENTORY`) | In PR | 14–20 rules per pair; held-out-fitted joint quorum best or tied in 7/10 rows (UD English n/v 93.7 %, de n/v 98.8 %); summed quorum fails on correlated voters (de n/v 84.9 %); Animal Farm adj/adv and German adj/adv stay best with position × frequency. Entry `2026-10-04-deepnsm-v2-rule-quorum.md` | +| **D-LXC-20** | Frageprobe through the Satzklammer; `answered_questions` returns `causal_edge::CausalMask` | In PR | German predicative at the copula clause's right edge 88.5 %, manner adverb before the right bracket 90.0 % (vs after-copula 35.3 %); German adj/adv joint quorum 81.6 → 85.1 % (disable: 81.9 %). Entry `2026-10-04-deepnsm-v2-satzklammer-frageprobe.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-satzklammer-frageprobe.md b/.claude/board/entries/2026-10-04-deepnsm-v2-satzklammer-frageprobe.md new file mode 100644 index 000000000..150bfa574 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-satzklammer-frageprobe.md @@ -0,0 +1,52 @@ +# 2026-10-04 — deepnsm-v2: the Frageprobe through the Satzklammer; the question mask is CausalMask (D-LXC-20) + +**Status:** MEASURED (`UD_RULES=1 UD_DE_INVENTORY= cargo run --release +--example ud_pos_eval -- de_gsd-ud-train.conllu de_gsd-ud-test.conllu`; debug-0) +and TEST-PINNED (`fsm::tests::answered_questions_follow_the_clause`). + +Operator (2026-10-04): the 2³ ladder lives in CausalEdge64. Does *schnell* +with a question word mean the manner of an act (*schnell denken*) or a +property (*schnell sein*)? Satzklammer was the other heuristic used. + +## The mask has one bit order + +`fsm::answered_questions` now returns `causal_edge::pearl::CausalMask` +(S = `0b100`, P = `0b010`, O = `0b001`; the field CausalEdge64 packs at bits +[40:42]). The private `ANSWERED_*` constants used the reverse order and are +gone. deepnsm-v2 gains a path dependency on the zero-dependency `causal-edge`. +The tables only relabel their keys: Animal Farm adj/adv in `q` mode is still +91.8 % on 255 decided tokens. + +## The Frageprobe needs the verb, and German puts the answer at the bracket + +*Wie denkt er? — schnell* (adverb, Modal) and *Wie ist er? — schnell* +(predicative adjective) share the question word. What separates them is the +verb the question asks about. In German the predicative stands at the right +edge of a copula clause, and the manner adverb stands in the Mittelfeld just +before the right bracket (the participle or infinitive). Three rules, scored +in the D-LXC-19 quorum: + +| German rule (test tokens) | fires | right | +|---|---|---| +| after a copula → adj (D-LXC-19) | 17 | 35.3 % | +| copula clause, right edge → adj | 26 | **88.5 %** | +| full-verb clause, before the bracket → adv | 20 | **90.0 %** | +| full-verb clause, clause-final → adv | 15 | 53.3 % (votes no) | + +German adj/adv joint quorum: 81.6 → **85.1 %**. On the same tokens, frequency +is 79.3 %; position × frequency was 82.9 % on its own token set. Disable run: +with the first two rules forced silent it falls to 81.9 %. UD English (COCA) +adj/adv 92.0 → 93.0 %. In English the full-verb rules rarely fire, because +English verbs are rarely verb-only in the lexicon. + +## OPEN + +- No qualia-adjacency signal exists for word senses: the qualia modules + (`contract::qualia`, `cognitive::grammar::qualia`) are felt-state + coordinates with keyword heuristics, not a word × head compatibility table. + The measured stand-in is the verb class the word attaches to. A real + adjacency (does *schnell* combine with events or with entities) would need a + head-word table mined from the treebank's `advmod` / `amod` / `xcomp` + heads, and it is not built. +- `ontology_vocab.rs` (WordNet) was not found in lance-graph, OGAR or ndarray + on any branch, or by org code search. Its repo is unknown. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 5bf155ac0..6d699451c 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -193 entries, 2026-08-06 .. 2026-10-04. +194 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| +| 2026-10-04 | `D-LXC-20` | | [2026-10-04-deepnsm-v2-satzklammer-frageprobe.md](2026-10-04-deepnsm-v2-satzklammer-frageprobe.md) | | 2026-10-04 | `D-LXC-19` | | [2026-10-04-deepnsm-v2-rule-quorum.md](2026-10-04-deepnsm-v2-rule-quorum.md) | | 2026-10-04 | `D-LXC-18` | | [2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md](2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md) | | 2026-10-04 | `D-LXC-13` | | [2026-10-04-deepnsm-v2-predicate-by-position.md](2026-10-04-deepnsm-v2-predicate-by-position.md) | diff --git a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md index e3b7eaca6..360773027 100644 --- a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md +++ b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md @@ -504,3 +504,11 @@ weights instead fails, because the position tables are correlated voters. German inflection against the lemma replaces the English noun-adjacency rule as the German attribute signal. Board entry `2026-10-04-deepnsm-v2-rule-quorum.md`. + +## D-LXC-20: the Frageprobe through the Satzklammer (2026-10-04) + +The question mask is `causal_edge::CausalMask`, the ladder CausalEdge64 packs. +The German adjective/adverb question is answered by where the word sits +against the Satzklammer: a predicative at the right edge of a copula clause, +a manner adverb before the right bracket of a full-verb clause. German joint +quorum 85.1 %. Board entry `2026-10-04-deepnsm-v2-satzklammer-frageprobe.md`. diff --git a/crates/deepnsm-v2/Cargo.toml b/crates/deepnsm-v2/Cargo.toml index af2afd177..6881391e2 100644 --- a/crates/deepnsm-v2/Cargo.toml +++ b/crates/deepnsm-v2/Cargo.toml @@ -27,6 +27,9 @@ crate is untouched; this is the parallel v2. # `identity_v2()` (bytes 14..16) cannot read them back. Every key this crate # minted before 2026-08-22 was a V1 key for that reason. lance-graph-contract = { path = "../lance-graph-contract", features = ["guid-v3-tail"] } +# The Pearl 2³ ladder: `fsm::answered_questions` returns its `CausalMask`, the +# same eight masks CausalEdge64 packs at bits [40:42]. Zero-dependency crate. +causal-edge = { path = "../causal-edge" } # `test = true` makes `cargo test` run the example's own `#[cfg(test)]` tests # (the D-LXC-1 tagger). Cargo only compiles an example by default; its `main()` diff --git a/crates/deepnsm-v2/examples/ud_pos_eval.rs b/crates/deepnsm-v2/examples/ud_pos_eval.rs index c59e78175..ecdb9f4b1 100644 --- a/crates/deepnsm-v2/examples/ud_pos_eval.rs +++ b/crates/deepnsm-v2/examples/ud_pos_eval.rs @@ -57,8 +57,9 @@ use std::collections::HashMap; use deepnsm_v2::coca::fsm_pos_tag; use deepnsm_v2::fsm::{ answered_questions, attribute_rule, parse_readings_with, AdjectiveOrder, AttributeRule, Pos, - PosSet, Reading, Tagged, Typology, ANSWERED_S, + PosSet, Reading, Tagged, Typology, }; +use deepnsm_v2::CausalMask; /// One syntactic word of a UD sentence. struct Word { @@ -355,6 +356,9 @@ fn answered_masks(lex: &Lexicon, s: &[Word], cased: bool) -> Vec { }) .collect(); answered_questions(&tags) + .into_iter() + .map(|m| m as u8) + .collect() } fn bits(p: PosSet) -> u8 { @@ -589,6 +593,15 @@ struct Tok<'a> { ctx: Context, /// Lexicon counts of the pair's first and second reading for the word. counts: (usize, usize), + /// A copula stands elsewhere in the clause (between punctuation marks). + copula_clause: bool, + /// The clause has a word that can only be a verb, other than this one. + verb_clause: bool, + /// Nothing follows in the clause. + clause_final: bool, + /// Something follows in the clause, and all of it can only be a verb: the + /// word stands right before the Satzklammer's right bracket. + before_bracket: bool, } type Vote = Box Option>; @@ -711,7 +724,7 @@ fn rules( fixed.push(( "slot: subject answered, prev a noun -> verb", Box::new(|x: &Tok| { - (x.ctx.4 == ANSWERED_S && only(x.prev, Pos::Noun)).then_some(Pos::Verb) + (x.ctx.4 == CausalMask::S as u8 && only(x.prev, Pos::Noun)).then_some(Pos::Verb) }), )); fixed.push(( @@ -850,6 +863,24 @@ fn rules( }), )); } + fixed.push(( + "Frageprobe: copula clause, right edge -> adj", + Box::new(|x: &Tok| { + (x.copula_clause && (x.clause_final || x.before_bracket)).then_some(Pos::Adj) + }), + )); + fixed.push(( + "Frageprobe: full-verb clause, before the bracket -> adv", + Box::new(|x: &Tok| { + (!x.copula_clause && x.verb_clause && x.before_bracket).then_some(Pos::Adv) + }), + )); + fixed.push(( + "Frageprobe: full-verb clause, clause-final -> adv", + Box::new(|x: &Tok| { + (!x.copula_clause && x.verb_clause && x.clause_final).then_some(Pos::Adv) + }), + )); let c = cop.clone(); fixed.push(( "clause-final, no copula before -> adv", @@ -888,8 +919,19 @@ fn pair_tokens<'a>( .copied() .unwrap_or(0) }; + let is_break = |w: &Word| !w.surface.chars().any(char::is_alphanumeric); + let lo = (0..k).rev().find(|&j| is_break(&s[j])).map_or(0, |j| j + 1); + let hi = (k + 1..s.len()) + .find(|&j| is_break(&s[j])) + .unwrap_or(s.len()); + let others = || (lo..hi).filter(|&j| j != k); + let verb_only = |j: usize| set(j) == PosSet::single(Pos::Verb); out.push(( Tok { + copula_clause: others().any(|j| copulas.contains(&s[j].form.to_lowercase())), + verb_clause: others().any(verb_only), + clause_final: k + 1 == hi, + before_bracket: k + 1 < hi && (k + 1..hi).all(verb_only), this, prev: k.checked_sub(1).map_or(PosSet::EMPTY, &set), next: if k + 1 < s.len() { diff --git a/crates/deepnsm-v2/src/fsm.rs b/crates/deepnsm-v2/src/fsm.rs index 97206b740..7ef141d4f 100644 --- a/crates/deepnsm-v2/src/fsm.rs +++ b/crates/deepnsm-v2/src/fsm.rs @@ -41,6 +41,7 @@ use crate::spo::Spo; use crate::vocab::WordId; +use causal_edge::pearl::CausalMask; /// A coarse part-of-speech tag — the eight the FSM distinguishes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -307,37 +308,31 @@ pub fn parse_to_spo(tokens: &[Tagged]) -> Vec { out } -/// Question-test bits of [`answered_questions`]: which of the SPO questions -/// the clause has already answered — *who/what?* (subject), *does what?* -/// (predicate), *whom/what?* (object). The 8 values are the Pearl 2³ masks. -pub const ANSWERED_S: u8 = 1; -/// See [`ANSWERED_S`]. -pub const ANSWERED_P: u8 = 2; -/// See [`ANSWERED_S`]. -pub const ANSWERED_O: u8 = 4; - -/// For each token, the 2³ mask of SPO questions its clause has answered +/// For each token, the Pearl 2³ mask of SPO questions its clause has answered /// BEFORE it arrives — the school question test (*Frageprobe*) as a position: -/// a word arriving after "who?" is answered and before "does what?" is, is -/// asked "does what?". Steps the same transition table as [`parse_to_spo`]; -/// a `Stop` starts a new clause (mask 0). +/// a word arriving after *who/what?* (subject) is answered and before *does +/// what?* (predicate) is, is asked *does what?*; *whom/what?* is the object. +/// The mask is [`CausalMask`], the same eight values `CausalEdge64` packs, so +/// the ladder has one bit order (S = `0b100`, P = `0b010`, O = `0b001`). Steps +/// the same transition table as [`parse_to_spo`]; a `Stop` starts a new +/// clause ([`CausalMask::None`]). #[must_use] -pub fn answered_questions(tokens: &[Tagged]) -> Vec { +pub fn answered_questions(tokens: &[Tagged]) -> Vec { let mut out = Vec::with_capacity(tokens.len()); let mut core = Core::START; let mut emitted = false; for &t in tokens { let mut mask = 0; if matches!(core.state, State::HaveSubject | State::HaveVerb) { - mask |= ANSWERED_S; + mask |= CausalMask::S as u8; } if core.state == State::HaveVerb || emitted { - mask |= ANSWERED_P; + mask |= CausalMask::P as u8; } if emitted { - mask |= ANSWERED_O; + mask |= CausalMask::O as u8; } - out.push(mask); + out.push(CausalMask::from_bits(mask)); if t.pos == Pos::Stop { core.state = State::Start; core.rel = Rel::None; @@ -1252,8 +1247,8 @@ mod tests { fn answered_questions_follow_the_clause() { let toks = [n(1), v(2), n(3), Tagged::new(0, Pos::Stop), n(4)]; let m = answered_questions(&toks); - let spo = ANSWERED_S | ANSWERED_P | ANSWERED_O; - assert_eq!(m, vec![0, ANSWERED_S, ANSWERED_S | ANSWERED_P, spo, 0]); + use CausalMask::{None as No, S, SP, SPO}; + assert_eq!(m, vec![No, S, SP, SPO, No]); } /// T4: a single-reading word is never rejected, even where the licensing diff --git a/crates/deepnsm-v2/src/lib.rs b/crates/deepnsm-v2/src/lib.rs index b625f07fb..474a625e0 100644 --- a/crates/deepnsm-v2/src/lib.rs +++ b/crates/deepnsm-v2/src/lib.rs @@ -59,6 +59,7 @@ pub use basin::{ HeldOutGate, }; pub use belief::{Belief, BeliefArena, CStmt, Copula, ReviseOutcome, Stamp}; +pub use causal_edge::pearl::CausalMask; pub use codebook::{load_cam96_codes, load_cam96_space, CodebookError}; pub use evidence::{ evidence_basin, forward_gate, novelty_rate, open_question_yield, partial_spearman, From 488243eaefaaee12c49b4b14531cc8774b8cb5b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:47:21 +0000 Subject: [PATCH 09/33] deepnsm-v2: a floating quantifier keeps the verb slot open (Bugbot, #1321) Licensing dropped a homograph's verb reading after any determiner. A determiner that comes straight after the subject's head noun while the clause still waits for its verb ("the men all eat") is a quantifier floating off the subject, not an object's start, so the next word's verb reading now survives. A determiner after any other word ("when the land") still opens a group; an adjective after the subject still does too. Test a_floating_quantifier_keeps_the_verb_slot_open, disable-verified (red with the new flag forced false). KJV pins re-pinned with the difference recorded in place: 651 tokens stay ambiguous instead of being forced to a noun, legacy triples lost 1,231 -> 1,130, certain 57,350 -> 57,277. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- crates/deepnsm-v2/examples/bible_wave.rs | 8 +++- crates/deepnsm-v2/src/fsm.rs | 58 +++++++++++++++++++++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/crates/deepnsm-v2/examples/bible_wave.rs b/crates/deepnsm-v2/examples/bible_wave.rs index 801d5690b..6fe82ac22 100644 --- a/crates/deepnsm-v2/examples/bible_wave.rs +++ b/crates/deepnsm-v2/examples/bible_wave.rs @@ -265,6 +265,10 @@ impl LexicalDecodeReport { // decided by position (D-LXC-13, reopens D-LXC-3). Before D-LXC-13: // reading (771_176, 683_805, 3_363, 141, 84_008, 1_908, 1_455, 179, // 35), triples (70_393, 69_670, 1_716, 732, 113, 122, 1_131, 16). + // Floating quantifier (Bugbot on #1321): a determiner straight after + // the subject's head no longer licenses away the next verb reading. + // Before it: reading (…, 14_383, 20_441, 1_943, 162), triples + // (70_393, 57_350, 29_001, 12_984, 1_231, 1_190, 12_437, 256). // A deliberate decoder change re-pins these with the difference // reported. assert_eq!( @@ -279,7 +283,7 @@ impl LexicalDecodeReport { self.legacy_eliminated, self.eliminated_words.len(), ), - (771_176, 652_344, 34_824, 451, 84_008, 14_383, 20_441, 1_943, 162), + (771_176, 652_344, 34_824, 451, 84_008, 13_732, 21_092, 1_805, 155), "KILL D-LXC-2: reading accounting moved from the pinned KJV layout" ); assert_eq!( @@ -293,7 +297,7 @@ impl LexicalDecodeReport { self.verses_changed, self.peak_configs, ), - (70_393, 57_350, 29_001, 12_984, 1_231, 1_190, 12_437, 256), + (70_393, 57_277, 29_684, 13_151, 1_130, 1_183, 12_581, 256), "KILL D-LXC-2: triple accounting moved from the pinned KJV layout" ); println!("D-LXC-2 PASS accounting matches the pinned KJV layout"); diff --git a/crates/deepnsm-v2/src/fsm.rs b/crates/deepnsm-v2/src/fsm.rs index 7ef141d4f..1abc4285d 100644 --- a/crates/deepnsm-v2/src/fsm.rs +++ b/crates/deepnsm-v2/src/fsm.rs @@ -151,6 +151,15 @@ struct Core { /// The open nominal group was opened by an adjective, not a determiner. /// Licensing reads it only where [`Typology::adjective_opens_nominal`]. adjective_open: bool, + /// The previous token was a determiner that came straight after the + /// subject's head noun while the clause still waited for its verb: a + /// quantifier floating off the subject ("the men **all** eat"), not an + /// object's start. A determiner after any other word ("when **the** land") + /// opens a group as usual. Read only by licensing. + floating_quantifier: bool, + /// The previous token was a noun. Read only to recognise a floating + /// quantifier, which follows the subject's head directly. + after_noun: bool, /// The previous token opened the clause as its subject — a noun taken in /// [`State::Start`] with no determiner or adjective in front ("they", /// "men", "God"). An object carried into the subject slot ("gave him …") @@ -173,6 +182,8 @@ impl Core { antecedent: 0, nominal_open: false, adjective_open: false, + floating_quantifier: false, + after_noun: false, fresh_subject: false, predicated: false, }; @@ -185,6 +196,9 @@ impl Core { self.fresh_subject = false; self.nominal_open = matches!(t.pos, Pos::Det | Pos::Adj); self.adjective_open = t.pos == Pos::Adj; + self.floating_quantifier = + t.pos == Pos::Det && self.after_noun && matches!(self.state, State::HaveSubject); + self.after_noun = t.pos == Pos::Noun; // While a relative clause is open, its own tiny machine consumes the // embedded S-V-O; the matrix subject stays parked in `matrix`. @@ -581,8 +595,16 @@ impl Default for Typology { /// [`Typology`] says an adjective always opens one. [`parse_readings`] /// applies it relatively: it never removes a token's last admissible /// reading. +/// +/// Except a determiner that arrived while the clause held a subject and still +/// waited for its verb ([`Core::floating_quantifier`]): it is a quantifier +/// floating off the subject ("the men **all** eat"), not the start of an +/// object, so the verb slot is still open. An adjective there still opens a +/// group. const fn licensed(core: &Core, pos: Pos, typology: Typology) -> bool { - let open = core.nominal_open && (!core.adjective_open || typology.adjective_opens_nominal); + let open = core.nominal_open + && (!core.adjective_open || typology.adjective_opens_nominal) + && !core.floating_quantifier; !(open && matches!(pos, Pos::Verb)) } @@ -1108,6 +1130,40 @@ mod tests { assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Noun)); } + /// Bugbot (#1321): a quantifier floating after the subject ("the men + /// **all** eat", "they **both** agree") opens no object group: the clause + /// still waits for its verb, so licensing keeps the homograph's verb + /// reading. A determiner after the verb ("men saw the record") or at the + /// clause start ("the record") still drops it. + #[test] + fn a_floating_quantifier_keeps_the_verb_slot_open() { + let p = parse_readings(&[ + one(9, Pos::Det), + one(1, Pos::Noun), + one(8, Pos::Det), + noun_or_verb(2), + ]); + assert!(p.ambiguous[0].survived.contains(Pos::Verb)); + assert_eq!(p.unlicensed_dropped, 0); + let p = parse_readings(&[ + one(1, Pos::Noun), + one(4, Pos::Verb), + one(9, Pos::Det), + noun_or_verb(2), + ]); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Noun)); + let p = parse_readings(&[one(9, Pos::Det), noun_or_verb(2)]); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Noun)); + // "man when the land": a determiner after a non-noun opens a group. + let p = parse_readings(&[ + one(1, Pos::Noun), + one(7, Pos::Other), + one(9, Pos::Det), + noun_or_verb(2), + ]); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Noun)); + } + const ADJ_ADV: PosSet = PosSet::single(Pos::Adj).with(Pos::Adv); /// D-LXC-14: each attribute clause matches its own position, and none From e717e3de926e2ddc74c815b8d0c36abf35b1d552 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:57:58 +0000 Subject: [PATCH 10/33] deepnsm(-v2): restore the deleted grammar heuristics; WordNet identity register (D-LXC-21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restored, unchanged unless noted: - deepnsm-v2 toc/hydrate/promote (68955ecb): the KJV table of contents as an HHTL tree, triples addressed to their verse node, basins at TOC keys; toc_hydrate example runs green on the full KJV (32,357 nodes, 70,393 triples, 0 unaddressed, 1,227 basins). - deepnsm-v2 lexicon, loci, tekamolo (German right-corner lanes). - deepnsm persona_chain_replay (8c03aaa9): ContextChain replay with real candidates; measures the margin gate 335x too coarse for the rank metric. - deepnsm ontology_vocab (48405aa2), repurposed: the OBO half answers empty (98de36eb retracted the 0x03 rows; test-pinned), and a WordNet identity register is added -- WordNetRail (all senses, all hypernym edges, exact arity), is_a, and ConceptMask, which covers a sense when it or an ancestor is a mask root. No distances. Tested on real WordNet rows (whale/dolphin vs shark/manta vs octopus); both guards disable-verified. Ledger of which commit held what, and the merge queue: .claude/knowledge/deleted-grammar-heuristics-ledger.md. Open: tekamolo's V2StyleProvider (refused in deepnsm-v2 by §3o) and loci's missing agreement filter. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + .claude/board/SUPERSESSION-INDEX.md | 2 +- ...-10-04-deepnsm-deleted-grammar-restored.md | 45 ++ .claude/board/entries/README.md | 3 +- .../deleted-grammar-heuristics-ledger.md | 55 ++ crates/deepnsm-v2/examples/toc_hydrate.rs | 315 ++++++++ crates/deepnsm-v2/src/hydrate.rs | 370 +++++++++ crates/deepnsm-v2/src/lexicon.rs | 307 ++++++++ crates/deepnsm-v2/src/lib.rs | 7 + crates/deepnsm-v2/src/loci.rs | 357 +++++++++ crates/deepnsm-v2/src/promote.rs | 591 ++++++++++++++ crates/deepnsm-v2/src/tekamolo.rs | 718 ++++++++++++++++++ crates/deepnsm-v2/src/toc.rs | 426 +++++++++++ .../deepnsm/examples/persona_chain_replay.rs | 376 +++++++++ crates/deepnsm/src/lib.rs | 1 + crates/deepnsm/src/ontology_vocab.rs | 382 ++++++++++ crates/deepnsm/src/wordnet_excerpt.tsv | 41 + 17 files changed, 3995 insertions(+), 2 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-deleted-grammar-restored.md create mode 100644 .claude/knowledge/deleted-grammar-heuristics-ledger.md create mode 100644 crates/deepnsm-v2/examples/toc_hydrate.rs create mode 100644 crates/deepnsm-v2/src/hydrate.rs create mode 100644 crates/deepnsm-v2/src/lexicon.rs create mode 100644 crates/deepnsm-v2/src/loci.rs create mode 100644 crates/deepnsm-v2/src/promote.rs create mode 100644 crates/deepnsm-v2/src/tekamolo.rs create mode 100644 crates/deepnsm-v2/src/toc.rs create mode 100644 crates/deepnsm/examples/persona_chain_replay.rs create mode 100644 crates/deepnsm/src/ontology_vocab.rs create mode 100644 crates/deepnsm/src/wordnet_excerpt.tsv diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 9abd797d6..64b3e15df 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -44,6 +44,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-18** | question test as the Pearl 2³ mask (`fsm::answered_questions`, `UD_CTX`); blast radius of deleted grammar heuristics | In PR | 2³ context: Animal Farm adj/adv 91.8 % on decided (frequency 78.4 %), German adj/adv 85.1 % (best); noun/verb better with neighbours. German TEKAMOLO capability was lost in `68955ecb`; tenant writer reads the condemned v1 WordNet rail. Entry `2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md` | | **D-LXC-19** | literal rules, priorities and a quorum for word-class homographs (`ud_pos_eval` `UD_RULES`, `UD_DE_INVENTORY`) | In PR | 14–20 rules per pair; held-out-fitted joint quorum best or tied in 7/10 rows (UD English n/v 93.7 %, de n/v 98.8 %); summed quorum fails on correlated voters (de n/v 84.9 %); Animal Farm adj/adv and German adj/adv stay best with position × frequency. Entry `2026-10-04-deepnsm-v2-rule-quorum.md` | | **D-LXC-20** | Frageprobe through the Satzklammer; `answered_questions` returns `causal_edge::CausalMask` | In PR | German predicative at the copula clause's right edge 88.5 %, manner adverb before the right bracket 90.0 % (vs after-copula 35.3 %); German adj/adv joint quorum 81.6 → 85.1 % (disable: 81.9 %). Entry `2026-10-04-deepnsm-v2-satzklammer-frageprobe.md` | +| **D-LXC-21** | deleted grammar heuristics restored (deepnsm-v2 toc/hydrate/promote/lexicon/loci/tekamolo + toc_hydrate; deepnsm ontology_vocab + persona_chain_replay); WordNet identity register + ConceptMask | In PR | toc_hydrate on KJV: 32,357 HHTL nodes, 70,393 triples addressed, 1,227 basins at TOC keys; ContextChain margin 335× too coarse for the rank metric. Ledger `.claude/knowledge/deleted-grammar-heuristics-ledger.md`; entry `2026-10-04-deepnsm-deleted-grammar-restored.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/SUPERSESSION-INDEX.md b/.claude/board/SUPERSESSION-INDEX.md index 7d79f6c02..6e599dcd4 100644 --- a/.claude/board/SUPERSESSION-INDEX.md +++ b/.claude/board/SUPERSESSION-INDEX.md @@ -62,7 +62,7 @@ coverage instead. | `ResonanceDto` | REPURPOSE | `PerturbationDto` | 2 | 12 | 7 | | `BindSpace` | RETIRE | — | 68 | 48 | 42 | | `CollapseGateEmission` | RETIRE | — | 5 | 14 | 12 | -| `ThinkingStyle` | RETIRE-toward-contract | — | 52 | 28 | 24 | +| `ThinkingStyle` | RETIRE-toward-contract | — | 53 | 28 | 24 | ## Table 2 — plans naming a ruled symbol without citing the ruling (76) diff --git a/.claude/board/entries/2026-10-04-deepnsm-deleted-grammar-restored.md b/.claude/board/entries/2026-10-04-deepnsm-deleted-grammar-restored.md new file mode 100644 index 000000000..d811ba91f --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-deleted-grammar-restored.md @@ -0,0 +1,45 @@ +# 2026-10-04 — deleted grammar heuristics restored; WordNet identity register (D-LXC-21) + +**Status:** VERIFIED-IN-CODE and MEASURED (`cargo test --release` deepnsm-v2: 194 lib tests; deepnsm: 222; `toc_hydrate` and `persona_chain_replay` run on real data, debug-0). + +Operator (2026-10-04): restore the deleted grammar heuristics, repurpose or merge the superseded ones, drop none, and write down which commit had what. Ledger: `.claude/knowledge/deleted-grammar-heuristics-ledger.md`. + +## Restored + +**From `68955ecb`, into deepnsm-v2:** +- `toc.rs`, `hydrate.rs` and `promote.rs`: the KJV TOC as an HHTL tree, triples addressed to verse nodes, and basins promoted at TOC keys. +- `lexicon.rs`. +- `loci.rs`. +- `tekamolo.rs`: the German right-corner lanes. +- the `toc_hydrate` example. + +**Into deepnsm:** +- `ontology_vocab.rs`, from `48405aa2`. +- the `persona_chain_replay` example, from `8c03aaa9`. + +## Repurposed + +**`ontology_vocab.rs` now holds a WordNet identity register:** +- `WordNetRail` reads every sense and every hypernym edge. +- `ConceptMask` covers a sense when the sense or any ancestor is a mask root, so a mask can cut across the taxonomy (e.g. water animals). +- There are no distances, so A1 holds. +- Its OBO half answers empty, because `98de36eb` retracted the 0x03 rows; a test pins that. +- Disable runs: + - is_a without the ancestor walk → 3 of 5 tests red; + - every sense collapsed to number 1 → 2 red. + +## Measured + +**`toc_hydrate` on the KJV:** +- 32,357 nodes (66 books, 31,102 verses); +- 70,393 triples, 0 unaddressed; +- 38.8 % cross-verse links; +- 1,227 basins at TOC keys; +- loci: Recency binds 35,025 of 35,613 pronoun subjects, SelectionalFit binds 27,788. + +**`persona_chain_replay`:** the ContextChain margin gate is 335× too coarse for the rank metric (spread 0.0003 vs threshold 0.1), so every replay escalates. + +## OPEN + +- `tekamolo.rs` still carries `V2StyleProvider`. §3o refused thinking styles in deepnsm-v2; the operator decides whether to keep it here or move it to the planner. +- `loci.rs` is agreement-blind. The shipped agreement resolver scores 0.727 against 0.273 for the agreement-blind rule. Merge queue in the ledger. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 6d699451c..160f65cac 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -194 entries, 2026-08-06 .. 2026-10-04. +195 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| @@ -37,6 +37,7 @@ exactly one of them, never both. | 2026-10-04 | `D-LXC-16` | | [2026-10-04-deepnsm-v2-learned-position-table.md](2026-10-04-deepnsm-v2-learned-position-table.md) | | 2026-10-04 | `D-LXC-15` | | [2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md](2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md) | | 2026-10-04 | `D-LXC-17` | | [2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md](2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md) | +| 2026-10-04 | `D-LXC-21` | | [2026-10-04-deepnsm-deleted-grammar-restored.md](2026-10-04-deepnsm-deleted-grammar-restored.md) | | 2026-10-03 | `population-law-crosscheck` | | [2026-10-03-population-law-crosscheck.md](2026-10-03-population-law-crosscheck.md) | | 2026-10-03 | `orkg-reference-harvest` | | [2026-10-03-orkg-reference-harvest.md](2026-10-03-orkg-reference-harvest.md) | | 2026-10-03 | `known-is-not-alpha-premise-gate` | | [2026-10-03-known-is-not-alpha-premise-gate.md](2026-10-03-known-is-not-alpha-premise-gate.md) | diff --git a/.claude/knowledge/deleted-grammar-heuristics-ledger.md b/.claude/knowledge/deleted-grammar-heuristics-ledger.md new file mode 100644 index 000000000..56b05d733 --- /dev/null +++ b/.claude/knowledge/deleted-grammar-heuristics-ledger.md @@ -0,0 +1,55 @@ +# Deleted grammar heuristics — which commit had what, and where it is now + +READ BY: anyone touching deepnsm / deepnsm-v2 grammar, TEKAMOLO, loci / +anaphora, the TOC→HHTL tree, WordNet/ontology registers, or ContextChain +replay — BEFORE writing a "new" one. Also read before deleting any of these +again. + +Restored 2026-10-04 on operator instruction: *"restore the rest of the +deleted and repurpose or merge the ideas when you think they are superseded +but don't drop and create a document which commit had what so it can be used +for later reference."* + +## How to read a row + +- **commit**: where the file left the tree. Retrieve the last version with `git show ^:`. +- **born in**: the commits that built it. Read them for the design reasoning. +- **now**: one of + - RESTORED: the file is back unchanged; + - RESTORED+REPURPOSED: the file is back, with a stated change; + - SUPERSEDED-IN-PLACE: the idea lives elsewhere, which is named; + - HISTORY-ONLY: the code is not restored and the reason is given. +- **measured**: numbers the code itself produced, with where. + +## The ledger + +| path | lines | commit (date) | born in | what it held | why it was deleted | now | +|---|---|---|---|---|---|---| +| `crates/deepnsm-v2/src/toc.rs` | 426 | `68955ecb` (2026-08-22) | `abd331b9` | **The TOC as an HHTL tree of SoA node addresses.** Book:chapter:verse spawned as the full skeleton before any reasoning (operator: *"bevor wir über das Buch nachdenken wird ein Inhaltsverzeichnis als HHTL-Baum mit SoA-Knoten erstellt"*). Two nibbles per level. | "the hydrated table is rebuilt from scratch" | **RESTORED**. This is the HHTL literal chapter:verse address; the basins below are the cross-chapter nodes. | +| `crates/deepnsm-v2/src/hydrate.rs` | 370 | `68955ecb` | `bfc69bf8`, `6e385c88` | Every SPO triple joined to the verse node it was read in. The reading-order chain is kept (the Markov trajectory over the book) along with the scanpath residue. | as `toc.rs` | **RESTORED**. | +| `crates/deepnsm-v2/src/promote.rs` | 591 | `68955ecb` | `dd728f17`, `ec50f07b` | Basins written as SoA `NodeRow`s at TOC node keys: keyed by literary unit, not by bare subject (`E-HERMENEUTIK-RUNG-LADDER-1`). Also the TEKAMOLO tenant writer. | "also minted V1 keys" | **RESTORED**. The V1 claim was already false at deletion: `ec50f07b` made the promoter refuse the V1 tail (`mint_for` + read-back check, test-pinned). | +| `crates/deepnsm-v2/examples/toc_hydrate.rs` | 315 | `68955ecb` | `6e385c88` | The end-to-end run of tree → hydrate → promote → loci over the real KJV. | as above | **RESTORED**. Measured 2026-10-04: 32,357 nodes (66 books, 31,102 verses); 70,393 triples, 0 unaddressed; 27,298 cross-verse links (38.8 %); 1,227 basins promoted at TOC keys. | +| `crates/deepnsm-v2/src/lexicon.rs` | 307 | `68955ecb` | `ec50f07b` | One COCA lexicon: lemma → forms → archaic → Other, monotone. It replaced three hand-copied taggers. | "taggers inlined into the two remaining examples" | **RESTORED**. The inlined copies in `bible_wave` / `genre_shapes` remain (they are pin-guarded). Merging them back onto this module is a deliberate re-pin. Measured in the file: KJV verses with no triple went from 35.6 % to 12.2 % once the forms table was read. | +| `crates/deepnsm-v2/src/loci.rs` | 357 | `68955ecb` | `f47f0528` | Derives `Locus::Antecedent` (a 24×i4 `CausalWitnessFacet` nibble) from the SPO stream. Candidates are non-pronoun subjects within REACH = 8. The ranking is `SelectionalFit` (Cam96 similarity of each candidate to the pronoun clause's predicate) against `Recency`, with a margin gate. | "resolution AND binding already shipped" (`spo_anaphora_nibble`, `l9_loci_real_text`, `probe_antecedent_binder`) | **RESTORED**, with a known defect. It is agreement-blind. On Aesop the shipped agreement resolver scores 0.727 and the agreement-blind rule scores 0.273. **Merge pending:** take the gender/number/case agreement filter (and the German case table, see the research inventory) as the candidate filter, then keep SelectionalFit as the ranker. KJV numbers from `toc_hydrate`: Recency binds 35,025 of 35,613; SelectionalFit binds 27,788 (7,237 thin-margin). | +| `crates/deepnsm-v2/src/tekamolo.rs` | 718 | `f3d42b7c` (style half), `c57cc256` (reverted that revert), `68955ecb` (whole file) | `7bae6c84` | **German** right-corner TEKAMOLO lane reading. A left-corner hypothesis is opened per adverbial and committed at the right corner on margin. It reads Luther 1545 and Elberfelder 1905 as a two-translation control, uses `de/tekamolo.tsv`, and hydrates the Kausal/Modal/Lokal lanes. | "insight_coca_read already emits all four lanes" — but that replacement is **English COCA only**, so the German capability was lost (D-LXC-18 blast radius) | **RESTORED**, with a ruling conflict. `V2StyleProvider` (ThinkingStyle → FieldModulation knobs) is the half §3o of `alpha-channel-rung-overlay-v1` refused for deepnsm-v2: reasoning lives in the planner. **Open for the operator:** keep the knobs here, or move `V2StyleProvider` to lance-graph-planner and pass the scan parameters in. | +| `crates/deepnsm/src/ontology_vocab.rs` | 141 | `48405aa2` (2026-08-22) | `ae8e762e` (`5ff56bdf` is cited by `48405aa2` but not in this clone) | An OBO ontology concept register (`0x03XX`) over `ogar_codebook::concepts_in_domain`: name ↔ id, with the COCA rank space kept apart. Lookups only; it never used CAM-PQ itself. | "CAM-PQ is prohibited for ontologies", and it sat in v1 when the instruction named v2 | **RESTORED+REPURPOSED**. (1) The OBO half returns empty: `98de36eb` retracted the 14 0x03XX mirror rows as never minted, so the domain is reserved with zero rows, and a test pins that. (2) **New WordNet half:** `WordNetRail` (all senses, all hypernym edges from `wordnet31_isa_v2.tsv`, exact 7-column arity), `ancestors`, `is_a`, and `ConceptMask`, which matches by membership of the synset or any ancestor. Identity only, no distance, so the A1 ruling holds. Tested on real WordNet rows: whale/dolphin under `aquatic_mammal`, shark/manta under `cartilaginous_fish`, and a three-root water-animal mask. | +| `crates/deepnsm/examples/persona_chain_replay.rs` | 376 | `8c03aaa9` (revert of `a74c3bbd`, no reason given) | `a74c3bbd` (not in this clone; its file content survives at `8c03aaa9^`) | The first caller of `ContextChain::disambiguate_with` with REAL candidates (window NP heads), scored against the agreement resolver's gold. Pronoun case gate: Ambiguous never decides. | none recorded | **RESTORED**. Measured 2026-10-04: the margin gate and the rank metric are on incompatible scales. The widest coherence spread is 0.0003 against a threshold of 0.1, 335× too small, so every replay escalates. This is the baseline for the counterfactual "does this make sense" test. | +| `crates/deepnsm/src/markov_soa.rs` | 360 | `9a5f54c1` (2026-05-31) | — | The Markov wave over SoA SPO rows. | layer inversion: a core concern sat in a linguistics sensor | **SUPERSEDED-IN-PLACE** at `crates/lance-graph/src/graph/arigraph/markov_soa.rs` (vocabulary-agnostic, with an injected distance). | +| `crates/deepnsm/src/{content_fp,markov_bundle,trajectory}.rs` | 98 / 250 / 298 | `0ae9f906` (2026-04-24) | D5 | 10K-bit XOR content fingerprints, bit-rotation braiding, Hamming recovery margin. | wrong substrate: GF(2)/XOR where the stack uses ℝ multiply+add (I-VSA-IDENTITIES, CHANGELOG 2026-04-21) | `markov_bundle.rs` and `trajectory.rs` exist again on the ℝ substrate. **`content_fp.rs` is HISTORY-ONLY**, because its `Vsa10k` type was removed from the contract. Its idea (a deterministic per-rank content vector) lives in the ℝ switchboard. | + +## Not grammar, listed so they are not mistaken for losses + +| path | commit | now | +|---|---|---| +| `crates/lance-graph-planner/examples/data/coca/lexicon.tsv` | `4d00cba3` | moved to a release asset (data, not code) | +| `crates/deepnsm-v2/data/{bible_vocab.txt,cam96_*.bin}` | `73a8745f` | release `v0.1.0-cam96-data` | +| `crates/lance-graph-cognitive/src/learning/cognitive_frameworks.rs` | `6265ac7d` | moved to a standalone crate | +| `crates/jc/examples/ontology_locality_probe.rs` | `0b9860c3` | VSA vision corrected; not restored | +| `crates/lance-graph/examples/p_community_basin_agree.rs` | `5e1393a3` | S1 probe removed with the jc dev-coupling | + +## Merge queue (ideas restored but not yet merged) + +1. **loci.rs and the agreement resolver.** Add a gender/number/case agreement filter, then keep SelectionalFit as the ranker. Measure against `l9_loci_real_text` gold (agreement-only 0.727). +2. **tekamolo.rs and planner styles.** This waits for the operator's call on `V2StyleProvider`. +3. **lexicon.rs and the inlined taggers** in `bible_wave` / `genre_shapes`. This is a deliberate KJV re-pin. +4. **persona_chain_replay and ContextChain.** Fix the scale mismatch (a rank thermometer against a 0.1 margin) before the replay can decide anything. diff --git a/crates/deepnsm-v2/examples/toc_hydrate.rs b/crates/deepnsm-v2/examples/toc_hydrate.rs new file mode 100644 index 000000000..ea3757c73 --- /dev/null +++ b/crates/deepnsm-v2/examples/toc_hydrate.rs @@ -0,0 +1,315 @@ +//! End-to-end over the REAL corpus: spawn the TOC tree, hydrate the triple +//! stream over it, promote basins at TOC node keys. +//! +//! ```sh +//! cargo run --release --example toc_hydrate -- /path/to/pg10.txt +//! ``` +//! +//! Loading and tagging mirror `bible_wave` exactly (same vocab, same trained +//! Cam96 codebook, same COCA lemma lexicon + archaic fallback) so this cannot +//! silently disagree with the pipeline that already runs. + +use deepnsm_v2::basin::basin_self_code; +use deepnsm_v2::codebook::{load_cam96_codes, load_cam96_space}; +use deepnsm_v2::corpus::split_verses_detailed; +use deepnsm_v2::fsm::{parse_to_spo, Pos, Tagged}; +use deepnsm_v2::hydrate::hydrate; +use deepnsm_v2::lexicon::{normalise, Lexicon}; +use deepnsm_v2::loci::{loci_from_triples, AnteRule}; +use deepnsm_v2::promote::{key_at, promote, read_lane, row_of}; +use deepnsm_v2::spo::Spo; +use deepnsm_v2::vocab::PaletteVocab; +use lance_graph_contract::hhtl::NiblePath; +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +const BASIN: u8 = 3; +// A KJV corpus concept has NO mint — that is a real gap, not a detail. This +// borrows OSINT-V3 for ONE property only: it resolves to the V3 tail, so the +// key can carry `leaf` and read its own identity back. It says nothing about +// what a scripture node IS; the actual address needs an OGAR mint. +// +// The previous stand-in here was `0x0301_0000` (MONDO's block), which has no +// registry entry and therefore resolves to the V1 tail. `key_at` now refuses +// that outright — see its doc for how silently it used to succeed. +const CLASSID: u32 = lance_graph_contract::canonical_node::NodeGuid::CLASSID_OSINT_V3; + +fn data_file(name: &str) -> Vec { + let dir = std::env::var("DEEPNSM_V2_DATA") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data")); + std::fs::read(dir.join(name)).unwrap_or_else(|e| panic!("{name}: {e}")) +} + +fn main() { + let path = std::env::args() + .nth(1) + .expect("usage: toc_hydrate "); + let text = std::fs::read_to_string(&path).expect("corpus"); + + // ── verses + their (chapter, verse) markers, from ONE walk ── + // This used to be two walks: `split_verses` for the text, and a separate + // `split_whitespace().filter(is_verse_marker)` over the RAW text for the + // markers. The second walk did not apply the footer trim, so any `d+:d+` + // in the Gutenberg license would have been admitted as a verse marker and + // de-aligned every index after it. On the shipped file it happened to be + // clean (measured: 0 markers outside the body); nothing guaranteed it. + // `CorpusSplit::markers` is aligned with `verses` by construction. + let split = split_verses_detailed(&text); + let (verses, markers) = (split.verses, split.markers); + println!( + "corpus {} verses, {} markers", + verses.len(), + markers.len() + ); + assert_eq!( + markers.len(), + verses.len(), + "marker/verse misalignment — the join key would be wrong" + ); + + // ── vocab + trained codebook (the DOCUMENTED artifacts) ── + let vocab_text = String::from_utf8(data_file("bible_vocab.txt")).expect("utf8"); + let mut vocab = PaletteVocab::new(); + vocab.from_frequency_ranked(vocab_text.lines()); + let space = load_cam96_space(&data_file("cam96_codebook.bin")).expect("codebook"); + let codes = load_cam96_codes(&data_file("cam96_codes.bin")).expect("codes"); + assert_eq!(codes.len(), vocab.len(), "codes/vocab misaligned"); + println!("codebook {} words, {} codes", vocab.len(), codes.len()); + + // ── PoS lexicon: COCA lemmas + FORMS + archaic fallback ── + // One lexicon, in the crate. Reading `lemmas_5k.csv` alone left every + // inflected verb (`created`, `made`, `called`) untagged; see + // `deepnsm_v2::lexicon` for the measurement. + let lexicon = Lexicon::from_coca( + &std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../deepnsm/word_frequency/lemmas_5k.csv" + )) + .expect("lemmas_5k.csv"), + &std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../deepnsm/word_frequency/word_forms.csv" + )) + .expect("word_forms.csv"), + ); + println!( + "lexicon {} lemmas + {} forms", + lexicon.sizes().0, + lexicon.sizes().1 + ); + + // ── per-verse SPO, same tagging as bible_wave ── + let mut per_verse: Vec> = Vec::with_capacity(verses.len()); + let mut tagged: Vec = Vec::new(); + for verse in &verses { + tagged.clear(); + for tok in verse.split_whitespace() { + let Some(w) = normalise(tok) else { continue }; + let Some(id) = vocab.id(&w) else { continue }; + tagged.push(Tagged::new(id, lexicon.pos(&w))); + } + tagged.push(Tagged::new(0, Pos::Stop)); + per_verse.push(parse_to_spo(&tagged)); + } + let total: usize = per_verse.iter().map(Vec::len).sum(); + println!("triples {total} from {} verses", verses.len()); + + // ── SPAWN + HYDRATE ── + let h = hydrate(&markers, &per_verse, BASIN); + println!( + "tree {} nodes ({} books, {} verse nodes)", + h.entries.len(), + h.toc.book_count(), + h.verse_nodes() + ); + println!( + "hydrated {} addressed · {} barren verses · {} unaddressed", + h.triples.len(), + h.barren_verses, + h.unaddressed + ); + assert_eq!( + h.triples.len() + h.unaddressed, + total, + "triples vanished in the join" + ); + + let chain = h.chain(); + let scan = h.scanpath(); + println!( + "chain {} links, {} cross-verse ({:.1}%)", + chain.len(), + h.cross_verse_links(), + 100.0 * h.cross_verse_links() as f64 / chain.len().max(1) as f64 + ); + println!("scanpath {} entries (residue over the tree)", scan.len()); + + // Every address must be a minted verse node. + let minted: HashSet = h + .entries + .iter() + .filter(|e| e.level == deepnsm_v2::toc::TocLevel::Verse) + .map(|e| e.path) + .collect(); + let stray = h + .triples + .iter() + .filter(|t| !minted.contains(&t.path)) + .count(); + println!("addressing {stray} triples at an unminted address"); + assert_eq!( + stray, 0, + "a triple is at an address nothing can ascend from" + ); + + // Keys round-trip: fold a path into tiers, read it back through the canon. + let mut checked = 0usize; + for e in h.entries.iter().take(2000) { + let k = key_at(CLASSID, e.path, 0, 1).expect("V3 classid mints"); + assert_eq!( + NiblePath::from_guid_prefix_v3(&k).prefix(e.path.depth()), + Some(e.path), + "key does not carry its own address back" + ); + checked += 1; + } + println!("keys {checked} round-tripped through the v3 fold"); + + // ── BASINS from the REAL codebook, promoted at TOC keys ── + let mut by_subject: HashMap> = HashMap::new(); + for t in &h.triples { + by_subject + .entry(t.spo.subject) + .or_default() + .push((t.spo.predicate, t.spo.object)); + } + let mut promoted = 0usize; + let mut with_code = 0usize; + let mut first_verse: HashMap = HashMap::new(); + for t in &h.triples { + first_verse.entry(t.spo.subject).or_insert(t.path); + } + // The entry a path addresses, indexed once. Scanning `h.entries` inside the + // loop was O(subjects x entries) — 702 x 32,357 — and only tolerable while + // the loop was capped. + let entry_at: HashMap = + h.entries.iter().map(|e| (e.path, e)).collect(); + + // Deterministic order. `HashMap::iter` is randomised per run, so the old + // `.take(64)` promoted a DIFFERENT 64 subjects each time — which is fine + // for proving the path and useless for anything that has to be reproduced. + let mut subjects: Vec = by_subject.keys().copied().collect(); + subjects.sort_unstable(); + + let mut rows = Vec::new(); + let mut no_members = 0usize; + let mut no_code = 0usize; + for subject in subjects { + let edges = &by_subject[&subject]; + let members: Vec<[u8; 12]> = edges + .iter() + .filter_map(|(_, o)| codes.get(*o as usize).copied()) + .collect(); + if members.is_empty() { + no_members += 1; + continue; + } + let Some(code) = basin_self_code(&space, subject, &members, edges) else { + no_code += 1; + continue; + }; + if code.self_code != [0u8; 12] { + with_code += 1; + } + let path = first_verse[&subject]; + let entry = entry_at + .get(&path) + .expect("the address came from a minted node"); + // Identity is the SUBJECT, not a positional counter. The counter made + // the key depend on iteration order — two runs would address the same + // basin differently. The subject is the basin's own name, so the key + // is stable across runs and across corpus growth, and the lane + // round-trip below becomes a real check that key and payload agree. + let row = promote( + entry, + &row_of(&code, 0, verses.len() as u64), + CLASSID, + u32::from(subject), + ) + .expect("CLASSID resolves to the V3 tail, so the mint cannot be refused"); + assert_eq!(read_lane(&row).subject, subject, "lane round-trip"); + assert_eq!( + row.key.identity_v2(), + subject, + "key identity must be the basin's own subject" + ); + rows.push(row); + promoted += 1; + } + + // Every promoted basin must occupy its OWN address. Basins that first + // appear in the same verse share a path, so the identity half is what + // separates them — this is the assertion that proves it does. + let distinct: HashSet<_> = rows.iter().map(|r| r.key).collect(); + assert_eq!(distinct.len(), rows.len(), "two basins collided on one key"); + let shared_path = rows.len() + - rows + .iter() + .map(|r| NiblePath::from_guid_prefix_v3(&r.key)) + .collect::>() + .len(); + println!("skipped {no_members} with no coded member, {no_code} with no self-code"); + println!( + "collision {} distinct keys / {} rows · {shared_path} rows share a verse address", + distinct.len(), + rows.len() + ); + println!( + "basins {} subjects total · {promoted} promoted at TOC keys · {with_code} carry a real Cam96 self-code", + by_subject.len() + ); + println!( + "rows {} NodeRows, {} bytes each", + rows.len(), + core::mem::size_of::() + ); + // ── LOCI: the adapter wave.rs declares missing, on the real stream ── + let flat: Vec = h.triples.iter().map(|t| t.spo).collect(); + let pronoun_by_id: Vec = (0..vocab.len()) + .map(|i| vocab.word(i as u16).is_some_and(|w| lexicon.is_pronoun(w))) + .collect(); + let pcount = pronoun_by_id.iter().filter(|b| **b).count(); + println!("pronouns {pcount} of {} vocabulary ids", vocab.len()); + for rule in [AnteRule::Recency, AnteRule::SelectionalFit] { + let (stream, rep) = loci_from_triples(&space, &codes, &pronoun_by_id, &flat, rule); + // Ground every bound locus through wave.rs's own resolver. + let (mut causal, mut escalate, mut unbound) = (0usize, 0usize, 0usize); + for pos in 0..stream.len() { + match stream.ground_at( + pos, + lance_graph_contract::causal_witness::Locus::Antecedent, + u64::MAX, + 4, + ) { + lance_graph_contract::witness_fabric::WaveGrounding::Causal => causal += 1, + lance_graph_contract::witness_fabric::WaveGrounding::Escalate => escalate += 1, + lance_graph_contract::witness_fabric::WaveGrounding::Unbound => unbound += 1, + } + } + println!( + "loci {:<15} {} pronoun subjects · {} bound · {} thin-margin · {} beyond reach", + format!("{rule:?}"), + rep.pronoun_subjects, + rep.bound, + rep.refused_thin_margin, + rep.beyond_reach + ); + println!( + " wave grounding {causal} Causal · {escalate} Escalate · {unbound} Unbound (of {} events)", + stream.len() + ); + } + + println!("\nEND-TO-END GREEN over the real corpus."); +} diff --git a/crates/deepnsm-v2/src/hydrate.rs b/crates/deepnsm-v2/src/hydrate.rs new file mode 100644 index 000000000..21f15da22 --- /dev/null +++ b/crates/deepnsm-v2/src/hydrate.rs @@ -0,0 +1,370 @@ +//! `hydrate` — laying the triple stream over the spawned TOC tree. +//! +//! [`crate::toc`] mints the skeleton; [`crate::promote`] writes basins at its +//! keys. This is the middle step: every SPO triple gets the **address of the +//! verse it was read in**, so a triple is no longer a free-floating +//! `(s, p, o)` but an observation at a literary unit. +//! +//! # What this adds, precisely +//! +//! The triples already exist — `fsm::parse_to_spo` produces them per verse. +//! What did not exist is the JOIN to the tree. Without it a triple's only +//! coordinate is its index in a flat stream, which is exactly the rung-0 +//! representation `E-HERMENEUTIK-RUNG-LADDER-1` diagnoses: with canonical +//! position discarded there is no non-size structure left to find. +//! +//! # The chain is the reading order, and it is preserved +//! +//! The triples are kept in corpus order and [`Hydration::chain`] exposes +//! consecutive pairs — the Markov trajectory over the book. Order is therefore +//! load-bearing, not incidental: a sort or a shuffle anywhere in this module +//! destroys the chain while leaving every triple individually correct, which is +//! the kind of break a count-based test cannot see. Hence +//! `the_chain_follows_reading_order_not_the_tree`. +//! +//! # The nodes are where the residue lands +//! +//! The TOC nodes are not only addresses. The alpha-layer eyetracker cascade and +//! the HHTL nodes are what carry the **residue** — the record of what the +//! thinking actually looked at — so a scanpath over this tree is the thinking +//! made visible rather than inferred. [`Hydration::scanpath`] is that sequence +//! in its rawest form: the addresses visited, in reading order. +//! +//! It is a TRAJECTORY, not a set: only consecutive repeats collapse, so a +//! return to a verse later in the stream shows up again. Deduplicating globally +//! would turn the residue into coverage and throw away the revisit — which is +//! the part that carries the thinking. +//! +//! # Scope +//! +//! It ADDRESSES; it does not tag or parse. The per-verse triples are an input, +//! produced by the caller's existing FSM path, so this module has no opinion +//! about tokenization and cannot silently disagree with the pipeline that +//! already runs. + +use crate::spo::Spo; +use crate::toc::{spawn, CorpusToc, TocEntry, TocLevel}; +use lance_graph_contract::hhtl::NiblePath; + +/// One triple, addressed at the verse node it was read in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AddressedTriple { + /// The verse node's HHTL address. + pub path: NiblePath, + /// The triple. + pub spo: Spo, + /// Position in the corpus-order stream — the Markov index. + pub order: u32, +} + +/// The tree plus the stream laid over it. +#[derive(Debug, Clone, Default)] +pub struct Hydration { + /// The outline the tree was spawned from. + pub toc: CorpusToc, + /// Every minted node, top-down. + pub entries: Vec, + /// Every triple, in corpus reading order. + pub triples: Vec, + /// Verses that produced no triple — reported, never hidden. + pub barren_verses: usize, + /// Triples whose verse had no addressable node (a level over capacity). + pub unaddressed: usize, +} + +impl Hydration { + /// Verse nodes in the tree. + #[must_use] + pub fn verse_nodes(&self) -> usize { + self.entries + .iter() + .filter(|e| e.level == TocLevel::Verse) + .count() + } + + /// Consecutive `(from, to)` pairs in reading order — the Markov chain over + /// the book. `n` triples yield `n − 1` links; fewer than two yield none. + #[must_use] + pub fn chain(&self) -> Vec<(&AddressedTriple, &AddressedTriple)> { + self.triples.windows(2).map(|w| (&w[0], &w[1])).collect() + } + + /// The **scanpath**: addresses in reading order, consecutive repeats + /// collapsed. + /// + /// This is the residue in its rawest form — what a focus mask over the tree + /// is built from. Only CONSECUTIVE repeats collapse: a verse revisited + /// later appears again, because a revisit is the signal, not noise. A + /// global dedup would report coverage and lose the trajectory. + #[must_use] + pub fn scanpath(&self) -> Vec { + let mut out: Vec = Vec::new(); + for t in &self.triples { + if out.last() != Some(&t.path) { + out.push(t.path); + } + } + out + } + + /// How many chain links cross a verse boundary — the structure a flat + /// stream cannot report, and the reason the addressing is worth its cost. + #[must_use] + pub fn cross_verse_links(&self) -> usize { + self.chain() + .iter() + .filter(|(a, b)| a.path != b.path) + .count() + } +} + +/// Lay `per_verse` triples over a tree spawned from `markers`. +/// +/// `markers[i]` and `per_verse[i]` describe the SAME verse — index `i` is the +/// join key, so the two must come from one walk of the corpus. A length +/// mismatch is not an error here: the walk stops at the shorter of the two and +/// the shortfall shows up as untouched verse nodes, which a caller can see. +/// +/// Verses whose node is missing (a level over the tree's per-level capacity) +/// have their triples counted in [`Hydration::unaddressed`] rather than +/// attached to a neighbour — the same refusal-not-fold rule the spawn uses. +#[must_use] +pub fn hydrate(markers: &[(u16, u16)], per_verse: &[Vec], basin: u8) -> Hydration { + let toc = CorpusToc::from_markers(markers); + let entries = spawn(&toc, basin); + + // Verse nodes in the order the spawn emitted them, which is the order the + // markers were read — so the i-th verse marker is the i-th verse node. + let verse_paths: Vec = entries + .iter() + .filter(|e| e.level == TocLevel::Verse) + .map(|e| e.path) + .collect(); + + let mut triples = Vec::new(); + let mut barren_verses = 0usize; + let mut unaddressed = 0usize; + let mut order = 0u32; + + for (i, spos) in per_verse.iter().enumerate().take(markers.len()) { + if spos.is_empty() { + barren_verses += 1; + continue; + } + match verse_paths.get(i) { + Some(&path) => { + for spo in spos { + triples.push(AddressedTriple { + path, + spo: *spo, + order, + }); + order = order.saturating_add(1); + } + } + None => unaddressed += spos.len(), + } + } + + Hydration { + toc, + entries, + triples, + barren_verses, + unaddressed, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + /// Two books: 2 chapters (2 + 1 verses), then 1 chapter (2 verses) = 5 verses. + fn markers() -> Vec<(u16, u16)> { + vec![(1, 1), (1, 2), (2, 1), (1, 1), (1, 2)] + } + + fn spo(n: u16) -> Spo { + Spo::new(n, n + 100, n + 200) + } + + /// **Every triple lands on a real minted verse node.** The join is the whole + /// deliverable, so it is the test: an address that is not in the tree means + /// a triple nothing can ascend from. + #[test] + fn every_triple_is_addressed_at_a_minted_verse_node() { + let per_verse = vec![ + vec![spo(1), spo(2)], + vec![spo(3)], + vec![], + vec![spo(4)], + vec![spo(5)], + ]; + let h = hydrate(&markers(), &per_verse, 3); + + assert_eq!(h.verse_nodes(), 5, "the outline has 5 verses"); + assert_eq!(h.triples.len(), 5, "2+1+0+1+1 triples"); + assert_eq!( + h.barren_verses, 1, + "the empty verse is REPORTED, not hidden" + ); + assert_eq!(h.unaddressed, 0); + + let minted: HashSet = h + .entries + .iter() + .filter(|e| e.level == TocLevel::Verse) + .map(|e| e.path) + .collect(); + for t in &h.triples { + assert!(minted.contains(&t.path), "{t:?} is at an unminted address"); + } + // Anti-vacuity: the triples do NOT all share one address, so + // "contains" is checking a real spread. + let used: HashSet = h.triples.iter().map(|t| t.path).collect(); + assert!(used.len() >= 3, "only {} distinct addresses", used.len()); + } + + /// Triples from ONE verse share that verse's address; triples from + /// different verses do not. Both halves — otherwise a constant address + /// would pass the test above. + #[test] + fn triples_share_an_address_exactly_when_they_share_a_verse() { + let per_verse = vec![vec![spo(1), spo(2)], vec![spo(3)], vec![], vec![], vec![]]; + let h = hydrate(&markers(), &per_verse, 2); + assert_eq!(h.triples.len(), 3); + assert_eq!( + h.triples[0].path, h.triples[1].path, + "same verse ⇒ same address" + ); + assert_ne!( + h.triples[1].path, h.triples[2].path, + "different verse ⇒ different address" + ); + } + + /// **The chain is READING order, not tree order.** A sort or shuffle would + /// leave every triple individually correct and silently destroy the Markov + /// trajectory, which no count-based assertion could see. + #[test] + fn the_chain_follows_reading_order_not_the_tree() { + // **The subjects are DELIBERATELY not ascending.** A first version used + // 1,2,3,4,5 and a "sort the stream by subject" mutation did not fire — + // sorted and reading order were the same sequence, so the fixture could + // not tell them apart. The fixture's SHAPE is part of the coverage. + let per_verse = vec![ + vec![spo(50), spo(10)], + vec![spo(40)], + vec![], + vec![spo(20)], + vec![spo(30)], + ]; + let h = hydrate(&markers(), &per_verse, 4); + + // `order` is dense and ascending across the whole stream. + let orders: Vec = h.triples.iter().map(|t| t.order).collect(); + assert_eq!(orders, (0..h.triples.len() as u32).collect::>()); + // The subjects come back in CORPUS order — the join did not re-sort. + let subs: Vec = h.triples.iter().map(|t| t.spo.subject).collect(); + assert_eq!(subs, vec![50, 10, 40, 20, 30]); + let mut sorted = subs.clone(); + sorted.sort_unstable(); + assert_ne!( + subs, sorted, + "anti-vacuity: the fixture must distinguish reading order from sorted order" + ); + + let chain = h.chain(); + assert_eq!(chain.len(), h.triples.len() - 1, "n triples ⇒ n−1 links"); + for (a, b) in &chain { + assert_eq!(b.order, a.order + 1, "a link must be consecutive"); + } + + // Cross-verse links are the structure only the addressing can report: + // 4 links here, of which the 1→2 link is WITHIN a verse. + assert_eq!(h.cross_verse_links(), 3, "3 of 4 links cross a verse"); + assert!( + h.cross_verse_links() < chain.len(), + "anti-vacuity: not every link crosses, so the measure discriminates" + ); + } + + /// **The scanpath is a trajectory, not a set.** A revisit must survive; a + /// consecutive repeat must collapse. Both halves, because collapsing + /// everything and collapsing nothing both "pass" a length-only check. + #[test] + fn the_scanpath_collapses_repeats_but_keeps_revisits() { + // v0 gets two triples (a consecutive repeat), v3 is revisited after v4 + // by putting a later triple back on it — the stream order is what + // decides, so this is expressed through the marker/verse pairing. + let per_verse = vec![ + vec![spo(1), spo(2)], // verse 0 twice in a row -> ONE scan entry + vec![spo(3)], // verse 1 + vec![], + vec![spo(4)], // verse 3 + vec![spo(5)], // verse 4 + ]; + let h = hydrate(&markers(), &per_verse, 6); + let sp = h.scanpath(); + + assert_eq!(sp.len(), 4, "5 triples over 4 distinct consecutive verses"); + assert!(sp.windows(2).all(|w| w[0] != w[1]), "no consecutive repeat"); + // Anti-vacuity: it really did collapse something. + assert!( + sp.len() < h.triples.len(), + "the consecutive repeat was not collapsed" + ); + // …and it is not merely the distinct set: the scan follows the stream. + let distinct: std::collections::HashSet = sp.iter().copied().collect(); + assert_eq!(distinct.len(), sp.len(), "no revisit in THIS fixture"); + + // The revisit half: a stream that returns to an earlier address keeps + // both visits, so the scanpath is longer than the distinct set. + let revisit = Hydration { + triples: vec![ + h.triples[0], + h.triples[2], + h.triples[0], // back to the first verse + ], + ..Hydration::default() + }; + let rsp = revisit.scanpath(); + assert_eq!(rsp.len(), 3, "a revisit must appear again"); + let rdistinct: std::collections::HashSet = rsp.iter().copied().collect(); + assert_eq!(rdistinct.len(), 2, "…while the distinct set stays 2"); + assert!(rsp.len() > rdistinct.len(), "trajectory, not set"); + } + + /// Fewer than two triples is no chain — and one triple is still addressed. + #[test] + fn a_single_triple_has_an_address_but_no_chain() { + let per_verse = vec![vec![spo(9)], vec![], vec![], vec![], vec![]]; + let h = hydrate(&markers(), &per_verse, 1); + assert_eq!(h.triples.len(), 1); + assert!(h.chain().is_empty(), "one triple cannot form a link"); + assert_eq!(h.cross_verse_links(), 0); + // …and the empty stream is safe too. + let e = hydrate(&markers(), &[], 1); + assert!(e.triples.is_empty() && e.chain().is_empty()); + assert_eq!(e.verse_nodes(), 5, "the tree still spawned"); + } + + /// A verse whose node is missing has its triples COUNTED, never attached to + /// a neighbour — the same refusal-not-fold rule the spawn uses. + #[test] + fn triples_of_an_unaddressable_verse_are_counted_not_reattached() { + // One book, one chapter, 300 verses — 45 past the 255 capacity. + let m: Vec<(u16, u16)> = (1..=300).map(|v| (1u16, v)).collect(); + let per_verse: Vec> = (1..=300).map(|v| vec![spo(v as u16)]).collect(); + let h = hydrate(&m, &per_verse, 5); + + assert_eq!(h.verse_nodes(), 255, "only the addressable verses minted"); + assert_eq!(h.triples.len(), 255); + assert_eq!(h.unaddressed, 45, "the overflow is reported"); + assert_eq!(h.triples.len() + h.unaddressed, 300, "nothing vanished"); + // No address carries two different verses' triples. + let used: HashSet = h.triples.iter().map(|t| t.path).collect(); + assert_eq!(used.len(), 255, "an address was reused by a folded verse"); + } +} diff --git a/crates/deepnsm-v2/src/lexicon.rs b/crates/deepnsm-v2/src/lexicon.rs new file mode 100644 index 000000000..0d0a239e0 --- /dev/null +++ b/crates/deepnsm-v2/src/lexicon.rs @@ -0,0 +1,307 @@ +//! One POS lexicon, loaded from the COCA tables — replacing three hand-copied +//! taggers. +//! +//! # Why this module exists +//! +//! `bible_wave`, `genre_shapes` and `toc_hydrate` each carried their own +//! `coca_pos` + `archaic_pos` + `pos_of` map, hand-copied between them. That +//! has already cost real yield twice: +//! +//! 1. A paraphrase of `archaic_pos` in one copy dropped the explicit KJV word +//! list and silently lost ~6.5k triples (34,277 vs 40,767) before the corpus +//! caught it. +//! 2. **All three copies read `lemmas_5k.csv` only.** That table is keyed by +//! LEMMA (`create`, `make`, `call`) while the KJV text is INFLECTED +//! (`created`, `made`, `called`), so every ordinary past-tense verb fell to +//! [`Pos::Other`] and the FSM skipped it. `word_forms.csv` — the COCA +//! form→lemma+PoS table, 11,460 rows, sitting in the same directory — +//! answers exactly that and was never opened. +//! +//! Measured on the 31,102-verse KJV: consulting the forms table takes verses +//! that yield no triple from **11,080 (35.6%) to 3,803 (12.2%)**. Genesis 1:1 +//! — *"In the beginning God created the heaven and the earth"*, every word in +//! vocabulary — was one of the barren ones, because `created` had no tag. +//! +//! # Resolution order, and why it is monotone +//! +//! `lemma table → forms table → archaic → Other`. +//! +//! Lemma-first is deliberate: it means **no word that already had a tag gets a +//! different one**. The forms table can only fill silence, never overrule. So +//! adding it is strictly additive against every prior measurement, which is +//! what makes the 11,080 → 3,803 delta attributable to coverage rather than to +//! a re-tagging. [`tests::adding_forms_never_retags_a_word_the_lemmas_knew`] +//! pins that property. + +use crate::fsm::Pos; +use std::collections::HashMap; + +/// Map a COCA part-of-speech letter to a [`Pos`]. +/// +/// `n`/`p` are noun-ish, `v` verbal, `j` adjectival, `a`/`d` determiners. +/// Everything else is [`Pos::Other`] — the FSM skips it. +#[must_use] +pub fn coca_pos(letter: &str) -> Pos { + match letter { + "n" | "p" => Pos::Noun, + "v" => Pos::Verb, + "j" => Pos::Adj, + "a" | "d" => Pos::Det, + _ => Pos::Other, + } +} + +/// Early-modern English forms COCA does not carry. +/// +/// The explicit word list is load-bearing and must not be paraphrased down to +/// the `-eth`/`-est` rule: `thou`/`hath`/`shall`/`saith` are among the most +/// frequent tokens in the corpus and none of them matches that suffix. +/// Early-modern personal pronouns. COCA tags pronouns `p` but carries none of +/// these, so they are named ONCE here and read by both [`archaic_pos`] (which +/// gives them a noun slot) and [`Lexicon::is_pronoun`] (which remembers the +/// slot holds a POINTER, not a referent). +pub const ARCHAIC_PRONOUNS: [&str; 3] = ["thou", "thee", "ye"]; + +#[must_use] +pub fn archaic_pos(w: &str) -> Option { + if ARCHAIC_PRONOUNS.contains(&w) { + return Some(Pos::Noun); + } + match w { + "thy" | "thine" => Some(Pos::Det), + "shalt" | "hath" | "doth" | "saith" | "spake" | "begat" | "art" | "wilt" | "hast" + | "shall" | "cometh" | "wast" => Some(Pos::Verb), + "unto" | "thereof" | "wherefore" | "verily" | "yea" | "lo" => Some(Pos::Other), + _ => { + if w.ends_with("eth") || w.ends_with("est") { + Some(Pos::Verb) + } else { + None + } + } + } +} + +/// A word → [`Pos`] lexicon over the COCA lemma and form tables. +#[derive(Debug, Clone, Default)] +pub struct Lexicon { + lemmas: HashMap, + forms: HashMap, + /// Words COCA tags `p`. [`coca_pos`] maps `p` to [`Pos::Noun`] — correct, a + /// pronoun fills a noun slot — which discards the fact that the slot holds + /// a POINTER. Keeping the set is what lets `loci` bind an antecedent + /// instead of committing `(he, said, them)` as though `he` named someone. + pronouns: std::collections::HashSet, +} + +impl Lexicon { + /// Build from the two COCA CSVs, passed as file CONTENTS. + /// + /// Contents rather than paths so the crate stays free of I/O and a test can + /// hand it three lines. Both tables take the FIRST row for a given key — + /// they are frequency-ranked, so the first is the dominant reading. + /// + /// * `lemmas_csv` — `rank,lemma,PoS,…` (`lemmas_5k.csv`) + /// * `forms_csv` — `lemRank,lemma,PoS,lemFreq,wordFreq,word` (`word_forms.csv`) + #[must_use] + pub fn from_coca(lemmas_csv: &str, forms_csv: &str) -> Self { + Self::from_columns(lemmas_csv, 1, 2).with_forms(forms_csv) + } + + /// Build the base (lemma-keyed) table from `academic_20k.csv`. + /// + /// A different vocabulary, the SAME defect: its `word` column is lemmas + /// (`be,v`), so it needs [`Self::with_forms`] layered under it exactly as + /// `lemmas_5k.csv` does. + #[must_use] + pub fn from_academic_20k(csv: &str) -> Self { + // header: ID,band,status,word,Pos,... + Self::from_columns(csv, 3, 4) + } + + /// The base table from any frequency-ranked CSV, given the column indices + /// of the word and its PoS letter. First row per key wins. + #[must_use] + pub fn from_columns(csv: &str, word_col: usize, pos_col: usize) -> Self { + let mut lemmas = HashMap::new(); + let mut pronouns: std::collections::HashSet = + ARCHAIC_PRONOUNS.iter().map(|s| (*s).to_string()).collect(); + for line in csv.lines().skip(1) { + let f: Vec<&str> = line.split(',').collect(); + let (Some(word), Some(pos)) = (f.get(word_col), f.get(pos_col)) else { + continue; + }; + if word.is_empty() { + continue; + } + let w = word.to_lowercase(); + if *pos == "p" { + pronouns.insert(w.clone()); + } + lemmas.entry(w).or_insert_with(|| coca_pos(pos)); + } + Self { + lemmas, + forms: HashMap::new(), + pronouns, + } + } + + /// Layer the COCA form table (`word_forms.csv`) UNDER the base table. + /// + /// `lemRank,lemma,PoS,lemFreq,wordFreq,word` — the PoS is the lemma's, at + /// column 2; the surface form is column 5. + #[must_use] + pub fn with_forms(mut self, forms_csv: &str) -> Self { + for line in forms_csv.lines().skip(1) { + let f: Vec<&str> = line.split(',').collect(); + let (Some(pos), Some(word)) = (f.get(2), f.get(5)) else { + continue; + }; + let w = word.to_lowercase(); + if *pos == "p" { + self.pronouns.insert(w.clone()); + } + self.forms.entry(w).or_insert_with(|| coca_pos(pos)); + } + self + } + + /// The part of speech for `w`, which must already be lowercased and + /// stripped to ascii letters. + /// + /// See the module doc for why the order is lemma → form → archaic. + #[must_use] + pub fn pos(&self, w: &str) -> Pos { + self.lemmas + .get(w) + .or_else(|| self.forms.get(w)) + .copied() + .or_else(|| archaic_pos(w)) + .unwrap_or(Pos::Other) + } + + /// Did the lemma table alone know `w`? Used by the monotonicity test and + /// by any caller reporting where its coverage comes from. + #[must_use] + pub fn lemma_knows(&self, w: &str) -> bool { + self.lemmas.contains_key(w) + } + + /// Is `w` a pronoun — a word whose noun slot holds a POINTER, not a + /// referent? Derived from COCA's `p` tag plus [`ARCHAIC_PRONOUNS`]; there + /// is no hand-written list. + #[must_use] + pub fn is_pronoun(&self, w: &str) -> bool { + self.pronouns.contains(w) + } + + /// How many pronouns the tables yielded. + #[must_use] + pub fn pronoun_count(&self) -> usize { + self.pronouns.len() + } + + /// `(lemma rows, form rows)`. + #[must_use] + pub fn sizes(&self) -> (usize, usize) { + (self.lemmas.len(), self.forms.len()) + } +} + +/// Normalise one raw token: ascii letters only, lowercased. `None` for +/// anything under two letters, which the taggers all skipped. +#[must_use] +pub fn normalise(tok: &str) -> Option { + let w: String = tok + .chars() + .filter(char::is_ascii_alphabetic) + .collect::() + .to_lowercase(); + (w.len() >= 2).then_some(w) +} + +#[cfg(test)] +mod tests { + use super::*; + + const LEMMAS: &str = "rank,lemma,PoS,freq\n1,create,v,9\n2,god,n,8\n3,light,n,7\n"; + const FORMS: &str = + "lemRank,lemma,PoS,lemFreq,wordFreq,word\n1,create,v,9,4,created\n2,be,v,9,4,is\n\ + 3,god,n,8,2,god\n4,he,p,9,9,he\n"; + + #[test] + fn the_forms_table_tags_what_the_lemma_table_cannot() { + let lx = Lexicon::from_coca(LEMMAS, FORMS); + // The measured defect, in miniature: `create` is a lemma, `created` is + // not, and before the forms table `created` was Pos::Other. + assert_eq!(lx.pos("create"), Pos::Verb); + assert!(!lx.lemma_knows("created")); + assert_eq!(lx.pos("created"), Pos::Verb); + // Anti-vacuity: without the forms table the same word is untagged, so + // the assertion above is discriminating. + let lemmas_only = Lexicon::from_coca(LEMMAS, ""); + assert_eq!(lemmas_only.pos("created"), Pos::Other); + } + + #[test] + fn adding_forms_never_retags_a_word_the_lemmas_knew() { + // The monotonicity property the module doc claims. A forms row that + // disagrees with the lemma table must lose. + let conflicting = "lemRank,lemma,PoS,lemFreq,wordFreq,word\n1,x,n,9,4,create\n"; + let lemmas_only = Lexicon::from_coca(LEMMAS, ""); + let both = Lexicon::from_coca(LEMMAS, conflicting); + assert_eq!(lemmas_only.pos("create"), Pos::Verb); + assert_eq!(both.pos("create"), Pos::Verb, "the lemma table must win"); + // ...and the conflicting row IS loaded, so the test is not passing + // merely because the parse dropped it. + assert_eq!(both.sizes().1, 1); + } + + #[test] + fn archaic_still_fires_below_both_tables() { + let lx = Lexicon::from_coca(LEMMAS, FORMS); + assert_eq!(lx.pos("hath"), Pos::Verb); + assert_eq!(lx.pos("thou"), Pos::Noun); + assert_eq!(lx.pos("thy"), Pos::Det); + assert_eq!(lx.pos("walketh"), Pos::Verb); + // ...and it can stay silent: an unknown word is not smuggled to Verb. + assert_eq!(lx.pos("zzzz"), Pos::Other); + } + + #[test] + fn the_academic_table_is_lemma_keyed_too_and_takes_the_same_forms() { + // ID,band,status,word,Pos,... + let acad = "ID,band,status,word,Pos,x\n1,1,c,be,v,0\n2,1,c,god,n,0\n"; + let bare = Lexicon::from_academic_20k(acad); + assert_eq!(bare.pos("be"), Pos::Verb); + assert_eq!(bare.pos("is"), Pos::Other, "the lemma table alone is blind"); + let layered = Lexicon::from_academic_20k(acad).with_forms(FORMS); + assert_eq!(layered.pos("is"), Pos::Verb); + } + + #[test] + fn pronoun_hood_comes_from_the_table_not_a_hand_list() { + let lx = Lexicon::from_coca(LEMMAS, FORMS); + assert!(lx.is_pronoun("he"), "COCA tags `he` as p"); + assert!( + lx.is_pronoun("thou"), + "archaic, named once in ARCHAIC_PRONOUNS" + ); + // Stays silent on nouns and verbs — a set that flags everything carries + // as much information as one that flags nothing. + assert!(!lx.is_pronoun("god")); + assert!(!lx.is_pronoun("create")); + // A pronoun still fills a NOUN slot; the two facts coexist. + assert_eq!(lx.pos("he"), Pos::Noun); + assert_eq!(lx.pos("thou"), Pos::Noun); + } + + #[test] + fn normalise_strips_and_drops_the_short() { + assert_eq!(normalise("God,").as_deref(), Some("god")); + assert_eq!(normalise("(LORD)").as_deref(), Some("lord")); + assert_eq!(normalise("a"), None, "one letter was always skipped"); + assert_eq!(normalise("1:1"), None); + } +} diff --git a/crates/deepnsm-v2/src/lib.rs b/crates/deepnsm-v2/src/lib.rs index 474a625e0..7615414df 100644 --- a/crates/deepnsm-v2/src/lib.rs +++ b/crates/deepnsm-v2/src/lib.rs @@ -42,12 +42,18 @@ pub mod codebook; pub mod corpus; pub mod evidence; pub mod fsm; +pub mod hydrate; pub mod introspect; pub mod lexical; +pub mod lexicon; +pub mod loci; +pub mod promote; pub mod reason; pub mod shape; pub mod space; pub mod spo; +pub mod tekamolo; +pub mod toc; pub mod vocab; pub mod wave; @@ -78,6 +84,7 @@ pub use lexical::{ load_word_forms_csv, EvidenceError, LemmaEntry, LemmaRef, LexicalEvidence, LexicalEvidenceBuilder, LexicalReading, PosCode, WordFormsReport, }; +pub use loci::{loci_from_triples, AnteRule, LociReport}; pub use shape::{ detect, detect_all, detect_all_measured, detect_measured, MeasuredShape, Representation, ShapeClass, ShapeReport, diff --git a/crates/deepnsm-v2/src/loci.rs b/crates/deepnsm-v2/src/loci.rs new file mode 100644 index 000000000..6e5de84b6 --- /dev/null +++ b/crates/deepnsm-v2/src/loci.rs @@ -0,0 +1,357 @@ +//! `loci` — the adapter [`crate::wave`] declares missing: deriving +//! [`CausalWitnessFacet`] loci FROM the [`Spo`] triple stream. +//! +//! `wave.rs` states the gap twice, in its own words: +//! +//! > "Deriving loci FROM `Spo` triples is a separate adapter and is +//! > intentionally NOT bundled here, so this stays a clean resolver." +//! +//! > "**nothing in this repository populates a `WitnessStream` from text or +//! > from a real `NodeRow`.** Every `WitnessStream::new()` call site is +//! > `#[cfg(test)]` — there is no production adapter feeding real loci into +//! > this stream today." +//! +//! This module is that adapter. It reads the triple stream and writes ONE +//! locus — [`Locus::Antecedent`] — leaving all 23 others UNBOUND, because this +//! crate has grounded no evidence for them and a fabricated pointer is worse +//! than an absent one (`0` is the zero-fallback: *not consulted*). +//! +//! ## What decides the antecedent +//! +//! The candidate set for a pronoun subject is the non-pronoun subjects inside +//! the `-8` reach. Ranking uses **this crate's own trained meaning space** — +//! [`Cam96Space::similarity`] between a candidate's code and the pronoun +//! clause's PREDICATE code. That is selectional fit: of the referents in view, +//! which one is distributionally the kind of thing that does this verb. +//! +//! [`AnteRule::Recency`] (nearest candidate, ignoring meaning) is carried as +//! the declared BASELINE so the meaning-based rule is never reported without +//! the control it has to beat — the house pattern. +//! +//! ## Why the reach is 8 and not a tunable +//! +//! `E-ANAPHORA-BEYOND-I4-IS-A-BASIN-EDGE-1`: a locus nibble is a signed `i4`, +//! so it addresses `-8..=+7`, and that boundary is a TYPE boundary rather than +//! a size limit — inside it, position discriminates; beyond it, identity does, +//! so a wider pointer would still address a position when the discriminator +//! has changed kind. An antecedent further back is a BASIN EDGE, not an +//! overflow, so this adapter leaves the locus UNBOUND rather than clamping to +//! `-8` and naming the wrong event. +//! +//! ## Margin, and who decides +//! +//! When the best and second-best candidate score within [`MIN_MARGIN`], the +//! locus stays UNBOUND: the adapter refuses rather than committing on a +//! coin-flip, and `wave.rs`'s resolver then reads it as +//! `WaveGrounding::Unbound`. The adapter never invents an answer the evidence +//! did not separate. + +use crate::space::{Cam96, Cam96Space}; +use crate::spo::Spo; +use crate::wave::WitnessStream; +use lance_graph_contract::causal_witness::{CausalWitnessFacet, Locus}; + +/// The `i4` locus reach — a TYPE boundary, not a tunable (see module doc). +pub const REACH: usize = 8; + +/// Minimum separation between best and second-best candidate before the +/// adapter will bind a locus. Below this the evidence did not choose. +pub const MIN_MARGIN: f32 = 0.01; + +/// Which rule ranks the candidate antecedents. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AnteRule { + /// Nearest candidate wins. The declared BASELINE — carries no meaning. + Recency, + /// Highest [`Cam96Space::similarity`] between the candidate's code and the + /// pronoun clause's PREDICATE code: selectional fit. + SelectionalFit, +} + +/// What the adapter did, so a caller never has to infer it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct LociReport { + /// Triples whose subject is a pronoun — a slot holding a pointer. + pub pronoun_subjects: usize, + /// …that had at least one candidate inside the reach. + pub with_candidates: usize, + /// …that bound `Locus::Antecedent`. + pub bound: usize, + /// …left UNBOUND because best and second-best were within `MIN_MARGIN`. + pub refused_thin_margin: usize, + /// …left UNBOUND because no candidate sat inside the `-8` reach (the + /// antecedent is a BASIN EDGE, not an overflow). + pub beyond_reach: usize, +} + +/// Score one candidate under `rule`. Higher wins. +/// +/// `Recency` scores by proximity alone. `SelectionalFit` reads the trained +/// space; a candidate or predicate without a code scores `f32::NEG_INFINITY` +/// so it can never win by default — an absent code is absent evidence. +fn score( + rule: AnteRule, + space: &Cam96Space, + codes: &[Cam96], + cand_subject: u16, + predicate: u16, + back: usize, +) -> f32 { + match rule { + // Nearest = highest. `back` is 1..=REACH. + AnteRule::Recency => -(back as f32), + AnteRule::SelectionalFit => { + match ( + codes.get(cand_subject as usize), + codes.get(predicate as usize), + ) { + (Some(c), Some(p)) => space.similarity(c, p), + _ => f32::NEG_INFINITY, + } + } + } +} + +/// Derive loci from the triple stream and return the populated +/// [`WitnessStream`] plus what the derivation did. +/// +/// One event per triple, at version = its stream index, so the append index IS +/// the `±8` offset space `wave.rs` resolves against. +/// +/// `pronoun_by_id[id]` says whether vocabulary id `id` is a pronoun. An empty +/// slice means no pronoun information, and then nothing is bound — absent +/// evidence is not a licence to guess. +#[must_use] +pub fn loci_from_triples( + space: &Cam96Space, + codes: &[Cam96], + pronoun_by_id: &[bool], + triples: &[Spo], + rule: AnteRule, +) -> (WitnessStream, LociReport) { + let is_pronoun = |id: u16| pronoun_by_id.get(id as usize).copied().unwrap_or(false); + let mut stream = WitnessStream::new(); + let mut rep = LociReport::default(); + + for (i, t) in triples.iter().enumerate() { + let mut reg = CausalWitnessFacet::ZERO; + + if is_pronoun(t.subject) { + rep.pronoun_subjects += 1; + + // Candidates: non-pronoun subjects inside the -8 reach, nearest first. + let mut scored: Vec<(usize, f32)> = (1..=REACH) + .filter_map(|back| i.checked_sub(back).map(|j| (back, j))) + .filter(|&(_, j)| !is_pronoun(triples[j].subject)) + .map(|(back, j)| { + ( + back, + score(rule, space, codes, triples[j].subject, t.predicate, back), + ) + }) + .filter(|&(_, s)| s > f32::NEG_INFINITY) + .collect(); + + if scored.is_empty() { + rep.beyond_reach += 1; + } else { + rep.with_candidates += 1; + // Descending by score; ties keep the nearer candidate because + // the source iteration is nearest-first and the sort is stable. + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + let margin = if scored.len() >= 2 { + scored[0].1 - scored[1].1 + } else { + f32::INFINITY + }; + if margin < MIN_MARGIN { + rep.refused_thin_margin += 1; + } else { + let back = i8::try_from(scored[0].0).unwrap_or(i8::MAX); + // Negative: `-` is before / antecedent. + reg = reg.with(Locus::Antecedent, -back); + rep.bound += 1; + } + } + } + + stream.push(i as u64, reg); + } + + (stream, rep) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A codebook where similarity is controllable: code `[k;12]` reconstructs + /// to a point whose distance from `[m;12]` grows with `|k-m|`. + fn space() -> Cam96Space { + Cam96Space::demo(4) + } + + fn codes(n: usize) -> Vec { + (0..n).map(|i| [(i * 7 % 251) as u8; 12]).collect() + } + + fn mask(n: usize, ps: &[u16]) -> Vec { + (0..n).map(|i| ps.contains(&(i as u16))).collect() + } + + /// The guard FIRES: a pronoun subject one step after a real subject binds + /// `Antecedent = -1`, and `wave`'s own resolver grounds it Causal. + #[test] + fn a_pronoun_subject_binds_and_the_wave_grounds_it() { + let triples = vec![Spo::new(1, 100, 200), Spo::new(9, 101, 201)]; + let (stream, rep) = loci_from_triples( + &space(), + &codes(256), + &mask(16, &[9]), + &triples, + AnteRule::Recency, + ); + assert_eq!(rep.pronoun_subjects, 1); + assert_eq!(rep.bound, 1); + // The stream `wave.rs` owns resolves the locus this adapter wrote. + assert_eq!( + stream.ground_at(1, Locus::Antecedent, 99, 4), + lance_graph_contract::witness_fabric::WaveGrounding::Causal + ); + let r = stream + .resolve_at(1, Locus::Antecedent, 99, 1) + .expect("focal visible"); + assert_eq!(r.final_offset, Some(-1)); + } + + /// The guard STAYS SILENT on non-trivial input: a stream of real subjects + /// binds nothing. Not an empty stream — an empty one would prove only that + /// the loop handles emptiness. + #[test] + fn real_subjects_bind_nothing() { + let triples = vec![ + Spo::new(1, 100, 200), + Spo::new(2, 101, 201), + Spo::new(3, 102, 202), + ]; + let (stream, rep) = loci_from_triples( + &space(), + &codes(256), + &mask(16, &[9]), + &triples, + AnteRule::Recency, + ); + assert_eq!(triples.len(), 3, "anti-vacuity: there IS a stream"); + assert_eq!(rep.pronoun_subjects, 0); + assert_eq!(rep.bound, 0); + for pos in 0..3 { + assert_eq!( + stream.ground_at(pos, Locus::Antecedent, 99, 4), + lance_graph_contract::witness_fabric::WaveGrounding::Unbound + ); + } + } + + /// Past the `i4` reach the locus stays UNBOUND rather than clamping to -8 + /// and naming the wrong event: the antecedent has become a BASIN EDGE. + #[test] + fn beyond_the_reach_is_unbound_not_clamped() { + // One real subject, then 9 pronoun subjects. The 9th cannot see it. + let mut triples = vec![Spo::new(1, 100, 200)]; + triples.extend((0..9).map(|i| Spo::new(9, 110 + i, 210 + i))); + let (stream, rep) = loci_from_triples( + &space(), + &codes(256), + &mask(16, &[9]), + &triples, + AnteRule::Recency, + ); + assert_eq!(rep.pronoun_subjects, 9); + assert_eq!(rep.bound, 8, "the eight that fit"); + assert_eq!(rep.beyond_reach, 1, "the ninth is a basin edge"); + assert_eq!( + stream.ground_at(9, Locus::Antecedent, 99, 4), + lance_graph_contract::witness_fabric::WaveGrounding::Unbound + ); + } + + /// No pronoun information binds nothing — and the SAME input with a mask + /// does bind, so this is a real difference and not an unbindable stream. + #[test] + fn absent_evidence_is_not_a_licence_to_guess() { + let triples = vec![Spo::new(1, 100, 200), Spo::new(9, 101, 201)]; + let (_, bare) = loci_from_triples(&space(), &codes(256), &[], &triples, AnteRule::Recency); + assert_eq!(bare.bound, 0); + assert_eq!(bare.pronoun_subjects, 0); + let (_, with) = loci_from_triples( + &space(), + &codes(256), + &mask(16, &[9]), + &triples, + AnteRule::Recency, + ); + assert_eq!(with.bound, 1); + } + + /// Only `Antecedent` is written. A fabricated TEKAMOLO or Kausal pointer + /// would be indistinguishable downstream from a grounded one. + #[test] + fn no_locus_but_antecedent_is_ever_bound() { + let triples = vec![Spo::new(1, 100, 200), Spo::new(9, 101, 201)]; + let (stream, _) = loci_from_triples( + &space(), + &codes(256), + &mask(16, &[9]), + &triples, + AnteRule::Recency, + ); + let w = stream.window_range(lance_graph_contract::temporal_pov::VersionRange::new(0, 99)); + assert!(w.iter().any(|(_, f)| f.bound_count() == 1), "one is bound"); + for (_, f) in &w { + for l in Locus::ALL { + if l != Locus::Antecedent { + assert_eq!(f.at(l), 0, "{l:?} must stay unbound"); + } + } + } + } + + /// The two rules are genuinely different: on an input where the nearest + /// candidate is NOT the best selectional fit, they pick different offsets. + /// Without this the baseline would be decorative. + #[test] + fn selectional_fit_can_disagree_with_recency() { + let sp = space(); + let cds = codes(256); + // predicate id 5; candidate A (id 3) nearer, candidate B (id 40) farther. + // Whichever the trained space prefers, the two rules must be capable of + // disagreeing — assert that they DO on at least one arrangement. + let mut disagreed = false; + for far in [20u16, 40, 60, 80, 100] { + let triples = vec![ + Spo::new(far, 1, 2), + Spo::new(3, 1, 2), + Spo::new(9, 5, 2), // pronoun subject, predicate 5 + ]; + let m = mask(256, &[9]); + let (a, _) = loci_from_triples(&sp, &cds, &m, &triples, AnteRule::Recency); + let (b, _) = loci_from_triples(&sp, &cds, &m, &triples, AnteRule::SelectionalFit); + let ra = a + .resolve_at(2, Locus::Antecedent, 99, 1) + .unwrap() + .final_offset; + let rb = b + .resolve_at(2, Locus::Antecedent, 99, 1) + .unwrap() + .final_offset; + if ra != rb { + disagreed = true; + break; + } + } + assert!( + disagreed, + "the rules never disagreed — the baseline would be decorative" + ); + } +} diff --git a/crates/deepnsm-v2/src/promote.rs b/crates/deepnsm-v2/src/promote.rs new file mode 100644 index 000000000..42ba62476 --- /dev/null +++ b/crates/deepnsm-v2/src/promote.rs @@ -0,0 +1,591 @@ +//! `promote` — writing basins as SoA rows **at TOC node keys**. +//! +//! The step `D-ACR-6` was blocked on, now that its rail is minted and the tree +//! is spawned. A basin stops being a partition held in RAM and becomes a row +//! addressed at a literary unit. +//! +//! # Why the key is the whole point +//! +//! `E-HERMENEUTIK-RUNG-LADDER-1`: a basin keyed by a bare SUBJECT is rung-0 — +//! it keeps grammatical adjacency and discards genre, pericope and canonical +//! position, leaving SIZE as the only between-basin signal (which is what three +//! independent gates then measured). Promoting at a [`TocEntry`]'s address +//! keys the basin by **literary unit**, and it needs no change to the rail, +//! because a node's cascade position lives in its own key. +//! +//! So this module writes TWO things into one row and they answer different +//! questions: +//! +//! | half | carries | question | +//! |---|---|---| +//! | the KEY | classid + the TOC path folded into HEEL/HIP/TWIG/LEAF | *where in the book* | +//! | the LANE | [`BasinRow`] — subject, count, self-code, version range | *what was observed there* | +//! +//! # The tier fold is generic, not hand-carved +//! +//! [`tiers_of`] packs a [`NiblePath`]'s nibbles **left-aligned** into the +//! 16-nibble space the canon's tiers span, so +//! `NiblePath::from_guid_prefix_v3(key).prefix(depth)` returns the original +//! path by construction. Carving book/chapter/verse into bytes by hand would +//! split a chapter across two tiers and would be a second home for an address +//! the path already holds. +//! +//! # What this module does not decide +//! +//! **The classid is a parameter.** A corpus/book concept is a MINT, and minting +//! is not this module's call — it takes the classid it is given and refuses +//! nothing about it. Likewise `self_code` is copied from the basin's own +//! [`Cam96`](crate::space::Cam96) when one exists and left zero when it does +//! not; a fabricated centroid would be worse than an honest absence. + +use crate::basin::BasinCode; +use crate::toc::TocEntry; +use lance_graph_contract::canonical_node::{ + classid_read_mode, EdgeBlock, NodeGuid, NodeRow, TailVariant, ValueTenant, +}; +use lance_graph_contract::episodic_basin::{BasinRow, BASIN_ROW_BYTES}; +use lance_graph_contract::facet::FacetCascade; +use lance_graph_contract::hhtl::{NiblePath, MAX_DEPTH}; +use lance_graph_contract::tekamolo_facet::TekamoloFacet; + +/// The TEKAMOLO facet's on-row width — the 16-byte V3 4+12 facet. +pub const TEKAMOLO_BYTES: usize = 16; + +/// Fold a path into the four canonical tiers, **left-aligned**. +/// +/// The v3 read (`NiblePath::from_guid_prefix_v3`) assembles +/// `heel<<48 | hip<<32 | twig<<16 | leaf` as a full 16-nibble path, so a path +/// shorter than 16 must occupy the HIGH nibbles for its prefix to survive the +/// round trip. `packed()` right-aligns, hence the shift. +#[must_use] +pub fn tiers_of(path: NiblePath) -> (u16, u16, u16, u16) { + let (bits, depth) = path.packed(); + let shift = 4 * u32::from(MAX_DEPTH - depth); + let full = bits << shift; + ( + ((full >> 48) & 0xFFFF) as u16, + ((full >> 32) & 0xFFFF) as u16, + ((full >> 16) & 0xFFFF) as u16, + (full & 0xFFFF) as u16, + ) +} + +/// The node key for a TOC address under `classid`, or `None` if `classid` +/// cannot carry one. +/// +/// Minted through `mint_for(classid_read_mode(classid).tail_variant, …)` — the +/// symmetric spine — never `NodeGuid::new`. +/// +/// # Why this REFUSES the V1 tail instead of minting it +/// +/// The canon forbids the V1 tail for new units, and this is a promoter for new +/// units only. Refusing matters because the fallback is otherwise **silent** in +/// two stacked ways: +/// +/// 1. A classid with no registry entry resolves to `ReadMode::DEFAULT`, whose +/// `tail_variant` is `V1`. +/// 2. `mint_for` itself has a `#[cfg(not(feature = "guid-v2-tail"))]` arm that +/// maps V2/V3 back onto the V1 layout "so the crate compiles". +/// +/// Either path produces a key that looks fine — `Debug` prints, `Eq` works, +/// the HHTL prefix even round-trips while `family == 0` — but has dropped +/// `leaf` and written the tail as `family:u24 ++ identity:u24` at bytes 10..16, +/// so `identity_v2()` (bytes 14..16) reads something else entirely. That was +/// live in this crate until an identity assertion in `toc_hydrate` caught it. +/// +/// `None` means *this concept has no mint yet* — an OGAR question, not +/// something to paper over with a V1 key. +#[must_use] +pub fn key_at(classid: u32, path: NiblePath, family: u32, identity: u32) -> Option { + let tail = classid_read_mode(classid).tail_variant; + if tail == TailVariant::V1 { + return None; + } + let (heel, hip, twig, leaf) = tiers_of(path); + let key = NodeGuid::mint_for(tail, classid, heel, hip, twig, leaf, family, identity); + // The fallback arm above is a cfg, not a runtime branch, so a build without + // the feature would return a V1 key from a V3 request and this is the only + // place that can still see it. + debug_assert_eq!( + key.identity_v2(), + u16::try_from(identity).unwrap_or(u16::MAX), + "minted key does not read its own identity back — V1 fallback compiled in?" + ); + Some(key) +} + +/// A [`BasinRow`] from this crate's own [`BasinCode`], over a version range. +/// +/// `member_count` saturates rather than wraps: a basin with more than 65_535 +/// members reports the ceiling, which is visibly wrong, instead of a small +/// number that looks plausible. +#[must_use] +pub fn row_of(code: &BasinCode, version_from: u64, version_to: u64) -> BasinRow { + BasinRow { + subject: code.subject, + member_count: u16::try_from(code.members).unwrap_or(u16::MAX), + self_code: code.self_code, + version_from, + version_to, + } +} + +/// Write a basin into a row's [`ValueTenant::EpisodicBasin`] lane, in place. +/// +/// The offset is **derived** from the tenant descriptor +/// ([`ValueTenant::value_offset`]), never written as a literal — the table's own +/// rule, after a reservation was recorded as an absolute offset three times +/// running and went stale each time. +pub fn write_lane(row: &mut NodeRow, basin: &BasinRow) { + let off = ValueTenant::EpisodicBasin.value_offset(); + row.value[off..off + BASIN_ROW_BYTES].copy_from_slice(&basin.to_le_bytes()); +} + +/// Read the lane back. Returns [`BasinRow::EMPTY`] for an unwritten lane — +/// zero-fallback, so "no basin promoted here" is a value, not an error. +#[must_use] +pub fn read_lane(row: &NodeRow) -> BasinRow { + let off = ValueTenant::EpisodicBasin.value_offset(); + let mut b = [0u8; BASIN_ROW_BYTES]; + b.copy_from_slice(&row.value[off..off + BASIN_ROW_BYTES]); + BasinRow::from_le_bytes(&b) +} + +/// Promote one basin to a full row at a TOC address. +/// +/// `identity` distinguishes rows that share an address — the caller's +/// discriminator, not one this module invents. +#[must_use] +pub fn promote(entry: &TocEntry, basin: &BasinRow, classid: u32, identity: u32) -> Option { + let mut row = NodeRow { + key: key_at(classid, entry.path, 0, identity)?, + edges: EdgeBlock::default(), + value: [0u8; 480], + }; + write_lane(&mut row, basin); + Some(row) +} + +/// Promote many, pairing each basin with the TOC entry it was observed at. +/// +/// Pairs are `(entry, basin)` because a basin has no opinion about where it +/// belongs — the promoter's caller decides that, and this signature makes the +/// decision explicit instead of inferring it. +#[must_use] +pub fn promote_all(pairs: &[(TocEntry, BasinRow)], classid: u32) -> Vec { + pairs + .iter() + .enumerate() + .filter_map(|(i, (e, b))| promote(e, b, classid, u32::try_from(i).unwrap_or(0))) + .collect() +} + +// ── TEKAMOLO: the tenant every SoA row carries ───────────────────────────── + +/// The TEKAMOLO facet for a TOC address — the *when* lane, hydrated. +/// +/// `ValueTenant::Tekamolo` is a lane on EVERY SoA row, not a property of the +/// promoted basins: every node and every addressed triple carries its own +/// when/why/how/where address. This builds the one lane this crate has +/// evidence for. +/// +/// - **Temporal (when)** = `[book, chapter, verse]`, the node's own place in +/// corpus reading order, read as the lane's `256:256:256` coarse→fine +/// cascade. Narrative position IS the temporal address of a scripture node, +/// and it is exact — no inference. +/// - **Kausal / Modal / Lokal** stay ALL-ZERO. The tenant's own doc fixes the +/// meaning of that: *"Zero-fallback: an all-zero facet reads as unaddressed +/// (no lane asserted), never a wrong circumstance."* This crate extracts no +/// causal connective, no modal auxiliary and no place, so asserting any of +/// the three would be a fabricated circumstance — worse than an absent one. +/// +/// Returns `None` if a tier exceeds `u8` — a refusal, never a fold, matching +/// [`crate::toc`]'s rule that silently wrapping two literary units onto one +/// address is the failure to avoid. +#[must_use] +pub fn tekamolo_of(entry: &TocEntry, facet_classid: u32) -> Option { + let b = u8::try_from(entry.book).ok()?; + let c = u8::try_from(entry.chapter).ok()?; + let v = u8::try_from(entry.verse).ok()?; + Some(TekamoloFacet::from_lanes( + facet_classid, + [b, c, v], // temporal — the only lane with evidence + [0, 0, 0], // kausal — unaddressed + [0, 0, 0], // modal — unaddressed + [0, 0, 0], // lokal — unaddressed + )) +} + +/// Write the TEKAMOLO facet into a row's [`ValueTenant::Tekamolo`] lane. +/// +/// Offset derived from the tenant descriptor, never a literal — same rule as +/// [`write_lane`]. +pub fn write_tekamolo(row: &mut NodeRow, facet: &TekamoloFacet) { + let off = ValueTenant::Tekamolo.value_offset(); + row.value[off..off + TEKAMOLO_BYTES].copy_from_slice(&facet.facet().to_bytes()); +} + +/// Read the TEKAMOLO lane back. An unwritten lane reads all-zero — the +/// documented zero-fallback, i.e. *unaddressed*, not an error. +#[must_use] +pub fn read_tekamolo(row: &NodeRow) -> TekamoloFacet { + let off = ValueTenant::Tekamolo.value_offset(); + let mut b = [0u8; TEKAMOLO_BYTES]; + b.copy_from_slice(&row.value[off..off + TEKAMOLO_BYTES]); + TekamoloFacet::new(FacetCascade::from_bytes(&b)) +} + +/// A row for EVERY TOC entry, each with its TEKAMOLO lane hydrated. +/// +/// The basin lane is left at its zero-fallback here; [`promote`] writes that +/// for the subset of addresses a basin was measured at. TEKAMOLO is not that +/// subset — it is on every row. +#[must_use] +pub fn rows_with_tekamolo(entries: &[TocEntry], classid: u32) -> Vec { + entries + .iter() + .enumerate() + .filter_map(|(i, e)| { + let facet = tekamolo_of(e, classid)?; + let mut row = NodeRow { + key: key_at(classid, e.path, 0, u32::try_from(i).unwrap_or(0))?, + edges: EdgeBlock::default(), + value: [0u8; 480], + }; + write_tekamolo(&mut row, &facet); + Some(row) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::toc::{spawn, CorpusToc, TocLevel}; + use std::collections::HashSet; + + const CLASSID: u32 = NodeGuid::CLASSID_OSINT_V3; + + fn toy() -> Vec { + spawn( + &CorpusToc { + chapters: vec![vec![3, 2], vec![1]], + }, + 3, + ) + } + + fn basin(subject: u16, members: u32) -> BasinRow { + BasinRow { + subject, + member_count: u16::try_from(members).unwrap_or(u16::MAX), + self_code: [9; 12], + version_from: 4, + version_to: 11, + } + } + + /// **The load-bearing claim: the key still IS the TOC address.** + /// + /// Folding a path into tiers and reading it back through the canon's own + /// v3 fold must return the path. If it does not, a promoted basin is at an + /// address nobody can ascend from — which is precisely the gap the tree was + /// spawned to close. + #[test] + fn the_key_round_trips_to_the_toc_path() { + let entries = toy(); + assert!(!entries.is_empty(), "anti-vacuity: there are entries"); + let mut checked = 0usize; + for e in &entries { + let key = key_at(CLASSID, e.path, 0, 1).expect("V3 classid mints"); + let back = NiblePath::from_guid_prefix_v3(&key); + assert_eq!( + back.prefix(e.path.depth()), + Some(e.path), + "{e:?}: the key does not carry its own address back" + ); + checked += 1; + } + assert!(checked >= 9, "only {checked} addresses round-tripped"); + } + + /// Distinct literary units get distinct keys. Two basins sharing a key + /// would be unreadable apart once written. + #[test] + fn distinct_toc_entries_get_distinct_keys() { + let entries = toy(); + let keys: HashSet<[u8; 16]> = entries + .iter() + .map(|e| { + *key_at(CLASSID, e.path, 0, 0) + .expect("V3 classid mints") + .as_bytes() + }) + .collect(); + assert_eq!(keys.len(), entries.len(), "an address collided"); + // …and the ancestry survives the fold: a verse's key folds to a path + // whose parent-prefix is its chapter's path. + let verse = entries + .iter() + .find(|e| e.level == TocLevel::Verse) + .expect("a verse exists"); + let chapter = entries + .iter() + .find(|e| { + e.level == TocLevel::Chapter && e.book == verse.book && e.chapter == verse.chapter + }) + .expect("its chapter exists"); + assert!( + chapter.path.is_ancestor_of(verse.path), + "the chapter must remain an ancestor of its verse" + ); + } + + /// TEKAMOLO round-trips through a real row, and writing it touches NO + /// other byte — the field-isolation half for the second lane. + #[test] + fn tekamolo_round_trips_and_disturbs_nothing_else() { + let e = toy()[0]; + let f = tekamolo_of(&e, CLASSID).expect("toy tiers fit u8"); + let mut row = NodeRow { + key: key_at(CLASSID, e.path, 0, 1).expect("V3 mints"), + edges: EdgeBlock::default(), + value: [0u8; 480], + }; + write_tekamolo(&mut row, &f); + assert_eq!(read_tekamolo(&row), f, "the lane must read back"); + + let off = ValueTenant::Tekamolo.value_offset(); + for (i, b) in row.value.iter().enumerate() { + if i < off || i >= off + TEKAMOLO_BYTES { + assert_eq!(*b, 0, "byte {i} outside the TEKAMOLO lane was written"); + } + } + // Anti-vacuity: the lane itself is NOT all-zero, so the sweep above is + // discriminating rather than asserting a zeroed row is zeroed. + assert!( + row.value[off..off + TEKAMOLO_BYTES].iter().any(|b| *b != 0), + "the lane must carry something" + ); + } + + /// The two lanes do not overlap: writing one leaves the other intact. + #[test] + fn tekamolo_and_basin_lanes_are_disjoint() { + let e = toy()[0]; + let b = basin(77, 5); + let f = tekamolo_of(&e, CLASSID).expect("tiers fit"); + let mut row = promote(&e, &b, CLASSID, 3).expect("V3 mints"); + write_tekamolo(&mut row, &f); + assert_eq!(read_lane(&row), b, "basin survived the TEKAMOLO write"); + assert_eq!(read_tekamolo(&row), f, "TEKAMOLO survived the basin write"); + } + + /// Only the temporal lane is asserted. The other three stay at the + /// documented zero-fallback — "unaddressed, never a wrong circumstance". + #[test] + fn only_the_temporal_lane_is_asserted() { + let entries = toy(); + let verse = entries + .iter() + .find(|e| e.level == TocLevel::Verse) + .expect("a verse exists"); + let f = tekamolo_of(verse, CLASSID).expect("tiers fit"); + assert_eq!( + f.temporal(), + [ + u8::try_from(verse.book).unwrap(), + u8::try_from(verse.chapter).unwrap(), + u8::try_from(verse.verse).unwrap() + ] + ); + assert_eq!(f.causal(), [0, 0, 0]); + assert_eq!(f.modal(), [0, 0, 0]); + assert_eq!(f.local(), [0, 0, 0]); + } + + /// The temporal lane is a REAL prefix-routing axis: two verses in the same + /// chapter share more coarse→fine temporal prefix than two in different + /// books. Without this the lane would be stored and meaningless. + #[test] + fn temporal_prefix_routing_separates_near_from_far() { + use lance_graph_contract::tekamolo_facet::TekamoloRole; + let mk = |b: u16, c: u16, v: u16| { + tekamolo_of( + &TocEntry { + path: toy()[0].path, + level: TocLevel::Verse, + book: b, + chapter: c, + verse: v, + }, + CLASSID, + ) + .expect("tiers fit") + }; + let a = mk(1, 2, 3); + let same_chapter = mk(1, 2, 9); + let same_book = mk(1, 7, 3); + let other_book = mk(5, 2, 3); + let s_ch = a.shared(&same_chapter, TekamoloRole::Temporal); + let s_bk = a.shared(&same_book, TekamoloRole::Temporal); + let s_ot = a.shared(&other_book, TekamoloRole::Temporal); + assert!( + s_ch > s_bk && s_bk > s_ot, + "temporal prefix must order near→far: same-chapter {s_ch} > same-book {s_bk} > other-book {s_ot}" + ); + // Identity is the maximum, and it is reachable — the scale is not flat. + assert_eq!(a.shared(&a, TekamoloRole::Temporal), 3); + } + + /// A tier past `u8` is REFUSED, not folded onto a wrong address. + #[test] + fn a_tier_past_u8_is_refused_not_folded() { + let bad = TocEntry { + path: toy()[0].path, + level: TocLevel::Verse, + book: 0, + chapter: 300, // > 255 + verse: 1, + }; + assert!(tekamolo_of(&bad, CLASSID).is_none()); + // …and a legal one right at the boundary still mints, so the guard is + // not simply always-None. + let ok = TocEntry { + chapter: 255, + ..bad + }; + assert!(tekamolo_of(&ok, CLASSID).is_some()); + } + + /// EVERY tree node gets a row with a hydrated lane — TEKAMOLO is not the + /// basin subset. + #[test] + fn every_toc_entry_gets_a_hydrated_row() { + let entries = toy(); + let rows = rows_with_tekamolo(&entries, CLASSID); + assert_eq!(rows.len(), entries.len(), "one row per entry"); + for (r, e) in rows.iter().zip(&entries) { + assert_eq!( + read_tekamolo(r).temporal(), + [ + u8::try_from(e.book).unwrap(), + u8::try_from(e.chapter).unwrap(), + u8::try_from(e.verse).unwrap() + ] + ); + } + } + + /// A classid with no registry entry resolves to the V1 tail, and the + /// promoter must REFUSE it rather than mint the forbidden shape. + /// + /// This is the can-fire half; `distinct_toc_entries_get_distinct_keys` + /// above is the can-stay-silent half — it mints on a real V3 classid and + /// gets `Some` for every entry, so the guard is not simply always-None. + #[test] + fn an_unminted_classid_is_refused_not_silently_downgraded() { + let e = toy()[0]; + // 0x0301_0000 — MONDO's block, which is exactly what this crate used + // as a corpus stand-in. It has no read-mode entry, so it resolves to + // ReadMode::DEFAULT (V1) and the old code minted a V1 key from it. + let unminted = 0x0301_0000; + assert_eq!( + classid_read_mode(unminted).tail_variant, + TailVariant::V1, + "anti-vacuity: this classid really does resolve to V1" + ); + assert!(key_at(unminted, e.path, 0, 1).is_none()); + assert!(promote(&e, &basin(1, 1), unminted, 1).is_none()); + // ...and the V3 classid the tests use is genuinely different. + assert_ne!(classid_read_mode(CLASSID).tail_variant, TailVariant::V1); + assert!(key_at(CLASSID, e.path, 0, 1).is_some()); + } + + /// The lane round-trips through a real row, and writing it **touches no + /// other byte of the 480-byte slab** — the row-level field-isolation half. + #[test] + fn the_lane_round_trips_and_disturbs_nothing_else() { + let e = toy()[0]; + let b = basin(77, 5); + let row = promote(&e, &b, CLASSID, 3).expect("V3 classid mints"); + assert_eq!( + read_lane(&row), + b, + "the lane must read back what was written" + ); + + // Every byte outside the lane is still zero. + let off = ValueTenant::EpisodicBasin.value_offset(); + for (i, byte) in row.value.iter().enumerate() { + if (off..off + BASIN_ROW_BYTES).contains(&i) { + continue; + } + assert_eq!(*byte, 0, "byte {i} outside the lane was written"); + } + // Anti-vacuity: the lane itself is NOT all zero, so the loop above is + // not trivially true of the whole slab. + assert!(row.value[off..off + BASIN_ROW_BYTES] + .iter() + .any(|b| *b != 0)); + + // An unwritten row reads as "no basin", not as an error. + let bare = NodeRow { + key: row.key, + edges: EdgeBlock::default(), + value: [0u8; 480], + }; + assert!(read_lane(&bare).is_empty()); + } + + /// `row_of` carries the basin's own Cam96 through unchanged, and saturates + /// the member count rather than wrapping. + #[test] + fn row_of_carries_the_self_code_and_saturates_the_count() { + let code = BasinCode { + subject: 42, + self_code: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + width: 0.5, + members: 7, + contradiction: 0.25, + }; + let r = row_of(&code, 2, 8); + assert_eq!(r.subject, 42); + assert_eq!(r.self_code, code.self_code, "the Cam96 passes through"); + assert_eq!(r.member_count, 7); + assert_eq!((r.version_from, r.version_to), (2, 8)); + + // Saturation, not wrap: 70_000 members must not report 4_464. + let huge = BasinCode { + members: 70_000, + ..code + }; + assert_eq!(row_of(&huge, 0, 1).member_count, u16::MAX); + } + + /// `promote_all` keeps each basin with the entry it was paired to — a + /// reordering would put a basin at the wrong literary unit. + #[test] + fn promote_all_keeps_each_basin_at_its_own_entry() { + let entries = toy(); + let pairs: Vec<(TocEntry, BasinRow)> = entries + .iter() + .take(4) + .enumerate() + .map(|(i, e)| (*e, basin(u16::try_from(i).unwrap() + 1, i as u32 + 1))) + .collect(); + let rows = promote_all(&pairs, CLASSID); + assert_eq!(rows.len(), pairs.len()); + for (row, (e, b)) in rows.iter().zip(&pairs) { + assert_eq!(read_lane(row), *b, "basin landed on the wrong row"); + assert_eq!( + NiblePath::from_guid_prefix_v3(&row.key).prefix(e.path.depth()), + Some(e.path), + "row is keyed at the wrong literary unit" + ); + } + // Anti-vacuity: the subjects really differ, so a shuffle would show. + let subs: HashSet = pairs.iter().map(|(_, b)| b.subject).collect(); + assert_eq!(subs.len(), pairs.len()); + } +} diff --git a/crates/deepnsm-v2/src/tekamolo.rs b/crates/deepnsm-v2/src/tekamolo.rs new file mode 100644 index 000000000..44b2331cb --- /dev/null +++ b/crates/deepnsm-v2/src/tekamolo.rs @@ -0,0 +1,718 @@ +//! `tekamolo` — hydrating all FOUR lanes of the `ValueTenant::Tekamolo` tenant +//! (**Temporal · Kausal · Modal · Lokal**) by a **left-corner**, **style-driven** +//! read of German scripture. +//! +//! The tenant is a 16-byte V3 facet on EVERY SoA row — `classid(4) + 6×(u8:u8)` +//! read in `G4D3` as four `256:256:256` coarse→fine lanes, answering *when / +//! why / how / where*. Its contract fixes the zero-fallback: **an all-zero lane +//! reads as UNADDRESSED — never a wrong circumstance.** +//! +//! [`crate::promote::tekamolo_of`] hydrates **Temporal** from the TOC address +//! (`book:chapter:verse` — narrative position, exact). This module hydrates the +//! other three from German **grammar heuristics**. +//! +//! ## The heuristics are German, so the corpus is German +//! +//! `de/tekamolo.tsv` (German codebook, UD German-GSD + German-HDT): `lane · +//! lemma · relation · count`, 163 rows — which adverbial form, in which +//! syntactic relation, lands in which lane. German forms ground German text and +//! nothing else: pointed at an English lane they match nothing and the tenant +//! would read "unaddressed" for a corpus that merely had the wrong lexicon. +//! +//! Two German versions are carried on purpose (`luther1545`, +//! `elberfelder1905`). A lane rate that holds across two independent +//! translations of the SAME verses is a property of the language; one that +//! appears in a single lane is a property of that translator. The pair is the +//! control, not a bigger sample. +//! +//! ## Left-corner: carry the incomplete hypothesis, commit at the right corner +//! +//! `.claude/knowledge/left-corner-grammar-tree-pointer-fabric.md` and PR #849 +//! both name the failure mode: **premature commitment**, not a missing rule. +//! The obvious scan — take the FIRST adverbial that matches a lane — IS that +//! bug: it commits a left-corner reading before the clause has shown what it +//! is. A verse opening *"Da sprach er, weil …"* would bind Kausal to `da` +//! (which is also Lokal, at identical corpus count) and never see `weil`, the +//! unambiguous causal marker two words later. +//! +//! So a match OPENS a [`LaneHypothesis`] instead of committing one. Hypotheses +//! accumulate to the clause's **right corner** (a verse or punctuation +//! boundary), and only there does the lane resolve — by the heuristics' own +//! corpus evidence, with the runner-up's margin deciding commit-vs-abstain. +//! +//! [`Commit::LeftCorner`] keeps the premature behaviour available as the +//! declared BASELINE, because a right-corner claim with no left-corner control +//! measures nothing. +//! +//! ## The thinking style drives the read +//! +//! The knobs are not constants. `ThinkingStyle → FieldModulation → ScanParams` +//! is the shipped read-parameterization, and this module consumes it: +//! +//! | knob | what it sets here | +//! |---|---| +//! | `fan_out` | how many competing hypotheses a lane carries to the right corner | +//! | `depth_bias` | ≥ `LEFT_CORNER_AT` commits at the left corner (depth-first = decide now) | +//! | `resonance_threshold` | the margin the winner must beat the runner-up by | +//! | `noise_tolerance` | whether an ambiguous form (`da`) may win a lane at all | +//! +//! [`V2StyleProvider`] supplies the modulation. The contract declares +//! [`ThinkingStyleProvider`] and ships **no implementation** — the provider is +//! the consumer's job, and this is v2's. +//! +//! ## `da` is genuinely ambiguous and is not silently resolved +//! +//! The heuristics list `da` under BOTH Kausal and Lokal at identical counts +//! (*because* / *there*). This module never breaks that tie by fiat: an +//! ambiguous form opens a hypothesis in EVERY lane it is listed under, and +//! whether it may ultimately win one is `noise_tolerance`'s call, reported per +//! read. [`GrammarHeuristics::ambiguous`] counts such forms. + +use lance_graph_contract::mul::{DkPosition, FlowState, MulAssessment}; +use lance_graph_contract::tekamolo_facet::TekamoloRole; +use lance_graph_contract::thinking::{ + FieldModulation, ScanParams, SparseVec, StyleCluster, ThinkingStyle, ThinkingStyleProvider, +}; +use std::collections::HashMap; + +/// `depth_bias` at or above this commits at the LEFT corner (decide now). +pub const LEFT_CORNER_AT: f64 = 0.7; + +/// `noise_tolerance` at or above this lets an ambiguous form win a lane. +pub const AMBIGUOUS_OK_AT: f64 = 0.5; + +/// A form's syntactic relation class — the COARSE tier of a hydrated lane. +/// Ordinals start at 1 so `0` stays the zero-fallback (unaddressed). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum RelationClass { + /// `advmod` — adverbial modifier. + Advmod = 1, + /// `mark` — subordinating marker. + Mark = 2, + /// `cc` — coordinating conjunction. + Cc = 3, + /// `obl` — oblique nominal. + Obl = 4, + /// `advcl` — adverbial clause. + Advcl = 5, +} + +impl RelationClass { + /// Parse the UD relation the heuristics carry; `None` for an unknown one — + /// refused, never folded onto a neighbouring class. + #[must_use] + pub fn parse(s: &str) -> Option { + Some(match s.trim() { + "advmod" => Self::Advmod, + "mark" => Self::Mark, + "cc" => Self::Cc, + "obl" => Self::Obl, + "advcl" => Self::Advcl, + _ => return None, + }) + } +} + +/// One hydrated lane address: coarse relation class + rank within the lane. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LaneAddress { + /// Coarse tier — which relation carried the form. + pub relation: RelationClass, + /// Fine tier — the form's rank inside its lane, 1-based, by corpus count. + /// Saturates at 255 rather than wrapping onto a different form. + pub rank: u8, +} + +impl LaneAddress { + /// The lane's 3-byte `256:256:256` cascade, coarse→fine. Tier 2 is reserved + /// `0`: this read has two graded levels of evidence, not three, and padding + /// it with a third would fake a depth the heuristics do not carry. + #[must_use] + pub const fn bytes(self) -> [u8; 3] { + [self.relation as u8, self.rank, 0] + } +} + +/// One carried, uncommitted lane reading — the left-corner hypothesis. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct LaneHypothesis { + /// Where it would land if it wins. + pub addr: LaneAddress, + /// The heuristics' corpus count for this (form, lane, relation) — the + /// evidence the right corner adjudicates on. + pub weight: f64, + /// Token index that opened it, so a caller can see WHERE the circumstance + /// was stated rather than only that it was. + pub at: usize, + /// Is the opening form listed under more than one lane (`da`)? + pub ambiguous: bool, +} + +/// Where a lane commits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Commit { + /// Commit at the FIRST match — the premature left-corner reading the + /// left-corner literature names as the failure mode. Kept as the declared + /// BASELINE so the right-corner result has a control. + LeftCorner, + /// Carry every hypothesis to the clause boundary and adjudicate there. + RightCorner, +} + +/// German grammar heuristics: which adverbial form, in which relation, lands in +/// which TEKAMOLO lane. +#[derive(Debug, Clone, Default)] +pub struct GrammarHeuristics { + /// `form -> [(lane, address, count)]`; a form listed under two lanes keeps + /// BOTH entries — the ambiguity is data, not noise to be cleaned. + by_form: HashMap>, + ambiguous: usize, + rows: usize, +} + +impl GrammarHeuristics { + /// Parse `de/tekamolo.tsv` — `lane\tlemma\trelation\tcount`, `#` comments. + /// + /// Ranks are per lane by descending count (ties by form, so the ranking is + /// stable across runs — a rank that moves between runs is not an address). + #[must_use] + pub fn parse(tsv: &str) -> Self { + let mut per_lane: HashMap> = HashMap::new(); + let mut rows = 0usize; + for line in tsv.lines() { + if line.starts_with('#') || line.trim().is_empty() { + continue; + } + let f: Vec<&str> = line.split('\t').collect(); + let (Some(lane), Some(lemma), Some(rel), Some(count)) = + (f.first(), f.get(1), f.get(2), f.get(3)) + else { + continue; + }; + let role = match lane.trim() { + "Temporal" => TekamoloRole::Temporal, + "Kausal" => TekamoloRole::Kausal, + "Modal" => TekamoloRole::Modal, + "Lokal" => TekamoloRole::Lokal, + _ => continue, + }; + let (Some(relation), Ok(n)) = (RelationClass::parse(rel), count.trim().parse::()) + else { + continue; + }; + rows += 1; + per_lane.entry(role as u8).or_default().push(( + lemma.trim().to_lowercase(), + relation, + n, + )); + } + + let mut by_form: HashMap> = HashMap::new(); + for role in TekamoloRole::ALL { + let Some(mut v) = per_lane.remove(&(role as u8)) else { + continue; + }; + v.sort_by(|a, b| b.2.cmp(&a.2).then_with(|| a.0.cmp(&b.0))); + for (i, (lemma, relation, n)) in v.into_iter().enumerate() { + let rank = u8::try_from(i + 1).unwrap_or(u8::MAX); + by_form + .entry(lemma) + .or_default() + .push((role, LaneAddress { relation, rank }, n)); + } + } + let ambiguous = by_form.values().filter(|v| lanes_of(v).len() > 1).count(); + Self { + by_form, + ambiguous, + rows, + } + } + + /// Entries for a lowercased form — empty if the heuristics do not carry it. + #[must_use] + pub fn lookup(&self, form: &str) -> &[(TekamoloRole, LaneAddress, u64)] { + self.by_form.get(form).map_or(&[], Vec::as_slice) + } + + /// Is this form listed under more than one lane (`da`)? + #[must_use] + pub fn is_ambiguous(&self, form: &str) -> bool { + lanes_of(self.lookup(form)).len() > 1 + } + + /// How many forms carry more than one lane. + #[must_use] + pub const fn ambiguous(&self) -> usize { + self.ambiguous + } + + /// `(distinct forms, parsed rows)`. + #[must_use] + pub fn sizes(&self) -> (usize, usize) { + (self.by_form.len(), self.rows) + } +} + +/// Distinct lanes among a form's entries. +fn lanes_of(entries: &[(TekamoloRole, LaneAddress, u64)]) -> Vec { + let mut l: Vec = entries.iter().map(|(r, _, _)| *r as u8).collect(); + l.sort_unstable(); + l.dedup(); + l +} + +/// deepnsm-v2's [`ThinkingStyleProvider`] — the contract declares the trait and +/// ships no implementation, because the modulation table is the consumer's. +/// +/// The projection is **7D, not 23D**, and says so: the 23D cognitive vector +/// belongs to the persona-modeling storyline, which `.claude/v3/knowledge/ +/// persona-vs-rung-ladder.md` records as deliberately unwired ("carried and +/// displayed, not acted on"). Claiming 23 dimensions here would import that +/// storyline into a read path that has no business asserting it. +#[derive(Debug, Clone, Copy, Default)] +pub struct V2StyleProvider; + +impl ThinkingStyleProvider for V2StyleProvider { + /// The 7 [`FieldModulation`] knobs as a sparse vector — this crate's honest + /// projection, indices `0..7` in `to_fingerprint` order. + fn style_vector(&self, style: ThinkingStyle) -> SparseVec { + let m = self.default_modulation(style); + vec![ + (0, m.resonance_threshold as f32), + (1, m.fan_out as f32), + (2, m.depth_bias as f32), + (3, m.breadth_bias as f32), + (4, m.noise_tolerance as f32), + (5, m.speed_bias as f32), + (6, m.exploration as f32), + ] + } + + /// Per-CLUSTER modulation, not per-style: this crate has evidence for how a + /// cluster reads (converge vs diverge vs attend vs hurry) and none for how + /// `Blunt` differs from `Frank`. Six rows are what is grounded; 36 would be + /// 30 invented ones. + fn default_modulation(&self, style: ThinkingStyle) -> FieldModulation { + match style.cluster() { + // Decide now, narrowly, on strong evidence — the LEFT-corner reader. + StyleCluster::Analytical => FieldModulation { + resonance_threshold: 0.6, + fan_out: 2, + depth_bias: 0.85, + breadth_bias: 0.15, + noise_tolerance: 0.15, + speed_bias: 0.4, + exploration: 0.2, + }, + // Same haste, less rigour: commits early AND tolerates a thin margin. + StyleCluster::Direct => FieldModulation { + resonance_threshold: 0.3, + fan_out: 2, + depth_bias: 0.9, + breadth_bias: 0.1, + noise_tolerance: 0.4, + speed_bias: 0.9, + exploration: 0.1, + }, + // Carry everything to the right corner; admit ambiguous forms. + StyleCluster::Exploratory => FieldModulation { + resonance_threshold: 0.15, + fan_out: 8, + depth_bias: 0.2, + breadth_bias: 0.8, + noise_tolerance: 0.75, + speed_bias: 0.2, + exploration: 0.9, + }, + StyleCluster::Creative => FieldModulation { + resonance_threshold: 0.2, + fan_out: 6, + depth_bias: 0.3, + breadth_bias: 0.7, + noise_tolerance: 0.7, + speed_bias: 0.3, + exploration: 0.8, + }, + // Wide attention, but unwilling to assert on a thin margin. + StyleCluster::Empathic => FieldModulation { + resonance_threshold: 0.5, + fan_out: 5, + depth_bias: 0.35, + breadth_bias: 0.65, + noise_tolerance: 0.55, + speed_bias: 0.3, + exploration: 0.5, + }, + // Deliberate: carries the most, demands the most before committing. + StyleCluster::Meta => FieldModulation { + resonance_threshold: 0.7, + fan_out: 8, + depth_bias: 0.25, + breadth_bias: 0.75, + noise_tolerance: 0.3, + speed_bias: 0.1, + exploration: 0.6, + }, + } + } + + /// Style from the MUL assessment. The mapping is deliberately coarse and + /// states its own ground: an overconfident reader is pushed to the style + /// that REFUSES early commitment, not the one that indulges it. + fn select_from_assessment(&self, a: &MulAssessment) -> ThinkingStyle { + match a.dk_position { + // Mount Stupid commits early and is wrong: force the right-corner + // reader on it. This is the whole point of measuring DK at all. + DkPosition::MountStupid => ThinkingStyle::Metacognitive, + DkPosition::ValleyOfDespair => ThinkingStyle::Curious, + DkPosition::SlopeOfEnlightenment => ThinkingStyle::Analytical, + DkPosition::Plateau => match a.homeostasis.flow_state { + FlowState::Flow => ThinkingStyle::Direct, + _ => ThinkingStyle::Systematic, + }, + } + } +} + +/// A style-parameterized read of one clause. +#[derive(Debug, Clone, Copy)] +pub struct StyleRead { + /// Where the lanes commit — derived from `depth_bias`. + pub commit: Commit, + /// Hypotheses a lane carries — `fan_out`. + pub fan_out: usize, + /// Margin the winner must beat the runner-up by, as a fraction of the + /// winner's weight — `resonance_threshold`. + pub margin: f64, + /// May an ambiguous form (`da`) win a lane — `noise_tolerance`. + pub admit_ambiguous: bool, + /// The SIMD scan params the same modulation yields, carried so a caller + /// reads ONE object rather than re-deriving them. + pub scan: ScanParams, +} + +impl StyleRead { + /// Derive the read from a style through the provider — never hand-set. + #[must_use] + pub fn of(style: ThinkingStyle) -> Self { + let m = V2StyleProvider.default_modulation(style); + Self { + commit: if m.depth_bias >= LEFT_CORNER_AT { + Commit::LeftCorner + } else { + Commit::RightCorner + }, + fan_out: m.fan_out.max(1), + margin: m.resonance_threshold, + admit_ambiguous: m.noise_tolerance >= AMBIGUOUS_OK_AT, + scan: m.to_scan_params(), + } + } +} + +/// What one clause's read produced. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct ClauseRead { + /// The committed address per lane, in Te-Ka-Mo-Lo order. `None` = + /// UNADDRESSED (the tenant's zero-fallback), never a guess. + pub lanes: [Option; 4], + /// Hypotheses opened across all lanes — the evidence the read SAW. + pub opened: usize, + /// Lanes that had a hypothesis but committed nothing (margin too thin, or + /// an ambiguous form the style would not admit). The abstention count is + /// the honest half of the report. + pub abstained: usize, + /// Lanes whose winner was an ambiguous form. + pub ambiguous_wins: usize, +} + +/// Read ONE clause's tokens under `style`. +/// +/// Every matching form opens a [`LaneHypothesis`]. Under +/// [`Commit::RightCorner`] the hypotheses are carried to the end of the token +/// slice — the clause's right corner — and only then adjudicated. Under +/// [`Commit::LeftCorner`] each lane locks on its first hypothesis, which is the +/// premature reading kept as the baseline. +#[must_use] +pub fn read_clause(h: &GrammarHeuristics, tokens: &[&str], style: ThinkingStyle) -> ClauseRead { + let sr = StyleRead::of(style); + let mut carried: [Vec; 4] = [vec![], vec![], vec![], vec![]]; + let mut out = ClauseRead::default(); + + for (i, tok) in tokens.iter().enumerate() { + let w: String = tok + .chars() + .filter(|c| c.is_alphabetic()) + .collect::() + .to_lowercase(); + if w.is_empty() { + continue; + } + let amb = h.is_ambiguous(&w); + for &(role, addr, n) in h.lookup(&w) { + let lane = &mut carried[role as usize]; + // LEFT corner: the first hypothesis for a lane locks it. This is + // the documented premature commitment, reproduced deliberately. + if sr.commit == Commit::LeftCorner && !lane.is_empty() { + continue; + } + if lane.len() >= sr.fan_out { + continue; // fan_out bounds what a lane may carry + } + lane.push(LaneHypothesis { + addr, + weight: n as f64, + at: i, + ambiguous: amb, + }); + out.opened += 1; + } + } + + for (li, lane) in carried.iter_mut().enumerate() { + if lane.is_empty() { + continue; + } + // Adjudicate on the heuristics' own corpus evidence; ties broken by the + // EARLIER token, because with equal evidence the framing adverbial is + // the one that opened the clause. + lane.sort_by(|a, b| { + b.weight + .partial_cmp(&a.weight) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.at.cmp(&b.at)) + }); + let best = lane[0]; + if best.ambiguous && !sr.admit_ambiguous { + out.abstained += 1; + continue; + } + // Margin as a FRACTION of the winner's weight, so the threshold means + // the same thing for a 3,916-count form and a 5-count one. + let runner = lane.get(1).map_or(0.0, |h| h.weight); + let sep = if best.weight > 0.0 { + (best.weight - runner) / best.weight + } else { + 0.0 + }; + if lane.len() >= 2 && sep < sr.margin { + out.abstained += 1; + continue; + } + if best.ambiguous { + out.ambiguous_wins += 1; + } + out.lanes[li] = Some(best.addr); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A slice of the real German heuristics, verbatim from `de/tekamolo.tsv` + /// (including `da` under BOTH lanes at identical counts). + const H: &str = "# lane\tlemma\trelation\tcount\n\ + Lokal\tzu\tmark\t3916\n\ + Temporal\tnoch\tadvmod\t2575\n\ + Modal\tso\tadvmod\t1703\n\ + Kausal\tdamit\tadvmod\t986\n\ + Kausal\tda\tmark\t437\n\ + Lokal\tda\tmark\t437\n\ + Kausal\tweil\tmark\t273\n\ + Lokal\thier\tadvmod\t440\n\ + Temporal\theute\tadvmod\t637\n"; + + fn h() -> GrammarHeuristics { + GrammarHeuristics::parse(H) + } + + #[test] + fn heuristics_parse_and_rank_per_lane() { + let g = h(); + assert_eq!(g.sizes().1, 9, "9 data rows"); + // Kausal ranks by descending count: damit(986) > da(437) > weil(273). + let damit = g.lookup("damit")[0].1; + let weil = g.lookup("weil")[0].1; + assert_eq!(damit.rank, 1); + assert_eq!(weil.rank, 3); + assert_eq!(damit.relation, RelationClass::Advmod); + assert_eq!(weil.relation, RelationClass::Mark); + // `da` carries TWO lanes and is reported ambiguous. + assert_eq!(g.lookup("da").len(), 2); + assert!(g.is_ambiguous("da")); + assert!(!g.is_ambiguous("weil")); + assert_eq!(g.ambiguous(), 1); + } + + /// **The left-corner bug, reproduced and then fixed.** + /// + /// *"Da sprach er, weil …"* — `da` (ambiguous, 437) opens Kausal first; + /// `weil` (unambiguous, 273) arrives two words later. A LEFT-corner reader + /// locks `da` and never sees `weil`. A RIGHT-corner reader carries both and + /// adjudicates. This is the whole point of the module. + #[test] + fn left_corner_locks_the_first_form_right_corner_carries_both() { + let g = h(); + let toks = ["Da", "sprach", "er", "weil", "Gott", "rief"]; + + // Direct: depth_bias 0.9 → LeftCorner. Only `da` was ever opened. + let left = read_clause(&g, &toks, ThinkingStyle::Direct); + assert_eq!( + StyleRead::of(ThinkingStyle::Direct).commit, + Commit::LeftCorner + ); + assert_eq!(left.opened, 2, "da opened Kausal AND Lokal, then locked"); + + // Exploratory: depth_bias 0.2 → RightCorner, fan_out 8. + let right = read_clause(&g, &toks, ThinkingStyle::Exploratory); + assert_eq!( + StyleRead::of(ThinkingStyle::Exploratory).commit, + Commit::RightCorner + ); + assert_eq!(right.opened, 3, "da x2 + weil — weil was SEEN"); + assert!( + right.opened > left.opened, + "the right-corner read must see strictly more evidence: {} vs {}", + right.opened, + left.opened + ); + } + + /// The two readers disagree about the Kausal ADDRESS, not merely about how + /// much they looked at. Without this the distinction would be cosmetic. + #[test] + fn the_two_corners_commit_different_kausal_addresses() { + let g = h(); + // `weil` first (273), then `damit` (986) — the LATER form has the + // stronger corpus evidence, so left-corner and right-corner must differ. + let toks = ["weil", "er", "rief", "damit", "es", "geschah"]; + let left = read_clause(&g, &toks, ThinkingStyle::Direct); + let right = read_clause(&g, &toks, ThinkingStyle::Exploratory); + let k = TekamoloRole::Kausal as usize; + assert_eq!( + left.lanes[k].map(|a| a.rank), + Some(3), + "left corner locks `weil` (rank 3) because it came first" + ); + assert_eq!( + right.lanes[k].map(|a| a.rank), + Some(1), + "right corner adjudicates to `damit` (rank 1) on the evidence" + ); + assert_ne!(left.lanes[k], right.lanes[k]); + } + + /// An ambiguous form may win only where the style admits it — and the + /// abstention is REPORTED, not silently dropped. + #[test] + fn ambiguity_is_admitted_by_style_never_by_fiat() { + let g = h(); + let toks = ["Da", "stand", "er"]; // only `da` — ambiguous, both lanes + // Analytical: noise_tolerance 0.15 → refuses ambiguous forms. + let strict = read_clause(&g, &toks, ThinkingStyle::Analytical); + assert!(!StyleRead::of(ThinkingStyle::Analytical).admit_ambiguous); + assert_eq!(strict.lanes[TekamoloRole::Kausal as usize], None); + assert_eq!(strict.lanes[TekamoloRole::Lokal as usize], None); + assert_eq!(strict.abstained, 2, "both lanes abstained, and said so"); + assert_eq!(strict.ambiguous_wins, 0); + + // Exploratory: noise_tolerance 0.75 → admits it, in BOTH lanes, and the + // tie is never broken by fiat. + let open = read_clause(&g, &toks, ThinkingStyle::Exploratory); + assert!(StyleRead::of(ThinkingStyle::Exploratory).admit_ambiguous); + assert!(open.lanes[TekamoloRole::Kausal as usize].is_some()); + assert!(open.lanes[TekamoloRole::Lokal as usize].is_some()); + assert_eq!(open.ambiguous_wins, 2); + } + + /// A clause with no German adverbial leaves every lane UNADDRESSED — the + /// tenant's zero-fallback, and the guard can stay silent on real input. + #[test] + fn a_clause_without_cues_addresses_nothing() { + let g = h(); + let toks = ["Gott", "sprach", "und", "es", "ward", "Licht"]; + for style in [ThinkingStyle::Direct, ThinkingStyle::Exploratory] { + let r = read_clause(&g, &toks, style); + assert_eq!(r.opened, 0); + assert_eq!(r.abstained, 0); + assert!(r.lanes.iter().all(Option::is_none), "{style:?}"); + } + } + + /// `fan_out` actually bounds what a lane carries: raising it strictly + /// increases the evidence a right-corner read opens. + #[test] + fn fan_out_bounds_the_carried_hypotheses() { + let g = h(); + // Four Kausal forms in one clause. + let toks = ["damit", "da", "weil", "damit", "x", "y"]; + let creative = read_clause(&g, &toks, ThinkingStyle::Creative); // fan_out 6 + let meta = read_clause(&g, &toks, ThinkingStyle::Metacognitive); // fan_out 8 + assert!( + StyleRead::of(ThinkingStyle::Creative).fan_out + < StyleRead::of(ThinkingStyle::Metacognitive).fan_out + ); + assert!( + meta.opened >= creative.opened, + "a wider fan_out cannot see less: {} vs {}", + meta.opened, + creative.opened + ); + } + + /// The provider is real: every cluster yields a distinct modulation, and + /// the derived commit corner splits the six clusters into both kinds. + #[test] + fn the_provider_separates_the_clusters() { + let mut corners = std::collections::HashSet::new(); + let mut fingerprints = std::collections::HashSet::new(); + for s in ThinkingStyle::ALL { + let sr = StyleRead::of(s); + corners.insert(sr.commit); + fingerprints.insert(V2StyleProvider.default_modulation(s).to_fingerprint()); + } + assert_eq!(corners.len(), 2, "both commit corners must be reachable"); + assert_eq!( + fingerprints.len(), + 6, + "one modulation per cluster, all distinct" + ); + // The 7D projection is the modulation, not a fabricated 23D vector. + assert_eq!(V2StyleProvider.style_vector(ThinkingStyle::Wise).len(), 7); + } + + /// An overconfident reader is pushed AWAY from early commitment — the + /// reason to measure Dunning-Kruger at all. + #[test] + fn mount_stupid_is_routed_to_the_right_corner_reader() { + use lance_graph_contract::mul::{Homeostasis, TrustQualia, TrustTexture}; + let mk = |dk: DkPosition| MulAssessment { + trust: TrustQualia { + value: 0.5, + texture: TrustTexture::Calibrated, + }, + dk_position: dk, + homeostasis: Homeostasis { + flow_state: FlowState::Flow, + allostatic_load: 0.2, + }, + complexity_mapped: true, + free_will_modifier: 0.5, + }; + let overconfident = V2StyleProvider.select_from_assessment(&mk(DkPosition::MountStupid)); + assert_eq!( + StyleRead::of(overconfident).commit, + Commit::RightCorner, + "Mount Stupid must NOT be handed the early-commit reader" + ); + // …and the expert in flow IS allowed to decide fast, so the mapping is + // discriminating rather than always-RightCorner. + let expert = V2StyleProvider.select_from_assessment(&mk(DkPosition::Plateau)); + assert_eq!(StyleRead::of(expert).commit, Commit::LeftCorner); + } +} diff --git a/crates/deepnsm-v2/src/toc.rs b/crates/deepnsm-v2/src/toc.rs new file mode 100644 index 000000000..01cc9e261 --- /dev/null +++ b/crates/deepnsm-v2/src/toc.rs @@ -0,0 +1,426 @@ +//! `toc` — spawning the table of contents as an HHTL tree of SoA nodes. +//! +//! Operator: *"Brutal heißt, bevor wir über das Buch nachdenken wird ein +//! Inhaltsverzeichnis als HHTL-Baum mit SoA-Knoten erstellt."* "Brutal" names +//! the SEQUENCING: the TOC is minted as the FULL tree skeleton **before** any +//! triple-level reasoning touches a verse — no incremental, content-driven +//! tree-growing. +//! +//! # Why the tree has to exist first — two reasons, not one +//! +//! 1. **Addressing.** `D-ACR-12`'s ascent primitive walks `NiblePath::parent()` +//! until it finds content. Over an unspawned tree an ascent from a verse +//! witness terminates at an unminted gap. Top-down construction makes every +//! ancestor real by definition. +//! 2. **Structure-bearing basins.** `E-HERMENEUTIK-RUNG-LADDER-1`: a basin +//! keyed by a bare SUBJECT is rung-0 — it keeps grammatical adjacency and +//! discards genre, pericope and canonical position, leaving SIZE as the only +//! between-basin signal (which is what three independent gates then +//! measured). A basin promoted at a TOC node's key IS keyed by literary +//! unit. The tree is what makes that possible; the rail already supports it +//! with no byte change, because a node's cascade position is in its own key. +//! +//! # Two nibbles per level, and why that is the canon rather than a workaround +//! +//! `FAN_OUT = 16` — one nibble addresses 0..15. The KJV does not fit that at any +//! level: 66 books, chapters to 150 (Psalms), verses to 176 (Psalm 119). So each +//! level spends **two** nibbles (capacity 256), which is exactly the canon's +//! *"scale is the next cascade level, never field-widening"* — the answer to +//! "more than 16 children" is more levels, never a wider nibble. +//! +//! Depth cost: the basin root nibble (1) + book 2 + chapter 2 + verse 2 = +//! **7 of `MAX_DEPTH`'s 16**, so the whole KJV addresses inside one `u64` path +//! with room left over. The root nibble is easy to forget — +//! [`NiblePath::root`] already consumes depth 1, so a book lands at depth 3, +//! not 2; the depth assertion in the tests is what caught that. +//! +//! # What this module does NOT do +//! +//! It mints ADDRESSES, not rows: a `TocEntry` is a path plus the coordinate it +//! encodes. Writing SoA rows at those paths is the promoter's job, and reading +//! them is gated by the `le-contract.md` §3b jc-pillar rule like any lane. + +use lance_graph_contract::hhtl::{NiblePath, FAN_OUT, MAX_DEPTH}; + +/// Nibbles one TOC level occupies. Two ⇒ 256 children per level, which covers +/// every KJV level with headroom (max is 176 verses in Psalm 119). +pub const NIBBLES_PER_LEVEL: u8 = 2; + +/// Largest ordinal one level can address (`16² − 1`). +pub const MAX_PER_LEVEL: u16 = (FAN_OUT as u16) * (FAN_OUT as u16) - 1; + +// ── Compile-time sizing guarantees ─────────────────────────────────────── +// The KJV's real extremes must fit the two-nibble carve, and a verse address +// must leave depth for deeper routing. These are properties of the CONSTANTS, +// so they are asserted at compile time for every build — not in a test that +// only runs under `cargo test`. (Same precedent as `canonical_node.rs`'s +// `const _` row-size asserts.) +const _: () = assert!(66 <= MAX_PER_LEVEL, "66 books must address in one level"); +const _: () = assert!(150 <= MAX_PER_LEVEL, "Psalms has 150 chapters"); +const _: () = assert!(176 <= MAX_PER_LEVEL, "Psalm 119 has 176 verses"); +const _: () = assert!( + TocLevel::Verse.depth() + NIBBLES_PER_LEVEL <= MAX_DEPTH, + "a verse address must leave depth for a level below it" +); + +/// Which tier of the book tree an entry sits at. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum TocLevel { + /// A book (Genesis …) — depth 3 (basin 1 + 2). + Book, + /// A chapter within a book — depth 5. + Chapter, + /// A verse within a chapter — depth 7. + Verse, +} + +impl TocLevel { + /// Path depth in nibbles at this level, **including the basin root + /// nibble** — the off-by-one that a depth assertion caught during the + /// build: `NiblePath::root` is already depth 1. + #[must_use] + pub const fn depth(self) -> u8 { + match self { + TocLevel::Book => 1 + NIBBLES_PER_LEVEL, + TocLevel::Chapter => 1 + 2 * NIBBLES_PER_LEVEL, + TocLevel::Verse => 1 + 3 * NIBBLES_PER_LEVEL, + } + } +} + +/// One minted node of the skeleton: its address and the coordinate it encodes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TocEntry { + /// The HHTL address. + pub path: NiblePath, + /// Tier. + pub level: TocLevel, + /// 0-based book ordinal. + pub book: u16, + /// 1-based chapter, `0` at [`TocLevel::Book`]. + pub chapter: u16, + /// 1-based verse, `0` above [`TocLevel::Verse`]. + pub verse: u16, +} + +/// Extend `base` by `value` encoded in [`NIBBLES_PER_LEVEL`] nibbles, high +/// nibble first. `None` if `value` exceeds [`MAX_PER_LEVEL`] or the path is +/// full — a **refusal, never a fold**: silently wrapping a 256th chapter onto +/// chapter 0 would address two literary units with one node. +#[must_use] +pub fn descend(base: NiblePath, value: u16) -> Option { + if value > MAX_PER_LEVEL || base.depth() + NIBBLES_PER_LEVEL > MAX_DEPTH { + return None; + } + let hi = u8::try_from(value >> 4).ok()?; + let lo = u8::try_from(value & 0x0F).ok()?; + base.try_child(hi)?.try_child(lo) +} + +/// The corpus outline the spawn walks: per book, the verse count of each of its +/// chapters. `chapters[b][c]` = verses in book `b`, chapter `c`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CorpusToc { + /// Per book, per chapter, the verse count. + pub chapters: Vec>, +} + +impl CorpusToc { + /// Derive the outline from an ordered `(chapter, verse)` marker stream — + /// what [`crate::corpus`]'s `d+:d+` markers give. + /// + /// **A book opens at `1:1`.** Every book starts at chapter 1 verse 1, and + /// `(1,1)` never recurs inside a book, so that marker is exactly the book + /// boundary. An inference from the corpus's own numbering, not an external + /// table. + /// + /// **⊘ CORRECTED 2026-08-22 by the real corpus.** The first rule here was + /// "a chapter RESET opens a book" — chapter 1 following a higher chapter. + /// Run against the real KJV it found **61 books, not 66**, and the missing + /// five are exactly the single-chapter books (Obadiah, Philemon, 2 John, + /// 3 John, Jude): such a book is followed by another that also starts at + /// chapter 1, so `prev_chapter == 1`, no reset is seen, and the two books + /// MERGE. The merge then costs verses too (the merged chapter keeps the + /// max verse count, not the sum), which is where 78 missing verse nodes and + /// 88 unaddressed triples came from. + /// + /// The synthetic fixtures could not catch it: none of them contained a + /// single-chapter book. It took the corpus. + #[must_use] + pub fn from_markers(markers: &[(u16, u16)]) -> Self { + let mut chapters: Vec> = Vec::new(); + for &(ch, vs) in markers { + if ch == 1 && vs == 1 { + chapters.push(Vec::new()); + } + let book = match chapters.last_mut() { + Some(b) => b, + None => { + chapters.push(Vec::new()); + chapters.last_mut().expect("just pushed") + } + }; + while book.len() < ch as usize { + book.push(0); + } + if ch >= 1 { + let slot = &mut book[ch as usize - 1]; + *slot = (*slot).max(vs); + } + } + Self { chapters } + } + + /// Books in the outline. + #[must_use] + pub fn book_count(&self) -> usize { + self.chapters.len() + } + + /// Total verses across the outline. + #[must_use] + pub fn verse_count(&self) -> usize { + self.chapters + .iter() + .flat_map(|b| b.iter()) + .map(|v| *v as usize) + .sum() + } +} + +/// Spawn the FULL skeleton, top-down: every book, then its chapters, then their +/// verses — in that order, so no entry is ever emitted before its parent. +/// +/// `basin` is the root nibble the whole corpus hangs under. +/// +/// Entries whose ordinal exceeds [`MAX_PER_LEVEL`] are **skipped, with their +/// subtree**, rather than folded onto a sibling. A dropped node is a gap an +/// ascent can detect; a folded one is two literary units sharing an address, +/// which nothing downstream could detect. +#[must_use] +pub fn spawn(toc: &CorpusToc, basin: u8) -> Vec { + let root = NiblePath::root(basin); + if root == NiblePath::EMPTY { + return Vec::new(); + } + let mut out = Vec::new(); + for (bi, book) in toc.chapters.iter().enumerate() { + let Ok(b) = u16::try_from(bi) else { continue }; + let Some(bp) = descend(root, b) else { continue }; + out.push(TocEntry { + path: bp, + level: TocLevel::Book, + book: b, + chapter: 0, + verse: 0, + }); + for (ci, &verses) in book.iter().enumerate() { + let Ok(c) = u16::try_from(ci + 1) else { + continue; + }; + let Some(cp) = descend(bp, c) else { continue }; + out.push(TocEntry { + path: cp, + level: TocLevel::Chapter, + book: b, + chapter: c, + verse: 0, + }); + for v in 1..=verses { + let Some(vp) = descend(cp, v) else { continue }; + out.push(TocEntry { + path: vp, + level: TocLevel::Verse, + book: b, + chapter: c, + verse: v, + }); + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + /// A small outline: 2 books, chapters with differing verse counts. + fn toy() -> CorpusToc { + CorpusToc { + chapters: vec![vec![3, 2], vec![1]], + } + } + + /// **The reason the tree is spawned at all: every ancestor is real.** + /// + /// `D-ACR-12`'s ascent walks `parent()` until it finds content. This asserts + /// the walk can never leave the minted set — from EVERY node, every ancestor + /// up to the basin is itself minted. Over an unspawned tree this is exactly + /// what fails. + #[test] + fn every_ancestor_of_every_node_is_itself_minted() { + let entries = spawn(&toy(), 3); + let minted: HashSet = entries.iter().map(|e| e.path).collect(); + let root = NiblePath::root(3); + assert!( + !entries.is_empty(), + "anti-vacuity: the spawn produced nodes" + ); + + let mut checked = 0usize; + for e in &entries { + let mut cur = e.path; + while let Some(p) = cur.parent() { + if p == root || p.depth() < 2 { + break; + } + // Only whole levels are nodes; the odd depths are interior + // nibbles of a two-nibble level, not addresses. + // Node depths are 3/5/7 (basin + whole levels); the even + // depths between them are interior nibbles, not addresses. + if (p.depth() - 1) % NIBBLES_PER_LEVEL == 0 { + assert!( + minted.contains(&p), + "{:?} at depth {} has an unminted ancestor at depth {}", + e.level, + e.path.depth(), + p.depth() + ); + checked += 1; + } + cur = p; + } + } + assert!( + checked >= 6, + "anti-vacuity: only {checked} ancestor links were actually checked" + ); + } + + /// Top-down order: a parent is always emitted before any of its descendants. + /// "Brutal" is a sequencing claim, so it gets a sequencing test. + #[test] + fn the_skeleton_is_emitted_parent_before_child() { + let entries = spawn(&toy(), 1); + let mut seen: HashSet = HashSet::new(); + for e in &entries { + if e.level != TocLevel::Book { + let parent_depth = e.level.depth() - NIBBLES_PER_LEVEL; + let p = e.path.prefix(parent_depth).expect("parent prefix exists"); + assert!(seen.contains(&p), "{e:?} emitted before its parent"); + } + seen.insert(e.path); + } + // Anti-vacuity: there ARE children to order. + assert!(entries.iter().any(|e| e.level == TocLevel::Verse)); + } + + /// Counts and depths match the outline exactly — no node invented, none lost. + #[test] + fn the_skeleton_covers_the_outline_exactly() { + let t = toy(); + let entries = spawn(&t, 5); + let n = |lvl: TocLevel| entries.iter().filter(|e| e.level == lvl).count(); + assert_eq!(n(TocLevel::Book), t.book_count()); + assert_eq!(n(TocLevel::Chapter), 3, "2 + 1 chapters"); + assert_eq!(n(TocLevel::Verse), t.verse_count(), "3 + 2 + 1 verses"); + for e in &entries { + assert_eq!(e.path.depth(), e.level.depth(), "{e:?} depth mismatch"); + } + // Every address is distinct — two literary units may never share a node. + let uniq: HashSet = entries.iter().map(|e| e.path).collect(); + assert_eq!(uniq.len(), entries.len(), "an address was reused"); + } + + /// **Refusal, never a fold.** A level ordinal past 255 has no address; the + /// node and its subtree are skipped rather than wrapped onto a sibling. + /// Both halves: 255 addresses, 256 refuses. + #[test] + fn an_ordinal_past_the_level_capacity_is_refused_not_folded() { + let base = NiblePath::root(2); + assert_eq!(MAX_PER_LEVEL, 255); + assert!(descend(base, 255).is_some(), "255 is addressable"); + assert_eq!(descend(base, 256), None, "256 must refuse"); + // …and distinctly: 255 and 0 are different addresses, so nothing folded. + assert_ne!(descend(base, 255), descend(base, 0)); + + // A chapter past capacity drops its verses WITH it, rather than + // re-pointing them at chapter 0's node. + let over = CorpusToc { + chapters: vec![vec![1; 300]], + }; + let entries = spawn(&over, 2); + let chapters = entries + .iter() + .filter(|e| e.level == TocLevel::Chapter) + .count(); + assert_eq!(chapters, 255, "only the addressable chapters are minted"); + let uniq: HashSet = entries.iter().map(|e| e.path).collect(); + assert_eq!( + uniq.len(), + entries.len(), + "a dropped node was folded instead" + ); + } + + /// Book boundaries come from `1:1`, and — the case the real KJV caught — + /// **two single-chapter books in a row must still be two books.** The old + /// chapter-reset rule merged them, losing 5 books and 78 verses on the real + /// corpus. No synthetic fixture had a single-chapter book; this one does. + #[test] + fn two_single_chapter_books_in_a_row_stay_two_books() { + // Obadiah-shaped: one chapter, then another one-chapter book. + let t = CorpusToc::from_markers(&[(1, 1), (1, 2), (1, 1), (1, 2), (1, 3)]); + assert_eq!(t.book_count(), 2, "the chapter-reset rule merged these"); + assert_eq!(t.chapters[0], vec![2]); + assert_eq!(t.chapters[1], vec![3]); + assert_eq!(t.verse_count(), 5, "no verse may be lost to a merge"); + } + + /// Book boundaries come from `1:1`, and the derivation is falsifiable: a + /// stream with a second `1:1` yields two books, one without yields one. + #[test] + fn book_boundaries_come_from_chapter_one_verse_one() { + let two = CorpusToc::from_markers(&[(1, 1), (1, 2), (2, 1), (1, 1), (1, 2)]); + assert_eq!( + two.book_count(), + 2, + "the reset at the 4th marker opens a book" + ); + assert_eq!(two.chapters[0], vec![2, 1]); + assert_eq!(two.chapters[1], vec![2]); + + // Silence half: no reset ⇒ one book, so the rule discriminates. + let one = CorpusToc::from_markers(&[(1, 1), (2, 1), (3, 1)]); + assert_eq!(one.book_count(), 1); + assert_eq!(one.verse_count(), 3); + } + + /// The level depths are what the module claims — the off-by-one the basin + /// root nibble causes, pinned. (The CAPACITY claims are `const _` asserts + /// at module level: they are properties of constants, so they hold at + /// compile time for every build, not only under `cargo test`.) + #[test] + fn the_level_depths_include_the_basin_root_nibble() { + assert_eq!(TocLevel::Book.depth(), 3, "1 basin + 2, not 2"); + assert_eq!(TocLevel::Chapter.depth(), 5); + assert_eq!(TocLevel::Verse.depth(), 7); + // Each level is exactly NIBBLES_PER_LEVEL below the next. + assert_eq!( + TocLevel::Chapter.depth() - TocLevel::Book.depth(), + NIBBLES_PER_LEVEL + ); + assert_eq!( + TocLevel::Verse.depth() - TocLevel::Chapter.depth(), + NIBBLES_PER_LEVEL + ); + // And a real spawned node agrees with the declared depth — the tie + // between the constant and the code that emits addresses. + let e = spawn(&toy(), 4); + for x in &e { + assert_eq!(x.path.depth(), x.level.depth()); + } + } +} diff --git a/crates/deepnsm/examples/persona_chain_replay.rs b/crates/deepnsm/examples/persona_chain_replay.rs new file mode 100644 index 000000000..4a10a04c2 --- /dev/null +++ b/crates/deepnsm/examples/persona_chain_replay.rs @@ -0,0 +1,376 @@ +//! PROBE PERSONA-CHAIN-REPLAY — drive `ContextChain` replay with REAL +//! CANDIDATES, and measure it against the shipped agreement resolver's gold. +//! +//! ## The gap this closes +//! +//! `disambiguator_glue::disambiguate_with_trajectory` shipped the fingerprint +//! bridge, but **all three of its round-trip tests pass +//! `std::iter::empty::()`** — they exercise the +//! empty-candidates SENTINEL path only. The contract's replay machinery +//! (`ContextChain::disambiguate_with` → winner / margin / dispersion / +//! `escalate_to_llm`) has therefore never had a caller that supplied real +//! candidates. This probe is that caller. +//! +//! Nothing here is new machinery. Every piece is shipped: +//! +//! | piece | where it already lives | +//! |---|---| +//! | ±5 replay, margin, escalation | `contract::grammar::context_chain` | +//! | f32 bundle → `Binary16K` | `deepnsm::disambiguator_glue::sign_binarize_to_binary16k` | +//! | pronoun case, `Ambiguous` never decides | `contract::grammar::clause_cues::pronoun_case` | +//! | where the answer is recorded | `contract::causal_witness::{CausalWitnessFacet, Locus::Antecedent}` | +//! | the gold + the agreement resolver | `deepnsm::examples::spo_anaphora_nibble` | +//! +//! ## The candidate set IS the persona register +//! +//! One surface form addresses several referents ("not all Marys are the mother +//! of Jesus"). The register supplies N candidates for one form; the chain +//! replay picks by coherence; the margin decides commit-vs-escalate. Here the +//! candidates are the window's non-pronoun NP heads — the same set the shipped +//! agreement resolver ranks — so the two doors are compared on ONE input. +//! +//! ## The content fingerprint is the DOCUMENTED metric, not an invention +//! +//! `E-FREQ-IS-COSINE-REPLACEMENT-1` (probe `freq_is_cosine`, rho 0.762) makes +//! frequency-rank distance the metric; `l9_loci_real_text` uses `|delta rank|/16` +//! on the same COCA table. A THERMOMETER code carries that metric exactly into +//! the `Binary16K` Hamming carrier: bit `i` is set iff `i < rank/16`, so +//! +//! ```text +//! hamming(fp(a), fp(b)) == |rank(a)/16 - rank(b)/16| +//! ``` +//! +//! — the documented distance, byte-exact, not a re-derivation. Gate G3 asserts it. +//! +//! ## Registered gates (fixed before the first run) +//! +//! 1. **G1 REAL-CANDIDATES** — every disambiguation call has +//! `candidate_count >= 2` (the sentinel path is NOT what is being measured). +//! 2. **G2 CASE-GATE** — an `Ambiguous` pronoun (`you`/`it`/`her`) is never +//! committed on case; #849's "no parse beats a wrong parse". +//! 3. **G3 METRIC-EXACT** — the thermometer Hamming equals the documented +//! rank distance for every pair in the fixture. +//! 4. **G4 REPORTED, NOT TUNED** — agreement with the shipped resolver's gold +//! is printed whatever it is. A worse number is a finding (cf. #852's +//! closed-class detectors measured LOSING to `rank<=150` and recorded as +//! "stop searching"), never a reason to tune the kernel until it passes. +//! +//! ## Run +//! +//! ```bash +//! cargo run --manifest-path crates/deepnsm/Cargo.toml --example persona_chain_replay +//! ``` + +use deepnsm::disambiguator_glue::sign_binarize_to_binary16k; +use lance_graph_contract::causal_witness::{CausalWitnessFacet, Locus}; +use lance_graph_contract::crystal::fingerprint::CrystalFingerprint; +use lance_graph_contract::grammar::clause_cues::{pronoun_case, PronounCase}; +use lance_graph_contract::grammar::context_chain::{ + ContextChain, DisambiguateOpts, WeightingKernel, DISAMBIGUATION_MARGIN_THRESHOLD, +}; +use std::collections::HashMap; + +/// Bits in a `Binary16K`. +const BITS: usize = 16_384; +/// The documented rank-distance scale (`l9_loci_real_text`: `|delta rank|/16`). +const RANK_SCALE: u32 = 16; +/// The i4 locus reach — `E-ANAPHORA-BEYOND-I4-IS-A-BASIN-EDGE-1`. Beyond this a +/// reference is a BASIN EDGE (a different KIND of reference), never a wider +/// pointer, so the window is a type boundary and not a tunable. +const WINDOW: usize = 8; + +/// `lemma -> rank` from the committed COCA table — the same loader shape +/// `spo_anaphora_nibble` uses, on the same file. +fn load_ranks(csv_path: &str) -> HashMap { + let text = std::fs::read_to_string(csv_path).unwrap_or_default(); + let mut m = HashMap::new(); + for line in text.lines().skip(1) { + let f: Vec<&str> = line.split(',').collect(); + if f.len() < 2 { + continue; + } + if let Ok(r) = f[0].trim().parse::() { + m.entry(f[1].to_ascii_lowercase()).or_insert(r); + } + } + m +} + +/// The thermometer level for a rank: `rank / RANK_SCALE`, capped at `BITS`. +fn level(rank: u32) -> usize { + ((rank / RANK_SCALE) as usize).min(BITS) +} + +/// A f32 content vector whose sign-binarization is the thermometer code. +/// +/// Bit `i` set iff `i < level(rank)`, so Hamming between two words is exactly +/// `|level(a) - level(b)|` — the documented `|delta rank|/16`. +fn thermometer_content(rank: u32) -> Vec { + let lv = level(rank); + (0..BITS).map(|i| if i < lv { 1.0 } else { -1.0 }).collect() +} + +/// A word's `Binary16K` fingerprint: the thermometer content, sign-binarized +/// by the shipped glue. +/// +/// **Deliberately NOT routed through `MarkovBundler`'s role slice.** The +/// obvious-looking move — build a one-token `WindowedSentence` so the call +/// "uses the shipped bundler" — would lay the thermometer into a 2000-wide +/// SUBJECT slice and silently CLAMP every `level(rank) > 2000`, destroying the +/// exact `|delta rank|/16` identity that G3 pins. The bundler's job is +/// superposing a WINDOW of sentences; a single candidate word is not that, and +/// calling it here would be decoration that breaks the metric. The chain below +/// is over TOKEN positions — the same ±8 token window `spo_anaphora_nibble` +/// and `l9_loci_real_text` both use. +fn word_fp(rank: u32) -> CrystalFingerprint { + CrystalFingerprint::Binary16K(sign_binarize_to_binary16k(&thermometer_content(rank))) +} + +/// Is this (lowercased) token a pronoun the case catalogue knows? +fn is_pronoun(tok: &str) -> bool { + pronoun_case(tok).is_some() +} + +/// The candidate antecedents for the pronoun at `pos`: the non-pronoun tokens +/// with a known rank inside the `-WINDOW` reach, nearest first. +/// +/// This set IS the persona register for this probe: N referents addressable +/// from one pronoun. The chain replay's job is to pick among them. +fn candidates_at(stream: &[&str], pos: usize, ranks: &HashMap) -> Vec<(usize, u32)> { + let lo = pos.saturating_sub(WINDOW); + (lo..pos) + .rev() + .filter(|&k| !is_pronoun(stream[k])) + .filter_map(|k| ranks.get(stream[k]).map(|&r| (k, r))) + .collect() +} + +/// Fill an 11-slot `ContextChain` centred on `pos`: slot `focal + delta` carries +/// the fingerprint of the token at `pos + delta`, or `None` where the token has +/// no rank (out of the COCA table) or falls outside the stream. +/// +/// The focal slot is left EMPTY — that is the position the replay writes each +/// candidate into. +fn chain_around(stream: &[&str], pos: usize, ranks: &HashMap) -> ContextChain { + let mut chain = ContextChain::new(); + let focal = ContextChain::focal_index() as isize; + for slot in 0..chain.fingerprints.len() { + let delta = slot as isize - focal; + if delta == 0 { + continue; // the replay slot + } + let idx = pos as isize + delta; + if idx < 0 || idx as usize >= stream.len() { + continue; + } + if let Some(&r) = ranks.get(stream[idx as usize]) { + chain.fingerprints[slot] = Some(word_fp(r)); + } + } + chain +} + +fn main() { + let csv = concat!(env!("CARGO_MANIFEST_DIR"), "/word_frequency/lemmas_5k.csv"); + let ranks = load_ranks(csv); + assert!(!ranks.is_empty(), "COCA table must load"); + + // The fixture is `spo_anaphora_nibble`'s, VERBATIM — same stream, same gold. + // Comparing two doors on one input is the point; a fresh fixture would + // measure nothing. + let stream: Vec<&str> = vec![ + "the", "man", "read", "a", "book", ".", // 0..6 + "he", "liked", "it", ".", // 6:he->man, 8:it->book + "the", "girls", "played", ".", // 10..14 + "they", "won", ".", // 14:they->girls + "the", "car", "that", "broke", ".", // 18..23 19:that->car + "it", "rained", ".", // 22:it -> UNRESOLVED (pleonastic) + ]; + let gold: &[(usize, Option<&str>)] = &[ + (6, Some("man")), + (8, Some("book")), + (14, Some("girls")), + (19, Some("car")), + (22, None), + ]; + + println!("PERSONA-CHAIN-REPLAY — real candidates through ContextChain replay\n"); + + let mut fail: Vec = Vec::new(); + let mut agree = 0usize; + let mut scored = 0usize; + let mut out_of_catalogue: Vec = Vec::new(); + let mut min_candidates = usize::MAX; + let mut widest_spread = 0.0f32; + + for &(pos, want) in gold { + let tok = stream[pos]; + let Some(case) = pronoun_case(tok) else { + // `that` is a RELATIVE pronoun; `clause_cues::pronoun_case` is the + // PERSONAL-pronoun case catalogue and does not carry it. Reported, + // not patched: extending a Core catalogue to make a probe pass is + // the adapter-hack the core-gap doctrine forbids. + out_of_catalogue.push(pos); + println!(" '{tok}'@{pos} OUT-OF-CATALOGUE (relative pronoun; personal-case catalogue only)"); + continue; + }; + + let cands = candidates_at(&stream, pos, &ranks); + if cands.len() < 2 { + println!(" '{tok}'@{pos} <2 candidates in the window — no replay"); + continue; + } + + min_candidates = min_candidates.min(cands.len()); + let chain = chain_around(&stream, pos, &ranks); + let res = chain.disambiguate_with( + ContextChain::focal_index(), + cands.iter().map(|&(_, r)| word_fp(r)).collect::>(), + DisambiguateOpts { + kernel: Some(WeightingKernel::MexicanHat), + sentinel_fp: None, + }, + ); + + // G1: this is the REAL-candidate path, not the sentinel. + if res.candidate_count < 2 { + fail.push(format!( + "@{pos} G1: candidate_count={}", + res.candidate_count + )); + } + + // The DYNAMIC RANGE of the metric, against the threshold that gates a + // commit. If the widest coherence spread the candidates can produce is + // orders of magnitude under the threshold, the replay cannot commit on + // ANY input — which is a property of the pairing, not of this fixture. + let spread = res + .alternatives + .first() + .zip(res.alternatives.last()) + .map_or(0.0, |(hi, lo)| hi.1 - lo.1); + widest_spread = widest_spread.max(spread); + + // G2: `Ambiguous` case never decides (#849 — no parse beats a wrong parse). + let case_blocked = case == PronounCase::Ambiguous; + + let committed = if case_blocked || res.escalate_to_llm { + None + } else { + cands.get(res.winner_index).copied() + }; + + let witness = match committed { + Some((k, _)) => { + let off = i8::try_from(pos - k).map(|d| -d).unwrap_or(0); + CausalWitnessFacet::ZERO.with(Locus::Antecedent, off) + } + None => CausalWitnessFacet::ZERO, + }; + + let picked = committed.map(|(k, _)| stream[k]); + let reason = if case_blocked { + "case=Ambiguous -> withheld" + } else if res.escalate_to_llm { + "margin below threshold -> escalate" + } else { + "committed" + }; + println!( + " '{tok}'@{pos} case={case:?} n={} margin={:.4} disp={:.4} ante={:+} -> {:?} [{reason}] (gold {:?})", + res.candidate_count, + res.margin, + res.dispersion, + witness.antecedent(), + picked, + want + ); + + // Only the case-decidable pronouns are SCORED against gold; a withheld + // Ambiguous is correct behaviour, not a wrong answer. + if !case_blocked { + scored += 1; + if picked == want { + agree += 1; + } + } else if want.is_some() { + // An Ambiguous pronoun whose gold IS resolvable is exactly the + // information the case gate costs. Recorded, not hidden. + println!(" ^ the case gate withheld a resolvable one — the price of G2"); + } + } + + // ── G3: the thermometer carries the DOCUMENTED metric exactly ── + let words: Vec<&str> = ["man", "book", "girls", "car", "read", "played"] + .into_iter() + .filter(|w| ranks.contains_key(*w)) + .collect(); + assert!(words.len() >= 3, "anti-vacuity: G3 needs real pairs"); + let mut g3_pairs = 0usize; + let mut g3_nonzero = 0usize; + for (i, a) in words.iter().enumerate() { + for b in &words[i + 1..] { + let (ra, rb) = (ranks[*a], ranks[*b]); + let expect = level(ra).abs_diff(level(rb)) as u32; + let got: u32 = match (word_fp(ra), word_fp(rb)) { + (CrystalFingerprint::Binary16K(x), CrystalFingerprint::Binary16K(y)) => x + .iter() + .zip(y.iter()) + .map(|(p, q)| (p ^ q).count_ones()) + .sum(), + _ => unreachable!("word_fp always builds Binary16K"), + }; + if got != expect { + fail.push(format!("G3 {a}/{b}: hamming {got} != |dRank|/16 {expect}")); + } + g3_pairs += 1; + if expect > 0 { + g3_nonzero += 1; + } + } + } + // Anti-vacuity: if every pair had distance 0 the identity would hold + // trivially and prove nothing about the carrier. + if g3_nonzero * 2 < g3_pairs { + fail.push(format!( + "G3 is near-vacuous: only {g3_nonzero}/{g3_pairs} pairs have non-zero distance" + )); + } + + // ── report ── + println!(); + println!("G1 REAL-CANDIDATES min candidates per replay = {min_candidates} (sentinel path never taken)"); + println!("G2 CASE-GATE Ambiguous pronouns withheld, never committed on case"); + println!("G3 METRIC-EXACT {g3_pairs} pairs, {g3_nonzero} with non-zero distance, hamming == |dRank|/{RANK_SCALE}"); + println!( + "G4 REPORTED chain-replay agreement with the shipped resolver's gold: {agree}/{scored}" + ); + if !out_of_catalogue.is_empty() { + println!( + " {} gold item(s) out of the personal-case catalogue: {out_of_catalogue:?}", + out_of_catalogue.len() + ); + } + println!(" escalation threshold = {DISAMBIGUATION_MARGIN_THRESHOLD} (contract constant, untouched)"); + println!( + "G5 SCALE widest coherence spread across candidates = {widest_spread:.6}; \n\ + \x20 threshold = {DISAMBIGUATION_MARGIN_THRESHOLD} -> ratio {:.0}x too small. The rank metric\n\ + \x20 and the margin gate are on INCOMPATIBLE SCALES: common-word\n\ + \x20 ranks differ by ~15 bits out of 16,384, so every replay\n\ + \x20 escalates regardless of input. Reported, not tuned.", + f64::from(DISAMBIGUATION_MARGIN_THRESHOLD) / f64::from(widest_spread.max(1e-9)) + ); + + if fail.is_empty() { + println!("\nGATES PASS — the real-candidate replay path runs end to end: candidates from"); + println!("the window, coherence from the documented rank metric, commit-vs-escalate from"); + println!("the contract's own margin, and the answer recorded as a Locus::Antecedent"); + println!("offset. G4 is a MEASUREMENT, not a target: see the note below."); + } else { + println!("\nGATES FAILED:"); + for f in &fail { + println!(" - {f}"); + } + std::process::exit(1); + } +} diff --git a/crates/deepnsm/src/lib.rs b/crates/deepnsm/src/lib.rs index 99be66ce8..cfcdd0472 100644 --- a/crates/deepnsm/src/lib.rs +++ b/crates/deepnsm/src/lib.rs @@ -63,6 +63,7 @@ pub mod vocabulary; pub mod markov_bundle; pub mod nsm_primes; +pub mod ontology_vocab; pub mod trajectory; // E-ENGLISH-BIFURCATES — two SEPARATE faculties (don't fuse them): diff --git a/crates/deepnsm/src/ontology_vocab.rs b/crates/deepnsm/src/ontology_vocab.rs new file mode 100644 index 000000000..c48fd303e --- /dev/null +++ b/crates/deepnsm/src/ontology_vocab.rs @@ -0,0 +1,382 @@ +//! The **second codebook**: the OBO Ontology vocabulary, alongside COCA. +//! +//! DeepNSM's own [`Vocabulary`](crate::vocabulary::Vocabulary) is the academic +//! frequency codebook — COCA ranks in a 12-bit space. This module is the other +//! one a reasoning caller needs: the public OBO biomedical reference +//! (`ConceptDomain::Ontology`, `0x03XX`), read through the contract's zero-dep +//! wire mirror. +//! +//! # No new dependency, and that is the point +//! +//! `lance-graph-contract` is already a hard dep of this crate (the canonical +//! `RoleKeySlice` constants), so reaching the ontology costs nothing here. The +//! alternative — deping the producer `ogar-obo` — is what the plug-and-play +//! posture exists to avoid, and it would pull an OBO bake into a crate whose +//! job is distributional semantics. +//! +//! # Two codebooks, two address spaces — never silently merged +//! +//! | codebook | address | width | source | +//! |---|---|---|---| +//! | COCA | frequency rank | 12-bit (`VOCAB_SIZE`) | [`crate::vocabulary`] | +//! | Ontology | canonical concept id | 16-bit (`0x03XX`) | the OGAR mint, mirrored | +//! +//! A rank and a concept id are not interchangeable and this module does not +//! offer a conversion. Fusing them into one integer space would make +//! `rank == concept` collisions unnoticeable, and the two spaces have entirely +//! different owners: a rank moves when the corpus is re-counted, a concept id +//! is minted once and never moves. +//! +//! # Why this could not be written before 2026-08-22 +//! +//! `concepts_in_domain(ConceptDomain::Ontology)` returned **empty**. The OBO +//! concept ids lived only in the producer crates, so the shared codebook had +//! nothing in that domain — and an empty enumeration reads exactly like "this +//! domain has nothing to reason about". The operator's ruling that the domains +//! are minted in `ogar-vocab` is what gave this module something to return. +//! +//! # WordNet: the same law, a second register (restored 2026-10-04) +//! +//! [`WordNetRail`] reads the WordNet 3.1 is-a rail (`wordnet31_isa_v2.tsv`, +//! codebook release `v0.1.0-codebooks-2026-07-26`, WordNet License). A synset +//! offset is an IDENTITY exactly like an ontology concept id: minted by +//! WordNet, never moved, with no distance semantics. So the register offers +//! lookup, ancestry and membership — `is_a` and [`ConceptMask`] — and no +//! distance of any kind. That is the line `48405aa2` drew (CAM-PQ is +//! prohibited for ontologies), kept here: the hypernym chain is a parent/child +//! ADDRESS (the HHTL reading), never a metric. +//! +//! A mask is a set of synsets; a sense is covered when it or any ancestor is in +//! the set. That expresses categories that cut ACROSS the taxonomy — water +//! animals = {aquatic_mammal, aquatic_vertebrate, cephalopod} covers whale, +//! dolphin, shark, manta and octopus, whose only common ancestor is `animal`. +//! Senses are explicit: `dolphin` sense 1 is a fish and `octopus` sense 1 is +//! seafood, so a mask asks about a SENSE, never a word. + +use lance_graph_contract::ogar_codebook::{concepts_in_domain, ConceptDomain}; +use std::collections::{HashMap, HashSet, VecDeque}; + +/// One ontology concept: its canonical name and the id it is minted at. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OntologyConcept { + /// Canonical concept name as minted (`"mondo"`, `"uberon"`, `"bfo"`, …). + pub name: &'static str, + /// The canonical hi-u16 concept id (`0x03XX`). + pub concept_id: u16, +} + +/// The whole Ontology vocabulary, in codebook order. +/// +/// Derived from the mirror on every call — never cached into a second table, +/// so it cannot drift from the mint the way a local copy would. +#[must_use] +pub fn ontology_vocabulary() -> Vec { + concepts_in_domain(ConceptDomain::Ontology) + .map(|(name, concept_id)| OntologyConcept { name, concept_id }) + .collect() +} + +/// Resolve a concept name to its id, or `None` if the Ontology domain does not +/// mint it — a refusal, never a guess. +#[must_use] +pub fn concept_id(name: &str) -> Option { + concepts_in_domain(ConceptDomain::Ontology) + .find(|(n, _)| *n == name) + .map(|(_, id)| id) +} + +/// The inverse: which ontology concept a `0x03XX` id names, or `None` outside +/// the domain. +#[must_use] +pub fn concept_at(concept_id: u16) -> Option<&'static str> { + concepts_in_domain(ConceptDomain::Ontology) + .find(|(_, id)| *id == concept_id) + .map(|(n, _)| n) +} + +// ═══════════════════════════════════════════════════════════════════════════ +// WordNet — an identity register over synset offsets +// ═══════════════════════════════════════════════════════════════════════════ + +/// A WordNet synset offset: an identity, never a coordinate in a metric space. +pub type Synset = u32; + +/// One sense of a word: its synset and WordNet's own sense rank (1-based). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Sense { + /// The synset this sense names. + pub synset: Synset, + /// WordNet's sense number for the lemma (1 = most frequent). + pub sense_num: u16, +} + +/// Why a rail line was refused. A malformed line is an error, never skipped: +/// a reader that silently drops lines reports a smaller taxonomy as if it were +/// the whole one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RailError { + /// The line does not have exactly 7 tab-separated columns. + Arity { line: usize, cols: usize }, + /// A sense number or synset offset is not a number. + Number { line: usize }, +} + +/// The WordNet is-a rail: senses by `(lemma, pos)`, hypernym edges by synset. +/// All senses and every hypernym edge are kept (multiple inheritance included); +/// nothing is collapsed to a first sense. +#[derive(Debug, Default)] +pub struct WordNetRail { + senses: HashMap<(String, char), Vec>, + parents: HashMap>, + names: HashMap, +} + +impl WordNetRail { + /// Parse the 7-column rail (`word pos sense_num synset kind hypernym + /// hypernym_offset`). `#` lines are comments. Exact arity, per the + /// release manifest: a line with any other column count is refused. + pub fn parse(text: &str) -> Result { + let mut rail = Self::default(); + for (n, line) in text.lines().enumerate() { + let line_no = n + 1; + if line.is_empty() || line.starts_with('#') { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() != 7 { + return Err(RailError::Arity { + line: line_no, + cols: c.len(), + }); + } + let num = |s: &str| { + s.parse::() + .map_err(|_| RailError::Number { line: line_no }) + }; + let pos = c[1] + .chars() + .next() + .ok_or(RailError::Number { line: line_no })?; + let sense_num = + u16::try_from(num(c[2])?).map_err(|_| RailError::Number { line: line_no })?; + let synset = num(c[3])?; + let senses = rail.senses.entry((c[0].to_string(), pos)).or_default(); + if !senses.iter().any(|s| s.synset == synset) { + senses.push(Sense { synset, sense_num }); + senses.sort_by_key(|s| s.sense_num); + } + rail.names.entry(synset).or_insert_with(|| c[0].to_string()); + if !c[6].is_empty() { + let parent = num(c[6])?; + // The hypernym column carries the parent's canonical name. + rail.names.insert(parent, c[5].to_string()); + let ps = rail.parents.entry(synset).or_default(); + if !ps.contains(&parent) { + ps.push(parent); + } + } + } + Ok(rail) + } + + /// Every sense of `word` as part of speech `pos` (`'n'`, `'v'`, `'a'`, + /// `'r'`), in sense order. Empty when the rail does not know the word. + #[must_use] + pub fn senses(&self, word: &str, pos: char) -> &[Sense] { + self.senses + .get(&(word.to_string(), pos)) + .map_or(&[], Vec::as_slice) + } + + /// The synset of one numbered sense, or `None`. + #[must_use] + pub fn synset(&self, word: &str, pos: char, sense_num: u16) -> Option { + self.senses(word, pos) + .iter() + .find(|s| s.sense_num == sense_num) + .map(|s| s.synset) + } + + /// A synset's name: the hypernym name the rail gives it, else the first + /// word read for it. + #[must_use] + pub fn name(&self, synset: Synset) -> Option<&str> { + self.names.get(&synset).map(String::as_str) + } + + /// Every ancestor of `synset` through all hypernym edges, nearest first, + /// each once. The synset itself is not included. + #[must_use] + pub fn ancestors(&self, synset: Synset) -> Vec { + let mut out = Vec::new(); + let mut seen: HashSet = HashSet::from([synset]); + let mut queue: VecDeque = VecDeque::from([synset]); + while let Some(s) = queue.pop_front() { + for &p in self.parents.get(&s).map_or(&[][..], Vec::as_slice) { + if seen.insert(p) { + out.push(p); + queue.push_back(p); + } + } + } + out + } + + /// `synset` is `ancestor` or lies below it. + #[must_use] + pub fn is_a(&self, synset: Synset, ancestor: Synset) -> bool { + synset == ancestor || self.ancestors(synset).contains(&ancestor) + } +} + +/// A category as a set of synsets: a sense is covered when it or any of its +/// ancestors is in the set. Membership only — two senses are never "close", +/// they are inside or outside. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConceptMask { + roots: Vec, +} + +impl ConceptMask { + /// A mask over the given root synsets. + #[must_use] + pub fn new(roots: impl IntoIterator) -> Self { + Self { + roots: roots.into_iter().collect(), + } + } + + /// The root synsets of the mask. + #[must_use] + pub fn roots(&self) -> &[Synset] { + &self.roots + } + + /// `synset` falls under one of the mask's roots. + #[must_use] + pub fn covers(&self, rail: &WordNetRail, synset: Synset) -> bool { + self.roots.iter().any(|&r| rail.is_a(synset, r)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The OBO half is superseded, not dropped: `98de36eb` retracted the 14 + /// 0x03XX mirror rows as never minted by ogar-vocab, so the Ontology + /// domain is reserved with zero rows. The lookups answer empty and refuse, + /// which is the honest state. When ogar-vocab mints 0x03 rows, this test + /// fails and the original round-trip tests come back + /// (`git show 48405aa2^:crates/deepnsm/src/ontology_vocab.rs`). + #[test] + fn the_ontology_domain_is_reserved_and_answers_empty() { + assert!(ontology_vocabulary().is_empty()); + assert_eq!(concept_id("mondo"), None); + assert_eq!(concept_at(0x0301), None); + } + + /// Real rows from the WordNet 3.1 rail (WordNet License): the chains of + /// whale, dolphin, shark, manta and octopus, all senses that appear. + const EXCERPT: &str = include_str!("wordnet_excerpt.tsv"); + + fn rail() -> WordNetRail { + WordNetRail::parse(EXCERPT).expect("excerpt parses") + } + + fn sense(r: &WordNetRail, w: &str, n: u16) -> Synset { + r.synset(w, 'n', n) + .unwrap_or_else(|| panic!("{w} sense {n}")) + } + + fn named(r: &WordNetRail, name: &str) -> Synset { + r.synset(name, 'n', 1) + .or_else(|| r.names.iter().find(|(_, n)| *n == name).map(|(s, _)| *s)) + .unwrap_or_else(|| panic!("no synset named {name}")) + } + + /// The hypernym chain is an address: whale (sense 2) reaches `animal` + /// through cetacean and aquatic_mammal, nearest first. + #[test] + fn ancestry_walks_every_hypernym_nearest_first() { + let r = rail(); + let whale = sense(&r, "whale", 2); + let up: Vec<&str> = r + .ancestors(whale) + .iter() + .filter_map(|&s| r.name(s)) + .collect(); + assert_eq!(&up[..3], &["cetacean", "aquatic_mammal", "placental"]); + assert!(up.contains(&"animal")); + assert!(r.is_a(whale, named(&r, "mammal"))); + assert!(!r.is_a(whale, named(&r, "fish"))); + } + + /// A mask cuts across the taxonomy. Aquatic mammals take whale and + /// dolphin; cartilaginous fish take shark and manta; neither takes the + /// other's members. Water animals need three roots from three branches, + /// because the only common ancestor is `animal`. + #[test] + fn masks_cut_across_the_taxonomy() { + let r = rail(); + let whale = sense(&r, "whale", 2); + let dolphin = sense(&r, "dolphin", 2); + let shark = sense(&r, "shark", 1); + let manta = sense(&r, "manta", 2); + let octopus = sense(&r, "octopus", 2); + let mammals = ConceptMask::new([named(&r, "aquatic_mammal")]); + let cartilaginous = ConceptMask::new([named(&r, "cartilaginous_fish")]); + let water = ConceptMask::new([ + named(&r, "aquatic_mammal"), + named(&r, "aquatic_vertebrate"), + named(&r, "cephalopod"), + ]); + for s in [whale, dolphin] { + assert!(mammals.covers(&r, s) && !cartilaginous.covers(&r, s)); + } + for s in [shark, manta] { + assert!(cartilaginous.covers(&r, s) && !mammals.covers(&r, s)); + } + assert!(!mammals.covers(&r, octopus) && !cartilaginous.covers(&r, octopus)); + for s in [whale, dolphin, shark, manta, octopus] { + assert!(water.covers(&r, s)); + } + // The aquatic_vertebrate root alone does not reach the whale: WordNet + // files whales under mammal, not under aquatic_vertebrate. + assert!(!ConceptMask::new([named(&r, "aquatic_vertebrate")]).covers(&r, whale)); + } + + /// A mask asks about a sense, never a word: dolphin sense 1 is a + /// percoid fish and octopus sense 1 is seafood. + #[test] + fn a_mask_reads_the_sense_not_the_word() { + let r = rail(); + let mammals = ConceptMask::new([named(&r, "aquatic_mammal")]); + let water = ConceptMask::new([ + named(&r, "aquatic_mammal"), + named(&r, "aquatic_vertebrate"), + named(&r, "cephalopod"), + ]); + assert!(!mammals.covers(&r, sense(&r, "dolphin", 1))); + assert!( + water.covers(&r, sense(&r, "dolphin", 1)), + "a fish is a water animal" + ); + assert!( + !water.covers(&r, sense(&r, "octopus", 1)), + "seafood is food" + ); + assert_eq!(r.senses("dolphin", 'n').len(), 2); + } + + /// Exact arity: a 6-column line is refused, never skipped. + #[test] + fn a_line_with_the_wrong_arity_is_refused() { + let bad = "whale\tn\t2\t02065397\tisa\tcetacean"; + assert_eq!( + WordNetRail::parse(bad).unwrap_err(), + RailError::Arity { line: 1, cols: 6 } + ); + assert!(WordNetRail::parse("# comment only\n").is_ok()); + } +} diff --git a/crates/deepnsm/src/wordnet_excerpt.tsv b/crates/deepnsm/src/wordnet_excerpt.tsv new file mode 100644 index 000000000..2ec17358d --- /dev/null +++ b/crates/deepnsm/src/wordnet_excerpt.tsv @@ -0,0 +1,41 @@ +# Excerpt of wordnet31_isa_v2.tsv (codebook release v0.1.0-codebooks-2026-07-26): the hypernym chains of whale, dolphin, shark, manta, octopus. +# Derived from Princeton WordNet 3.1. WordNet License (BSD-style, attribution): Princeton University WordNet. +# columns: word pos sense_num synset_offset kind hypernym_word hypernym_offset +animal n 1 00015568 isa organism 00004475 +aquatic_mammal n 1 02064670 isa placental 01889397 +aquatic_vertebrate n 1 01476447 isa vertebrate 01474323 +bony_fish n 1 02517456 isa fish 02514684 +cartilaginous_fish n 1 01483157 isa fish 02514684 +cephalopod n 1 01970956 isa mollusk 01943377 +cetacean n 1 02065083 isa aquatic_mammal 02064670 +chordate n 1 01468898 isa animal 00015568 +dolphin n 1 02584588 isa percoid_fish 02557361 +dolphin n 2 02071627 isa toothed_whale 02069360 +dolphinfish n 2 02584588 isa percoid_fish 02557361 +elasmobranch n 1 01484712 isa cartilaginous_fish 01483157 +fish n 1 02514684 isa aquatic_vertebrate 01476447 +food n 2 07571428 isa solid 15071467 +invertebrate n 1 01908302 isa animal 00015568 +living_thing n 1 00004258 isa whole 00003553 +mammal n 1 01864419 isa vertebrate 01474323 +manta n 2 01502732 isa ray 01498342 +matter n 3 00021007 isa physical_entity 00001930 +mollusk n 1 01943377 isa invertebrate 01908302 +object n 1 00002684 isa physical_entity 00001930 +octopod n 1 01972367 isa cephalopod 01970956 +octopus n 1 07798341 isa seafood 07792842 +octopus n 2 01972805 isa octopod 01972367 +organism n 1 00004475 isa living_thing 00004258 +percoid_fish n 1 02557361 isa spiny-finned_fish 02554802 +physical_entity n 1 00001930 isa entity 00001740 +placental n 1 01889397 isa mammal 01864419 +ray n 7 01498342 isa elasmobranch 01484712 +seafood n 1 07792842 isa food 07571428 +shark n 1 01484971 isa elasmobranch 01484712 +solid n 1 15071467 isa matter 00021007 +spiny-finned_fish n 1 02554802 isa teleost_fish 02530794 +teleost_fish n 1 02530794 isa bony_fish 02517456 +toothed_whale n 1 02069360 isa whale 02065397 +vertebrate n 1 01474323 isa chordate 01468898 +whale n 2 02065397 isa cetacean 02065083 +whole n 2 00003553 isa object 00002684 From 558e802a4529a2bd8ce08ec9a913bf2081bd0939 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:02:51 +0000 Subject: [PATCH 11/33] =?UTF-8?q?coresearch:=20German=20grammar=20as=20pos?= =?UTF-8?q?ition=20+=20morphology=20evidence=20=E2=80=94=20exploration=20m?= =?UTF-8?q?ap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One /coresearch council (5 scouts, 23-row crosswalk, 3 co-architects). The premise gate split the question: Q1 word-class voters, Q2 case, Q3 anaphora, Q4 TEKAMOLO tendency, Q5 plausibility. Only Q1 rules vote in the quorum. The map ranks 19 rules with source grades, firewall verdicts and a pre-registered kill probe each: .claude/knowledge/german-grammar-rule-inventory.md. Five mined German tables have no consumer yet; UD German has no iobj and ADP carries no Case; GSD Case gold is ~10 % noisy, so Case kills must hold on HDT. The operator's additions (belief transitions as 24×i4 loci, WordNet with CLAM/CHAODA) are mapped with the scope decision they need. Nothing is adopted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- ...0-04-coresearch-german-grammar-evidence.md | 49 ++++++++ .claude/board/entries/README.md | 3 +- .../german-grammar-rule-inventory.md | 107 ++++++++++++++++++ 3 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md create mode 100644 .claude/knowledge/german-grammar-rule-inventory.md diff --git a/.claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md b/.claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md new file mode 100644 index 000000000..d7236acac --- /dev/null +++ b/.claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md @@ -0,0 +1,49 @@ +# 2026-10-04 — /coresearch: German grammar as position + morphology evidence (exploration map) + +**Status:** OPEN — an exploration map. Nothing is adopted; the operator chooses. +Map: `.claude/knowledge/german-grammar-rule-inventory.md`. + +**Council:** +- Brief and premise gate: PREMISE-SPLIT into Q1 word class, Q2 case, Q3 anaphora, Q4 TEKAMOLO, Q5 plausibility. +- 5 scouts: code, internal prior art, literature, linguistic resources, grammar concepts. +- A 23-row crosswalk. +- 3 co-architects: bridge, firewall, falsifier. + +## Key results + +**Already have.** +- Satzklammer adj/adv rules: 88.5 % / 90.0 %. +- German adj/adv quorum: 85.1 %. +- The TEKAMOLO order law is killed as a decider; the cue-by-verb-position rule scores 88.5 %. + +**Inventories without consumers.** +- Five mined TSVs have no Rust reader: `article_decidability`, `valency`, `wechsel`, `relative_pronoun`, `reflexive`. +- They become one Q2 case-constraint evaluator (`ud_case_eval`), not five voters. +- Shared precondition: `next_form` in `Tok`, plus the tables re-mined on the fit split. + +**Data facts that change the design.** +- UD German has `iobj` = 0: a dative object is obj/obl + Case=Dat. +- ADP carries no Case. +- GSD Case gold is about 10 % noisy, so Case kills must hold on HDT. + +**Cheapest probes first.** +- Fixed-case prepositions. +- der/die/das DET vs PRON. +- Relative-pronoun case. +- Paradigm-exact monoflexion (read on HDT). +- Expletive es. +- Separable particles. +- Auxiliary selection. + +**Operator additions (2026-10-04).** +- **Belief transitions on the AriGraph tenant as 24×i4 loci:** CONFLICT S4. The rule lives in the planner and the tenant holds offsets only. Probe: the transition record must predict later contradiction beyond the final TruthValue. +- **WordNet → ndarray CLAM/CHAODA:** outside the letter of `48405aa2`, inside its spirit. Clean only offline, under a recorded DECISION; the draft wording is in the map, §6. +- **hydrate = node creation:** ALREADY-HAVE (restored, green on the KJV). + +## Not searched + +- adverbial vs predicative ADJD accuracy; +- a computational Wechselpräposition accuracy; +- German Winograd beyond Wino-X; +- adverbial-order corpus studies; +- left-corner parsing for German. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 160f65cac..65b468d46 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -195 entries, 2026-08-06 .. 2026-10-04. +196 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| @@ -38,6 +38,7 @@ exactly one of them, never both. | 2026-10-04 | `D-LXC-15` | | [2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md](2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md) | | 2026-10-04 | `D-LXC-17` | | [2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md](2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md) | | 2026-10-04 | `D-LXC-21` | | [2026-10-04-deepnsm-deleted-grammar-restored.md](2026-10-04-deepnsm-deleted-grammar-restored.md) | +| 2026-10-04 | `coresearch-german-grammar-evidence` | | [2026-10-04-coresearch-german-grammar-evidence.md](2026-10-04-coresearch-german-grammar-evidence.md) | | 2026-10-03 | `population-law-crosscheck` | | [2026-10-03-population-law-crosscheck.md](2026-10-03-population-law-crosscheck.md) | | 2026-10-03 | `orkg-reference-harvest` | | [2026-10-03-orkg-reference-harvest.md](2026-10-03-orkg-reference-harvest.md) | | 2026-10-03 | `known-is-not-alpha-premise-gate` | | [2026-10-03-known-is-not-alpha-premise-gate.md](2026-10-03-known-is-not-alpha-premise-gate.md) | diff --git a/.claude/knowledge/german-grammar-rule-inventory.md b/.claude/knowledge/german-grammar-rule-inventory.md new file mode 100644 index 000000000..4d21f39df --- /dev/null +++ b/.claude/knowledge/german-grammar-rule-inventory.md @@ -0,0 +1,107 @@ +# German grammar as position + morphology evidence — rule inventory + +READ BY: anyone adding a voter to `crates/deepnsm-v2/examples/ud_pos_eval.rs`; +anyone building a German case, anaphora, TEKAMOLO or plausibility evaluator; +anyone consuming `build_de_codebook.py`'s TSVs. + +**Status:** an exploration map from one `/coresearch` council (2026-10-04). +Every rule below is graded by its source. Nothing is adopted. Numbers marked +OURS were measured in this workspace; all others are from the cited source +at the grade shown. Board entry: `.claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md`. + +## 0. The premise split (read first) + +A quorum voter has one signature: `Fn(&Tok) -> Option`, one word class per token, scored on UPOS. Only Q1 rules are voters. The other four questions get their own evaluators. They may feed Q1 voters as cues; they never become voters (premise-auditor, 2026-10-04). + +| Q | question | output | gold | evaluator | +|---|---|---|---|---| +| Q1 | word class (adj/adv, noun/verb, DET/PRON) | Pos per token | UPOS | `ud_pos_eval` quorum | +| Q2 | case and government | Case= per nominal | UD Feats `Case=` on the governed DET/NOUN (ADP carries none) | new `ud_case_eval` | +| Q3 | anaphora | a link per anaphor | coreference (CorefUD PotsdamCC; TüBa-D/Z academic-only) | its own | +| Q4 | TEKAMOLO / topological fields | an order preference | rates with bootstrap CIs | its own | +| Q5 | "does this make sense" | a graded score | AUC original vs swapped | its own | + +Naming trap: `contract/grammar/wechsel.rs` means dual-role TOKENS. The Dat/Akk Wechselpräposition data is `build_de_codebook.py`'s `wechsel.tsv`. + +## 1. Data facts (measured on UD German-GSD train, 263,791 tokens) + +- Case is present on NOUN 98.3 %, DET 99.4 % and PRON 98.1 %. **ADP has no Case** (1 of 29,374). +- **`iobj` = 0** in GSD (110 in 3.4M HDT tokens). A dative object is `obj`/`obl` with Case=Dat. +- ADV carries no Degree (1 of 11,900); ADJ does (19,452 of 19,489). That is a gold fact, not a run-time cue. +- GSD Case gold is noisy at about 10 %: `obj` + Case=Nom 740 against 6,180 Acc. **Case kills must hold on HDT**, which is CC BY-SA, 3.4M tokens and converted from manual annotation. +- `build_de_codebook.py` mines every table from UD **gold**. That is allowed only on the fit split, with scoring on test only, and the tables must be re-mined on fit before held-out weighting. Five of them (`article_decidability`, `valency`, `wechsel`, `relative_pronoun`, `reflexive`) **have no Rust consumer**. + +## 2. The ranked inventory + +**Columns:** + +- Verdicts: bridge = OPPORTUNITY / WORTH-EXPLORING / DROP; firewall = PASS / CONFLICT (seam) / TRAP. +- "Measured / published" gives OURS numbers or the source's number. +- Precision comes from the published source or from OURS. Everything else is CONJECTURE until its probe runs. + +**Seams:** + +- **S1 (licence):** a table mined from UD is a CC BY-SA derived database. It ships as a fetched, built artefact under its own share-alike notice, never as a `const` or `include_bytes!` table in a code crate, and never in `lance-graph-contract`. +- **S2:** mine on train, score on test, and assert the split in code. +- **S3:** a lexicon is injected; it is never a SoA fact or a contract type. +- **S4:** plausibility and discourse resolution live in `lance-graph-planner`. deepnsm-v2 emits only candidates and agreement features. + +| rank | Q | rule (testable) | signal | gold | measured / published | bridge | firewall | kill probe (pre-registered) | +|---|---|---|---|---|---|---|---|---| +| 1 | Q1 | **der/die/das + NOUN/ADJ → DET; + finite verb, or closing a verb-final clause → PRON** | next form/readings; clause end | UPOS of der/die/das | none yet | OPPORTUNITY (needs `next_form` in `Tok`) | PASS | KILL if: PRON precision < 0.80; or PRON recall < 0.50; or fires > 2× gold PRON. S | +| 2 | Q1 | **Copula clause, word at the right edge → ADJ (predicative); full-verb clause, word before the right bracket → ADV** | Satzklammer position + copula | UPOS (ADJ+advmod folds to ADV) | OURS: 88.5 % (26 fires), 90.0 % (20); German adj/adv joint quorum 81.6 → 85.1 % (disable: 81.9 %) (D-LXC-20) | ALREADY-HAVE | PASS | done | +| 3 | Q1 | **Monoflexion: stem + paradigm ending (-e/-en/-em/-er/-es) after DET/ADP and before a NOUN → ADJ (attributive). A bare stem is never attributive.** Exceptions: indeclinables (lila, rosa), city -er adjectives, nominalised adjectives (NOUN), `am -sten` (adverbial) | form vs lemma + position | UPOS, deprel amod | OURS: the broad ending rule is 70.6 % on 17 fires; "uninflected before a noun → adv" 80.0 % | OPPORTUNITY | PASS (hand-written endings) | KILL if: precision ≤ 70.6 % or ≤ frequency on the same tokens (read on HDT); or fires on < 1 % of tokens. Silence: indeclinables give 0 ADJ votes. S | +| 4 | Q1 | **A clause-final particle after a V2 finite verb → ADP (compound:prt)** | position + closed particle list | deprel compound:prt | Batinić & Schmidt GSCL 2017 (READ-IN-FULL slides): 99 % precision on FOLK; 11 % of finite verbs separated | OPPORTUNITY | PASS as a rule (the 7,658-verb list has an unknown licence: measure against it only) | KILL if: precision < 0.95; or > 2 % false fires on `case` ADP / pronominal adverbs; demote to a cue if frequency is already ≥ 98 %. S | +| 5 | Q2 | **Fixed-case prepositions** (Acc: durch/für/gegen/ohne/um; Dat: aus/bei/mit/nach/seit/von/zu/gegenüber; Gen: wegen/trotz/während/statt/innerhalb) | closed list | Case= of the governed DET/NOUN | Duden (SECONDHAND) | OPPORTUNITY | PASS (hand-authored list) | KILL if: precision ≤ the noun form's train-majority case on the same tokens; or Acc/Dat < 0.90 on HDT; or Gen < 0.70. S | +| 6 | Q2 | **Relative pronoun case = its role in the relative clause; gender/number from the nearest NP to the left** | form + in-clause elimination | Case= of PronType=Rel | Klenner & Tuggener RANLP 2011 (READ-IN-FULL): relative → next NP left | OPPORTUNITY | CONFLICT S1–S3 (mined TSV) | KILL if: form-only ≤ form frequency; or elimination lifts syncretic die/das/der by < 10 points. S | +| 7 | Q2 | **One nominative per finite clause** (elimination over syncretic articles; decide only when complete) | article candidate sets + clause segmentation | Case= on nsubj/obj | WOGLI arXiv 2306.04523 (SECTION-READ): only masc sg der/den, ein/einen, dieser/diesen and weak nouns mark Nom vs Acc; Table 7 lists 8 surface-identical patterns | OPPORTUNITY | PASS as Q2; **TRAP as a UPOS voter** | KILL if: precision ≤ the article-majority case; or it completes on < 10 % of syncretic NPs. Silence: the WOGLI residue abstains ≥ 90 %. M | +| 8 | Q2 | **Wechselpräposition: the article's case decides; Dat = location (wo), Acc = goal (wohin); verb-pair prior (stellen/legen/setzen → Acc, stehen/liegen/sitzen → Dat)** | article + verb lemma | Case= of the governed NP (no Wo/Wohin gold exists) | No computational accuracy is published (OPEN); Draye 2014 (ABSTRACT-ONLY) | OPPORTUNITY | CONFLICT S1–S3 | KILL if: verb-prior precision ≤ the preposition's majority case on ambiguous forms (den); or fires < 5 % of Wechsel NPs. M | +| 9 | Q2 | **Valency + uniqueness:** a verb frame licenses ≤ 1 argument per case | verb lemma (particles joined via rank 4) | Case= (Dat = obj/obl + Case=Dat) | Bader & Häussler 2006 (READ-IN-FULL): OS in the Mittelfeld is 93.7 % Dat; Somers et al.: Dat-first 42 % for alternating verbs | WORTH-EXPLORING | CONFLICT S1–S3; E-VALBU measure-against only | KILL if: gain < 1 point over rank 7; or precision ≤ the article baseline. M | +| 10 | Q2 cue | **Auxiliary selection: sein + participle → unaccusative/motion; a directional (Acc) PP co-occurs with sein-motion** | aux form + participle | Acc rate of Wechsel NPs, sein vs haben | Keller & Sorace 2003 (SECTION-READ): the Auxiliary Selection Hierarchy; reflexives always take haben | WORTH-EXPLORING | PASS as a cue; TRAP as a voter | KILL if: the Acc-rate difference is < 10 points; or Fisher p > 0.01; or there are < 200 PPs on HDT. S | +| 11 | Q1 cue | **Strong/weak ending agrees with the article class** (an ending/article mismatch means not attributive to that noun) | ending × article | amod of the next noun | Eisenberg (SECONDHAND) | WORTH-EXPLORING | CONFLICT S1 if baked from DEMorphy/Wiktionary; PASS as hand-written Rust rules | KILL if: precision ≤ base + 5 points; or fires < 1 %. Silence: ≤ 5 % false mismatches. M | +| 12 | Q1 cue | **Topological field anchors: LK (finite verb), C (complementizer), VC (verb cluster)** | finite-verb / complementizer position | downstream gain of the Satzklammer voters | Cheung & Penn ACL 2009 (READ-IN-FULL): LK F1 99.75, C 98.98, VC 98.56, MF 94.99, NF 82.73 (TüBa-D/Z) | WORTH-EXPLORING | PASS | KILL if: the Satzklammer voters gain < 20 % more fires; or their precision drops > 2 points. M | +| 13 | Q3 | **Agreement filter (gender/number/case) + Binding A (sich → subject of the same clause) + B (a pronoun is not bound by its own co-arguments); salience subject > object** | lexicon features, clause boundaries | coreference | Klenner & Tuggener (READ-IN-FULL): TüBa-D/Z CEAF salience-only 51.41; per type personal 58.86, relative 55.97, reflexive 54.16. OURS (English, Aesop, small n): agreement 0.727, noun-only 0.455, agreement-blind 0.273 | OPPORTUNITY | CONFLICT S4 (ranking in the planner) + A6 (`loci.rs` SelectionalFit is float: quarantine it) | KILL if: ≤ nearest-NP + 3 points; or the filter drops the gold antecedent > 5 %; or it removes < 30 % of candidates. Binding A ≤ nearest-NP. L | +| 14 | Q3 | **Expletive es** (weather/impersonal verb, or Vorfeld with a postponed subject) | verb lemma + position | deprel expl (GSD: 373, plus 206 expl:pv to check) | UD counts (READ-IN-FULL) | OPPORTUNITY | PASS as a Q3 filter; TRAP as a voter (es is PRON either way) | KILL if: precision < 0.70; or recall < 0.30; or referential > 20 % of fires. S | +| 15 | Q4 | **TEKAMOLO order is a tendency, not a decision**; Luther vs edited text differ | cue lexicon + Mittelfeld | rates with bootstrap CIs | OURS: the order law 60.3/65.2 % (Mittelfeld 45.7/58.8 %), killed as a decider; cue-by-verb-position 88.5 % vs 84.1 % (D-LXC-17) | ALREADY-HAVE | PASS (Luther 1545/1912 public domain only) | KILL if: \|T 25 % of the gain. M | +| — | ceiling | Higher-order CRF on TIGER: POS 97.44 %, POS+MORPH 88.58 % (91.65 % with an analyser); case needs long context | — | — | Mueller/Schmid/Schütze EMNLP 2013 (SECTION-READ) | DROP as a design | PASS as a citation | none | + +## 3. Shared precondition (bridge 0) + +Most of ranks 1–9 need the same two changes: + +- `next_form` added to `Tok`; +- `Inventory` loading `article_case` / `article_decidability` / `wechsel` / `valency` / `relative_pronoun`, re-mined on the fit split. + +Ranks 5–9 are one case-constraint intersection, i.e. one Q2 evaluator rather than five voters. + +## 4. Resources (licence decides what may be baked) + +- **May be built and fetched:** UD German GSD + HDT (CC BY-SA), the de-codebook TSVs (CC BY-SA, derived), Wiktionary / DEMorphy paradigms (CC BY-SA), Wino-X (MIT), WordNet 3.1 (BSD-style). +- **Measure against only, never bake:** TüBa-D/Z, TIGER, GermaNet, E-VALBU (licence unconfirmed), the Batinić particle list. Check WOGLI and PEP-3k before use. + +## 5. Not searched / OPEN + +- adverbial vs predicative ADJD accuracy in the literature; +- a computational Wechselpräposition accuracy (rank 8 would be the first); +- German noun/verb homographs by position; +- per-case accuracy on UD; +- German Winograd beyond Wino-X; +- adverbial (TEKAMOLO) order corpus studies (only argument-order studies were found); +- left-corner parsing for German (leads only: arXiv 2109.04939, 2311.16258); +- whether the 85.1 % adj/adv figure used lexicons mined only on train. `UD_DE_INVENTORY` was built from GSD train, so yes, but the D-LXC-19 `Inventory` lemma map is mined from all of train, not the fit split. + +## 6. Recorded for the operator, not decided + +- **A1 scope for rank 18.** The firewall critic's draft wording: + + > **DECISION:** ontology concept ids remain identities; CAM-PQ and any learned/embedded distance over them stay prohibited (`48405aa2` unchanged). + > + > **SCOPE:** an exact graph metric derived solely from the ontology's own edges (WordNet hypernym shortest path, minimised over multiple inheritance) MAY be used as a `Distance` for ndarray ClamTree/CHAODA offline, as an evaluation or research instrument. It is never stored as a concept attribute, never mixed into the CAM-PQ/palette space, and never consulted on the substrate hot path. + > + > **REVISIT WHEN:** a proposal promotes it to run time, to a quorum voter, or to a non-WordNet ontology; or the swap AUC fails to beat frequency. +- **Rank 19.** Should the transition record use one extra 24×i4 register per node (within the existing ruling), or the 8 reserved loci of the `CausalWitness` facet (a new decision)? +- **`hydrate` = node creation.** It is restored and green. The invariants worth pinning as tests: one verse node per triple, 0 orphans, idempotent re-hydration (32,357 nodes), and every promoted basin key resolving. From 35383c99349ace27a3359c759baf7d5409bb12f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:03:13 +0000 Subject: [PATCH 12/33] coresearch map: re-file ranks 18 and 19 per the second premise gate A hypernym path distance is symmetric, so CHAODA cannot see an S<->O swap: rank 18 becomes a density evaluator over role-indexed points (or verb swaps only). Belief-state transitions are magnitudes ordered across revisions, not +-8 position loci: rank 19 moves to the planner's belief arena; only a peer pointer within +-8 stays a CausalWitness locus. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .../2026-10-04-coresearch-german-grammar-evidence.md | 4 ++-- .claude/knowledge/german-grammar-rule-inventory.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md b/.claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md index d7236acac..d120cffe1 100644 --- a/.claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md +++ b/.claude/board/entries/2026-10-04-coresearch-german-grammar-evidence.md @@ -36,8 +36,8 @@ Map: `.claude/knowledge/german-grammar-rule-inventory.md`. - Auxiliary selection. **Operator additions (2026-10-04).** -- **Belief transitions on the AriGraph tenant as 24×i4 loci:** CONFLICT S4. The rule lives in the planner and the tenant holds offsets only. Probe: the transition record must predict later contradiction beyond the final TruthValue. -- **WordNet → ndarray CLAM/CHAODA:** outside the letter of `48405aa2`, inside its spirit. Clean only offline, under a recorded DECISION; the draft wording is in the map, §6. +- **Belief transitions:** re-filed by the second premise gate. Revision history is a planner belief-arena record (magnitudes ordered across books), not 24×i4 loci; only a peer pointer within ±8 may be a locus. Probe: the history must predict later contradiction beyond the final TruthValue. +- **WordNet → ndarray CLAM/CHAODA:** outside the letter of `48405aa2`, inside its spirit. Clean only offline, under a recorded DECISION (draft wording in the map, §6). Second premise gate: a path distance is symmetric, so S↔O swaps are invisible; points must be role-indexed `(verb, role, synset)`, or the items restricted to verb swaps. - **hydrate = node creation:** ALREADY-HAVE (restored, green on the KJV). ## Not searched diff --git a/.claude/knowledge/german-grammar-rule-inventory.md b/.claude/knowledge/german-grammar-rule-inventory.md index 4d21f39df..d4ddf4a40 100644 --- a/.claude/knowledge/german-grammar-rule-inventory.md +++ b/.claude/knowledge/german-grammar-rule-inventory.md @@ -65,8 +65,8 @@ Naming trap: `contract/grammar/wechsel.rs` means dual-role TOKENS. The Dat/Akk W | 15 | Q4 | **TEKAMOLO order is a tendency, not a decision**; Luther vs edited text differ | cue lexicon + Mittelfeld | rates with bootstrap CIs | OURS: the order law 60.3/65.2 % (Mittelfeld 45.7/58.8 %), killed as a decider; cue-by-verb-position 88.5 % vs 84.1 % (D-LXC-17) | ALREADY-HAVE | PASS (Luther 1545/1912 public domain only) | KILL if: \|T 25 % of the gain. M | +| 18 | Q5 (density evaluator, re-filed by the second premise gate) | **WordNet hypernym path metric → ndarray ClamTree + CHAODA density over ROLE-INDEXED points `(verb, role, synset)`**. A plain path distance is symmetric (d(S,O) = d(O,S)), so an S↔O swap is invisible to it. Density measures rarity, and rare is not implausible. Items: verb swaps only, unless a role-indexed point is defined | exact graph metric over synsets | the same swap AUC | CHAODA: Ishaq et al. 2021 (SECONDHAND via `ndarray/src/hpc/clam.rs`); LFD alone ≈ 0.62 ROC-AUC on a synthetic mixture | NEW (operator 2026-10-04) | CONFLICT: outside the letter of `48405aa2`, inside its spirit; **clean only offline as an evaluator, under a recorded DECISION**; TRAP on the hot path | Items: WOGLI swaps (minus those morphology separates) + frequency-matched verb swaps; senses = first sense or the minimum over senses, never gold. KILL if: AUC ≤ 0.55; or ≤ the 95th percentile of a lemma→synset shuffle null; or ≤ frequency; or ≤ the ConceptMask rule (rank 17) + 0.02. Silence: symmetric predicates (heiraten, treffen) within 0.45–0.55; identical-address swaps abstain ≥ 90 %. Coverage: WordNet resolves S and O on < 40 % → KILL. Spot-check the triangle inequality on 10⁴ triples first; any violation kills the CLAM premise. M | +| 19 | planner / belief (re-filed by the second premise gate) | **Belief-state transitions (Admitted / Revised / Chosen, `belief::ReviseOutcome`) as a revision-history evaluator in the planner's belief arena.** Counts and \|f₁−f₂\| are magnitudes ordered across revisions, not ±8 position offsets, so they are not 24×i4 loci. Only a peer pointer within ±8 (supported_by / contradiction) may be written as a locus on the CausalWitness tenant | revision events | does the transition record predict later contradiction/revision beyond the final TruthValue? | none | NEW (operator 2026-10-04) | CONFLICT S4: the transition rule lives in the planner; loci are offsets only; no support/refute mask layer returns (removed 2026-09-02); one extra 24×i4 register per node is allowed, the 8 reserved loci need a decision | KJV arena revised in book order: prefix books 1–33, suffix 34–66. Baseline = final (f,c); treatment adds Admitted/Revised/Chosen counts, max \|f₁−f₂\| and loci peers. Target: a later contradicting revision or a side flip in the suffix. KILL if: AUC gain < 0.02; or gain ≤ the 95th percentile of a null that shuffles transition features within (f,c) bins. Silence: single-Admitted statements predict identically. Coverage: < 500 multi-event statements → UNTESTED. Kill too if i4 quantisation (±8 reach) loses > 25 % of the gain. M | | — | ceiling | Higher-order CRF on TIGER: POS 97.44 %, POS+MORPH 88.58 % (91.65 % with an analyser); case needs long context | — | — | Mueller/Schmid/Schütze EMNLP 2013 (SECTION-READ) | DROP as a design | PASS as a citation | none | ## 3. Shared precondition (bridge 0) @@ -103,5 +103,5 @@ Ranks 5–9 are one case-constraint intersection, i.e. one Q2 evaluator rather t > **SCOPE:** an exact graph metric derived solely from the ontology's own edges (WordNet hypernym shortest path, minimised over multiple inheritance) MAY be used as a `Distance` for ndarray ClamTree/CHAODA offline, as an evaluation or research instrument. It is never stored as a concept attribute, never mixed into the CAM-PQ/palette space, and never consulted on the substrate hot path. > > **REVISIT WHEN:** a proposal promotes it to run time, to a quorum voter, or to a non-WordNet ontology; or the swap AUC fails to beat frequency. -- **Rank 19.** Should the transition record use one extra 24×i4 register per node (within the existing ruling), or the 8 reserved loci of the `CausalWitness` facet (a new decision)? +- **Rank 19.** The register question lapses (second premise gate). Transition history is a planner belief-arena record; the CausalWitness loci carry only peers within ±8. Open instead: does the AriGraph tenant need a planner-written revision-history lane at all, or is the belief arena enough? Rank 19's probe answers it. - **`hydrate` = node creation.** It is restored and green. The invariants worth pinning as tests: one verse node per triple, 0 orphans, idempotent re-hydration (32,357 nodes), and every promoted basin key resolving. From 873699b152e3493b398266e231e048ba32d3eac4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:30:13 +0000 Subject: [PATCH 13/33] deepnsm-v2: grammar reads by ReadParams, not thinking styles; WordNet CLAM/CHAODA scope (D-LXC-22) Grammar is rung 2 of the content ladder (the 144 universal-grammar verbs and verb_table's TEKAMOLO priors). Neither the 36 ThinkingStyle adjectives (the unwired persona storyline) nor the 12 StyleFamily macros hold a grammar style, so tekamolo::read_clause now takes ReadParams (commit point, fan-out, margin, admit-ambiguous) with LEFT_CORNER and RIGHT_CORNER presets. The style mapping is kept as the adapter ReadParams::from_style; its destination is the planner, or a thinking dialect on the ogar-loco IR. A test pins the presets to the adapter's rows (disable-verified). Recorded DECISION (operator, 2026-10-04): an exact WordNet hypernym metric may serve as the Distance for ndarray ClamTree/CHAODA offline, as an evaluation instrument. Concept ids stay identities; 48405aa2 stands. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...am-chaoda-scope-and-grammar-read-params.md | 32 ++++ .claude/board/entries/README.md | 3 +- .../deleted-grammar-heuristics-ledger.md | 4 +- .../german-grammar-rule-inventory.md | 6 +- crates/deepnsm-v2/src/tekamolo.rs | 140 +++++++++++++----- 6 files changed, 148 insertions(+), 38 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 64b3e15df..ef811aebe 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -45,6 +45,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-19** | literal rules, priorities and a quorum for word-class homographs (`ud_pos_eval` `UD_RULES`, `UD_DE_INVENTORY`) | In PR | 14–20 rules per pair; held-out-fitted joint quorum best or tied in 7/10 rows (UD English n/v 93.7 %, de n/v 98.8 %); summed quorum fails on correlated voters (de n/v 84.9 %); Animal Farm adj/adv and German adj/adv stay best with position × frequency. Entry `2026-10-04-deepnsm-v2-rule-quorum.md` | | **D-LXC-20** | Frageprobe through the Satzklammer; `answered_questions` returns `causal_edge::CausalMask` | In PR | German predicative at the copula clause's right edge 88.5 %, manner adverb before the right bracket 90.0 % (vs after-copula 35.3 %); German adj/adv joint quorum 81.6 → 85.1 % (disable: 81.9 %). Entry `2026-10-04-deepnsm-v2-satzklammer-frageprobe.md` | | **D-LXC-21** | deleted grammar heuristics restored (deepnsm-v2 toc/hydrate/promote/lexicon/loci/tekamolo + toc_hydrate; deepnsm ontology_vocab + persona_chain_replay); WordNet identity register + ConceptMask | In PR | toc_hydrate on KJV: 32,357 HHTL nodes, 70,393 triples addressed, 1,227 basins at TOC keys; ContextChain margin 335× too coarse for the rank metric. Ledger `.claude/knowledge/deleted-grammar-heuristics-ledger.md`; entry `2026-10-04-deepnsm-deleted-grammar-restored.md` | +| **D-LXC-22** | DECISION: WordNet hypernym metric may feed ndarray CLAM/CHAODA offline as an evaluation instrument (identities unchanged, `48405aa2` stands); grammar is not a thinking style → `tekamolo::read_clause` takes `ReadParams`, style adapter kept for the planner / ogar-loco | In PR | presets pinned to the adapter rows (disable-verified); 195 lib tests. Entry `2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md b/.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md new file mode 100644 index 000000000..17f640d15 --- /dev/null +++ b/.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md @@ -0,0 +1,32 @@ +# 2026-10-04 — WordNet may feed CLAM/CHAODA; grammar reads by ReadParams, not thinking styles (D-LXC-22) + +## DECISION — WordNet hypernym metric for CLAM/CHAODA + +Operator, 2026-10-04: *"Yes Wordnet would allow also clam Chaoda."* The +operator gave this answer to the scope question put in +`.claude/knowledge/german-grammar-rule-inventory.md` §6. + +- **DECISION:** ontology concept ids remain identities. CAM-PQ and any learned or embedded distance over them stay prohibited (`48405aa2` unchanged). +- **SCOPE:** an exact graph metric derived solely from WordNet's own hypernym edges (shortest path, minimised over multiple inheritance) MAY serve as the `Distance` for `ndarray::hpc::clam::ClamTree` and its CHAODA anomaly scores. It is used offline, as an evaluation and research instrument. It is never stored as a concept attribute, never mixed into the CAM-PQ / palette space, and never consulted on the substrate hot path. +- **BASIS:** the metric is defined by the ontology's own edges, not trained. That is the difference from the CAM-PQ case `48405aa2` rejected. +- **REVISIT WHEN:** + - a proposal moves it to run time or into a quorum voter; + - a proposal extends it to a non-WordNet ontology; + - its swap AUC fails to beat frequency (the probe in the inventory, rank 18). +- **Carried constraint (second premise gate):** a path distance is symmetric. Points must be role-indexed, `(verb, role, synset)`, or the items restricted to verb swaps. + +## Grammar is not a thinking style — `tekamolo.rs` reads by `ReadParams` + +**Checked:** +- The 36 `ThinkingStyle` adjectives and the 12 `StyleFamily` macros contain no grammar style. +- Grammar is rung 2 of the content ladder: the 144 universal-grammar verbs and `verb_table`'s TEKAMOLO slot priors (`.claude/v3/knowledge/persona-vs-rung-ladder.md`). +- The restored `tekamolo.rs` drove its reading knobs from the persona adjectives through `V2StyleProvider`, which is the wrong ladder. + +**Changed:** +- `read_clause` takes `ReadParams` (commit point, fan-out, margin, admit-ambiguous), with the presets `ReadParams::LEFT_CORNER` and `ReadParams::RIGHT_CORNER`. +- The style mapping is kept, not dropped, as the adapter `ReadParams::from_style` (plus `scan_for_style`). +- Its destination is the planner, and long term a thinking dialect on `ogar-loco`. The operator named `ogar-loco` / `ogar-r2il` as the long-term IR substrate; `ogar-loco`'s basin doc already treats a thinking IR as a caller that plugs in its vocabulary. + +**Test:** `the_presets_equal_the_adapter_rows` pins the presets to the adapter's Analytical and Exploratory rows. Disable-verified (a changed preset fails it). deepnsm-v2 has 195 lib tests; clippy is clean. + +**OPEN:** `V2StyleProvider` still lives in deepnsm-v2. Moving it out is its own change, once the planner side or a loco dialect exists to receive it. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 65b468d46..c76c304cc 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -196 entries, 2026-08-06 .. 2026-10-04. +197 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| +| 2026-10-04 | `D-LXC-22` | | [2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md](2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md) | | 2026-10-04 | `D-LXC-20` | | [2026-10-04-deepnsm-v2-satzklammer-frageprobe.md](2026-10-04-deepnsm-v2-satzklammer-frageprobe.md) | | 2026-10-04 | `D-LXC-19` | | [2026-10-04-deepnsm-v2-rule-quorum.md](2026-10-04-deepnsm-v2-rule-quorum.md) | | 2026-10-04 | `D-LXC-18` | | [2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md](2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md) | diff --git a/.claude/knowledge/deleted-grammar-heuristics-ledger.md b/.claude/knowledge/deleted-grammar-heuristics-ledger.md index 56b05d733..501914927 100644 --- a/.claude/knowledge/deleted-grammar-heuristics-ledger.md +++ b/.claude/knowledge/deleted-grammar-heuristics-ledger.md @@ -31,7 +31,7 @@ for later reference."* | `crates/deepnsm-v2/examples/toc_hydrate.rs` | 315 | `68955ecb` | `6e385c88` | The end-to-end run of tree → hydrate → promote → loci over the real KJV. | as above | **RESTORED**. Measured 2026-10-04: 32,357 nodes (66 books, 31,102 verses); 70,393 triples, 0 unaddressed; 27,298 cross-verse links (38.8 %); 1,227 basins promoted at TOC keys. | | `crates/deepnsm-v2/src/lexicon.rs` | 307 | `68955ecb` | `ec50f07b` | One COCA lexicon: lemma → forms → archaic → Other, monotone. It replaced three hand-copied taggers. | "taggers inlined into the two remaining examples" | **RESTORED**. The inlined copies in `bible_wave` / `genre_shapes` remain (they are pin-guarded). Merging them back onto this module is a deliberate re-pin. Measured in the file: KJV verses with no triple went from 35.6 % to 12.2 % once the forms table was read. | | `crates/deepnsm-v2/src/loci.rs` | 357 | `68955ecb` | `f47f0528` | Derives `Locus::Antecedent` (a 24×i4 `CausalWitnessFacet` nibble) from the SPO stream. Candidates are non-pronoun subjects within REACH = 8. The ranking is `SelectionalFit` (Cam96 similarity of each candidate to the pronoun clause's predicate) against `Recency`, with a margin gate. | "resolution AND binding already shipped" (`spo_anaphora_nibble`, `l9_loci_real_text`, `probe_antecedent_binder`) | **RESTORED**, with a known defect. It is agreement-blind. On Aesop the shipped agreement resolver scores 0.727 and the agreement-blind rule scores 0.273. **Merge pending:** take the gender/number/case agreement filter (and the German case table, see the research inventory) as the candidate filter, then keep SelectionalFit as the ranker. KJV numbers from `toc_hydrate`: Recency binds 35,025 of 35,613; SelectionalFit binds 27,788 (7,237 thin-margin). | -| `crates/deepnsm-v2/src/tekamolo.rs` | 718 | `f3d42b7c` (style half), `c57cc256` (reverted that revert), `68955ecb` (whole file) | `7bae6c84` | **German** right-corner TEKAMOLO lane reading. A left-corner hypothesis is opened per adverbial and committed at the right corner on margin. It reads Luther 1545 and Elberfelder 1905 as a two-translation control, uses `de/tekamolo.tsv`, and hydrates the Kausal/Modal/Lokal lanes. | "insight_coca_read already emits all four lanes" — but that replacement is **English COCA only**, so the German capability was lost (D-LXC-18 blast radius) | **RESTORED**, with a ruling conflict. `V2StyleProvider` (ThinkingStyle → FieldModulation knobs) is the half §3o of `alpha-channel-rung-overlay-v1` refused for deepnsm-v2: reasoning lives in the planner. **Open for the operator:** keep the knobs here, or move `V2StyleProvider` to lance-graph-planner and pass the scan parameters in. | +| `crates/deepnsm-v2/src/tekamolo.rs` | 718 | `f3d42b7c` (style half), `c57cc256` (reverted that revert), `68955ecb` (whole file) | `7bae6c84` | **German** right-corner TEKAMOLO lane reading. A left-corner hypothesis is opened per adverbial and committed at the right corner on margin. It reads Luther 1545 and Elberfelder 1905 as a two-translation control, uses `de/tekamolo.tsv`, and hydrates the Kausal/Modal/Lokal lanes. | "insight_coca_read already emits all four lanes" — but that replacement is **English COCA only**, so the German capability was lost (D-LXC-18 blast radius) | **RESTORED**, with a ruling conflict. `V2StyleProvider` (ThinkingStyle → FieldModulation knobs) is the half §3o of `alpha-channel-rung-overlay-v1` refused for deepnsm-v2: reasoning lives in the planner. **Resolved 2026-10-04 (D-LXC-22):** grammar is not a thinking style, so `read_clause` now takes `ReadParams` (`LEFT_CORNER` / `RIGHT_CORNER` presets). The style mapping is kept as the adapter `ReadParams::from_style`; its destination is the planner, or a thinking dialect on `ogar-loco` (the long-term IR substrate). | | `crates/deepnsm/src/ontology_vocab.rs` | 141 | `48405aa2` (2026-08-22) | `ae8e762e` (`5ff56bdf` is cited by `48405aa2` but not in this clone) | An OBO ontology concept register (`0x03XX`) over `ogar_codebook::concepts_in_domain`: name ↔ id, with the COCA rank space kept apart. Lookups only; it never used CAM-PQ itself. | "CAM-PQ is prohibited for ontologies", and it sat in v1 when the instruction named v2 | **RESTORED+REPURPOSED**. (1) The OBO half returns empty: `98de36eb` retracted the 14 0x03XX mirror rows as never minted, so the domain is reserved with zero rows, and a test pins that. (2) **New WordNet half:** `WordNetRail` (all senses, all hypernym edges from `wordnet31_isa_v2.tsv`, exact 7-column arity), `ancestors`, `is_a`, and `ConceptMask`, which matches by membership of the synset or any ancestor. Identity only, no distance, so the A1 ruling holds. Tested on real WordNet rows: whale/dolphin under `aquatic_mammal`, shark/manta under `cartilaginous_fish`, and a three-root water-animal mask. | | `crates/deepnsm/examples/persona_chain_replay.rs` | 376 | `8c03aaa9` (revert of `a74c3bbd`, no reason given) | `a74c3bbd` (not in this clone; its file content survives at `8c03aaa9^`) | The first caller of `ContextChain::disambiguate_with` with REAL candidates (window NP heads), scored against the agreement resolver's gold. Pronoun case gate: Ambiguous never decides. | none recorded | **RESTORED**. Measured 2026-10-04: the margin gate and the rank metric are on incompatible scales. The widest coherence spread is 0.0003 against a threshold of 0.1, 335× too small, so every replay escalates. This is the baseline for the counterfactual "does this make sense" test. | | `crates/deepnsm/src/markov_soa.rs` | 360 | `9a5f54c1` (2026-05-31) | — | The Markov wave over SoA SPO rows. | layer inversion: a core concern sat in a linguistics sensor | **SUPERSEDED-IN-PLACE** at `crates/lance-graph/src/graph/arigraph/markov_soa.rs` (vocabulary-agnostic, with an injected distance). | @@ -50,6 +50,6 @@ for later reference."* ## Merge queue (ideas restored but not yet merged) 1. **loci.rs and the agreement resolver.** Add a gender/number/case agreement filter, then keep SelectionalFit as the ranker. Measure against `l9_loci_real_text` gold (agreement-only 0.727). -2. **tekamolo.rs and planner styles.** This waits for the operator's call on `V2StyleProvider`. +2. **tekamolo.rs and planner styles.** The grammar path no longer uses styles (D-LXC-22). Open: moving `V2StyleProvider` out of deepnsm-v2 once the planner side or an `ogar-loco` dialect can receive it. 3. **lexicon.rs and the inlined taggers** in `bible_wave` / `genre_shapes`. This is a deliberate KJV re-pin. 4. **persona_chain_replay and ContextChain.** Fix the scale mismatch (a rank thermometer against a 0.1 margin) before the replay can decide anything. diff --git a/.claude/knowledge/german-grammar-rule-inventory.md b/.claude/knowledge/german-grammar-rule-inventory.md index d4ddf4a40..774d1057b 100644 --- a/.claude/knowledge/german-grammar-rule-inventory.md +++ b/.claude/knowledge/german-grammar-rule-inventory.md @@ -94,7 +94,11 @@ Ranks 5–9 are one case-constraint intersection, i.e. one Q2 evaluator rather t - left-corner parsing for German (leads only: arXiv 2109.04939, 2311.16258); - whether the 85.1 % adj/adv figure used lexicons mined only on train. `UD_DE_INVENTORY` was built from GSD train, so yes, but the D-LXC-19 `Inventory` lemma map is mined from all of train, not the fit split. -## 6. Recorded for the operator, not decided +## 6. Operator answers + +**Recorded 2026-10-04 (D-LXC-22): the WordNet scope below is DECIDED as worded** (operator: *"Yes Wordnet would allow also clam Chaoda"*). See `.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md`. + +### The questions as they were put - **A1 scope for rank 18.** The firewall critic's draft wording: diff --git a/crates/deepnsm-v2/src/tekamolo.rs b/crates/deepnsm-v2/src/tekamolo.rs index 44b2331cb..ab0f08721 100644 --- a/crates/deepnsm-v2/src/tekamolo.rs +++ b/crates/deepnsm-v2/src/tekamolo.rs @@ -44,10 +44,25 @@ //! declared BASELINE, because a right-corner claim with no left-corner control //! measures nothing. //! -//! ## The thinking style drives the read +//! ## The read is parameterized by [`ReadParams`] — grammar's own knobs //! -//! The knobs are not constants. `ThinkingStyle → FieldModulation → ScanParams` -//! is the shipped read-parameterization, and this module consumes it: +//! Grammar is rung 2 of the content ladder (the 144 universal-grammar verbs and +//! `verb_table`'s TEKAMOLO slot priors), NOT a thinking style: neither the 36 +//! `ThinkingStyle` adjectives (the separate, unwired persona storyline) nor the +//! 12 `StyleFamily` macros (rung 4) contain one +//! (`.claude/v3/knowledge/persona-vs-rung-ladder.md`). So [`read_clause`] takes +//! [`ReadParams`] directly, with [`ReadParams::LEFT_CORNER`] and +//! [`ReadParams::RIGHT_CORNER`] as the two named readings. +//! +//! The older path — `ThinkingStyle → FieldModulation` through +//! [`V2StyleProvider`] — is kept, not dropped, as the adapter +//! [`ReadParams::from_style`]. It sits outside the grammar path because §3o of +//! `alpha-channel-rung-overlay-v1` keeps style selection out of deepnsm-v2; its +//! destination is the planner, and long term a thinking dialect on the +//! `ogar-loco` IR (the operator names `ogar-loco` / `ogar-r2il` as the +//! long-term IR substrate), which would hand this reader a `ReadParams`. +//! +//! What each knob sets (the adapter maps a style cluster onto the same four): //! //! | knob | what it sets here | //! |---|---| @@ -377,9 +392,9 @@ impl ThinkingStyleProvider for V2StyleProvider { } } -/// A style-parameterized read of one clause. -#[derive(Debug, Clone, Copy)] -pub struct StyleRead { +/// How one clause is read: grammar's own (rung-2) parameters. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ReadParams { /// Where the lanes commit — derived from `depth_bias`. pub commit: Commit, /// Hypotheses a lane carries — `fan_out`. @@ -389,15 +404,35 @@ pub struct StyleRead { pub margin: f64, /// May an ambiguous form (`da`) win a lane — `noise_tolerance`. pub admit_ambiguous: bool, - /// The SIMD scan params the same modulation yields, carried so a caller - /// reads ONE object rather than re-deriving them. - pub scan: ScanParams, } -impl StyleRead { - /// Derive the read from a style through the provider — never hand-set. +impl ReadParams { + /// The left-corner baseline: each lane locks on its first hypothesis, + /// carries at most two, needs a 0.6 margin and admits no ambiguous form. + /// (Equal to the adapter's Analytical cluster row.) + pub const LEFT_CORNER: Self = Self { + commit: Commit::LeftCorner, + fan_out: 2, + margin: 0.6, + admit_ambiguous: false, + }; + + /// The right-corner reading: carry up to eight hypotheses per lane to the + /// clause's right corner, commit on a 0.15 margin, admit ambiguous forms. + /// (Equal to the adapter's Exploratory cluster row.) + pub const RIGHT_CORNER: Self = Self { + commit: Commit::RightCorner, + fan_out: 8, + margin: 0.15, + admit_ambiguous: true, + }; + + /// Adapter, kept from the style-driven version: a thinking style's + /// cluster row through [`V2StyleProvider`]. Not on the grammar path; its + /// destination is the planner / an `ogar-loco` thinking dialect (see the + /// module doc). #[must_use] - pub fn of(style: ThinkingStyle) -> Self { + pub fn from_style(style: ThinkingStyle) -> Self { let m = V2StyleProvider.default_modulation(style); Self { commit: if m.depth_bias >= LEFT_CORNER_AT { @@ -408,9 +443,14 @@ impl StyleRead { fan_out: m.fan_out.max(1), margin: m.resonance_threshold, admit_ambiguous: m.noise_tolerance >= AMBIGUOUS_OK_AT, - scan: m.to_scan_params(), } } + + /// The scan parameters the adapter's modulation yields for `style`. + #[must_use] + pub fn scan_for_style(style: ThinkingStyle) -> ScanParams { + V2StyleProvider.default_modulation(style).to_scan_params() + } } /// What one clause's read produced. @@ -429,7 +469,7 @@ pub struct ClauseRead { pub ambiguous_wins: usize, } -/// Read ONE clause's tokens under `style`. +/// Read ONE clause's tokens under `params`. /// /// Every matching form opens a [`LaneHypothesis`]. Under /// [`Commit::RightCorner`] the hypotheses are carried to the end of the token @@ -437,8 +477,8 @@ pub struct ClauseRead { /// [`Commit::LeftCorner`] each lane locks on its first hypothesis, which is the /// premature reading kept as the baseline. #[must_use] -pub fn read_clause(h: &GrammarHeuristics, tokens: &[&str], style: ThinkingStyle) -> ClauseRead { - let sr = StyleRead::of(style); +pub fn read_clause(h: &GrammarHeuristics, tokens: &[&str], params: ReadParams) -> ClauseRead { + let sr = params; let mut carried: [Vec; 4] = [vec![], vec![], vec![], vec![]]; let mut out = ClauseRead::default(); @@ -561,17 +601,21 @@ mod tests { let toks = ["Da", "sprach", "er", "weil", "Gott", "rief"]; // Direct: depth_bias 0.9 → LeftCorner. Only `da` was ever opened. - let left = read_clause(&g, &toks, ThinkingStyle::Direct); + let left = read_clause(&g, &toks, ReadParams::from_style(ThinkingStyle::Direct)); assert_eq!( - StyleRead::of(ThinkingStyle::Direct).commit, + ReadParams::from_style(ThinkingStyle::Direct).commit, Commit::LeftCorner ); assert_eq!(left.opened, 2, "da opened Kausal AND Lokal, then locked"); // Exploratory: depth_bias 0.2 → RightCorner, fan_out 8. - let right = read_clause(&g, &toks, ThinkingStyle::Exploratory); + let right = read_clause( + &g, + &toks, + ReadParams::from_style(ThinkingStyle::Exploratory), + ); assert_eq!( - StyleRead::of(ThinkingStyle::Exploratory).commit, + ReadParams::from_style(ThinkingStyle::Exploratory).commit, Commit::RightCorner ); assert_eq!(right.opened, 3, "da x2 + weil — weil was SEEN"); @@ -591,8 +635,12 @@ mod tests { // `weil` first (273), then `damit` (986) — the LATER form has the // stronger corpus evidence, so left-corner and right-corner must differ. let toks = ["weil", "er", "rief", "damit", "es", "geschah"]; - let left = read_clause(&g, &toks, ThinkingStyle::Direct); - let right = read_clause(&g, &toks, ThinkingStyle::Exploratory); + let left = read_clause(&g, &toks, ReadParams::from_style(ThinkingStyle::Direct)); + let right = read_clause( + &g, + &toks, + ReadParams::from_style(ThinkingStyle::Exploratory), + ); let k = TekamoloRole::Kausal as usize; assert_eq!( left.lanes[k].map(|a| a.rank), @@ -614,8 +662,8 @@ mod tests { let g = h(); let toks = ["Da", "stand", "er"]; // only `da` — ambiguous, both lanes // Analytical: noise_tolerance 0.15 → refuses ambiguous forms. - let strict = read_clause(&g, &toks, ThinkingStyle::Analytical); - assert!(!StyleRead::of(ThinkingStyle::Analytical).admit_ambiguous); + let strict = read_clause(&g, &toks, ReadParams::from_style(ThinkingStyle::Analytical)); + assert!(!ReadParams::from_style(ThinkingStyle::Analytical).admit_ambiguous); assert_eq!(strict.lanes[TekamoloRole::Kausal as usize], None); assert_eq!(strict.lanes[TekamoloRole::Lokal as usize], None); assert_eq!(strict.abstained, 2, "both lanes abstained, and said so"); @@ -623,8 +671,12 @@ mod tests { // Exploratory: noise_tolerance 0.75 → admits it, in BOTH lanes, and the // tie is never broken by fiat. - let open = read_clause(&g, &toks, ThinkingStyle::Exploratory); - assert!(StyleRead::of(ThinkingStyle::Exploratory).admit_ambiguous); + let open = read_clause( + &g, + &toks, + ReadParams::from_style(ThinkingStyle::Exploratory), + ); + assert!(ReadParams::from_style(ThinkingStyle::Exploratory).admit_ambiguous); assert!(open.lanes[TekamoloRole::Kausal as usize].is_some()); assert!(open.lanes[TekamoloRole::Lokal as usize].is_some()); assert_eq!(open.ambiguous_wins, 2); @@ -637,7 +689,7 @@ mod tests { let g = h(); let toks = ["Gott", "sprach", "und", "es", "ward", "Licht"]; for style in [ThinkingStyle::Direct, ThinkingStyle::Exploratory] { - let r = read_clause(&g, &toks, style); + let r = read_clause(&g, &toks, ReadParams::from_style(style)); assert_eq!(r.opened, 0); assert_eq!(r.abstained, 0); assert!(r.lanes.iter().all(Option::is_none), "{style:?}"); @@ -651,11 +703,15 @@ mod tests { let g = h(); // Four Kausal forms in one clause. let toks = ["damit", "da", "weil", "damit", "x", "y"]; - let creative = read_clause(&g, &toks, ThinkingStyle::Creative); // fan_out 6 - let meta = read_clause(&g, &toks, ThinkingStyle::Metacognitive); // fan_out 8 + let creative = read_clause(&g, &toks, ReadParams::from_style(ThinkingStyle::Creative)); // fan_out 6 + let meta = read_clause( + &g, + &toks, + ReadParams::from_style(ThinkingStyle::Metacognitive), + ); // fan_out 8 assert!( - StyleRead::of(ThinkingStyle::Creative).fan_out - < StyleRead::of(ThinkingStyle::Metacognitive).fan_out + ReadParams::from_style(ThinkingStyle::Creative).fan_out + < ReadParams::from_style(ThinkingStyle::Metacognitive).fan_out ); assert!( meta.opened >= creative.opened, @@ -672,7 +728,7 @@ mod tests { let mut corners = std::collections::HashSet::new(); let mut fingerprints = std::collections::HashSet::new(); for s in ThinkingStyle::ALL { - let sr = StyleRead::of(s); + let sr = ReadParams::from_style(s); corners.insert(sr.commit); fingerprints.insert(V2StyleProvider.default_modulation(s).to_fingerprint()); } @@ -706,13 +762,29 @@ mod tests { }; let overconfident = V2StyleProvider.select_from_assessment(&mk(DkPosition::MountStupid)); assert_eq!( - StyleRead::of(overconfident).commit, + ReadParams::from_style(overconfident).commit, Commit::RightCorner, "Mount Stupid must NOT be handed the early-commit reader" ); // …and the expert in flow IS allowed to decide fast, so the mapping is // discriminating rather than always-RightCorner. let expert = V2StyleProvider.select_from_assessment(&mk(DkPosition::Plateau)); - assert_eq!(StyleRead::of(expert).commit, Commit::LeftCorner); + assert_eq!(ReadParams::from_style(expert).commit, Commit::LeftCorner); + } + + /// The grammar presets are the adapter's Analytical and Exploratory rows, + /// so taking the style out of the grammar path changed no reading. If the + /// adapter's table moves, this fails instead of the two drifting apart. + #[test] + fn the_presets_equal_the_adapter_rows() { + assert_eq!( + ReadParams::from_style(ThinkingStyle::Analytical), + ReadParams::LEFT_CORNER + ); + assert_eq!( + ReadParams::from_style(ThinkingStyle::Exploratory), + ReadParams::RIGHT_CORNER + ); + assert_ne!(ReadParams::LEFT_CORNER, ReadParams::RIGHT_CORNER); } } From 45d341d2b41b7b6b58e9e46da3773c50563271e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:30:46 +0000 Subject: [PATCH 14/33] board: WordNet gives the address, COCA + tokens give the torque (D-LXC-22 note) Operator framing recorded with its consequence for probes 17/18: WordNet never judges plausibility alone; CHAODA points combine the WordNet address with COCA frequency band and slot position. Consistent with D-LXC-15. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- ...clam-chaoda-scope-and-grammar-read-params.md | 17 +++++++++++++++++ .../knowledge/german-grammar-rule-inventory.md | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md b/.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md index 17f640d15..e817e5599 100644 --- a/.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md +++ b/.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md @@ -30,3 +30,20 @@ operator gave this answer to the scope question put in **Test:** `the_presets_equal_the_adapter_rows` pins the presets to the adapter's Analytical and Exploratory rows. Disable-verified (a changed preset fails it). deepnsm-v2 has 195 lib tests; clippy is clean. **OPEN:** `V2StyleProvider` still lives in deepnsm-v2. Moving it out is its own change, once the planner side or a loco dialect exists to receive it. + +## WordNet gives the address, COCA + tokens give the torque (operator, 2026-10-04) + +*"The 'negative' about WordNet is that it doesn't contain linguistic torque. It +needs COCA and tokens to have a qualia about the text. In return it offers a +Cartesian address."* + +| source | gives | does not give | +|---|---|---| +| WordNet | the Cartesian address: parent/child, the HHTL DN path, masks across branches | torque: frequency, collocation, position, how a word moves in a sentence | +| COCA + tokens | the torque: frequency, word forms, position, context, i.e. the qualia of the text | a stable concept address | + +**Consistent with the measured record.** D-LXC-15: WordNet helped noun/verb neither as a chooser nor as a reading filter, while COCA frequency × position did. + +**Consequence for the probes.** WordNet never judges alone. +- Rank 17's animacy mask is the coordinate. The plausibility signal comes from COCA/token counts and the slot position. +- Rank 18's CHAODA points combine both: (WordNet address of S/O, verb, COCA frequency band, slot position). Density over addresses alone would measure an unusual concept location, not an implausible sentence. diff --git a/.claude/knowledge/german-grammar-rule-inventory.md b/.claude/knowledge/german-grammar-rule-inventory.md index 774d1057b..502402dbc 100644 --- a/.claude/knowledge/german-grammar-rule-inventory.md +++ b/.claude/knowledge/german-grammar-rule-inventory.md @@ -98,6 +98,13 @@ Ranks 5–9 are one case-constraint intersection, i.e. one Q2 evaluator rather t **Recorded 2026-10-04 (D-LXC-22): the WordNet scope below is DECIDED as worded** (operator: *"Yes Wordnet would allow also clam Chaoda"*). See `.claude/board/entries/2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md`. +**WordNet gives the address, COCA + tokens give the torque (operator, 2026-10-04).** +- WordNet supplies a Cartesian address but no linguistic torque (frequency, collocation, position). +- COCA and the tokens supply the qualia of the text. +- So rank 17's mask is only a coordinate, and the plausibility signal comes from COCA/token counts and the slot position. +- Rank 18's CHAODA points must combine both: (WordNet address of S/O, verb, COCA frequency band, slot position). +- This agrees with D-LXC-15: WordNet did not help noun/verb as a chooser or a filter. + ### The questions as they were put - **A1 scope for rank 18.** The firewall critic's draft wording: From 098d49d2a670953eb0153ec1e1bc0191cea97e9b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:33:02 +0000 Subject: [PATCH 15/33] knowledge: Bible lanes are not independent witnesses (Aramaic-derived properties in Koine Greek) Operator reference recorded in the German grammar inventory, joined to the PROBE-BABEL-STANCES translationese finding (inherited calque across en-kjv / de-luther1545 / la-vulgate / el-lxx). Consequences: Bible-lane agreement is not independent evidence; TEKAMOLO is tested on edited text with the Bible as the control; rank 15 predicts source-shaped deviation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/knowledge/german-grammar-rule-inventory.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.claude/knowledge/german-grammar-rule-inventory.md b/.claude/knowledge/german-grammar-rule-inventory.md index 502402dbc..e86aca1fb 100644 --- a/.claude/knowledge/german-grammar-rule-inventory.md +++ b/.claude/knowledge/german-grammar-rule-inventory.md @@ -83,6 +83,17 @@ Ranks 5–9 are one case-constraint intersection, i.e. one Q2 evaluator rather t - **May be built and fetched:** UD German GSD + HDT (CC BY-SA), the de-codebook TSVs (CC BY-SA, derived), Wiktionary / DEMorphy paradigms (CC BY-SA), Wino-X (MIT), WordNet 3.1 (BSD-style). - **Measure against only, never bake:** TüBa-D/Z, TIGER, GermaNet, E-VALBU (licence unconfirmed), the Batinić particle list. Check WOGLI and PEP-3k before use. +## 4a. Bible lanes are not independent witnesses (operator, 2026-10-04) + +**The operator's point:** for the Bible context it was established that Greek and Aramaic, and Koine Greek itself, carry Aramaic-derived properties. + +**It agrees with the measured record.** PROBE-BABEL-STANCES (`E-TWO-ROSETTA-STONES-AND-THE-FOUR-CHANNEL-SPLIT-1`) found the pragmatic channel in coherent antiphase across every verified lane (en-kjv, de-luther1545, la-vulgate, el-lxx). It read that as inherited calque, not independent convergence; the Aramaic and Czech rows read the same way but are reconstructions, so they are reported, not claimed. + +**Consequences:** +1. **Agreement among Bible lanes is not independent evidence.** The Semitic source shows through Greek into Latin, Luther and the KJV. Bible-based cross-language checks, including D-LXC-15's KJV↔Luther silver labels, carry this caveat. +2. **TEKAMOLO is tested on edited text only.** Semitic parataxis and verb-early clauses carry into Luther and the KJV, so the edited-text order law is not expected there. Edited text (UD GSD, Animal Farm) tests the tendency; the Bible is the control. +3. **Rank 15 sharpens.** The prediction is that Luther's deviation from the tendency is source-shaped (Semitic order), not noise. The probe should compare Luther with a non-biblical German text of the same period, not only with modern edited German. + ## 5. Not searched / OPEN - adverbial vs predicative ADJD accuracy in the literature; From eb5aed7fe2dcaaa5fcc6668e6abd23354585a5fb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:33:29 +0000 Subject: [PATCH 16/33] knowledge: why the TEKAMOLO reader is right-corner, even for German Links 7bae6c84's pinned Luther case (da/weil) to today's measured right-edge rules (D-LXC-17, D-LXC-20): the Satzklammer puts the deciding verb at the right edge, and Semitic parataxis in biblical text adds to it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/knowledge/german-grammar-rule-inventory.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.claude/knowledge/german-grammar-rule-inventory.md b/.claude/knowledge/german-grammar-rule-inventory.md index e86aca1fb..9069dfe1a 100644 --- a/.claude/knowledge/german-grammar-rule-inventory.md +++ b/.claude/knowledge/german-grammar-rule-inventory.md @@ -94,6 +94,14 @@ Ranks 5–9 are one case-constraint intersection, i.e. one Q2 evaluator rather t 2. **TEKAMOLO is tested on edited text only.** Semitic parataxis and verb-early clauses carry into Luther and the KJV, so the edited-text order law is not expected there. Edited text (UD GSD, Animal Farm) tests the tendency; the Bible is the control. 3. **Rank 15 sharpens.** The prediction is that Luther's deviation from the tendency is source-shaped (Semitic order), not noise. The probe should compare Luther with a non-biblical German text of the same period, not only with modern edited German. +**Why the reader is right-corner, even for German (operator, 2026-10-04; built in `7bae6c84`).** +- In Luther's *"Da sprach er, weil …"* the ambiguous *da* (Kausal and Lokal, 437 each) opens first and the deciding *weil* arrives two words later. A left-corner reader locks *da* and never sees *weil*. +- The restored `tekamolo.rs` therefore opens hypotheses and adjudicates them at the clause's right corner, with `ReadParams::LEFT_CORNER` kept as the baseline. +- German needs this even outside the Bible: the Satzklammer puts the deciding verb at the right edge. Today's measurements agree: + - the strongest German adj/adv rules are right-edge rules (88.5 % / 90.0 %, D-LXC-20); + - the cue reader decides by the segment-final finite verb (88.5 %, `da` 87.5 % vs 37.5 %, D-LXC-17). +- Semitic parataxis in biblical text adds a further reason not to commit early. + ## 5. Not searched / OPEN - adverbial vs predicative ADJD accuracy in the literature; From e3ec0f6fa8153149012a4d28fb31df9fdc083266 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:40:52 +0000 Subject: [PATCH 17/33] deepnsm-v2: first probes from the German grammar map (D-LXC-23) - ud_pos_eval: der/die/das article vs pronoun as quorum pairs (Det, Rel) and (Det, Noun) (German GSD carries no PronType=Rel). German 92.3 -> 96.0 %, pronoun precision 90.0 % / recall 54.1 %: PASS. The report now gives second-reading precision/recall for every pair. - ud_case_eval (new): Case= from form + position + train tables. On HDT: Acc/Dat prepositions 90.0 / 92.5 % PASS, Gen KILL; Wechselpraeposition decided by the article 99.7 %, verb prior KILL (63.0 vs 63.9, the GSD pass did not hold); relative pronoun after a comma 80.1 vs 63.8 % PASS. - tekamolo_corners (new): the right-corner reader overturns left-corner commitments less on Luther (7.1 %) than on edited German (12.3 %), with the commit point isolated in a post-hoc matched arm: the pre-registered prediction is KILL. Workers wrote the two new examples; gated and run centrally. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...sm-v2-first-probes-from-the-grammar-map.md | 53 ++ .claude/board/entries/README.md | 3 +- .../german-grammar-rule-inventory.md | 10 + .../deepnsm-v2/examples/tekamolo_corners.rs | 323 ++++++++++ crates/deepnsm-v2/examples/ud_case_eval.rs | 552 ++++++++++++++++++ crates/deepnsm-v2/examples/ud_pos_eval.rs | 59 +- 7 files changed, 999 insertions(+), 2 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md create mode 100644 crates/deepnsm-v2/examples/tekamolo_corners.rs create mode 100644 crates/deepnsm-v2/examples/ud_case_eval.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index ef811aebe..5ed04810f 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -46,6 +46,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-20** | Frageprobe through the Satzklammer; `answered_questions` returns `causal_edge::CausalMask` | In PR | German predicative at the copula clause's right edge 88.5 %, manner adverb before the right bracket 90.0 % (vs after-copula 35.3 %); German adj/adv joint quorum 81.6 → 85.1 % (disable: 81.9 %). Entry `2026-10-04-deepnsm-v2-satzklammer-frageprobe.md` | | **D-LXC-21** | deleted grammar heuristics restored (deepnsm-v2 toc/hydrate/promote/lexicon/loci/tekamolo + toc_hydrate; deepnsm ontology_vocab + persona_chain_replay); WordNet identity register + ConceptMask | In PR | toc_hydrate on KJV: 32,357 HHTL nodes, 70,393 triples addressed, 1,227 basins at TOC keys; ContextChain margin 335× too coarse for the rank metric. Ledger `.claude/knowledge/deleted-grammar-heuristics-ledger.md`; entry `2026-10-04-deepnsm-deleted-grammar-restored.md` | | **D-LXC-22** | DECISION: WordNet hypernym metric may feed ndarray CLAM/CHAODA offline as an evaluation instrument (identities unchanged, `48405aa2` stands); grammar is not a thinking style → `tekamolo::read_clause` takes `ReadParams`, style adapter kept for the planner / ogar-loco | In PR | presets pinned to the adapter rows (disable-verified); 195 lib tests. Entry `2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md` | +| **D-LXC-23** | first probes from the German grammar map: der/die/das DET vs PRON voters (`ud_pos_eval`), `ud_case_eval` (fixed-case prepositions, article decidability, Wechselpräposition, relative pronoun), `tekamolo_corners` | In PR | DET/PRON German 92.3 → 96.0 % (pronoun P 90.0 / R 54.1) PASS; case on HDT: Acc 90.0 / Dat 92.5 % PASS, Gen KILL, Wechsel article-decided 99.7 %, verb prior KILL (63.0 vs 63.9), relative pronoun 80.1 vs 63.8 % PASS; right-corner-on-Bible prediction KILL (Luther 7.1 % vs GSD 12.3 %). Entry `2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md b/.claude/board/entries/2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md new file mode 100644 index 000000000..a209b36ad --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md @@ -0,0 +1,53 @@ +# 2026-10-04 — the first probes from the German grammar map: DET/PRON, case, right corner (D-LXC-23) + +**Status:** MEASURED (debug-0, release). Runs: +- `ud_pos_eval` on UD r2.15 de/en/fr GSD/EWT; +- `ud_case_eval` on GSD train/test and HDT train-a-1/test, all fetched, not committed; +- `tekamolo_corners` on GSD test, Luther 1545 and Elberfelder 1905. + +Each kill bar was fixed before its run. The source is `.claude/knowledge/german-grammar-rule-inventory.md`. Workers wrote `tekamolo_corners.rs` and `ud_case_eval.rs`; the orchestrator gated and ran them. + +## Rank 1 — der/die/das, article vs pronoun (`ud_pos_eval`, new pairs (Det, Rel) and (Det, Noun)) + +German GSD has no `PronType=Rel`; its relative der is PRON, folded to Noun. So the German pair is (Det, Noun). The voters are "followed by a noun/adjective → det", "after a comma → pronoun", "followed by a determiner / verb-only word → pronoun" and the position tables. + +| | frequency | joint quorum | pronoun precision | pronoun recall | +|---|---|---|---|---| +| German GSD (1,719 tokens) | 92.3 % | **96.0 %** | 90.0 % (72/80) | 54.1 % (72/133) | +| UD English | 95.1 % | 96.7 % | 83.0 % | 73.7 % | +| French GSD | 97.0 % | 98.4 % | 89.2 % | 63.5 % | + +**PASS** on all three bars: precision ≥ 0.80, recall ≥ 0.50, fires ≤ 2× gold. German recall only just clears its bar. `ud_pos_eval` now prints second-reading precision and recall for every pair. + +## Ranks 5–8 — case (`ud_case_eval`, new) + +Each rule predicts Case= from form, position and train-mined tables. Gold is read only for scoring. + +| rule | GSD (7,780 scored) | HDT (103,276 scored) | verdict | +|---|---|---|---| +| R1 Acc prepositions | 96.9 % on 322 vs 32.3 % | 90.0 % on 3,345 vs 40.8 % | PASS (HDT exactly at the 0.90 bar) | +| R1 Dat prepositions | 95.9 % on 895 | 92.5 % on 10,642 vs 57.7 % | PASS | +| R1 Gen prepositions | 58.2 % on 55 | 68.4 % vs 70.9 % baseline | KILL (Dat drift) | +| R3 Wechsel, the article decides | 98.7 % on 315 | **99.7 % on 6,458** | — | +| R3 Wechsel, verb prior vs prep majority (same tokens) | 74.8 % vs 68.0 % | **63.0 % vs 63.9 %** | **KILL** (the GSD pass did not hold on HDT) | +| R4 relative pronoun after a comma | 83.1 % on 118 vs 64.4 % | 80.1 % on 2,545 vs 63.8 % | PASS | +| R5 combined, first rule wins | 79.3 % vs 61.3 %, coverage 50.8 % | 80.0 % vs 66.2 %, coverage 58.0 % | — | + +**Findings.** +- **Wo/Wohin is decided by the article's case.** Wherever the article form leaves one of Dat/Acc, it is right 99.7 % of the time. The verb prior adds nothing measurable on the cleaner gold. +- **R4 initially fired on nothing.** The spec mined on `PronType=Rel`, which GSD lacks. It now mines train under the same surface condition the test uses. +- **The largest residual confusion on HDT is Nom predicted as Gen** (2,665). It comes from R2 voting the train majority for the ambiguous `der`. Next: the one-nominative elimination (rank 7). + +## Rank 15 — right corner vs left corner (`tekamolo_corners`, new) + +Pre-registered prediction: right-corner reading overturns left-corner commitments more often on biblical text. + +| corpus | overturn rate (95 % CI) | matched arm, post hoc: only the commit point moved | right-only lanes | +|---|---|---|---| +| UD German-GSD test | 12.3 % (10.3–14.3) | 12.2 % | 31 | +| Luther 1545 | 7.1 % (6.9–7.3) | 7.0 % | 10,861 | +| Elberfelder 1905 | 7.9 % (7.6–8.1) | 8.0 % | 5,171 | + +**Verdict: KILL.** The prediction is falsified. The matched arm, post hoc, rules out the confound between commit point and ambiguity admission. On the Bible, the right reader's extra lanes come from admitting the ambiguous *da* (10,296 ambiguous wins on Luther), not from where it commits. + +**OPEN.** Why does edited German overturn more? Candidate: GSD clauses are longer and carry more competing cues per clause. This is not measured, and the reason is left unexplained. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index c76c304cc..df9a70837 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -197 entries, 2026-08-06 .. 2026-10-04. +198 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| @@ -38,6 +38,7 @@ exactly one of them, never both. | 2026-10-04 | `D-LXC-16` | | [2026-10-04-deepnsm-v2-learned-position-table.md](2026-10-04-deepnsm-v2-learned-position-table.md) | | 2026-10-04 | `D-LXC-15` | | [2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md](2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md) | | 2026-10-04 | `D-LXC-17` | | [2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md](2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md) | +| 2026-10-04 | `D-LXC-23` | | [2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md](2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md) | | 2026-10-04 | `D-LXC-21` | | [2026-10-04-deepnsm-deleted-grammar-restored.md](2026-10-04-deepnsm-deleted-grammar-restored.md) | | 2026-10-04 | `coresearch-german-grammar-evidence` | | [2026-10-04-coresearch-german-grammar-evidence.md](2026-10-04-coresearch-german-grammar-evidence.md) | | 2026-10-03 | `population-law-crosscheck` | | [2026-10-03-population-law-crosscheck.md](2026-10-03-population-law-crosscheck.md) | diff --git a/.claude/knowledge/german-grammar-rule-inventory.md b/.claude/knowledge/german-grammar-rule-inventory.md index 9069dfe1a..772eb4944 100644 --- a/.claude/knowledge/german-grammar-rule-inventory.md +++ b/.claude/knowledge/german-grammar-rule-inventory.md @@ -69,6 +69,16 @@ Naming trap: `contract/grammar/wechsel.rs` means dual-role TOKENS. The Dat/Akk W | 19 | planner / belief (re-filed by the second premise gate) | **Belief-state transitions (Admitted / Revised / Chosen, `belief::ReviseOutcome`) as a revision-history evaluator in the planner's belief arena.** Counts and \|f₁−f₂\| are magnitudes ordered across revisions, not ±8 position offsets, so they are not 24×i4 loci. Only a peer pointer within ±8 (supported_by / contradiction) may be written as a locus on the CausalWitness tenant | revision events | does the transition record predict later contradiction/revision beyond the final TruthValue? | none | NEW (operator 2026-10-04) | CONFLICT S4: the transition rule lives in the planner; loci are offsets only; no support/refute mask layer returns (removed 2026-09-02); one extra 24×i4 register per node is allowed, the 8 reserved loci need a decision | KJV arena revised in book order: prefix books 1–33, suffix 34–66. Baseline = final (f,c); treatment adds Admitted/Revised/Chosen counts, max \|f₁−f₂\| and loci peers. Target: a later contradicting revision or a side flip in the suffix. KILL if: AUC gain < 0.02; or gain ≤ the 95th percentile of a null that shuffles transition features within (f,c) bins. Silence: single-Admitted statements predict identically. Coverage: < 500 multi-event statements → UNTESTED. Kill too if i4 quantisation (±8 reach) loses > 25 % of the gain. M | | — | ceiling | Higher-order CRF on TIGER: POS 97.44 %, POS+MORPH 88.58 % (91.65 % with an analyser); case needs long context | — | — | Mueller/Schmid/Schütze EMNLP 2013 (SECTION-READ) | DROP as a design | PASS as a citation | none | +## 2a. Measured (2026-10-04, D-LXC-23 — entry `2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md`) + +| rank | result | verdict | +|---|---|---| +| 1 DET vs PRON | German 92.3 → 96.0 %, pronoun P 90.0 % / R 54.1 % | PASS | +| 5 fixed-case prepositions | HDT Acc 90.0 %, Dat 92.5 %, Gen 68.4 % (< baseline 70.9 %) | Acc/Dat PASS, Gen KILL | +| 6 relative pronoun | HDT 80.1 % vs 63.8 % | PASS | +| 8 Wechselpräposition | article decides 99.7 % on HDT; verb prior 63.0 % vs prep majority 63.9 % | verb prior KILL | +| 15 right corner on the Bible | Luther overturns 7.1 % vs GSD 12.3 % (matched arm the same) | KILL | + ## 3. Shared precondition (bridge 0) Most of ranks 1–9 need the same two changes: diff --git a/crates/deepnsm-v2/examples/tekamolo_corners.rs b/crates/deepnsm-v2/examples/tekamolo_corners.rs new file mode 100644 index 000000000..477d2772c --- /dev/null +++ b/crates/deepnsm-v2/examples/tekamolo_corners.rs @@ -0,0 +1,323 @@ +//! `tekamolo_corners` — how often does the RIGHT-corner TEKAMOLO read overturn +//! what the LEFT-corner read commits, per corpus? +//! +//! Pre-registered prediction: the overturn rate is higher on biblical German +//! (Luther 1545, Elberfelder 1905 — Semitic-shaped order inherited through Koine +//! Greek) than on edited German (UD German-GSD test text). +//! +//! Per clause the program reads the same tokens twice with +//! [`ReadParams::LEFT_CORNER`] and [`ReadParams::RIGHT_CORNER`]. For each of the +//! four lanes: a "left commitment" is a lane the left read committed; an +//! "overturn" is a left commitment whose right-read address differs (a different +//! address, or an abstention); "right-only" is a lane the left read left empty +//! and the right read committed. +//! +//! Inputs (positional): +//! 1. `TEKAMOLO.tsv` — the heuristics, passed straight to `GrammarHeuristics::parse`. +//! 2. `GSD.conllu` — UD German-GSD test; only the FORM column is read, never a +//! gold column. +//! 3. one or more Bible JSON files (`abbreviation`, `books[].chapters[].verses[].text`). +//! +//! Clauses: every text is split ONLY at `, ; : . ! ? ( )` by one shared splitter; +//! tokens are the whitespace split of the clause. +//! +//! KILL rule (pre-registered): the line `verdict:` is `KILL` if the first Bible +//! file's overturn rate is not above GSD's beyond the bootstrap 95% CI (Bible CI +//! lower bound not greater than GSD CI upper bound), else `PASS`. +//! +//! Matched arm (added after the first run, so post hoc — reported, never the +//! verdict): the two presets differ in TWO knobs, the commit point and whether +//! an ambiguous form (`da`) may win. The matched arm reads with the right +//! preset's fan-out, margin and ambiguity admission and only the commit point +//! moved to the left corner, so its overturn rate isolates "where it commits". +//! +//! ```text +//! cargo run --release --example tekamolo_corners -- TEKAMOLO.tsv de_gsd-ud-test.conllu bible_luther1545.json bible_elberfelder1905.json +//! ``` + +use deepnsm_v2::tekamolo::{read_clause, Commit, GrammarHeuristics, ReadParams}; +use std::collections::HashMap; + +const LANE_NAMES: [&str; 4] = ["Temporal", "Kausal", "Modal", "Lokal"]; +const BOOTSTRAP_ROUNDS: usize = 1000; + +/// `(rate, ci_lo, ci_hi)`. +type Rate = (f64, f64, f64); + +/// The right preset with only the commit point moved to the left corner. +const LEFT_MATCHED: ReadParams = ReadParams { + commit: Commit::LeftCorner, + ..ReadParams::RIGHT_CORNER +}; + +/// Per-clause counts, kept so clauses can be resampled. +#[derive(Clone, Copy, Default)] +struct ClauseStat { + left_commits: usize, + overturns: usize, + right_only: usize, + has_hypothesis: bool, + right_abstained: usize, + right_ambiguous_wins: usize, + /// Matched arm: commitments and overturns of [`LEFT_MATCHED`]. + matched_commits: usize, + matched_overturns: usize, +} + +/// One corpus's accumulated measurement. +struct Corpus { + label: String, + stats: Vec, + /// Form that opened an overturned left commitment -> count. + forms: HashMap, +} + +impl Corpus { + fn new(label: &str) -> Self { + Self { + label: label.to_string(), + stats: Vec::new(), + forms: HashMap::new(), + } + } +} + +/// The normalisation `read_clause` applies to a token before lookup. +fn normalise(tok: &str) -> String { + tok.chars() + .filter(|c| c.is_alphabetic()) + .collect::() + .to_lowercase() +} + +/// The ONE clause splitter: split only at `, ; : . ! ? ( )`, then on +/// whitespace; empty clauses and empty tokens are dropped. +fn for_each_clause<'a>(text: &'a str, mut f: impl FnMut(&[&'a str])) { + for clause in text.split([',', ';', ':', '.', '!', '?', '(', ')']) { + let tokens: Vec<&str> = clause.split_whitespace().collect(); + if !tokens.is_empty() { + f(&tokens); + } + } +} + +/// Read one text, adding its clauses to `corpus`. +fn measure_text(h: &GrammarHeuristics, text: &str, corpus: &mut Corpus) { + for_each_clause(text, |tokens| { + let l = read_clause(h, tokens, ReadParams::LEFT_CORNER); + let r = read_clause(h, tokens, ReadParams::RIGHT_CORNER); + let mut st = ClauseStat { + has_hypothesis: r.opened > 0, + right_abstained: r.abstained, + right_ambiguous_wins: r.ambiguous_wins, + ..ClauseStat::default() + }; + for lane in 0..4 { + match (l.lanes[lane], r.lanes[lane]) { + (Some(la), ra) => { + st.left_commits += 1; + if ra != Some(la) { + st.overturns += 1; + // The left winner is the FIRST token with an entry for + // this lane. + let opener = tokens.iter().map(|t| normalise(t)).find(|w| { + !w.is_empty() && h.lookup(w).iter().any(|e| e.0 as usize == lane) + }); + if let Some(w) = opener { + *corpus.forms.entry(w).or_default() += 1; + } + } + } + (None, Some(_)) => st.right_only += 1, + (None, None) => {} + } + } + let m = read_clause(h, tokens, LEFT_MATCHED); + for lane in 0..4 { + if let Some(ma) = m.lanes[lane] { + st.matched_commits += 1; + st.matched_overturns += usize::from(r.lanes[lane] != Some(ma)); + } + } + corpus.stats.push(st); + }); +} + +/// GSD: rebuild each sentence from FORM tokens (col 2 only), then run it +/// through the shared clause splitter. +fn measure_gsd(h: &GrammarHeuristics, path: &str) -> Corpus { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut corpus = Corpus::new("GSD"); + let mut forms: Vec<&str> = Vec::new(); + for line in text.lines() { + if line.trim().is_empty() { + if !forms.is_empty() { + measure_text(h, &forms.join(" "), &mut corpus); + forms.clear(); + } + continue; + } + if line.starts_with('#') { + continue; + } + let cols: Vec<&str> = line.split('\t').collect(); + if cols.len() < 2 || cols[0].contains(['-', '.']) { + continue; + } + forms.push(cols[1]); + } + if !forms.is_empty() { + measure_text(h, &forms.join(" "), &mut corpus); + } + corpus +} + +/// Bible JSON: `abbreviation` + `books[].chapters[].verses[].text`. +fn measure_bible(h: &GrammarHeuristics, path: &str) -> Corpus { + let raw = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let json: serde_json::Value = + serde_json::from_str(&raw).unwrap_or_else(|e| panic!("{path}: invalid JSON: {e}")); + let label = json + .get("abbreviation") + .and_then(serde_json::Value::as_str) + .unwrap_or_else(|| panic!("{path}: missing string \"abbreviation\"")); + let mut corpus = Corpus::new(label); + let books = json + .get("books") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("{path}: missing array \"books\"")); + for book in books { + let chapters = book.get("chapters").and_then(serde_json::Value::as_array); + for chapter in chapters.into_iter().flatten() { + let verses = chapter.get("verses").and_then(serde_json::Value::as_array); + for verse in verses.into_iter().flatten() { + if let Some(t) = verse.get("text").and_then(serde_json::Value::as_str) { + measure_text(h, t, &mut corpus); + } + } + } + } + corpus +} + +/// xorshift64, seeded 0x9E3779B97F4A7C15. +struct XorShift64(u64); + +impl XorShift64 { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } +} + +fn rate(overturns: usize, commits: usize) -> f64 { + if commits == 0 { + 0.0 + } else { + overturns as f64 / commits as f64 + } +} + +/// 95% bootstrap CI of the overturn rate, resampling clauses with replacement. +fn bootstrap_ci(stats: &[ClauseStat], pick: fn(&ClauseStat) -> (usize, usize)) -> (f64, f64) { + if stats.is_empty() { + return (0.0, 0.0); + } + let mut rng = XorShift64(0x9E37_79B9_7F4A_7C15); + let n = stats.len() as u64; + let mut rates = Vec::with_capacity(BOOTSTRAP_ROUNDS); + for _ in 0..BOOTSTRAP_ROUNDS { + let (mut o, mut c) = (0usize, 0usize); + for _ in 0..stats.len() { + let (so, sc) = pick(&stats[(rng.next() % n) as usize]); + o += so; + c += sc; + } + rates.push(rate(o, c)); + } + rates.sort_by(f64::total_cmp); + ( + rates[BOOTSTRAP_ROUNDS * 25 / 1000], + rates[BOOTSTRAP_ROUNDS * 975 / 1000], + ) +} + +/// Print one corpus block; returns `(rate, ci_lo, ci_hi)` for the +/// pre-registered arm and for the matched arm. +fn report(c: &Corpus) -> (Rate, Rate) { + let sum = |f: fn(&ClauseStat) -> usize| c.stats.iter().map(f).sum::(); + let commits = sum(|s| s.left_commits); + let overturns = sum(|s| s.overturns); + let r = rate(overturns, commits); + let (lo, hi) = bootstrap_ci(&c.stats, |s| (s.overturns, s.left_commits)); + let m_commits = sum(|s| s.matched_commits); + let m_overturns = sum(|s| s.matched_overturns); + let mr = rate(m_overturns, m_commits); + let (mlo, mhi) = bootstrap_ci(&c.stats, |s| (s.matched_overturns, s.matched_commits)); + println!("== {} ==", c.label); + println!(" clauses: {}", c.stats.len()); + println!( + " clauses with >=1 hypothesis (right read): {}", + c.stats.iter().filter(|s| s.has_hypothesis).count() + ); + println!(" left commitments: {commits}"); + println!(" overturns: {overturns}"); + println!(" overturn rate: {r:.4} (95% bootstrap CI {lo:.4} .. {hi:.4})"); + println!( + " matched arm (post hoc): {m_overturns}/{m_commits} = {mr:.4} (95% CI {mlo:.4} .. {mhi:.4})" + ); + println!(" right-only lanes: {}", sum(|s| s.right_only)); + println!(" right abstained total: {}", sum(|s| s.right_abstained)); + println!( + " ambiguous_wins total (right read): {}", + sum(|s| s.right_ambiguous_wins) + ); + let mut top: Vec<(&String, &usize)> = c.forms.iter().collect(); + top.sort_by(|a, b| b.1.cmp(a.1).then_with(|| a.0.cmp(b.0))); + println!(" top forms opening an overturned left commitment:"); + for (form, n) in top.into_iter().take(10) { + println!(" {form:12} {n}"); + } + ((r, lo, hi), (mr, mlo, mhi)) +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + if args.len() < 3 { + eprintln!("usage: tekamolo_corners TEKAMOLO.tsv GSD.conllu BIBLE.json [BIBLE.json ...]"); + std::process::exit(2); + } + let tsv = std::fs::read_to_string(&args[0]).unwrap_or_else(|e| panic!("{}: {e}", args[0])); + let h = GrammarHeuristics::parse(&tsv); + let (forms, rows) = h.sizes(); + println!( + "heuristics: {forms} forms, {rows} rows, {} ambiguous; lanes {}", + h.ambiguous(), + LANE_NAMES.join("/") + ); + + let gsd = measure_gsd(&h, &args[1]); + let bibles: Vec = args[2..].iter().map(|p| measure_bible(&h, p)).collect(); + + let ((gsd_rate, _gsd_lo, gsd_hi), (gsd_m, gsd_mlo, gsd_mhi)) = report(&gsd); + let mut first: Option<(Rate, Rate)> = None; + for b in &bibles { + let r = report(b); + first.get_or_insert(r); + } + let ((b_rate, b_lo, _b_hi), (b_m, b_mlo, b_mhi)) = + first.unwrap_or_else(|| panic!("no Bible corpus given")); + println!( + "comparison: {} rate {b_rate:.4} (CI lower {b_lo:.4}) vs GSD rate {gsd_rate:.4} (CI upper {gsd_hi:.4})", + bibles[0].label + ); + println!( + "matched arm (post hoc): {} {b_m:.4} (CI {b_mlo:.4} .. {b_mhi:.4}) vs GSD {gsd_m:.4} (CI {gsd_mlo:.4} .. {gsd_mhi:.4})", + bibles[0].label + ); + println!("verdict: {}", if b_lo > gsd_hi { "PASS" } else { "KILL" }); +} diff --git a/crates/deepnsm-v2/examples/ud_case_eval.rs b/crates/deepnsm-v2/examples/ud_case_eval.rs new file mode 100644 index 000000000..338d381a9 --- /dev/null +++ b/crates/deepnsm-v2/examples/ud_case_eval.rs @@ -0,0 +1,552 @@ +//! `ud_case_eval` — the German CASE evaluator (Q2 of +//! `.claude/knowledge/german-grammar-rule-inventory.md`, ranks 5-8). +//! +//! Rules predict the UD feature `Case=` (Nom / Acc / Dat / Gen) of a token from +//! its surface form, its position and tables mined from **train only**. Gold +//! (`Case=`, `PronType=`, UPOS, LEMMA) is read from train to mine the tables and +//! from test **only to score**; no test-time prediction reads it. A rule either +//! predicts a case for a token or abstains. The scored unit is a test token that +//! carries `Case=` in gold. +//! +//! Rules (each reported separately): +//! +//! - **R1** fixed-case prepositions (closed Acc / Dat / Gen lists); the case is +//! predicted for the tokens in the noun-phrase window after the preposition. +//! - **R2** article decidability: per lowercased DET/PRON form, train purity of +//! its case, tiered Decisive (>= 0.95), Dominant (>= 0.80), Ambiguous. +//! - **R3** Wechselpraeposition: contractions decide alone (am/im = Dat, +//! ans/ins/aufs = Acc); otherwise the article decides when the first following +//! token has exactly one of Acc/Dat in train, else a (preposition, verb lemma) +//! prior, else the preposition's own majority. +//! - **R4** relative pronoun after a comma, case taken from train tokens under +//! the SAME surface condition (a relativizer form right after a comma), plus +//! any token marked `PronType=Rel`. Mining on `PronType=Rel` alone fires +//! nothing on German GSD, which carries no `PronType=Rel` at all. +//! - **R5** combined, first rule that fires in the order R3-contraction, R1, R3, +//! R4, R2; coverage, precision, baseline and the Nom/Acc/Dat/Gen confusion. +//! +//! The baseline everywhere is the train-majority case of the token's lowercased +//! form over all train tokens with a case; an unseen form abstains and counts +//! wrong. Note: in GSD, contractions such as `zum`, `am` are multiword tokens +//! whose syntactic words are `zu dem`, `an dem`; this reader keeps the words and +//! skips the ranges, so the contraction entries may never fire (reported). +//! +//! KILL bars (from the research map): +//! +//! - R1: precision <= baseline on the same tokens, or Acc/Dat < 0.90, or +//! Gen < 0.70. +//! - R3: verb-prior precision <= prep-majority on the same tokens, or verb-prior +//! fires on < 5 % of the Wechsel-governed scored tokens (the first following +//! token of a non-contraction Wechsel preposition). +//! - R4: precision <= the plain form baseline. +//! +//! ```text +//! cargo run --release --example ud_case_eval -- de_gsd-ud-train.conllu de_gsd-ud-test.conllu +//! ``` + +use std::collections::HashMap; + +const CASES: [&str; 4] = ["Nom", "Acc", "Dat", "Gen"]; + +const ACC: [&str; 5] = ["durch", "für", "gegen", "ohne", "um"]; +const DAT: [&str; 12] = [ + "aus", + "bei", + "mit", + "nach", + "seit", + "von", + "zu", + "gegenüber", + "zum", + "zur", + "beim", + "vom", +]; +const GEN: [&str; 7] = [ + "wegen", + "trotz", + "während", + "statt", + "anstatt", + "innerhalb", + "außerhalb", +]; +/// Wechsel prepositions that need the article or the verb to decide. +const WECHSEL: [&str; 9] = [ + "an", "auf", "hinter", "in", "neben", "über", "unter", "vor", "zwischen", +]; +const CONTRACTION_DAT: [&str; 2] = ["am", "im"]; +const CONTRACTION_ACC: [&str; 3] = ["ans", "ins", "aufs"]; +const REL: [&str; 13] = [ + "der", "die", "das", "dem", "den", "dessen", "deren", "denen", "welcher", "welche", "welches", + "welchem", "welchen", +]; + +const TIERS: [&str; 3] = ["Decisive", "Dominant", "Ambiguous"]; +const KINDS: [&str; 3] = ["article-decided", "verb-prior", "prep-majority"]; + +struct Tok { + /// Lowercased form. + form: String, + /// The surface form starts with an uppercase letter. + upper: bool, + /// The form has an alphanumeric character (not punctuation). + word: bool, + lemma: String, + upos: String, + case: Option, + /// `PronType=Rel` in feats. + rel: bool, +} + +fn case_of(feats: &str) -> Option { + feats + .split('|') + .find_map(|f| f.strip_prefix("Case=")) + .and_then(|v| CASES.iter().position(|c| *c == v)) +} + +fn read(path: &str) -> Vec> { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut out = Vec::new(); + let mut cur = Vec::new(); + for line in text.lines() { + if line.is_empty() { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if line.starts_with('#') || c.len() < 8 || c[0].contains(['-', '.']) { + continue; + } + cur.push(Tok { + form: c[1].to_lowercase(), + upper: c[1].chars().next().is_some_and(char::is_uppercase), + word: c[1].chars().any(char::is_alphanumeric), + lemma: c[2].to_string(), + upos: c[3].to_string(), + case: case_of(c[5]), + rel: c[5].split('|').any(|f| f == "PronType=Rel"), + }); + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +fn in_list(list: &[&str], form: &str) -> bool { + list.contains(&form) +} + +/// The noun-phrase window after the preposition at `i`: up to four following +/// tokens, never through punctuation, ending after the first capitalised token. +fn window(s: &[Tok], i: usize) -> Vec { + let mut out = Vec::new(); + let mut j = i + 1; + while j < s.len() && j - i <= 4 && s[j].word { + out.push(j); + if s[j].upper { + break; + } + j += 1; + } + out +} + +/// The most frequent case, ties to the lower index; `None` when empty. +fn majority(c: &[usize; 4]) -> Option { + if c.iter().sum::() == 0 { + return None; + } + (0..4).max_by_key(|&k| (c[k], std::cmp::Reverse(k))) +} + +/// Everything mined from train. +struct Tables { + /// Lowercased form → case counts over all train tokens with a case. + form_case: HashMap, + /// Form → (tier, majority case) for DET/PRON forms. + det_rule: HashMap, + /// Form → case counts over `PronType=Rel` train tokens. + rel_case: HashMap, + /// Verb form → majority lemma (upos VERB). + verb_lemma: HashMap, + /// (preposition, verb lemma) → case counts of the Wechsel-governed token. + wechsel_key: HashMap<(String, String), [usize; 4]>, + /// Preposition → case counts of the Wechsel-governed token. + wechsel_prep: HashMap, +} + +impl Tables { + fn mine(train: &[Vec]) -> Self { + let mut form_case: HashMap = HashMap::new(); + let mut det_case: HashMap = HashMap::new(); + let mut rel_case: HashMap = HashMap::new(); + let mut verb_counts: HashMap> = HashMap::new(); + for t in train.iter().flatten() { + if let Some(c) = t.case { + form_case.entry(t.form.clone()).or_default()[c] += 1; + if t.upos == "DET" || t.upos == "PRON" { + det_case.entry(t.form.clone()).or_default()[c] += 1; + } + } + if t.upos == "VERB" { + *verb_counts + .entry(t.form.clone()) + .or_default() + .entry(t.lemma.clone()) + .or_default() += 1; + } + } + let verb_lemma = verb_counts + .into_iter() + .filter_map(|(f, m)| { + m.into_iter() + .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(&a.0))) + .map(|(l, _)| (f, l)) + }) + .collect(); + let det_rule = det_case + .iter() + .filter_map(|(f, c)| { + let m = majority(c)?; + let purity = c[m] as f64 / c.iter().sum::() as f64; + let tier = if purity >= 0.95 { + 0 + } else if purity >= 0.80 { + 1 + } else { + 2 + }; + Some((f.clone(), (tier, m))) + }) + .collect(); + // R4 is mined under its own test-time surface condition. + for sent in train { + for (k, t) in sent.iter().enumerate() { + let after_comma = k > 0 && sent[k - 1].form == ","; + if let Some(c) = t.case { + if (after_comma && in_list(&REL, &t.form)) || t.rel { + rel_case.entry(t.form.clone()).or_default()[c] += 1; + } + } + } + } + let mut t = Self { + form_case, + det_rule, + rel_case, + verb_lemma, + wechsel_key: HashMap::new(), + wechsel_prep: HashMap::new(), + }; + t.mine_wechsel(train); + t + } + + /// Wechsel-governed counts from train: the first following token of a + /// Wechsel preposition, scored against train gold (Acc or Dat only). + fn mine_wechsel(&mut self, train: &[Vec]) { + for s in train { + for (i, t) in s.iter().enumerate() { + if !in_list(&WECHSEL, &t.form) { + continue; + } + let Some(&j) = window(s, i).first() else { + continue; + }; + let Some(c) = s[j].case.filter(|&c| c == 1 || c == 2) else { + continue; + }; + self.wechsel_prep.entry(t.form.clone()).or_default()[c] += 1; + if let Some(lemma) = self.nearest_verb(s, i) { + let key = (t.form.clone(), lemma.clone()); + self.wechsel_key.entry(key).or_default()[c] += 1; + } + } + } + } + + /// The lemma of the nearest preceding token that is a known verb form. + fn nearest_verb(&self, s: &[Tok], i: usize) -> Option<&String> { + (0..i).rev().find_map(|k| self.verb_lemma.get(&s[k].form)) + } + + fn baseline(&self, form: &str) -> Option { + self.form_case.get(form).and_then(majority) + } + + fn baseline_ok(&self, form: &str, gold: usize) -> bool { + self.baseline(form) == Some(gold) + } + + /// R3 decision for the Wechsel preposition at `i`, predicting token `j`: + /// (kind, case, prep-majority case). + fn wechsel(&self, s: &[Tok], i: usize, j: usize) -> Option<(usize, usize, Option)> { + let prep = &s[i].form; + let prep_major = self.wechsel_prep.get(prep).and_then(majority); + let seen = self.form_case.get(&s[j].form); + let cands: Vec = [1usize, 2] + .into_iter() + .filter(|&c| seen.is_some_and(|x| x[c] > 0)) + .collect(); + if let [only] = cands.as_slice() { + return Some((0, *only, prep_major)); + } + let by_verb = self + .nearest_verb(s, i) + .and_then(|l| self.wechsel_key.get(&(prep.clone(), l.clone()))) + .and_then(majority); + match (by_verb, prep_major) { + (Some(c), _) => Some((1, c, prep_major)), + (None, Some(c)) => Some((2, c, prep_major)), + (None, None) => None, + } + } + + /// Every rule's prediction per token of a test sentence. No gold is read. + fn predict(&self, s: &[Tok]) -> Vec { + let mut preds: Vec = s.iter().map(|_| Pred::default()).collect(); + for (i, tok) in s.iter().enumerate() { + let f = tok.form.as_str(); + let fixed = if in_list(&ACC, f) { + Some(1) + } else if in_list(&DAT, f) { + Some(2) + } else if in_list(&GEN, f) { + Some(3) + } else { + None + }; + let contraction = if in_list(&CONTRACTION_DAT, f) { + Some(2) + } else if in_list(&CONTRACTION_ACC, f) { + Some(1) + } else { + None + }; + if fixed.is_some() || contraction.is_some() { + for j in window(s, i) { + if preds[j].r1.is_none() { + preds[j].r1 = fixed; + } + if preds[j].r3c.is_none() { + preds[j].r3c = contraction; + } + } + } + if in_list(&WECHSEL, f) { + if let Some(&j) = window(s, i).first() { + preds[j].wech = true; + if preds[j].r3.is_none() { + preds[j].r3 = self.wechsel(s, i, j); + } + } + } + if i > 0 && s[i - 1].form == "," && in_list(&REL, f) { + preds[i].r4 = self.rel_case.get(f).and_then(majority); + } + preds[i].r2 = self.det_rule.get(f).copied(); + } + preds + } +} + +#[derive(Default)] +struct Pred { + r1: Option, + r3c: Option, + r3: Option<(usize, usize, Option)>, + r4: Option, + r2: Option<(usize, usize)>, + /// First following token of a non-contraction Wechsel preposition. + wech: bool, +} + +#[derive(Default, Clone, Copy)] +struct Tally { + fires: usize, + correct: usize, + base: usize, +} + +impl Tally { + fn add(&mut self, ok: bool, base_ok: bool) { + self.fires += 1; + self.correct += usize::from(ok); + self.base += usize::from(base_ok); + } + + fn precision(&self) -> f64 { + pct(self.correct, self.fires) + } + + fn baseline(&self) -> f64 { + pct(self.base, self.fires) + } + + fn line(&self) -> String { + format!( + "fires {:6} precision {:5.1}% baseline {:5.1}%", + self.fires, + self.precision(), + self.baseline() + ) + } +} + +fn pct(x: usize, d: usize) -> f64 { + if d == 0 { + 0.0 + } else { + 100.0 * x as f64 / d as f64 + } +} + +fn verdict(pass: bool) -> &'static str { + if pass { + "PASS" + } else { + "KILL" + } +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let [train_path, test_path] = args.as_slice() else { + eprintln!("usage: ud_case_eval TRAIN.conllu TEST.conllu"); + std::process::exit(2); + }; + let train = read(train_path); + let test = read(test_path); + let tables = Tables::mine(&train); + + let mut scored = 0usize; + let mut wech_scored = 0usize; + let mut r1 = [Tally::default(); 4]; + let mut r2 = [Tally::default(); 3]; + let mut r3c = Tally::default(); + let mut r3 = [Tally::default(); 3]; + let mut verb_prep_major = 0usize; + let mut r4 = Tally::default(); + let mut r5 = Tally::default(); + let mut confusion = [[0usize; 4]; 4]; + + for s in &test { + let preds = tables.predict(s); + for (t, p) in s.iter().zip(&preds) { + let Some(gold) = t.case else { + continue; + }; + scored += 1; + let base = tables.baseline_ok(&t.form, gold); + if p.wech { + wech_scored += 1; + } + if let Some(c) = p.r1 { + r1[c].add(c == gold, base); + } + if let Some(c) = p.r3c { + r3c.add(c == gold, base); + } + if let Some((kind, c, prep_major)) = p.r3 { + r3[kind].add(c == gold, base); + if kind == 1 && prep_major == Some(gold) { + verb_prep_major += 1; + } + } + if let Some(c) = p.r4 { + r4.add(c == gold, base); + } + if let Some((tier, c)) = p.r2 { + r2[tier].add(c == gold, base); + } + let pick = p + .r3c + .or(p.r1) + .or(p.r3.map(|d| d.1)) + .or(p.r4) + .or(p.r2.map(|d| d.1)); + if let Some(c) = pick { + r5.add(c == gold, base); + confusion[gold][c] += 1; + } + } + } + + let ntrain: usize = train.iter().map(Vec::len).sum(); + let ntest: usize = test.iter().map(Vec::len).sum(); + println!("train {ntrain} tokens, test {ntest} tokens, scored (Case= in gold) {scored}"); + println!( + "mined from train: {} forms with a case, {} DET/PRON forms, {} verb forms, {} (prep, lemma) keys", + tables.form_case.len(), + tables.det_rule.len(), + tables.verb_lemma.len(), + tables.wechsel_key.len() + ); + + println!( + "\nR1 fixed-case prepositions (KILL: precision <= baseline, Acc/Dat < 0.90, Gen < 0.70)" + ); + let mut r1_pass = true; + for c in [1usize, 2, 3] { + let t = r1[c]; + let floor = if c == 3 { 70.0 } else { 90.0 }; + let pass = t.fires > 0 && t.precision() > t.baseline() && t.precision() >= floor; + r1_pass &= pass; + println!(" {} list: {} {}", CASES[c], t.line(), verdict(pass)); + } + println!(" R1 {}", verdict(r1_pass)); + + println!("\nR2 article decidability (no KILL bar; baseline = form majority)"); + for (k, name) in TIERS.iter().enumerate() { + println!(" {name:9}: {}", r2[k].line()); + } + + println!( + "\nR3 Wechselpraeposition (KILL: verb-prior precision <= prep-majority, or verb-prior fires < 5% of {wech_scored} Wechsel-governed scored tokens)" + ); + println!(" contraction : {}", r3c.line()); + for (k, name) in KINDS.iter().enumerate() { + println!(" {name:15} : {}", r3[k].line()); + } + let verb = r3[1]; + let prep_major_precision = pct(verb_prep_major, verb.fires); + println!( + " prep-majority on the verb-prior tokens: precision {prep_major_precision:5.1}% verb-prior share of Wechsel-governed {:5.1}%", + pct(verb.fires, wech_scored) + ); + let r3_pass = + verb.fires > 0 && verb.precision() > prep_major_precision && verb.fires * 20 >= wech_scored; + println!(" R3 {}", verdict(r3_pass)); + + println!("\nR4 relative pronoun after a comma (KILL: precision <= form baseline)"); + println!(" {}", r4.line()); + println!( + " R4 {}", + verdict(r4.fires > 0 && r4.precision() > r4.baseline()) + ); + + println!("\nR5 combined (R3-contraction, R1, R3, R4, R2)"); + println!( + " coverage {:5.1}% ({} of {scored}) {}", + pct(r5.fires, scored), + r5.fires, + r5.line() + ); + println!(" confusion over covered tokens (rows gold, columns predicted)"); + println!( + " {:>7} {:>7} {:>7} {:>7}", + "Nom", "Acc", "Dat", "Gen" + ); + for (g, row) in confusion.iter().enumerate() { + println!( + " {:>5} {:>7} {:>7} {:>7} {:>7}", + CASES[g], row[0], row[1], row[2], row[3] + ); + } +} diff --git a/crates/deepnsm-v2/examples/ud_pos_eval.rs b/crates/deepnsm-v2/examples/ud_pos_eval.rs index ecdb9f4b1..9842bccae 100644 --- a/crates/deepnsm-v2/examples/ud_pos_eval.rs +++ b/crates/deepnsm-v2/examples/ud_pos_eval.rs @@ -763,6 +763,30 @@ fn rules( (cased && x.surface.starts_with(char::is_lowercase)).then_some(Pos::Verb) }), )); + } else if pair.0 == Pos::Det { + // der/die/das (and English "that"): an article opens a nominal group; + // the pronoun reading (`Rel` where the treebank marks relativizers, + // `Noun` where it folds a relative/demonstrative PRON, as German GSD + // does) opens a clause after a comma or stands without a noun. + let pron = pair.1; + fixed.push(( + "followed by a noun or adjective -> det", + Box::new(|x: &Tok| { + (x.next.contains(Pos::Noun) || x.next.contains(Pos::Adj)).then_some(Pos::Det) + }), + )); + fixed.push(( + "after a comma -> pronoun", + Box::new(move |x: &Tok| (x.prev_form == Some(",")).then_some(pron)), + )); + fixed.push(( + "followed by a determiner -> pronoun", + Box::new(move |x: &Tok| only(x.next, Pos::Det).then_some(pron)), + )); + fixed.push(( + "followed by a verb-only word -> pronoun", + Box::new(move |x: &Tok| only(x.next, Pos::Verb).then_some(pron)), + )); } else { for (name, rule) in [ ( @@ -1073,6 +1097,9 @@ fn quorum_report( let toks = pair_tokens(test, lex, copulas, cased, pair); let mut joint_ok = 0usize; + // Second-reading precision/recall of the joint quorum: (predicted b, + // predicted b and gold b, gold b). + let (mut pred_b, mut hit_b, mut gold_b) = (0usize, 0usize, 0usize); let (mut freq_ok, mut prio_ok, mut quorum_ok, mut silent) = (0usize, 0usize, 0usize, 0usize); let mut fired = vec![(0usize, 0usize); rules.len()]; for (t, g) in &toks { @@ -1101,7 +1128,11 @@ fn quorum_report( }; quorum_ok += usize::from(quorum == *g); let z: f64 = features(t).iter().zip(&w).map(|(xi, wi)| xi * wi).sum(); - joint_ok += usize::from(if z > 0.0 { a } else { b } == *g); + let joint = if z > 0.0 { a } else { b }; + joint_ok += usize::from(joint == *g); + pred_b += usize::from(joint == b); + hit_b += usize::from(joint == b && *g == b); + gold_b += usize::from(*g == b); } let pct = |x: usize, d: usize| { if d == 0 { @@ -1121,6 +1152,12 @@ fn quorum_report( pct(quorum_ok, n), pct(joint_ok, n) ); + println!( + " joint quorum on {:?}: precision {:.1}% ({hit_b}/{pred_b}), recall {:.1}% ({hit_b}/{gold_b})", + b, + pct(hit_b, pred_b), + pct(hit_b, gold_b) + ); if std::env::var_os("UD_RULES").is_some() { for i in 0..rules.len() { let voting = if order.contains(&i) { ' ' } else { 'x' }; @@ -1482,6 +1519,26 @@ fn main() { aa, "adjective/adverb", ); + // Article vs pronoun (der/die/das, "that"): the pronoun reading is `Rel` + // where the treebank marks relativizers, `Noun` where it folds PRON + // (German GSD carries no PronType=Rel). + for (pair, name) in [ + ((Pos::Det, Pos::Rel), "determiner/relativizer"), + ((Pos::Det, Pos::Noun), "determiner/pronoun"), + ] { + position_table_report(&train, &test, &lex, &copulas, cased, pair, name); + quorum_report( + &train, + &test, + &lex, + coca.is_some(), + &copulas, + cased, + typology, + pair, + name, + ); + } // Each attribute clause on its own: every adjective/adverb token whose // gold is one of the two, the clause that fires, and whether the clause From 915ed7e41507aba7f1b9e092bb99e0e36cf6814a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:48:46 +0000 Subject: [PATCH 18/33] deepnsm-v2 ud_case_eval: time vs place reading of Wechsel prepositions (R3t) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Wechsel phrase whose head noun is a time noun (mined from train as the heads after seit/während, reached through DET/ADJ only) takes its case from a (preposition, temporal) table. On HDT: - the time reading sharpens the case split (an: 76 -> 91.5 % Dat; in: 88 -> 95.4 %) but never changes which case wins, so R3t equals prep-majority (81.8 % vs 81.8 %): KILL; - on time heads the verb prior scores 72.4 % and R3t 80.2 % (116 tokens). The verb should not decide a temporal phrase, but the verb prior's HDT failure is mostly on place heads (62.3 %). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- ...sm-v2-first-probes-from-the-grammar-map.md | 21 ++++ crates/deepnsm-v2/examples/ud_case_eval.rs | 118 ++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md b/.claude/board/entries/2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md index a209b36ad..5f4e98a99 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md @@ -51,3 +51,24 @@ Pre-registered prediction: right-corner reading overturns left-corner commitment **Verdict: KILL.** The prediction is falsified. The matched arm, post hoc, rules out the confound between commit point and ambiguity admission. On the Bible, the right reader's extra lanes come from admitting the ambiguous *da* (10,296 ambiguous wins on Luther), not from where it commits. **OPEN.** Why does edited German overturn more? Candidate: GSD clauses are longer and carry more competing cues per clause. This is not measured, and the reason is left unexplained. + +## Addendum — time vs place reading of the Wechsel prepositions (R3t) + +Operator: the Wechsel ambiguity is often a time-vs-place ambiguity (*vor dem Haus* / *vor dem Essen*). + +**Method.** R3t marks a Wechsel phrase as temporal when its head noun is a time noun. Time nouns are mined from train as the heads after *seit* / *während*, the two prepositions that only read temporally. The head counts only when reached through DET/ADJ alone; the first mining run let numbers leak nouns from outside the phrase, e.g. *seit 1998 Mitbegründer*. The case then comes from a (preposition, temporal) table. + +| HDT train → time / place | % Dat, place head | % Dat, time head | +|---|---|---| +| *an* | 76 % | 91.5 % | +| *in* | 88 % | 95.4 % | +| *auf* | 50 % | 44 % (Acc leads: *auf Jahre*) | +| *über* | 11 % | 7 % (Acc: *über das Wochenende*) | + +**Results.** +- A time reading makes the case **more predictable but never changes which case wins**. R3t therefore equals prep-majority on HDT: 81.8 % vs 81.8 %, 302 fires, **KILL**. On GSD it is 88.2 % vs 82.4 % on 17 tokens, also KILL (the fire share is below 5 %). +- **The verb should not decide a temporal phrase.** On the time heads where the verb prior fires (HDT, 116 tokens), it scores 72.4 % and R3t scores 80.2 %. On place heads the verb prior is 62.3 %, so its HDT failure is mostly on place, and the time split recovers only about 9 of 1,760 tokens. + +**OPEN.** +- German UD has no time/place gold: GSD has 30 `obl:tmod`, HDT has 0. The time/place reading itself is therefore a TEKAMOLO-lane question (Q4) with no scorer here. +- The mined list still admits event nouns (*Fahrt*, *Hauptversammlung*). That reading is legitimately temporal, but it is not a pure time-noun list. diff --git a/crates/deepnsm-v2/examples/ud_case_eval.rs b/crates/deepnsm-v2/examples/ud_case_eval.rs index 338d381a9..645a6d0c9 100644 --- a/crates/deepnsm-v2/examples/ud_case_eval.rs +++ b/crates/deepnsm-v2/examples/ud_case_eval.rs @@ -18,6 +18,12 @@ //! ans/ins/aufs = Acc); otherwise the article decides when the first following //! token has exactly one of Acc/Dat in train, else a (preposition, verb lemma) //! prior, else the preposition's own majority. +//! - **R3t** time vs place: a Wechsel preposition whose head noun is a time +//! noun reads temporally (*in der Nacht*, *vor dem Essen*, *über das +//! Wochenende*), and the case comes from a (preposition, temporal) table +//! instead of the verb. Time nouns are mined from train as the head nouns +//! after *seit* / *während*, the prepositions with only a time reading; no +//! hand list. Scored on the Wechsel tokens the article leaves undecided. //! - **R4** relative pronoun after a comma, case taken from train tokens under //! the SAME surface condition (a relativizer form right after a comma), plus //! any token marked `PronType=Rel`. Mining on `PronType=Rel` alone fires @@ -38,6 +44,8 @@ //! - R3: verb-prior precision <= prep-majority on the same tokens, or verb-prior //! fires on < 5 % of the Wechsel-governed scored tokens (the first following //! token of a non-contraction Wechsel preposition). +//! - R3t: precision <= prep-majority on the same tokens, or fires on < 5 % of +//! the article-undecided Wechsel tokens. //! - R4: precision <= the plain form baseline. //! //! ```text @@ -83,6 +91,9 @@ const REL: [&str; 13] = [ "welchem", "welchen", ]; +/// Prepositions with only a time reading; their head nouns seed the time-noun set. +const TEMPORAL_ONLY: [&str; 2] = ["seit", "während"]; + const TIERS: [&str; 3] = ["Decisive", "Dominant", "Ambiguous"]; const KINDS: [&str; 3] = ["article-decided", "verb-prior", "prep-majority"]; @@ -179,6 +190,15 @@ struct Tables { wechsel_key: HashMap<(String, String), [usize; 4]>, /// Preposition → case counts of the Wechsel-governed token. wechsel_prep: HashMap, + /// Lowercased head-noun forms seen after a time-only preposition. + time_nouns: std::collections::HashSet, + /// (preposition, head is a time noun) → case counts of the governed token. + wechsel_time: HashMap<(String, bool), [usize; 4]>, +} + +/// The head noun of the window after `i`: its last token when capitalised. +fn head(s: &[Tok], i: usize) -> Option { + window(s, i).last().copied().filter(|&j| s[j].upper) } impl Tables { @@ -243,7 +263,26 @@ impl Tables { verb_lemma, wechsel_key: HashMap::new(), wechsel_prep: HashMap::new(), + time_nouns: Default::default(), + wechsel_time: HashMap::new(), }; + for s in train { + for (i, tok) in s.iter().enumerate() { + if in_list(&TEMPORAL_ONLY, &tok.form) { + // Only a noun reached through DET/ADJ, so a number or a + // verb never leaks a noun from outside the phrase. + let np = |j: usize| { + s[j].upos == "NOUN" + && s[i + 1..j] + .iter() + .all(|x| x.upos == "DET" || x.upos == "ADJ") + }; + if let Some(j) = head(s, i).filter(|&j| np(j)) { + t.time_nouns.insert(s[j].form.clone()); + } + } + } + } t.mine_wechsel(train); t } @@ -263,6 +302,8 @@ impl Tables { continue; }; self.wechsel_prep.entry(t.form.clone()).or_default()[c] += 1; + let time = self.is_time(s, i); + self.wechsel_time.entry((t.form.clone(), time)).or_default()[c] += 1; if let Some(lemma) = self.nearest_verb(s, i) { let key = (t.form.clone(), lemma.clone()); self.wechsel_key.entry(key).or_default()[c] += 1; @@ -271,6 +312,21 @@ impl Tables { } } + /// The Wechsel preposition at `i` heads a time noun. + fn is_time(&self, s: &[Tok], i: usize) -> bool { + head(s, i).is_some_and(|j| self.time_nouns.contains(&s[j].form)) + } + + /// R3t: the (preposition, time reading) majority, only for a time head. + fn wechsel_temporal(&self, s: &[Tok], i: usize) -> Option { + if !self.is_time(s, i) { + return None; + } + self.wechsel_time + .get(&(s[i].form.clone(), true)) + .and_then(majority) + } + /// The lemma of the nearest preceding token that is a known verb form. fn nearest_verb(&self, s: &[Tok], i: usize) -> Option<&String> { (0..i).rev().find_map(|k| self.verb_lemma.get(&s[k].form)) @@ -344,6 +400,8 @@ impl Tables { preds[j].wech = true; if preds[j].r3.is_none() { preds[j].r3 = self.wechsel(s, i, j); + preds[j].r3t = self.wechsel_temporal(s, i); + preds[j].time = self.is_time(s, i); } } } @@ -363,6 +421,10 @@ struct Pred { r3: Option<(usize, usize, Option)>, r4: Option, r2: Option<(usize, usize)>, + /// R3t prediction (time head only). + r3t: Option, + /// The Wechsel preposition heads a time noun. + time: bool, /// First following token of a non-contraction Wechsel preposition. wech: bool, } @@ -432,6 +494,14 @@ fn main() { let mut r3c = Tally::default(); let mut r3 = [Tally::default(); 3]; let mut verb_prep_major = 0usize; + // Article-undecided Wechsel tokens (R3 kind 1 or 2). + let mut undecided = 0usize; + let mut r3t = Tally::default(); + let mut r3t_prep_major = 0usize; + // Verb-prior split by reading: [place, time]. + let mut verb_by_reading = [Tally::default(); 2]; + // R3t on the time-head tokens where the verb prior fired. + let mut r3t_on_verb = Tally::default(); let mut r4 = Tally::default(); let mut r5 = Tally::default(); let mut confusion = [[0usize; 4]; 4]; @@ -458,6 +528,21 @@ fn main() { if kind == 1 && prep_major == Some(gold) { verb_prep_major += 1; } + if kind == 1 { + verb_by_reading[usize::from(p.time)].add(c == gold, base); + if let Some(t) = p.r3t { + r3t_on_verb.add(t == gold, base); + } + } + if kind != 0 { + undecided += 1; + if let Some(t) = p.r3t { + r3t.add(t == gold, base); + if prep_major == Some(gold) { + r3t_prep_major += 1; + } + } + } } if let Some(c) = p.r4 { r4.add(c == gold, base); @@ -524,6 +609,39 @@ fn main() { verb.fires > 0 && verb.precision() > prep_major_precision && verb.fires * 20 >= wech_scored; println!(" R3 {}", verdict(r3_pass)); + println!( + "\nR3t time vs place (KILL: precision <= prep-majority on the same tokens, or fires < 5% of {undecided} article-undecided Wechsel tokens)" + ); + println!(" time nouns mined from train: {}", tables.time_nouns.len()); + let mut keys: Vec<_> = tables.wechsel_time.iter().collect(); + keys.sort(); + for ((prep, time), c) in keys { + println!( + " {prep:9} {:5} Acc {:6} Dat {:6}", + if *time { "time" } else { "place" }, + c[1], + c[2] + ); + } + println!(" verb-prior, place head: {}", verb_by_reading[0].line()); + println!(" verb-prior, time head : {}", verb_by_reading[1].line()); + println!(" R3t on those time heads: {}", r3t_on_verb.line()); + let mut sample: Vec<&String> = tables.time_nouns.iter().collect(); + sample.sort(); + let step = (sample.len() / 15).max(1); + let shown: Vec<&str> = sample.iter().step_by(step).map(|w| w.as_str()).collect(); + println!(" time-noun sample: {}", shown.join(" ")); + let r3t_major = pct(r3t_prep_major, r3t.fires); + println!( + " R3t: {} prep-majority on the same tokens {r3t_major:5.1}% share {:5.1}%", + r3t.line(), + pct(r3t.fires, undecided) + ); + println!( + " R3t {}", + verdict(r3t.fires > 0 && r3t.precision() > r3t_major && r3t.fires * 20 >= undecided) + ); + println!("\nR4 relative pronoun after a comma (KILL: precision <= form baseline)"); println!(" {}", r4.line()); println!( From 88dbee98c7453c508d8c7eb55b4d0612ef6919ef Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:53:22 +0000 Subject: [PATCH 19/33] deepnsm-v2: prepositional object vs adverbial; abstract-noun class (D-LXC-24) ud_pp_arg_eval (new): a (verb lemma, preposition) object rate mined from HDT train decides whether a PP is verb-governed (sich freuen auf) or a free adverbial, by checking every verb in the clause. Gold is `obj` with an ADP child; German `obl:arg` marks bare datives, not PP objects. - A1: precision 77.2 %, recall 81.6 %: 896 of 1,098 objects leave the TEKAMOLO pool, 264 of 19,466 adverbials are lost. PASS; the rate check is disable-verified (7.7 % without it). - A2: the (verb, preposition) case of a detected Wechsel object scores 96.8 % vs 81.0 % for the preposition majority. PASS. Abstract heads (-ung, -heit, -keit, -schaft, -tion, ...) are prepositional objects twice as often (10.7 vs 4.6 %) and read as goal or topic (auf + abstract: Acc 243 vs Dat 77), not time. As a case rule on HDT, R3a equals prep-majority: KILL. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...eepnsm-v2-prepositional-object-detector.md | 42 ++ .claude/board/entries/README.md | 3 +- crates/deepnsm-v2/examples/ud_case_eval.rs | 101 +++-- crates/deepnsm-v2/examples/ud_pp_arg_eval.rs | 406 ++++++++++++++++++ 5 files changed, 521 insertions(+), 32 deletions(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-prepositional-object-detector.md create mode 100644 crates/deepnsm-v2/examples/ud_pp_arg_eval.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 5ed04810f..e0e926605 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -47,6 +47,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-21** | deleted grammar heuristics restored (deepnsm-v2 toc/hydrate/promote/lexicon/loci/tekamolo + toc_hydrate; deepnsm ontology_vocab + persona_chain_replay); WordNet identity register + ConceptMask | In PR | toc_hydrate on KJV: 32,357 HHTL nodes, 70,393 triples addressed, 1,227 basins at TOC keys; ContextChain margin 335× too coarse for the rank metric. Ledger `.claude/knowledge/deleted-grammar-heuristics-ledger.md`; entry `2026-10-04-deepnsm-deleted-grammar-restored.md` | | **D-LXC-22** | DECISION: WordNet hypernym metric may feed ndarray CLAM/CHAODA offline as an evaluation instrument (identities unchanged, `48405aa2` stands); grammar is not a thinking style → `tekamolo::read_clause` takes `ReadParams`, style adapter kept for the planner / ogar-loco | In PR | presets pinned to the adapter rows (disable-verified); 195 lib tests. Entry `2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md` | | **D-LXC-23** | first probes from the German grammar map: der/die/das DET vs PRON voters (`ud_pos_eval`), `ud_case_eval` (fixed-case prepositions, article decidability, Wechselpräposition, relative pronoun), `tekamolo_corners` | In PR | DET/PRON German 92.3 → 96.0 % (pronoun P 90.0 / R 54.1) PASS; case on HDT: Acc 90.0 / Dat 92.5 % PASS, Gen KILL, Wechsel article-decided 99.7 %, verb prior KILL (63.0 vs 63.9), relative pronoun 80.1 vs 63.8 % PASS; right-corner-on-Bible prediction KILL (Luther 7.1 % vs GSD 12.3 %). Entry `2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md` | +| **D-LXC-24** | prepositional object vs adverbial (`ud_pp_arg_eval`): the (verb, preposition) object rate takes verb-governed phrases out of the TEKAMOLO pool and fixes their case; abstract-noun suffix class | In PR | HDT (gold = `obj` + ADP; German `obl:arg` is bare datives): A1 P 77.2 / R 81.6 %, 896 of 1,098 objects removed, 264 of 19,466 adverbials lost, PASS; A2 case 96.8 vs 81.0 % PASS; abstract heads 10.7 % objects vs 4.6 %; R3a KILL on HDT. Entry `2026-10-04-deepnsm-v2-prepositional-object-detector.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-prepositional-object-detector.md b/.claude/board/entries/2026-10-04-deepnsm-v2-prepositional-object-detector.md new file mode 100644 index 000000000..96e514160 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-prepositional-object-detector.md @@ -0,0 +1,42 @@ +# 2026-10-04 — prepositional object vs adverbial: which phrases leave the TEKAMOLO puzzle (D-LXC-24) + +**Status:** MEASURED (debug-0, release) on UD r2.15 German HDT, train-a-1 → test. Kill bars were fixed before the run. Instrument: `crates/deepnsm-v2/examples/ud_pp_arg_eval.rs`. + +Operator: *sich freuen **auf*** vs *auf* as place. A verb-governed phrase is an argument, not an adverbial, so it fills no TEKAMOLO lane. Each one detected narrows the TEKAMOLO "Sudoku". The verb also fixes its case. + +## Gold, and a premise that was wrong at first + +- **German `obl:arg` is NOT the prepositional object.** It marks bare dative objects (*der Telekom*, *ihr*) and carries a preposition once in GSD train. +- The first run used it and scored 0 positives on HDT. +- **HDT encodes the prepositional object as `obj` with a `case` ADP child.** The top train pairs are *rechnen mit* (163), *gehören zu* (155), *führen zu* (125), *sorgen für* (116), *setzen auf* (105), *warten auf* (36). +- **German GSD has no prepositional-object label** (its PP objects are plain `obl`), so this probe runs on HDT only. + +## Results (HDT test, 20,564 verbal PPs, 1,098 prepositional objects = 5.3 %) + +**A1, detector.** Mines (verb lemma, preposition) → object rate from train. At test, every known verb form in the clause (the span between punctuation) votes; the best pair with ≥ 3 train occurrences and rate ≥ 0.5 decides. + +| | precision | recall | F1 | +|---|---|---|---| +| clause verb | **77.2 %** | **81.6 %** | 0.794 | +| preposition only | 0 % | 0 % | 0 | + +- **PASS.** The TEKAMOLO pool loses 896 of 1,098 objects and only 264 of 19,466 adverbials. +- By preposition: *über* F1 0.92, *mit* 0.88, *auf* 0.83 (recall 90.3 %). The weakest is *in* (0.58). +- Disable run: dropping the rate check takes precision to 7.7 %, so the guard is load-bearing. + +**A2, case of a detected object with a Wechsel preposition.** The (verb, preposition) table scores **96.8 %** against 81.0 % for the preposition's majority case, on 247 phrases: **PASS**. The phrase's case is the first `Case=` from the preposition to the head noun, because HDT often marks the article only; scoring on the noun fired 5 times. + +## Abstract vs concrete heads (operator: *vor **der Fahrt***) + +- The test is morphological: *-ung, -heit, -keit, -schaft, -tion, -nis, -ität, -ismus, -tum*. +- An abstract head is a prepositional object **10.7 %** of the time, against 4.6 % for other heads. +- In the `ud_case_eval` table (HDT train), abstract heads read as **goal or topic, not time**: + - *auf* + abstract: Acc 243 vs Dat 77 (other heads 1,376 vs 1,550); + - *über* + abstract: Acc 227 vs Dat 13; + - *an* + abstract: Dat 73 %, against 96 % for *an* + time. +- **R3a** (case from the (preposition, abstract) table) scores 84.1 % vs 79.5 % on GSD (44 tokens, PASS) and **75.7 % vs 75.7 % on HDT, KILL**. The *auf* flip leaves the case unchanged, because *auf*'s overall Wechsel majority is already Acc. +- The verb prior on abstract heads is 68.9 %, against 61.5 % on other heads. + +**OPEN.** +- A1 is not yet wired into a TEKAMOLO reader. The gain to lane accuracy is unmeasured: there is no German lane gold (see the D-LXC-23 addendum). +- Pronominal adverbs (*darauf*, *damit*) are the other half of the prepositional object and are not covered. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 72e0386a8..ff7d31795 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -199 entries, 2026-08-06 .. 2026-10-04. +200 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| @@ -33,6 +33,7 @@ exactly one of them, never both. | 2026-10-04 | `D-LXC-20` | | [2026-10-04-deepnsm-v2-satzklammer-frageprobe.md](2026-10-04-deepnsm-v2-satzklammer-frageprobe.md) | | 2026-10-04 | `D-LXC-19` | | [2026-10-04-deepnsm-v2-rule-quorum.md](2026-10-04-deepnsm-v2-rule-quorum.md) | | 2026-10-04 | `D-LXC-18` | | [2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md](2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md) | +| 2026-10-04 | `D-LXC-24` | | [2026-10-04-deepnsm-v2-prepositional-object-detector.md](2026-10-04-deepnsm-v2-prepositional-object-detector.md) | | 2026-10-04 | `D-LXC-13` | | [2026-10-04-deepnsm-v2-predicate-by-position.md](2026-10-04-deepnsm-v2-predicate-by-position.md) | | 2026-10-04 | `D-LXC-14` | | [2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md](2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md) | | 2026-10-04 | `D-LXC-16` | | [2026-10-04-deepnsm-v2-learned-position-table.md](2026-10-04-deepnsm-v2-learned-position-table.md) | diff --git a/crates/deepnsm-v2/examples/ud_case_eval.rs b/crates/deepnsm-v2/examples/ud_case_eval.rs index 645a6d0c9..e4a8a0923 100644 --- a/crates/deepnsm-v2/examples/ud_case_eval.rs +++ b/crates/deepnsm-v2/examples/ud_case_eval.rs @@ -24,6 +24,9 @@ //! instead of the verb. Time nouns are mined from train as the head nouns //! after *seit* / *während*, the prepositions with only a time reading; no //! hand list. Scored on the Wechsel tokens the article leaves undecided. +//! - **R3a** abstract head (*-ung, -heit, -keit, -schaft, -tion, -nis, -ität, +//! -ismus, -tum*): the same table keyed by the abstract class. Abstract +//! nouns read as goal or topic (*auf Erweiterung setzen*), not as time. //! - **R4** relative pronoun after a comma, case taken from train tokens under //! the SAME surface condition (a relativizer form right after a comma), plus //! any token marked `PronType=Rel`. Mining on `PronType=Rel` alone fires @@ -44,7 +47,7 @@ //! - R3: verb-prior precision <= prep-majority on the same tokens, or verb-prior //! fires on < 5 % of the Wechsel-governed scored tokens (the first following //! token of a non-contraction Wechsel preposition). -//! - R3t: precision <= prep-majority on the same tokens, or fires on < 5 % of +//! - R3t, R3a (each): precision <= prep-majority on the same tokens, or fires on < 5 % of //! the article-undecided Wechsel tokens. //! - R4: precision <= the plain form baseline. //! @@ -91,6 +94,17 @@ const REL: [&str; 13] = [ "welchem", "welchen", ]; +/// Derivational suffixes of German abstract nouns (inflected forms included). +const ABSTRACT_SUFFIXES: [&str; 18] = [ + "ung", "ungen", "heit", "heiten", "keit", "keiten", "schaft", "schaften", "tion", "tionen", + "nis", "nisse", "nissen", "ität", "itäten", "ismus", "ismen", "tum", +]; + +/// The lowercased noun form ends in an abstract-noun suffix. +fn abstract_noun(form: &str) -> bool { + form.chars().count() > 5 && ABSTRACT_SUFFIXES.iter().any(|x| form.ends_with(x)) +} + /// Prepositions with only a time reading; their head nouns seed the time-noun set. const TEMPORAL_ONLY: [&str; 2] = ["seit", "während"]; @@ -192,8 +206,9 @@ struct Tables { wechsel_prep: HashMap, /// Lowercased head-noun forms seen after a time-only preposition. time_nouns: std::collections::HashSet, - /// (preposition, head is a time noun) → case counts of the governed token. - wechsel_time: HashMap<(String, bool), [usize; 4]>, + /// (preposition, head class: 0 other, 1 time, 2 abstract) → case counts + /// of the governed token. + wechsel_time: HashMap<(String, usize), [usize; 4]>, } /// The head noun of the window after `i`: its last token when capitalised. @@ -302,8 +317,10 @@ impl Tables { continue; }; self.wechsel_prep.entry(t.form.clone()).or_default()[c] += 1; - let time = self.is_time(s, i); - self.wechsel_time.entry((t.form.clone(), time)).or_default()[c] += 1; + let class = self.head_class(s, i); + self.wechsel_time + .entry((t.form.clone(), class)) + .or_default()[c] += 1; if let Some(lemma) = self.nearest_verb(s, i) { let key = (t.form.clone(), lemma.clone()); self.wechsel_key.entry(key).or_default()[c] += 1; @@ -313,17 +330,31 @@ impl Tables { } /// The Wechsel preposition at `i` heads a time noun. + /// Head class of the Wechsel phrase at `i`: 1 time noun, 2 abstract noun + /// (by suffix, *vor der Fahrt*-type), 0 otherwise. Time wins. + fn head_class(&self, s: &[Tok], i: usize) -> usize { + if self.is_time(s, i) { + 1 + } else if head(s, i).is_some_and(|j| abstract_noun(&s[j].form)) { + 2 + } else { + 0 + } + } + fn is_time(&self, s: &[Tok], i: usize) -> bool { head(s, i).is_some_and(|j| self.time_nouns.contains(&s[j].form)) } - /// R3t: the (preposition, time reading) majority, only for a time head. + /// R3t / R3a: the (preposition, head class) majority, only for a time + /// (class 1) or abstract (class 2) head. fn wechsel_temporal(&self, s: &[Tok], i: usize) -> Option { - if !self.is_time(s, i) { + let class = self.head_class(s, i); + if class == 0 { return None; } self.wechsel_time - .get(&(s[i].form.clone(), true)) + .get(&(s[i].form.clone(), class)) .and_then(majority) } @@ -401,7 +432,7 @@ impl Tables { if preds[j].r3.is_none() { preds[j].r3 = self.wechsel(s, i, j); preds[j].r3t = self.wechsel_temporal(s, i); - preds[j].time = self.is_time(s, i); + preds[j].class = self.head_class(s, i); } } } @@ -423,8 +454,8 @@ struct Pred { r2: Option<(usize, usize)>, /// R3t prediction (time head only). r3t: Option, - /// The Wechsel preposition heads a time noun. - time: bool, + /// Head class of the Wechsel phrase (0 other, 1 time, 2 abstract). + class: usize, /// First following token of a non-contraction Wechsel preposition. wech: bool, } @@ -496,10 +527,11 @@ fn main() { let mut verb_prep_major = 0usize; // Article-undecided Wechsel tokens (R3 kind 1 or 2). let mut undecided = 0usize; - let mut r3t = Tally::default(); - let mut r3t_prep_major = 0usize; + // [time, abstract] head. + let mut r3t = [Tally::default(); 2]; + let mut r3t_prep_major = [0usize; 2]; // Verb-prior split by reading: [place, time]. - let mut verb_by_reading = [Tally::default(); 2]; + let mut verb_by_reading = [Tally::default(); 3]; // R3t on the time-head tokens where the verb prior fired. let mut r3t_on_verb = Tally::default(); let mut r4 = Tally::default(); @@ -529,17 +561,17 @@ fn main() { verb_prep_major += 1; } if kind == 1 { - verb_by_reading[usize::from(p.time)].add(c == gold, base); - if let Some(t) = p.r3t { + verb_by_reading[p.class].add(c == gold, base); + if let (1, Some(t)) = (p.class, p.r3t) { r3t_on_verb.add(t == gold, base); } } if kind != 0 { undecided += 1; if let Some(t) = p.r3t { - r3t.add(t == gold, base); + r3t[p.class - 1].add(t == gold, base); if prep_major == Some(gold) { - r3t_prep_major += 1; + r3t_prep_major[p.class - 1] += 1; } } } @@ -618,29 +650,36 @@ fn main() { for ((prep, time), c) in keys { println!( " {prep:9} {:5} Acc {:6} Dat {:6}", - if *time { "time" } else { "place" }, + ["other", "time", "abstract"][*time], c[1], c[2] ); } - println!(" verb-prior, place head: {}", verb_by_reading[0].line()); - println!(" verb-prior, time head : {}", verb_by_reading[1].line()); + println!(" verb-prior, other head : {}", verb_by_reading[0].line()); + println!(" verb-prior, time head : {}", verb_by_reading[1].line()); + println!(" verb-prior, abstract head: {}", verb_by_reading[2].line()); println!(" R3t on those time heads: {}", r3t_on_verb.line()); let mut sample: Vec<&String> = tables.time_nouns.iter().collect(); sample.sort(); let step = (sample.len() / 15).max(1); let shown: Vec<&str> = sample.iter().step_by(step).map(|w| w.as_str()).collect(); println!(" time-noun sample: {}", shown.join(" ")); - let r3t_major = pct(r3t_prep_major, r3t.fires); - println!( - " R3t: {} prep-majority on the same tokens {r3t_major:5.1}% share {:5.1}%", - r3t.line(), - pct(r3t.fires, undecided) - ); - println!( - " R3t {}", - verdict(r3t.fires > 0 && r3t.precision() > r3t_major && r3t.fires * 20 >= undecided) - ); + for (k, name) in ["R3t (time head)", "R3a (abstract head)"] + .iter() + .enumerate() + { + let t = r3t[k]; + let major = pct(r3t_prep_major[k], t.fires); + println!( + " {name}: {} prep-majority on the same tokens {major:5.1}% share {:5.1}%", + t.line(), + pct(t.fires, undecided) + ); + println!( + " {name} {}", + verdict(t.fires > 0 && t.precision() > major && t.fires * 20 >= undecided) + ); + } println!("\nR4 relative pronoun after a comma (KILL: precision <= form baseline)"); println!(" {}", r4.line()); diff --git a/crates/deepnsm-v2/examples/ud_pp_arg_eval.rs b/crates/deepnsm-v2/examples/ud_pp_arg_eval.rs new file mode 100644 index 000000000..5aac0b39f --- /dev/null +++ b/crates/deepnsm-v2/examples/ud_pp_arg_eval.rs @@ -0,0 +1,406 @@ +//! `ud_pp_arg_eval` — is a prepositional phrase governed by the verb +//! (*sich freuen **auf***, *bestehen **auf***, *denken **an***) or a free +//! adverbial (*auf dem Tisch*, *an dem Tag*)? +//! +//! A verb-governed phrase (a prepositional object) is an argument, so it fills no +//! TEKAMOLO lane. Every one detected takes a phrase out of the lane puzzle +//! (the TEKAMOLO "Sudoku"). Its case is also fixed by the verb, not by place +//! vs direction. +//! +//! Tables are mined from **train only**: +//! - preposition forms (majority UPOS ADP); +//! - verb form → lemma (UPOS VERB); +//! - (verb lemma, preposition) → (object count, adverbial count), read from +//! the gold tree: the ADP's head noun is `obj` (object) or `obl` +//! (adverbial), and that noun's head is the verb; +//! - (verb lemma, preposition) → case counts of the governed noun, over the +//! object phrases. +//! +//! **Gold.** German HDT encodes the prepositional object as `obj` with a +//! `case` ADP child (*rechnen mit*, *setzen auf*, *warten auf*). German +//! `obl:arg` is NOT this: it marks bare dative objects (*der Telekom*, +//! *ihr*), and carries a preposition only once in GSD train. German GSD has +//! no prepositional-object label at all (its PP objects are plain `obl`), so +//! this probe scores HDT only; on GSD the positive class is empty. +//! +//! Test-time prediction reads only surface forms. The clause is the span +//! between punctuation tokens. For a preposition, every known verb form in +//! that clause is a candidate, and the best candidate's argument rate +//! (>= `MIN_PAIR` train occurrences) decides. Gold (deprel, head, Case) is +//! read on test only to score. +//! +//! Units: +//! - **A1** every test token whose form is a mined preposition and whose gold +//! head noun is `obl` or `obj` (the verbal PPs, the TEKAMOLO pool). +//! Gold positive = `obj`. +//! - **A2** A1 positives whose preposition is a Wechsel preposition. Scored +//! on the phrase's case: the first `Case=` from the preposition to its gold +//! head noun (HDT often marks it on the article only). +//! +//! KILL bars (fixed before the run): +//! - A1: precision < 0.70, or recall < 0.30, or F1 <= the preposition-only +//! baseline (argument iff the preposition's own train rate >= 0.5). +//! - A2: precision <= the preposition's majority case on the same tokens. +//! +//! ```text +//! cargo run --release --example ud_pp_arg_eval -- de_hdt-ud-train-a-1.conllu de_hdt-ud-test.conllu +//! ``` + +use std::collections::HashMap; + +const CASES: [&str; 4] = ["Nom", "Acc", "Dat", "Gen"]; +const WECHSEL: [&str; 9] = [ + "an", "auf", "hinter", "in", "neben", "über", "unter", "vor", "zwischen", +]; +/// Derivational suffixes of German abstract nouns (inflected forms included). +const ABSTRACT_SUFFIXES: [&str; 18] = [ + "ung", "ungen", "heit", "heiten", "keit", "keiten", "schaft", "schaften", "tion", "tionen", + "nis", "nisse", "nissen", "ität", "itäten", "ismus", "ismen", "tum", +]; + +/// The lowercased noun form ends in an abstract-noun suffix. +fn abstract_noun(form: &str) -> bool { + form.chars().count() > 5 && ABSTRACT_SUFFIXES.iter().any(|x| form.ends_with(x)) +} + +/// Fewest train occurrences of a (verb, preposition) pair for it to vote. +const MIN_PAIR: usize = 3; +/// Argument rate at or above which a phrase is predicted verb-governed. +const ARG_RATE: f64 = 0.5; + +struct Tok { + form: String, + word: bool, + lemma: String, + upos: String, + case: Option, + /// Index of the head word, `None` for the root. + head: Option, + deprel: String, +} + +fn case_of(feats: &str) -> Option { + feats + .split('|') + .find_map(|f| f.strip_prefix("Case=")) + .and_then(|v| CASES.iter().position(|c| *c == v)) +} + +fn read(path: &str) -> Vec> { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut out = Vec::new(); + let mut cur = Vec::new(); + for line in text.lines() { + if line.is_empty() { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if line.starts_with('#') || c.len() < 8 || c[0].contains(['-', '.']) { + continue; + } + let head: usize = c[6].parse().unwrap_or(0); + cur.push(Tok { + form: c[1].to_lowercase(), + word: c[1].chars().any(char::is_alphanumeric), + lemma: c[2].to_string(), + upos: c[3].to_string(), + case: case_of(c[5]), + head: head.checked_sub(1), + deprel: c[7].to_string(), + }); + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +/// The most frequent case, ties to the lower index; `None` when empty. +fn majority(c: &[usize; 4]) -> Option { + if c.iter().sum::() == 0 { + return None; + } + (0..4).max_by_key(|&k| (c[k], std::cmp::Reverse(k))) +} + +/// Gold reading of the preposition at `i`: (head-noun index, is a +/// prepositional object, governing verb index), when the head noun is a +/// verbal `obl` (adverbial) or `obj` (object) dependent. +fn gold_pp(s: &[Tok], i: usize) -> Option<(usize, bool, usize)> { + if s[i].deprel != "case" { + return None; + } + let n = s[i].head?; + let rel = s[n].deprel.as_str(); + if rel != "obl" && rel != "obj" { + return None; + } + Some((n, rel == "obj", s[n].head?)) +} + +/// The phrase's case: the first `Case=` between the preposition at `i` and +/// its head noun `n`, inclusive. HDT often marks it on the article only. +fn phrase_case(s: &[Tok], i: usize, n: usize) -> Option { + if n <= i { + return s[n].case; + } + s[i + 1..=n].iter().find_map(|t| t.case) +} + +struct Tables { + preps: std::collections::HashSet, + verb_lemma: HashMap, + /// (verb lemma, preposition) → [object, adverbial]. + pair: HashMap<(String, String), [usize; 2]>, + /// Preposition → [object, adverbial]. + prep_rate: HashMap, + /// (verb lemma, preposition) → case counts over object phrases. + pair_case: HashMap<(String, String), [usize; 4]>, + /// Preposition → case counts over every verbal PP head noun. + prep_case: HashMap, +} + +impl Tables { + fn mine(train: &[Vec]) -> Self { + let mut adp: HashMap = HashMap::new(); + let mut verbs: HashMap> = HashMap::new(); + let mut pair: HashMap<(String, String), [usize; 2]> = HashMap::new(); + let mut prep_rate: HashMap = HashMap::new(); + let mut pair_case: HashMap<(String, String), [usize; 4]> = HashMap::new(); + let mut prep_case: HashMap = HashMap::new(); + for s in train { + for (i, t) in s.iter().enumerate() { + adp.entry(t.form.clone()).or_default()[usize::from(t.upos != "ADP")] += 1; + if t.upos == "VERB" { + *verbs + .entry(t.form.clone()) + .or_default() + .entry(t.lemma.clone()) + .or_default() += 1; + } + let Some((n, arg, v)) = gold_pp(s, i) else { + continue; + }; + let slot = usize::from(!arg); + prep_rate.entry(t.form.clone()).or_default()[slot] += 1; + if let Some(c) = phrase_case(s, i, n) { + prep_case.entry(t.form.clone()).or_default()[c] += 1; + } + if s[v].upos != "VERB" { + continue; + } + let key = (s[v].lemma.clone(), t.form.clone()); + pair.entry(key.clone()).or_default()[slot] += 1; + if let (true, Some(c)) = (arg, phrase_case(s, i, n)) { + pair_case.entry(key).or_default()[c] += 1; + } + } + } + let preps = adp + .into_iter() + .filter(|(_, c)| c[0] > c[1]) + .map(|(f, _)| f) + .collect(); + let verb_lemma = verbs + .into_iter() + .filter_map(|(f, m)| { + m.into_iter() + .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(&a.0))) + .map(|(l, _)| (f, l)) + }) + .collect(); + Self { + preps, + verb_lemma, + pair, + prep_rate, + pair_case, + prep_case, + } + } + + /// The best clause verb for the preposition at `i`: (lemma, argument + /// rate), over pairs seen at least `MIN_PAIR` times in train. + fn best_verb(&self, s: &[Tok], i: usize) -> Option<(&String, f64)> { + let lo = (0..i).rev().find(|&k| !s[k].word).map_or(0, |k| k + 1); + let hi = (i + 1..s.len()).find(|&k| !s[k].word).unwrap_or(s.len()); + (lo..hi) + .filter(|&k| k != i) + .filter_map(|k| self.verb_lemma.get(&s[k].form)) + .filter_map(|l| { + let c = self.pair.get(&(l.clone(), s[i].form.clone()))?; + let n = c[0] + c[1]; + (n >= MIN_PAIR).then(|| (l, c[0] as f64 / n as f64)) + }) + .max_by(|a, b| a.1.total_cmp(&b.1)) + } + + fn prep_says_arg(&self, prep: &str) -> bool { + self.prep_rate + .get(prep) + .is_some_and(|c| c[0] as f64 / (c[0] + c[1]).max(1) as f64 >= ARG_RATE) + } +} + +#[derive(Default)] +struct Pr { + tp: usize, + fp: usize, + fn_: usize, +} + +impl Pr { + fn add(&mut self, pred: bool, gold: bool) { + match (pred, gold) { + (true, true) => self.tp += 1, + (true, false) => self.fp += 1, + (false, true) => self.fn_ += 1, + _ => {} + } + } + fn p(&self) -> f64 { + self.tp as f64 / (self.tp + self.fp).max(1) as f64 + } + fn r(&self) -> f64 { + self.tp as f64 / (self.tp + self.fn_).max(1) as f64 + } + fn f1(&self) -> f64 { + let (p, r) = (self.p(), self.r()); + if p + r == 0.0 { + 0.0 + } else { + 2.0 * p * r / (p + r) + } + } + fn line(&self) -> String { + format!( + "precision {:5.1}% recall {:5.1}% F1 {:5.3} (tp {} fp {} fn {})", + 100.0 * self.p(), + 100.0 * self.r(), + self.f1(), + self.tp, + self.fp, + self.fn_ + ) + } +} + +fn verdict(pass: bool) -> &'static str { + if pass { + "PASS" + } else { + "KILL" + } +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let [train_path, test_path] = args.as_slice() else { + eprintln!("usage: ud_pp_arg_eval TRAIN.conllu TEST.conllu"); + std::process::exit(2); + }; + let train = read(train_path); + let test = read(test_path); + let t = Tables::mine(&train); + + let mut pool = 0usize; + let mut gold_args = 0usize; + let mut a1 = Pr::default(); + let mut base = Pr::default(); + let mut a2 = [0usize; 3]; // [fires, pair-case right, prep-majority right] + let mut per_prep: HashMap = HashMap::new(); + // [concrete, abstract] head → [objects, total]. + let mut by_head = [[0usize; 2]; 2]; + + for s in &test { + for i in 0..s.len() { + if !t.preps.contains(&s[i].form) { + continue; + } + let Some((n, gold, _)) = gold_pp(s, i) else { + continue; + }; + pool += 1; + gold_args += usize::from(gold); + let h = &mut by_head[usize::from(abstract_noun(&s[n].form))]; + h[0] += usize::from(gold); + h[1] += 1; + let best = t.best_verb(s, i); + let pred = best.is_some_and(|(_, r)| r >= ARG_RATE); + a1.add(pred, gold); + base.add(t.prep_says_arg(&s[i].form), gold); + per_prep + .entry(s[i].form.clone()) + .or_default() + .add(pred, gold); + if !(pred && gold && WECHSEL.contains(&s[i].form.as_str())) { + continue; + } + let Some(g) = phrase_case(s, i, n) else { + continue; + }; + let (lemma, _) = best.expect("pred implies a best verb"); + let by_pair = t + .pair_case + .get(&(lemma.clone(), s[i].form.clone())) + .and_then(majority); + let by_prep = t.prep_case.get(&s[i].form).and_then(majority); + if let Some(c) = by_pair { + a2[0] += 1; + a2[1] += usize::from(c == g); + a2[2] += usize::from(by_prep == Some(g)); + } + } + } + + println!( + "mined from train: {} prepositions, {} verb forms, {} (verb, prep) pairs", + t.preps.len(), + t.verb_lemma.len(), + t.pair.len() + ); + println!( + "test pool (verbal PPs, obl + obj): {pool}, of which prepositional objects {gold_args} ({:.1}%)", + 100.0 * gold_args as f64 / pool.max(1) as f64 + ); + + println!("\nA1 verb-governed vs adverbial (KILL: P < 0.70, R < 0.30, or F1 <= prep-only)"); + println!(" clause verb : {}", a1.line()); + println!(" prep only : {}", base.line()); + let a1_pass = a1.p() >= 0.70 && a1.r() >= 0.30 && a1.f1() > base.f1(); + println!(" A1 {}", verdict(a1_pass)); + println!( + " TEKAMOLO pool: {} of {} adverbials kept, {} of {} arguments removed", + pool - gold_args - a1.fp, + pool - gold_args, + a1.tp, + gold_args + ); + let mut rows: Vec<(&String, &Pr)> = per_prep.iter().collect(); + rows.sort_by_key(|(_, p)| std::cmp::Reverse(p.tp + p.fn_)); + println!(" by preposition (most gold arguments first):"); + for (prep, p) in rows.into_iter().take(8) { + println!(" {prep:8} {}", p.line()); + } + + for (k, name) in ["other head", "abstract head"].iter().enumerate() { + let [o, n] = by_head[k]; + println!( + " {name:13}: {n} phrases, {:.1}% prepositional objects", + 100.0 * o as f64 / n.max(1) as f64 + ); + } + + println!("\nA2 case of a verb-governed Wechsel phrase (KILL: precision <= prep majority)"); + let pct = |x: usize| 100.0 * x as f64 / a2[0].max(1) as f64; + println!( + " fires {} (verb, prep) case {:5.1}% prep majority {:5.1}%", + a2[0], + pct(a2[1]), + pct(a2[2]) + ); + println!(" A2 {}", verdict(a2[0] > 0 && a2[1] > a2[2])); +} From 04baf7c5039f1fb15839391eb2fa5d51a6346b3d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:00:27 +0000 Subject: [PATCH 20/33] deepnsm-v2: TIME / PLACE / FIG lane quorum for Wechsel phrases (D-LXC-25, in progress) ud_lane_quorum (new) classifies a Wechsel phrase as TIME, PLACE or FIG. The voters are noun class (time noun, abstract suffix, PROPN name), digit, temporal adverb, verb governance, TEKAMOLO position (Vorfeld, a later preposition) and article case, combined by multinomial logistic regression. Silver labels: 800 train + 400 test, two labelers per test item (kappa 0.958, model self-consistency rather than independent agreement). Ids only, no HDT text. - Operator hypothesis (an abstract head is never PLACE): 3.8 %, PASS. - Round 1 (pre-registered): quorum 72.7 % < noun voter 80.4 %, KILL. - Round 2 (exploratory, after test errors): name class, bare-obl time nouns, compound-head match. Noun voter 86.5 %, quorum 88.0 %. - Round 3 (fresh labels, bars fixed beforehand): pending. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...26-10-04-deepnsm-v2-wechsel-lane-quorum.md | 47 + .claude/board/entries/README.md | 3 +- .../examples/data/hdt_wechsel_lanes.BRIEF.md | 37 + .../examples/data/hdt_wechsel_lanes.tsv | 1204 +++++++++++++++++ crates/deepnsm-v2/examples/ud_lane_quorum.rs | 610 +++++++++ 6 files changed, 1901 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md create mode 100644 crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.BRIEF.md create mode 100644 crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.tsv create mode 100644 crates/deepnsm-v2/examples/ud_lane_quorum.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index e0e926605..554d0d30e 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -48,6 +48,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-22** | DECISION: WordNet hypernym metric may feed ndarray CLAM/CHAODA offline as an evaluation instrument (identities unchanged, `48405aa2` stands); grammar is not a thinking style → `tekamolo::read_clause` takes `ReadParams`, style adapter kept for the planner / ogar-loco | In PR | presets pinned to the adapter rows (disable-verified); 195 lib tests. Entry `2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md` | | **D-LXC-23** | first probes from the German grammar map: der/die/das DET vs PRON voters (`ud_pos_eval`), `ud_case_eval` (fixed-case prepositions, article decidability, Wechselpräposition, relative pronoun), `tekamolo_corners` | In PR | DET/PRON German 92.3 → 96.0 % (pronoun P 90.0 / R 54.1) PASS; case on HDT: Acc 90.0 / Dat 92.5 % PASS, Gen KILL, Wechsel article-decided 99.7 %, verb prior KILL (63.0 vs 63.9), relative pronoun 80.1 vs 63.8 % PASS; right-corner-on-Bible prediction KILL (Luther 7.1 % vs GSD 12.3 %). Entry `2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md` | | **D-LXC-24** | prepositional object vs adverbial (`ud_pp_arg_eval`): the (verb, preposition) object rate takes verb-governed phrases out of the TEKAMOLO pool and fixes their case; abstract-noun suffix class | In PR | HDT (gold = `obj` + ADP; German `obl:arg` is bare datives): A1 P 77.2 / R 81.6 %, 896 of 1,098 objects removed, 264 of 19,466 adverbials lost, PASS; A2 case 96.8 vs 81.0 % PASS; abstract heads 10.7 % objects vs 4.6 %; R3a KILL on HDT. Entry `2026-10-04-deepnsm-v2-prepositional-object-detector.md` | +| **D-LXC-25** | TIME / PLACE / FIG lane of a Wechsel phrase (`ud_lane_quorum`): voters for noun class, number, temporal adverb, verb governance, TEKAMOLO position and article case, combined by logistic regression; silver labels (κ 0.958) | In progress | operator hypothesis (abstract → never PLACE) PASS 3.8 %; round 1 KILL (72.7 % < noun 80.4 %); round 2 exploratory: noun 86.5 %, quorum 88.0 %; round 3 (fresh labels) pre-registered, pending. Entry `2026-10-04-deepnsm-v2-wechsel-lane-quorum.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md new file mode 100644 index 000000000..b83582d42 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md @@ -0,0 +1,47 @@ +# 2026-10-04 — TIME / PLACE / FIG for Wechsel phrases: a quorum over noun, adverb, verb and position evidence (D-LXC-25) + +**Status:** MEASURED (debug-0, release), German HDT. The first round is pre-registered (KILL). The second is exploratory, and a fresh confirmation round has been pre-registered and is pending. Instrument: `crates/deepnsm-v2/examples/ud_lane_quorum.rs`. + +Operator: +- *"Time/place Pronomen with abstract can only be time or figuratively"*; +- *"add tekamolo position and adverb and noun context and you have a quorum"*. + +## Labels (silver) + +- German UD has no lane gold. Independent Sonnet labelers read the full sentence and labelled each phrase TIME / PLACE / FIG, using `examples/data/hdt_wechsel_lanes.BRIEF.md`. +- Train has 800 items, one labeler each. Test has 400 items, two labelers each. +- **Cohen κ = 0.958** (98.0 % agreement): RELIABLE. +- The file `examples/data/hdt_wechsel_lanes.tsv` holds ids and labels only, no HDT text (HDT is CC BY-SA 4.0). +- **κ caveat.** The two labelers on a test item are two runs of the same model, not independent annotators. + - On the confirmation sample, fresh_1 A and B agreed 200/200 without reading each other's files; their transcripts were checked. + - So κ measures model self-consistency. It is an upper bound on label quality, not human-grade agreement. +- **One labeler discarded.** fresh_0 B overwrote its reading-based labels with a rule-based script, against the brief. A replacement labeler (C) was dispatched. + +## Results + +**Operator hypothesis: PASS.** Of 105 abstract-head items, 4 are PLACE (3.8 % ≤ 5 %): 95 FIG, 6 TIME. + +**Round 1 (pre-registered): KILL.** The quorum scores 72.7 %, below the noun voter alone (80.4 %). Every voter has PLACE F1 ≈ 0, because nothing separates a physical place from an institutional figurative one. + +**Round 2 (exploratory; changes made after seeing the test errors).** +- Added a `name` noun class: heads whose train majority UPOS is PROPN. +- Time nouns also mined from bare `obl` nouns: no `case` child and no `nummod` child. + - The first attempt used `obl*` and mined 1,052 nouns. That included the German `obl:arg` bare datives (*Kunden*) and measure phrases (*Prozent*, *Dollar*). + - The corrected version mines 411. +- Compound-head match: *Geschäftsquartal* → *quartal*. + +| | accuracy | TIME F1 | PLACE F1 | FIG F1 | +|---|---|---|---|---| +| noun voter | 86.5 % | 0.830 | 0.603 | 0.905 | +| quorum (all voters) | 88.0 % | 0.831 | 0.655 | 0.918 | + +- Ablation: dropping the noun voter costs 19 points. Dropping any other voter costs ≤ 1 point. +- The quorum misses its "+2 points over the best single voter" bar: noun context carries almost all the signal. +- TEKAMOLO position, adverb and verb add little on this sample. + +**Round 3 (pre-registered before its labels existed): pending.** +- Data: a fresh sample of 400 HDT test phrases, two labelers each. +- Bars: + - κ ≥ 0.6; + - noun voter TIME F1 ≥ 0.80 and PLACE F1 ≥ 0.55; + - quorum ≥ noun + 2 points. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index ff7d31795..2856e30e8 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -200 entries, 2026-08-06 .. 2026-10-04. +201 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| | 2026-10-04 | `D-LXC-22` | | [2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md](2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md) | +| 2026-10-04 | `D-LXC-25` | | [2026-10-04-deepnsm-v2-wechsel-lane-quorum.md](2026-10-04-deepnsm-v2-wechsel-lane-quorum.md) | | 2026-10-04 | `D-LXC-20` | | [2026-10-04-deepnsm-v2-satzklammer-frageprobe.md](2026-10-04-deepnsm-v2-satzklammer-frageprobe.md) | | 2026-10-04 | `D-LXC-19` | | [2026-10-04-deepnsm-v2-rule-quorum.md](2026-10-04-deepnsm-v2-rule-quorum.md) | | 2026-10-04 | `D-LXC-18` | | [2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md](2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md) | diff --git a/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.BRIEF.md b/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.BRIEF.md new file mode 100644 index 000000000..33542bc34 --- /dev/null +++ b/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.BRIEF.md @@ -0,0 +1,37 @@ +# Labeling brief: German Wechsel-preposition phrases → TIME / PLACE / FIG + +You are labeling German sentences. In each one, a prepositional phrase is +marked with [[ ... ]]. Its preposition is an, auf, hinter, in, neben, über, +unter, vor or zwischen. Give the marked phrase exactly one label. + +- **TIME**: the phrase locates or measures something in time. Examples: + - in 12 Monaten + - vor der Fahrt + - an dem Tag + - über das Wochenende + - in den letzten Jahren + - vor zwei Wochen + - im Jahr 2000 + - in Zukunft + - in der Nacht +- **PLACE**: a literal physical location or direction in the physical world. + - The test: it answers wo? or wohin? with a real place you could stand in, point at or travel to. + - Geographic names count: in Deutschland, nach … ins Ausland. + - So do buildings, rooms, objects, physical surfaces and vehicles: auf dem Tisch, in die Stadt, vor dem Haus, im Werk. +- **FIG**: everything else. + - The verb governs the preposition: setzen auf, warten auf, rechnen mit, reagieren auf, denken an. + - Metaphorical or abstract "places": in der Lage, unter Druck, auf dem Markt, im Internet, in der Branche, in Höhe von, auf Basis von, in diesem Fall, im Rahmen, unter den Anbietern, an der Börse (as an institution). + - Manner, cause or topic: über das Thema, in bar, in Euro. + +Decide from the whole sentence. When a phrase is clearly time or clearly physical place, use that label. When you are unsure between PLACE and FIG, ask whether a physical location is meant. + +## Rules + +- Read ONLY your input file. Do not open any other file in the repository. +- Do not run cargo, git, or any code. You read the sentences yourself. +- Label EVERY line. Do not skip any. +- Write your output file as TSV, one line per item: `idLABEL`, where LABEL is exactly TIME, PLACE or FIG. +- Write incrementally: + - after each batch of 25 items, append the lines to your output file with `cat >> FILE <<'EOF'` (use a heredoc, not a script); + - when you have finished, verify with `wc -l` that the output has the same number of lines as the input. +- Reply with the line count, your label distribution, and 3 items you found hardest, with their ids. Nothing else. diff --git a/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.tsv b/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.tsv new file mode 100644 index 000000000..ea3ce7416 --- /dev/null +++ b/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.tsv @@ -0,0 +1,1204 @@ +# Silver lane labels for German HDT (UD r2.15) Wechsel-preposition phrases: TIME / PLACE / FIG. +# Written by independent model labelers from the full sentence (brief: deepnsm-v2 D-LXC-25 entry). +# Train: one labeler (label_b = -). Test: two independent labelers. Ids only; no HDT text (CC BY-SA 4.0). +# columns: split, sent_id, token (1-based word index), label_a, label_b +train hdt-s31655 5 FIG - +train hdt-s23868 8 TIME - +train hdt-s39527 9 TIME - +train hdt-s45601 21 PLACE - +train hdt-s37741 2 FIG - +train hdt-s54381 20 FIG - +train hdt-s32512 1 PLACE - +train hdt-s28734 11 PLACE - +train hdt-s51278 6 FIG - +train hdt-s29870 18 FIG - +train hdt-s45653 13 PLACE - +train hdt-s58904 10 FIG - +train hdt-s43392 5 TIME - +train hdt-s33580 12 FIG - +train hdt-s33954 24 PLACE - +train hdt-s26619 9 TIME - +train hdt-s36123 19 FIG - +train hdt-s31901 37 FIG - +train hdt-s37203 8 TIME - +train hdt-s45960 6 PLACE - +train hdt-s36836 8 PLACE - +train hdt-s32459 10 FIG - +train hdt-s30490 13 FIG - +train hdt-s36051 9 FIG - +train hdt-s25304 3 FIG - +train hdt-s30061 1 FIG - +train hdt-s47588 12 FIG - +train hdt-s48135 4 FIG - +train hdt-s34321 21 FIG - +train hdt-s44038 9 TIME - +train hdt-s24364 25 FIG - +train hdt-s34897 10 FIG - +train hdt-s38024 10 FIG - +train hdt-s45193 6 FIG - +train hdt-s44015 1 FIG - +train hdt-s36433 7 FIG - +train hdt-s38133 20 FIG - +train hdt-s58832 2 TIME - +train hdt-s41212 11 FIG - +train hdt-s21569 1 FIG - +train hdt-s21251 6 FIG - +train hdt-s59954 1 FIG - +train hdt-s41651 3 PLACE - +train hdt-s58552 39 FIG - +train hdt-s32350 1 FIG - +train hdt-s28719 10 FIG - +train hdt-s38729 8 FIG - +train hdt-s55585 27 FIG - +train hdt-s59697 4 TIME - +train hdt-s48286 5 FIG - +train hdt-s28145 12 FIG - +train hdt-s34180 14 PLACE - +train hdt-s29034 9 FIG - +train hdt-s43218 7 FIG - +train hdt-s38712 3 FIG - +train hdt-s49210 11 TIME - +train hdt-s56199 7 FIG - +train hdt-s44425 14 PLACE - +train hdt-s22186 5 TIME - +train hdt-s33658 2 TIME - +train hdt-s24043 10 FIG - +train hdt-s59774 17 FIG - +train hdt-s58704 11 PLACE - +train hdt-s58031 22 TIME - +train hdt-s40484 11 TIME - +train hdt-s54691 5 TIME - +train hdt-s39756 1 FIG - +train hdt-s52925 17 FIG - +train hdt-s41686 1 TIME - +train hdt-s57232 15 FIG - +train hdt-s46149 6 FIG - +train hdt-s49127 9 PLACE - +train hdt-s50905 17 FIG - +train hdt-s55421 6 FIG - +train hdt-s54623 4 TIME - +train hdt-s39062 18 TIME - +train hdt-s45708 10 TIME - +train hdt-s33352 10 TIME - +train hdt-s56874 9 FIG - +train hdt-s40335 4 FIG - +train hdt-s47978 9 PLACE - +train hdt-s23664 4 FIG - +train hdt-s53696 2 TIME - +train hdt-s24292 14 TIME - +train hdt-s48264 31 TIME - +train hdt-s53732 4 FIG - +train hdt-s38860 6 TIME - +train hdt-s50375 5 PLACE - +train hdt-s39419 12 FIG - +train hdt-s56204 18 FIG - +train hdt-s20878 8 FIG - +train hdt-s30963 45 FIG - +train hdt-s29913 8 FIG - +train hdt-s57655 1 FIG - +train hdt-s60316 24 FIG - +train hdt-s42111 25 FIG - +train hdt-s59017 8 TIME - +train hdt-s53243 11 TIME - +train hdt-s53854 22 FIG - +train hdt-s39511 7 FIG - +train hdt-s41096 25 FIG - +train hdt-s43331 10 TIME - +train hdt-s47949 1 TIME - +train hdt-s37712 8 FIG - +train hdt-s34023 4 FIG - +train hdt-s34028 11 FIG - +train hdt-s39463 10 FIG - +train hdt-s51728 15 PLACE - +train hdt-s47767 9 FIG - +train hdt-s52157 7 TIME - +train hdt-s60404 39 TIME - +train hdt-s30134 23 FIG - +train hdt-s32747 1 FIG - +train hdt-s27975 4 PLACE - +train hdt-s60791 23 FIG - +train hdt-s47289 40 PLACE - +train hdt-s34113 9 FIG - +train hdt-s27378 7 FIG - +train hdt-s29107 17 FIG - +train hdt-s22128 5 PLACE - +train hdt-s56296 12 FIG - +train hdt-s34385 1 FIG - +train hdt-s37654 9 FIG - +train hdt-s23503 14 FIG - +train hdt-s34703 1 FIG - +train hdt-s24175 1 FIG - +train hdt-s43800 11 FIG - +train hdt-s28811 5 FIG - +train hdt-s23617 4 FIG - +train hdt-s36422 12 FIG - +train hdt-s44882 4 FIG - +train hdt-s51948 9 FIG - +train hdt-s29729 21 PLACE - +train hdt-s29541 5 TIME - +train hdt-s38987 17 FIG - +train hdt-s37712 29 FIG - +train hdt-s59464 20 PLACE - +train hdt-s28112 9 PLACE - +train hdt-s41970 11 TIME - +train hdt-s21818 12 TIME - +train hdt-s41840 10 PLACE - +train hdt-s25896 8 FIG - +train hdt-s60509 30 PLACE - +train hdt-s59779 4 FIG - +train hdt-s53394 3 FIG - +train hdt-s60029 1 TIME - +train hdt-s31798 1 TIME - +train hdt-s28137 4 TIME - +train hdt-s44583 7 FIG - +train hdt-s27238 27 FIG - +train hdt-s27212 14 PLACE - +train hdt-s51774 9 PLACE - +train hdt-s38540 2 TIME - +train hdt-s37381 1 FIG - +train hdt-s52908 14 FIG - +train hdt-s36869 1 TIME - +train hdt-s60191 3 FIG - +train hdt-s39470 4 PLACE - +train hdt-s36086 4 PLACE - +train hdt-s42973 1 FIG - +train hdt-s45083 37 FIG - +train hdt-s54983 3 FIG - +train hdt-s50090 5 FIG - +train hdt-s50378 10 FIG - +train hdt-s45130 5 FIG - +train hdt-s41451 1 PLACE - +train hdt-s46595 1 TIME - +train hdt-s27783 1 PLACE - +train hdt-s52336 21 FIG - +train hdt-s23979 9 FIG - +train hdt-s51098 4 PLACE - +train hdt-s21226 26 FIG - +train hdt-s33644 19 FIG - +train hdt-s60869 1 FIG - +train hdt-s42818 21 FIG - +train hdt-s33196 19 TIME - +train hdt-s30671 1 TIME - +train hdt-s34152 5 FIG - +train hdt-s57681 5 TIME - +train hdt-s28900 5 FIG - +train hdt-s36243 1 FIG - +train hdt-s46307 10 FIG - +train hdt-s52869 8 FIG - +train hdt-s24505 1 FIG - +train hdt-s41722 18 FIG - +train hdt-s21692 1 PLACE - +train hdt-s46087 2 FIG - +train hdt-s45941 18 FIG - +train hdt-s37143 20 TIME - +train hdt-s46105 12 PLACE - +train hdt-s31380 5 FIG - +train hdt-s22555 1 FIG - +train hdt-s30208 4 FIG - +train hdt-s24716 7 FIG - +train hdt-s25928 12 TIME - +train hdt-s42818 41 FIG - +train hdt-s41607 6 FIG - +train hdt-s22934 9 FIG - +train hdt-s56451 10 FIG - +train hdt-s52941 8 FIG - +train hdt-s60579 1 FIG - +train hdt-s40172 27 FIG - +train hdt-s41155 5 FIG - +train hdt-s52208 22 FIG - +train hdt-s26719 15 FIG - +train hdt-s59028 3 TIME - +train hdt-s40682 1 PLACE - +train hdt-s53073 26 FIG - +train hdt-s55590 1 FIG - +train hdt-s34672 11 FIG - +train hdt-s49468 14 FIG - +train hdt-s38925 7 FIG - +train hdt-s51548 24 FIG - +train hdt-s37121 6 PLACE - +train hdt-s33070 22 FIG - +train hdt-s29362 1 TIME - +train hdt-s28231 15 FIG - +train hdt-s25645 6 FIG - +train hdt-s49452 10 FIG - +train hdt-s48148 12 FIG - +train hdt-s35285 15 TIME - +train hdt-s43842 6 FIG - +train hdt-s52161 10 PLACE - +train hdt-s34684 12 FIG - +train hdt-s20938 7 FIG - +train hdt-s53274 17 FIG - +train hdt-s50808 21 FIG - +train hdt-s39253 6 FIG - +train hdt-s21524 6 TIME - +train hdt-s36557 6 PLACE - +train hdt-s47272 8 FIG - +train hdt-s58251 6 FIG - +train hdt-s56602 13 FIG - +train hdt-s38779 9 PLACE - +train hdt-s25721 1 FIG - +train hdt-s42861 8 FIG - +train hdt-s31567 3 TIME - +train hdt-s25550 1 FIG - +train hdt-s59839 1 FIG - +train hdt-s51627 20 FIG - +train hdt-s29770 1 TIME - +train hdt-s28711 5 FIG - +train hdt-s59511 7 TIME - +train hdt-s21992 21 FIG - +train hdt-s28469 13 PLACE - +train hdt-s29827 6 FIG - +train hdt-s20753 5 TIME - +train hdt-s49001 1 FIG - +train hdt-s24813 4 TIME - +train hdt-s59244 11 FIG - +train hdt-s29852 1 FIG - +train hdt-s59278 12 FIG - +train hdt-s53561 1 PLACE - +train hdt-s41109 4 FIG - +train hdt-s46510 13 TIME - +train hdt-s38543 1 FIG - +train hdt-s52992 1 FIG - +train hdt-s40582 1 FIG - +train hdt-s33071 26 FIG - +train hdt-s28656 1 FIG - +train hdt-s42700 13 FIG - +train hdt-s37119 1 PLACE - +train hdt-s30398 9 FIG - +train hdt-s33047 1 FIG - +train hdt-s40331 3 TIME - +train hdt-s36301 30 FIG - +train hdt-s53824 16 FIG - +train hdt-s42215 6 FIG - +train hdt-s58664 1 FIG - +train hdt-s49406 6 TIME - +train hdt-s30728 1 FIG - +train hdt-s46867 12 FIG - +train hdt-s27860 11 FIG - +train hdt-s54435 1 FIG - +train hdt-s41422 4 FIG - +train hdt-s54225 9 FIG - +train hdt-s54135 6 FIG - +train hdt-s37058 3 TIME - +train hdt-s27801 5 FIG - +train hdt-s58779 24 PLACE - +train hdt-s27880 7 FIG - +train hdt-s47321 8 FIG - +train hdt-s34382 8 FIG - +train hdt-s57800 1 TIME - +train hdt-s45685 6 TIME - +train hdt-s51494 10 PLACE - +train hdt-s27238 17 TIME - +train hdt-s21452 23 FIG - +train hdt-s46920 2 FIG - +train hdt-s50087 4 FIG - +train hdt-s58734 1 TIME - +train hdt-s38002 10 FIG - +train hdt-s23486 11 FIG - +train hdt-s38503 6 FIG - +train hdt-s39655 10 PLACE - +train hdt-s55129 1 FIG - +train hdt-s42590 7 PLACE - +train hdt-s21990 4 FIG - +train hdt-s55802 25 TIME - +train hdt-s43843 4 TIME - +train hdt-s40306 1 FIG - +train hdt-s21131 22 FIG - +train hdt-s36234 15 FIG - +train hdt-s57037 1 FIG - +train hdt-s30781 1 FIG - +train hdt-s34722 9 FIG - +train hdt-s46655 4 FIG - +train hdt-s50494 9 FIG - +train hdt-s44091 15 FIG - +train hdt-s48929 4 FIG - +train hdt-s29140 21 FIG - +train hdt-s40936 4 FIG - +train hdt-s26121 6 FIG - +train hdt-s53686 16 FIG - +train hdt-s26233 16 PLACE - +train hdt-s32062 1 FIG - +train hdt-s35661 5 FIG - +train hdt-s20032 7 FIG - +train hdt-s52664 17 TIME - +train hdt-s43372 2 PLACE - +train hdt-s21763 11 FIG - +train hdt-s26547 16 TIME - +train hdt-s31636 1 TIME - +train hdt-s37995 10 TIME - +train hdt-s36801 1 FIG - +train hdt-s29942 10 FIG - +train hdt-s45318 8 FIG - +train hdt-s46663 3 TIME - +train hdt-s59939 8 FIG - +train hdt-s56956 9 TIME - +train hdt-s34232 16 TIME - +train hdt-s57622 2 PLACE - +train hdt-s51661 15 TIME - +train hdt-s34280 1 FIG - +train hdt-s52975 19 FIG - +train hdt-s45005 4 FIG - +train hdt-s40283 30 FIG - +train hdt-s21267 1 FIG - +train hdt-s55942 5 FIG - +train hdt-s57459 7 FIG - +train hdt-s38906 9 FIG - +train hdt-s35304 20 FIG - +train hdt-s60510 13 PLACE - +train hdt-s32051 6 FIG - +train hdt-s35762 19 FIG - +train hdt-s36916 1 TIME - +train hdt-s60276 19 PLACE - +train hdt-s44004 1 TIME - +train hdt-s30686 29 FIG - +train hdt-s21927 6 TIME - +train hdt-s41075 2 FIG - +train hdt-s35418 2 FIG - +train hdt-s33591 6 FIG - +train hdt-s46795 3 FIG - +train hdt-s24078 31 FIG - +train hdt-s28761 5 TIME - +train hdt-s53887 12 PLACE - +train hdt-s36466 16 PLACE - +train hdt-s50651 9 FIG - +train hdt-s45767 17 FIG - +train hdt-s58602 7 FIG - +train hdt-s50436 6 PLACE - +train hdt-s36836 25 FIG - +train hdt-s54467 9 PLACE - +train hdt-s56193 4 FIG - +train hdt-s24951 1 FIG - +train hdt-s20356 8 PLACE - +train hdt-s43538 3 FIG - +train hdt-s41730 1 FIG - +train hdt-s26341 21 TIME - +train hdt-s51005 4 FIG - +train hdt-s25410 1 FIG - +train hdt-s28229 25 FIG - +train hdt-s51337 8 PLACE - +train hdt-s45493 10 FIG - +train hdt-s51984 6 TIME - +train hdt-s57985 4 FIG - +train hdt-s55153 18 FIG - +train hdt-s51627 14 TIME - +train hdt-s60808 1 FIG - +train hdt-s45005 12 PLACE - +train hdt-s48813 6 PLACE - +train hdt-s53718 3 TIME - +train hdt-s38580 4 FIG - +train hdt-s45404 6 FIG - +train hdt-s31093 11 PLACE - +train hdt-s58485 13 FIG - +train hdt-s58577 8 FIG - +train hdt-s40470 1 TIME - +train hdt-s56906 4 FIG - +train hdt-s27927 6 PLACE - +train hdt-s29619 15 FIG - +train hdt-s25458 1 FIG - +train hdt-s27955 9 FIG - +train hdt-s21565 18 FIG - +train hdt-s32123 24 FIG - +train hdt-s41292 31 FIG - +train hdt-s52181 12 FIG - +train hdt-s29888 19 FIG - +train hdt-s35359 12 FIG - +train hdt-s41055 4 FIG - +train hdt-s45954 3 FIG - +train hdt-s32875 24 FIG - +train hdt-s51806 1 TIME - +train hdt-s50059 8 TIME - +train hdt-s43115 3 TIME - +train hdt-s30523 11 FIG - +train hdt-s39549 10 TIME - +train hdt-s47994 3 FIG - +train hdt-s25649 6 PLACE - +train hdt-s52330 1 FIG - +train hdt-s42540 7 FIG - +train hdt-s37073 14 FIG - +train hdt-s30591 3 FIG - +train hdt-s23249 4 FIG - +train hdt-s29692 7 FIG - +train hdt-s30855 4 PLACE - +train hdt-s55371 8 FIG - +train hdt-s51761 5 FIG - +train hdt-s50730 13 FIG - +train hdt-s47437 3 TIME - +train hdt-s58775 4 FIG - +train hdt-s35917 7 TIME - +train hdt-s55298 4 FIG - +train hdt-s55611 6 TIME - +train hdt-s29275 7 FIG - +train hdt-s38302 12 TIME - +train hdt-s43486 3 TIME - +train hdt-s52079 8 FIG - +train hdt-s55362 15 FIG - +train hdt-s22190 19 FIG - +train hdt-s50271 7 FIG - +train hdt-s44934 7 TIME - +train hdt-s37082 7 FIG - +train hdt-s34525 16 FIG - +train hdt-s49703 7 TIME - +train hdt-s58032 1 FIG - +train hdt-s23293 5 TIME - +train hdt-s58711 5 FIG - +train hdt-s32747 12 FIG - +train hdt-s24590 7 FIG - +train hdt-s22625 13 PLACE - +train hdt-s60617 5 FIG - +train hdt-s32132 7 FIG - +train hdt-s28957 9 FIG - +train hdt-s56098 4 FIG - +train hdt-s27608 6 FIG - +train hdt-s56665 6 PLACE - +train hdt-s58506 8 FIG - +train hdt-s45889 27 FIG - +train hdt-s58240 24 FIG - +train hdt-s30330 1 TIME - +train hdt-s59925 27 TIME - +train hdt-s48288 15 FIG - +train hdt-s38386 23 PLACE - +train hdt-s40074 3 FIG - +train hdt-s43908 17 PLACE - +train hdt-s37080 17 FIG - +train hdt-s57506 6 FIG - +train hdt-s37030 6 PLACE - +train hdt-s42017 1 TIME - +train hdt-s35619 32 PLACE - +train hdt-s29449 7 FIG - +train hdt-s37910 11 PLACE - +train hdt-s58331 8 PLACE - +train hdt-s43477 5 FIG - +train hdt-s27777 6 FIG - +train hdt-s48285 24 FIG - +train hdt-s54551 11 FIG - +train hdt-s32472 10 FIG - +train hdt-s57968 1 FIG - +train hdt-s54784 12 TIME - +train hdt-s45471 10 FIG - +train hdt-s25866 10 PLACE - +train hdt-s50393 6 FIG - +train hdt-s48533 1 TIME - +train hdt-s33127 6 FIG - +train hdt-s57768 4 TIME - +train hdt-s45640 5 TIME - +train hdt-s43363 13 TIME - +train hdt-s38463 17 FIG - +train hdt-s34635 17 TIME - +train hdt-s60274 7 TIME - +train hdt-s54257 23 PLACE - +train hdt-s51505 1 TIME - +train hdt-s25120 1 FIG - +train hdt-s29116 9 PLACE - +train hdt-s59322 22 TIME - +train hdt-s43646 9 FIG - +train hdt-s39289 2 FIG - +train hdt-s37910 7 TIME - +train hdt-s28171 16 FIG - +train hdt-s39167 4 FIG - +train hdt-s56127 3 TIME - +train hdt-s37756 1 FIG - +train hdt-s50057 10 FIG - +train hdt-s59168 10 FIG - +train hdt-s27926 11 PLACE - +train hdt-s33397 7 TIME - +train hdt-s55599 1 TIME - +train hdt-s36511 1 TIME - +train hdt-s58285 1 FIG - +train hdt-s45537 11 FIG - +train hdt-s50159 14 FIG - +train hdt-s37606 28 FIG - +train hdt-s57497 5 TIME - +train hdt-s25030 15 FIG - +train hdt-s22458 12 FIG - +train hdt-s47195 17 FIG - +train hdt-s25322 13 TIME - +train hdt-s53113 24 PLACE - +train hdt-s49495 18 TIME - +train hdt-s57398 7 FIG - +train hdt-s32321 9 FIG - +train hdt-s48348 7 TIME - +train hdt-s35649 16 FIG - +train hdt-s47460 29 FIG - +train hdt-s24820 6 PLACE - +train hdt-s55188 4 TIME - +train hdt-s34645 4 TIME - +train hdt-s49160 16 FIG - +train hdt-s58107 13 FIG - +train hdt-s43708 5 FIG - +train hdt-s23512 4 TIME - +train hdt-s51865 15 FIG - +train hdt-s22975 11 FIG - +train hdt-s40818 17 FIG - +train hdt-s21799 14 PLACE - +train hdt-s55064 21 TIME - +train hdt-s25174 28 FIG - +train hdt-s46576 1 PLACE - +train hdt-s26242 1 TIME - +train hdt-s30170 7 TIME - +train hdt-s24826 1 TIME - +train hdt-s28298 4 FIG - +train hdt-s28634 16 FIG - +train hdt-s59427 18 PLACE - +train hdt-s48170 15 FIG - +train hdt-s45088 2 TIME - +train hdt-s44811 5 PLACE - +train hdt-s36890 4 FIG - +train hdt-s51387 29 PLACE - +train hdt-s20291 15 FIG - +train hdt-s32746 1 FIG - +train hdt-s55800 13 FIG - +train hdt-s41405 15 PLACE - +train hdt-s34516 17 FIG - +train hdt-s57903 10 FIG - +train hdt-s54596 19 FIG - +train hdt-s59566 5 FIG - +train hdt-s20713 22 TIME - +train hdt-s48670 4 FIG - +train hdt-s30716 13 TIME - +train hdt-s53733 18 FIG - +train hdt-s31876 5 PLACE - +train hdt-s41653 10 FIG - +train hdt-s45416 11 FIG - +train hdt-s36614 1 FIG - +train hdt-s37058 18 TIME - +train hdt-s58861 13 FIG - +train hdt-s29695 10 FIG - +train hdt-s54731 3 FIG - +train hdt-s22304 4 FIG - +train hdt-s27868 5 FIG - +train hdt-s21022 13 TIME - +train hdt-s32983 12 FIG - +train hdt-s47890 12 PLACE - +train hdt-s54822 14 FIG - +train hdt-s20747 2 TIME - +train hdt-s25745 2 TIME - +train hdt-s33316 17 PLACE - +train hdt-s33019 1 TIME - +train hdt-s29854 2 TIME - +train hdt-s58052 17 TIME - +train hdt-s50044 16 FIG - +train hdt-s55802 11 TIME - +train hdt-s50078 9 FIG - +train hdt-s47742 8 TIME - +train hdt-s56847 10 FIG - +train hdt-s42177 1 TIME - +train hdt-s32454 10 TIME - +train hdt-s53415 14 PLACE - +train hdt-s28312 14 TIME - +train hdt-s57064 11 PLACE - +train hdt-s47927 9 PLACE - +train hdt-s46759 2 TIME - +train hdt-s56366 14 FIG - +train hdt-s54410 8 PLACE - +train hdt-s38512 23 FIG - +train hdt-s46338 4 FIG - +train hdt-s52832 1 FIG - +train hdt-s51072 5 FIG - +train hdt-s41241 13 TIME - +train hdt-s35077 3 FIG - +train hdt-s36813 1 PLACE - +train hdt-s26283 27 FIG - +train hdt-s45438 10 FIG - +train hdt-s48686 6 TIME - +train hdt-s27198 19 PLACE - +train hdt-s49374 20 FIG - +train hdt-s60900 4 FIG - +train hdt-s54986 7 FIG - +train hdt-s44773 15 FIG - +train hdt-s29866 4 TIME - +train hdt-s22455 15 FIG - +train hdt-s49920 17 FIG - +train hdt-s44846 9 FIG - +train hdt-s55109 1 TIME - +train hdt-s43323 19 FIG - +train hdt-s23766 4 PLACE - +train hdt-s30242 17 FIG - +train hdt-s31360 16 TIME - +train hdt-s36444 8 TIME - +train hdt-s28129 10 TIME - +train hdt-s32909 37 FIG - +train hdt-s22587 11 FIG - +train hdt-s29961 8 FIG - +train hdt-s46126 33 FIG - +train hdt-s43629 10 FIG - +train hdt-s53468 11 TIME - +train hdt-s50350 15 TIME - +train hdt-s44450 5 FIG - +train hdt-s53401 12 PLACE - +train hdt-s60896 9 TIME - +train hdt-s54563 7 TIME - +train hdt-s49856 5 TIME - +train hdt-s21127 10 FIG - +train hdt-s40365 18 TIME - +train hdt-s34527 18 TIME - +train hdt-s28019 18 FIG - +train hdt-s48192 23 FIG - +train hdt-s43785 10 FIG - +train hdt-s59552 13 PLACE - +train hdt-s45498 14 FIG - +train hdt-s47145 11 PLACE - +train hdt-s46236 1 PLACE - +train hdt-s51276 5 FIG - +train hdt-s26357 6 PLACE - +train hdt-s51637 24 PLACE - +train hdt-s44441 1 FIG - +train hdt-s48938 3 FIG - +train hdt-s23346 19 PLACE - +train hdt-s42679 4 TIME - +train hdt-s58693 1 FIG - +train hdt-s23179 6 FIG - +train hdt-s24767 4 FIG - +train hdt-s48166 29 PLACE - +train hdt-s35474 32 FIG - +train hdt-s31759 20 PLACE - +train hdt-s42145 1 TIME - +train hdt-s22296 1 FIG - +train hdt-s52160 15 FIG - +train hdt-s59123 4 PLACE - +train hdt-s51641 5 TIME - +train hdt-s56084 12 FIG - +train hdt-s45543 3 PLACE - +train hdt-s30498 11 FIG - +train hdt-s39422 5 FIG - +train hdt-s53397 13 FIG - +train hdt-s50906 19 FIG - +train hdt-s59568 6 PLACE - +train hdt-s47168 13 FIG - +train hdt-s52598 20 FIG - +train hdt-s54646 6 TIME - +train hdt-s20549 4 TIME - +train hdt-s25992 15 FIG - +train hdt-s30580 15 FIG - +train hdt-s48777 14 FIG - +train hdt-s38357 18 FIG - +train hdt-s39692 10 PLACE - +train hdt-s25688 19 FIG - +train hdt-s29366 17 TIME - +train hdt-s51608 3 FIG - +train hdt-s57829 27 FIG - +train hdt-s23891 5 FIG - +train hdt-s40662 24 PLACE - +train hdt-s32437 25 FIG - +train hdt-s57368 20 FIG - +train hdt-s57760 9 TIME - +train hdt-s50781 5 TIME - +train hdt-s35467 6 FIG - +train hdt-s60222 14 FIG - +train hdt-s27537 10 TIME - +train hdt-s29482 2 TIME - +train hdt-s52838 9 FIG - +train hdt-s27044 18 FIG - +train hdt-s29458 6 PLACE - +train hdt-s51934 18 TIME - +train hdt-s30814 1 FIG - +train hdt-s57897 9 FIG - +train hdt-s59507 5 TIME - +train hdt-s50428 9 FIG - +train hdt-s35451 11 TIME - +train hdt-s27304 2 TIME - +train hdt-s48618 8 TIME - +train hdt-s55025 7 FIG - +train hdt-s30368 12 FIG - +train hdt-s44262 5 TIME - +train hdt-s38322 7 FIG - +train hdt-s32988 7 FIG - +train hdt-s58833 4 FIG - +train hdt-s25583 11 TIME - +train hdt-s57857 14 FIG - +train hdt-s56918 20 TIME - +train hdt-s39969 12 TIME - +train hdt-s38391 4 TIME - +train hdt-s38694 12 FIG - +train hdt-s42938 7 TIME - +train hdt-s53984 14 FIG - +train hdt-s20907 9 FIG - +train hdt-s23175 1 FIG - +train hdt-s56171 13 PLACE - +train hdt-s44786 1 FIG - +train hdt-s31804 10 TIME - +train hdt-s32470 5 TIME - +train hdt-s44538 8 FIG - +train hdt-s30487 9 FIG - +train hdt-s58498 1 FIG - +train hdt-s43451 9 PLACE - +train hdt-s53416 1 PLACE - +train hdt-s22077 1 FIG - +train hdt-s33652 2 FIG - +train hdt-s41439 15 FIG - +train hdt-s42256 1 TIME - +train hdt-s53784 14 FIG - +train hdt-s39681 14 PLACE - +train hdt-s55000 6 FIG - +train hdt-s28356 1 FIG - +train hdt-s41231 20 FIG - +train hdt-s26157 6 FIG - +train hdt-s20818 7 TIME - +train hdt-s26309 11 PLACE - +train hdt-s56233 1 TIME - +train hdt-s43983 20 PLACE - +train hdt-s41391 28 TIME - +train hdt-s30794 13 FIG - +train hdt-s30965 7 TIME - +train hdt-s60587 18 TIME - +train hdt-s55707 4 FIG - +train hdt-s34801 14 FIG - +train hdt-s33895 4 FIG - +train hdt-s60716 9 FIG - +train hdt-s22846 24 FIG - +train hdt-s28905 19 FIG - +train hdt-s54816 3 FIG - +train hdt-s24773 13 TIME - +train hdt-s39817 4 FIG - +train hdt-s30289 34 PLACE - +train hdt-s55557 13 FIG - +train hdt-s37300 20 TIME - +train hdt-s41809 5 FIG - +train hdt-s35963 18 FIG - +train hdt-s41323 1 PLACE - +train hdt-s53735 15 PLACE - +train hdt-s50240 11 FIG - +train hdt-s54800 17 PLACE - +train hdt-s31040 9 TIME - +train hdt-s60116 12 FIG - +train hdt-s20297 1 FIG - +train hdt-s41145 20 FIG - +train hdt-s36734 17 FIG - +train hdt-s45901 13 FIG - +train hdt-s60403 8 FIG - +train hdt-s42467 11 FIG - +train hdt-s47017 4 FIG - +train hdt-s42972 15 FIG - +train hdt-s57681 22 FIG - +train hdt-s25573 19 FIG - +train hdt-s41079 18 FIG - +train hdt-s48286 20 FIG - +train hdt-s41694 17 FIG - +train hdt-s39111 3 FIG - +train hdt-s30137 8 PLACE - +train hdt-s44543 6 FIG - +train hdt-s48765 5 TIME - +train hdt-s39990 2 TIME - +train hdt-s31524 11 FIG - +train hdt-s22657 5 TIME - +train hdt-s40618 4 TIME - +train hdt-s32751 24 FIG - +train hdt-s43322 2 PLACE - +train hdt-s35524 19 FIG - +train hdt-s48468 8 FIG - +train hdt-s48941 15 TIME - +train hdt-s32720 8 TIME - +train hdt-s38709 18 FIG - +train hdt-s29680 2 PLACE - +train hdt-s28972 1 FIG - +train hdt-s37675 4 FIG - +train hdt-s31443 3 FIG - +train hdt-s51479 5 FIG - +train hdt-s27913 4 PLACE - +train hdt-s22860 12 TIME - +train hdt-s31345 33 PLACE - +train hdt-s21347 24 FIG - +train hdt-s20185 1 TIME - +train hdt-s35863 22 FIG - +train hdt-s53470 3 FIG - +train hdt-s56596 16 FIG - +train hdt-s54309 2 TIME - +test hdt-s10941 23 TIME TIME +test hdt-s118689 6 TIME TIME +test hdt-s118259 1 TIME TIME +test hdt-s14329 12 FIG FIG +test hdt-s112892 11 FIG FIG +test hdt-s121740 23 TIME TIME +test hdt-s120319 17 FIG FIG +test hdt-s116606 10 TIME TIME +test hdt-s116465 6 FIG FIG +test hdt-s113969 10 FIG FIG +test hdt-s112269 11 FIG FIG +test hdt-s121429 10 TIME TIME +test hdt-s112750 4 TIME TIME +test hdt-s15686 12 TIME TIME +test hdt-s118029 9 FIG FIG +test hdt-s12300 22 FIG FIG +test hdt-s12903 9 TIME TIME +test hdt-s13023 1 TIME TIME +test hdt-s120204 2 PLACE PLACE +test hdt-s115544 18 FIG FIG +test hdt-s119975 10 FIG FIG +test hdt-s10107 5 FIG FIG +test hdt-s119532 5 FIG PLACE +test hdt-s112521 11 PLACE PLACE +test hdt-s15905 1 FIG FIG +test hdt-s116789 10 TIME TIME +test hdt-s15112 17 TIME TIME +test hdt-s116828 1 FIG FIG +test hdt-s12437 30 FIG FIG +test hdt-s120769 15 FIG FIG +test hdt-s12050 3 FIG FIG +test hdt-s19310 4 FIG FIG +test hdt-s10236 7 FIG FIG +test hdt-s18970 8 FIG FIG +test hdt-s113014 21 FIG FIG +test hdt-s17188 5 FIG FIG +test hdt-s112445 1 FIG FIG +test hdt-s119688 1 FIG FIG +test hdt-s113556 3 FIG FIG +test hdt-s116047 5 PLACE PLACE +test hdt-s13223 4 FIG FIG +test hdt-s10145 6 FIG FIG +test hdt-s117892 6 FIG FIG +test hdt-s19525 2 PLACE PLACE +test hdt-s120449 9 FIG FIG +test hdt-s118634 9 TIME TIME +test hdt-s120322 6 TIME TIME +test hdt-s120258 8 FIG FIG +test hdt-s16756 8 TIME TIME +test hdt-s19097 13 FIG FIG +test hdt-s112962 7 FIG FIG +test hdt-s118185 15 FIG FIG +test hdt-s11631 21 FIG FIG +test hdt-s14909 1 FIG FIG +test hdt-s19681 4 TIME TIME +test hdt-s17532 9 FIG FIG +test hdt-s114915 9 FIG FIG +test hdt-s114126 7 FIG FIG +test hdt-s12773 15 FIG FIG +test hdt-s120214 6 FIG FIG +test hdt-s13067 9 FIG FIG +test hdt-s114164 18 FIG FIG +test hdt-s17713 1 TIME TIME +test hdt-s116036 6 TIME TIME +test hdt-s119065 17 FIG FIG +test hdt-s11281 10 FIG FIG +test hdt-s18353 15 FIG FIG +test hdt-s112684 2 TIME TIME +test hdt-s15688 13 FIG FIG +test hdt-s112248 1 FIG FIG +test hdt-s14492 23 FIG FIG +test hdt-s121940 18 FIG FIG +test hdt-s118587 30 TIME TIME +test hdt-s16210 7 TIME TIME +test hdt-s114406 11 TIME TIME +test hdt-s14688 11 PLACE PLACE +test hdt-s114005 21 TIME TIME +test hdt-s118562 10 TIME TIME +test hdt-s120080 10 TIME TIME +test hdt-s119481 18 PLACE PLACE +test hdt-s113294 19 FIG FIG +test hdt-s120545 8 FIG FIG +test hdt-s112113 31 FIG FIG +test hdt-s121373 4 FIG FIG +test hdt-s118203 42 TIME TIME +test hdt-s15623 6 FIG FIG +test hdt-s10091 10 FIG FIG +test hdt-s121374 3 TIME TIME +test hdt-s15263 15 FIG FIG +test hdt-s13362 8 FIG FIG +test hdt-s10700 7 FIG FIG +test hdt-s12604 14 PLACE PLACE +test hdt-s11095 27 PLACE PLACE +test hdt-s14989 13 FIG FIG +test hdt-s115432 9 FIG FIG +test hdt-s113790 1 FIG FIG +test hdt-s119966 7 FIG FIG +test hdt-s116076 10 FIG FIG +test hdt-s19594 25 PLACE PLACE +test hdt-s15602 9 FIG FIG +test hdt-s10137 5 TIME TIME +test hdt-s115856 7 TIME TIME +test hdt-s12857 17 FIG FIG +test hdt-s12099 1 FIG FIG +test hdt-s13637 14 FIG FIG +test hdt-s12481 21 FIG FIG +test hdt-s19478 5 FIG FIG +test hdt-s118635 13 FIG FIG +test hdt-s18165 1 FIG FIG +test hdt-s12519 13 FIG FIG +test hdt-s10130 19 FIG FIG +test hdt-s120581 11 FIG FIG +test hdt-s12284 4 FIG FIG +test hdt-s18490 10 FIG FIG +test hdt-s114830 11 FIG FIG +test hdt-s17141 5 TIME TIME +test hdt-s15595 20 FIG FIG +test hdt-s10325 9 FIG FIG +test hdt-s115586 13 FIG FIG +test hdt-s119005 3 TIME TIME +test hdt-s13470 13 FIG FIG +test hdt-s112862 10 FIG FIG +test hdt-s114588 2 TIME TIME +test hdt-s120913 9 TIME TIME +test hdt-s120569 12 TIME TIME +test hdt-s12610 1 FIG FIG +test hdt-s117197 7 PLACE PLACE +test hdt-s19843 13 PLACE PLACE +test hdt-s14644 19 TIME TIME +test hdt-s10295 7 TIME TIME +test hdt-s10623 5 TIME TIME +test hdt-s16993 4 FIG FIG +test hdt-s118819 4 FIG FIG +test hdt-s121946 39 PLACE PLACE +test hdt-s120816 9 TIME TIME +test hdt-s10799 2 FIG FIG +test hdt-s119203 5 FIG FIG +test hdt-s16692 21 PLACE PLACE +test hdt-s18365 21 TIME TIME +test hdt-s114061 3 FIG FIG +test hdt-s114260 4 TIME TIME +test hdt-s115599 18 PLACE PLACE +test hdt-s112342 15 FIG FIG +test hdt-s113291 19 FIG FIG +test hdt-s13520 9 FIG FIG +test hdt-s117195 1 PLACE PLACE +test hdt-s15690 4 FIG FIG +test hdt-s13081 25 FIG FIG +test hdt-s17989 1 FIG FIG +test hdt-s13427 2 TIME TIME +test hdt-s10137 11 FIG FIG +test hdt-s113167 2 FIG FIG +test hdt-s13892 1 TIME TIME +test hdt-s15451 9 FIG FIG +test hdt-s113701 20 FIG FIG +test hdt-s121606 1 PLACE PLACE +test hdt-s13717 12 FIG PLACE +test hdt-s119376 19 FIG FIG +test hdt-s19312 4 TIME TIME +test hdt-s115968 13 FIG FIG +test hdt-s116034 15 FIG FIG +test hdt-s117800 18 FIG FIG +test hdt-s118130 7 PLACE PLACE +test hdt-s16077 5 FIG FIG +test hdt-s13986 1 FIG FIG +test hdt-s121133 6 FIG FIG +test hdt-s119560 8 FIG FIG +test hdt-s15798 22 TIME TIME +test hdt-s10123 6 FIG FIG +test hdt-s118685 1 FIG FIG +test hdt-s19035 11 FIG FIG +test hdt-s116126 13 FIG FIG +test hdt-s113655 18 FIG FIG +test hdt-s120345 20 FIG FIG +test hdt-s119852 21 FIG FIG +test hdt-s120599 15 FIG FIG +test hdt-s113003 8 FIG FIG +test hdt-s11170 5 TIME TIME +test hdt-s116869 7 FIG FIG +test hdt-s120902 12 FIG FIG +test hdt-s17599 22 TIME TIME +test hdt-s118706 5 FIG FIG +test hdt-s116047 13 FIG FIG +test hdt-s11259 4 FIG FIG +test hdt-s10802 18 TIME TIME +test hdt-s113260 13 FIG FIG +test hdt-s15986 12 FIG FIG +test hdt-s13973 12 TIME TIME +test hdt-s115239 1 PLACE PLACE +test hdt-s116299 3 TIME TIME +test hdt-s18083 4 TIME TIME +test hdt-s116143 9 FIG FIG +test hdt-s116169 4 FIG FIG +test hdt-s121321 5 FIG FIG +test hdt-s18077 18 FIG FIG +test hdt-s119232 25 TIME TIME +test hdt-s120310 13 TIME TIME +test hdt-s116799 1 FIG FIG +test hdt-s116719 16 PLACE PLACE +test hdt-s120323 25 FIG FIG +test hdt-s14865 13 FIG FIG +test hdt-s13087 11 FIG FIG +test hdt-s118788 4 PLACE PLACE +test hdt-s118230 10 PLACE PLACE +test hdt-s12164 1 FIG FIG +test hdt-s11411 1 FIG FIG +test hdt-s16602 3 FIG FIG +test hdt-s114019 27 FIG FIG +test hdt-s11438 18 FIG FIG +test hdt-s112548 17 PLACE PLACE +test hdt-s116375 17 FIG FIG +test hdt-s120653 18 FIG FIG +test hdt-s17890 4 PLACE PLACE +test hdt-s14858 1 TIME TIME +test hdt-s18517 6 PLACE PLACE +test hdt-s18972 18 FIG FIG +test hdt-s114845 7 FIG FIG +test hdt-s16938 3 TIME TIME +test hdt-s115642 1 FIG FIG +test hdt-s120756 11 FIG FIG +test hdt-s10015 22 TIME TIME +test hdt-s11441 5 TIME TIME +test hdt-s10590 7 FIG FIG +test hdt-s14061 1 PLACE PLACE +test hdt-s18505 8 FIG FIG +test hdt-s18992 9 FIG FIG +test hdt-s19493 3 FIG FIG +test hdt-s12086 1 FIG FIG +test hdt-s17732 9 FIG FIG +test hdt-s13279 1 TIME TIME +test hdt-s10784 4 TIME TIME +test hdt-s18778 6 TIME TIME +test hdt-s114528 23 PLACE PLACE +test hdt-s119975 24 TIME TIME +test hdt-s16937 17 FIG FIG +test hdt-s15315 7 FIG FIG +test hdt-s120652 14 FIG FIG +test hdt-s112868 4 FIG FIG +test hdt-s114001 5 FIG FIG +test hdt-s13990 7 FIG FIG +test hdt-s116232 2 TIME TIME +test hdt-s12041 11 FIG FIG +test hdt-s14221 4 TIME TIME +test hdt-s121838 1 FIG FIG +test hdt-s15104 13 FIG FIG +test hdt-s13548 20 FIG FIG +test hdt-s18568 17 FIG FIG +test hdt-s10339 12 FIG FIG +test hdt-s112671 11 FIG FIG +test hdt-s114937 9 FIG FIG +test hdt-s120615 16 TIME TIME +test hdt-s13017 36 PLACE FIG +test hdt-s120553 4 TIME TIME +test hdt-s121089 4 TIME TIME +test hdt-s116852 8 TIME TIME +test hdt-s19773 16 FIG FIG +test hdt-s15117 17 FIG FIG +test hdt-s119878 1 PLACE PLACE +test hdt-s117648 25 FIG FIG +test hdt-s119456 7 FIG FIG +test hdt-s117319 1 PLACE PLACE +test hdt-s117220 16 FIG FIG +test hdt-s10583 23 FIG FIG +test hdt-s17885 4 PLACE PLACE +test hdt-s17074 14 PLACE PLACE +test hdt-s18217 23 FIG FIG +test hdt-s113940 5 TIME TIME +test hdt-s13358 9 FIG FIG +test hdt-s114078 15 FIG FIG +test hdt-s15651 5 FIG FIG +test hdt-s13359 33 FIG FIG +test hdt-s117323 8 FIG FIG +test hdt-s13192 1 FIG FIG +test hdt-s15704 6 FIG FIG +test hdt-s15102 1 TIME TIME +test hdt-s18736 15 FIG FIG +test hdt-s16181 25 FIG FIG +test hdt-s14542 15 FIG FIG +test hdt-s119510 8 FIG FIG +test hdt-s112029 1 FIG FIG +test hdt-s113564 7 FIG FIG +test hdt-s119911 7 FIG FIG +test hdt-s117034 9 TIME TIME +test hdt-s118113 1 PLACE PLACE +test hdt-s17152 2 FIG FIG +test hdt-s11164 23 FIG FIG +test hdt-s117947 12 FIG FIG +test hdt-s113953 12 FIG FIG +test hdt-s112384 4 FIG FIG +test hdt-s14927 18 FIG FIG +test hdt-s112710 19 FIG FIG +test hdt-s15196 16 PLACE PLACE +test hdt-s120358 7 FIG FIG +test hdt-s119172 20 FIG FIG +test hdt-s118033 25 FIG FIG +test hdt-s118969 3 FIG FIG +test hdt-s112135 10 FIG FIG +test hdt-s10900 21 FIG FIG +test hdt-s121305 19 FIG FIG +test hdt-s12837 10 FIG FIG +test hdt-s115288 5 FIG FIG +test hdt-s13679 26 FIG FIG +test hdt-s19238 6 FIG FIG +test hdt-s115361 12 FIG FIG +test hdt-s118316 13 FIG FIG +test hdt-s17266 18 TIME TIME +test hdt-s119422 10 PLACE PLACE +test hdt-s118625 17 FIG FIG +test hdt-s119883 17 FIG FIG +test hdt-s13787 26 TIME TIME +test hdt-s118489 6 FIG FIG +test hdt-s17497 9 TIME TIME +test hdt-s120469 6 FIG FIG +test hdt-s112608 6 TIME TIME +test hdt-s15728 1 FIG FIG +test hdt-s112281 1 FIG FIG +test hdt-s113292 13 FIG FIG +test hdt-s15155 8 FIG FIG +test hdt-s18462 7 FIG FIG +test hdt-s119863 1 FIG FIG +test hdt-s115178 18 FIG FIG +test hdt-s117040 15 TIME TIME +test hdt-s118402 23 FIG FIG +test hdt-s15355 10 FIG FIG +test hdt-s115300 8 FIG FIG +test hdt-s117537 1 FIG PLACE +test hdt-s15861 28 TIME TIME +test hdt-s119795 13 FIG FIG +test hdt-s15922 18 PLACE FIG +test hdt-s115859 11 FIG FIG +test hdt-s19666 16 FIG FIG +test hdt-s15661 5 FIG FIG +test hdt-s113449 2 TIME TIME +test hdt-s19659 9 FIG FIG +test hdt-s118263 8 FIG FIG +test hdt-s13553 11 FIG FIG +test hdt-s113540 3 FIG FIG +test hdt-s12242 1 TIME TIME +test hdt-s120463 8 FIG FIG +test hdt-s120262 20 TIME TIME +test hdt-s13767 19 PLACE FIG +test hdt-s117417 1 FIG FIG +test hdt-s12809 11 FIG FIG +test hdt-s115887 33 TIME TIME +test hdt-s118372 25 FIG FIG +test hdt-s116588 7 FIG FIG +test hdt-s15098 8 TIME FIG +test hdt-s121549 1 TIME TIME +test hdt-s113794 1 TIME TIME +test hdt-s115040 8 FIG FIG +test hdt-s121448 11 FIG FIG +test hdt-s17078 11 FIG FIG +test hdt-s119220 1 FIG FIG +test hdt-s11266 6 PLACE PLACE +test hdt-s12703 1 PLACE PLACE +test hdt-s13565 8 FIG FIG +test hdt-s117754 22 FIG FIG +test hdt-s114170 8 FIG FIG +test hdt-s116420 24 PLACE FIG +test hdt-s18378 1 FIG FIG +test hdt-s113469 2 PLACE PLACE +test hdt-s120697 3 FIG FIG +test hdt-s118546 4 FIG FIG +test hdt-s15913 2 FIG FIG +test hdt-s114201 5 FIG FIG +test hdt-s114898 7 FIG FIG +test hdt-s14819 12 TIME TIME +test hdt-s117223 12 TIME TIME +test hdt-s15598 5 TIME TIME +test hdt-s114253 18 FIG FIG +test hdt-s114210 7 FIG FIG +test hdt-s113763 15 FIG FIG +test hdt-s121085 4 FIG FIG +test hdt-s16773 11 FIG FIG +test hdt-s112523 22 FIG FIG +test hdt-s16666 1 FIG FIG +test hdt-s12654 16 FIG FIG +test hdt-s121796 6 TIME TIME +test hdt-s16911 4 FIG FIG +test hdt-s119090 1 TIME TIME +test hdt-s112926 20 FIG FIG +test hdt-s121396 1 FIG FIG +test hdt-s120309 1 TIME TIME +test hdt-s119711 24 FIG FIG +test hdt-s120508 19 FIG FIG +test hdt-s118772 7 FIG FIG +test hdt-s113019 1 FIG FIG +test hdt-s119295 9 FIG FIG +test hdt-s18211 28 FIG FIG +test hdt-s120529 9 FIG FIG +test hdt-s121996 15 FIG FIG +test hdt-s116730 1 PLACE PLACE +test hdt-s15428 1 FIG FIG +test hdt-s113081 10 FIG FIG +test hdt-s118706 24 FIG FIG +test hdt-s117496 1 FIG FIG +test hdt-s17615 1 FIG FIG +test hdt-s19181 31 FIG FIG +test hdt-s15328 17 FIG FIG +test hdt-s121674 1 FIG FIG diff --git a/crates/deepnsm-v2/examples/ud_lane_quorum.rs b/crates/deepnsm-v2/examples/ud_lane_quorum.rs new file mode 100644 index 000000000..75480fff0 --- /dev/null +++ b/crates/deepnsm-v2/examples/ud_lane_quorum.rs @@ -0,0 +1,610 @@ +//! `ud_lane_quorum` — TIME vs PLACE vs FIG for a German Wechsel phrase, by a +//! quorum of position, adverb, noun and verb evidence. +//! +//! A Wechsel phrase (*an, auf, hinter, in, neben, über, unter, vor, +//! zwischen*) reads as one of three lanes: +//! - **TIME**: *vor der Fahrt*, *in 12 Monaten*; +//! - **PLACE**: a literal physical place or direction, *vor dem Haus*; +//! - **FIG**: everything else, i.e. a verb-governed object (*setzen auf*) or a +//! metaphorical place (*unter Druck*, *auf dem Markt*). +//! +//! Operator hypothesis: an abstract head is TIME or FIG, never PLACE. +//! +//! **Labels are silver**, written by independent model labelers reading the +//! full sentence (`examples/data/hdt_wechsel_lanes.tsv`, ids only, no HDT +//! text). Each test item has two labelers; only items they agree on are +//! scored, and Cohen's κ is reported. German UD has no lane gold. +//! +//! Voters, read from surface forms and tables mined from **train**: +//! - **noun**: the head (first capitalised token of the phrase) is a time +//! noun (heads after *seit* / *während* / *bis*, nouns used as bare +//! adverbials, or a compound ending in one), an abstract noun (suffix), a +//! name (train majority PROPN), or other; +//! - **number**: the phrase holds a digit; +//! - **adverb**: the token before the preposition is a temporal adverb +//! (adverbs right before *seit* / *während* / *bis* in train); +//! - **verb**: the (verb lemma, preposition) object rate from HDT `obj` + ADP, +//! checked against every verb in the clause (`ud_pp_arg_eval`); +//! - **position** (TEKAMOLO): Vorfeld (clause-initial), and whether another +//! preposition follows in the same clause (temporal phrases come earlier); +//! - **article case**: the article form leaves only Acc, only Dat, or both; +//! - **preposition**: one-hot (the prior). +//! +//! The quorum is a multinomial logistic regression over the one-hot voter +//! outputs, fitted on the train labels. Each single voter is fitted the same +//! way (that voter + preposition) as its own baseline. +//! +//! KILL bars (fixed before the labels existed): +//! - κ < 0.6: the silver gold is unreliable, and the result is reported as +//! such; +//! - quorum accuracy < best single voter + 2 points; +//! - TIME F1 < 0.80, or PLACE F1 < 0.60; +//! - hypothesis: more than 5 % of abstract-head items are labelled PLACE. +//! +//! `LANE_SPLIT=confirm` scores the fresh confirmation sample instead of the +//! exploratory `test` sample; `LANE_LAMBDA` overrides the L2 strength. +//! +//! ```text +//! cargo run --release --example ud_lane_quorum -- de_hdt-ud-train-a-1.conllu de_hdt-ud-test.conllu examples/data/hdt_wechsel_lanes.tsv +//! ``` + +use std::collections::{HashMap, HashSet}; + +const LANES: [&str; 3] = ["TIME", "PLACE", "FIG"]; +const WECHSEL: [&str; 9] = [ + "an", "auf", "hinter", "in", "neben", "über", "unter", "vor", "zwischen", +]; +const TEMPORAL_ONLY: [&str; 3] = ["seit", "während", "bis"]; +/// Derivational suffixes of German abstract nouns (inflected forms included). +const ABSTRACT_SUFFIXES: [&str; 18] = [ + "ung", "ungen", "heit", "heiten", "keit", "keiten", "schaft", "schaften", "tion", "tionen", + "nis", "nisse", "nissen", "ität", "itäten", "ismus", "ismen", "tum", +]; +const MIN_PAIR: usize = 3; + +struct Tok { + form: String, + upper: bool, + word: bool, + digit: bool, + lemma: String, + upos: String, + case: Option, + head: Option, + deprel: String, +} + +struct Sent { + id: String, + toks: Vec, +} + +fn read(path: &str) -> Vec { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut out = Vec::new(); + for block in text.split("\n\n") { + let mut id = String::new(); + let mut toks = Vec::new(); + for line in block.lines() { + if let Some(v) = line.strip_prefix("# sent_id = ") { + id = v.to_string(); + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if line.starts_with('#') || c.len() < 8 || c[0].contains(['-', '.']) { + continue; + } + let head: usize = c[6].parse().unwrap_or(0); + let case = c[5] + .split('|') + .find_map(|f| f.strip_prefix("Case=")) + .and_then(|v| ["Nom", "Acc", "Dat", "Gen"].iter().position(|x| *x == v)); + toks.push(Tok { + form: c[1].to_lowercase(), + upper: c[1].chars().next().is_some_and(char::is_uppercase), + word: c[1].chars().any(char::is_alphanumeric), + digit: c[1].chars().any(|ch| ch.is_ascii_digit()), + lemma: c[2].to_string(), + upos: c[3].to_string(), + case, + head: head.checked_sub(1), + deprel: c[7].to_string(), + }); + } + if !toks.is_empty() { + out.push(Sent { id, toks }); + } + } + out +} + +fn abstract_noun(form: &str) -> bool { + form.chars().count() > 5 && ABSTRACT_SUFFIXES.iter().any(|x| form.ends_with(x)) +} + +/// The phrase after the preposition at `i`: up to four tokens, never through +/// punctuation, ending after the first capitalised token. +fn phrase(s: &[Tok], i: usize) -> Vec { + let mut out = Vec::new(); + let mut j = i + 1; + while j < s.len() && j - i <= 4 && s[j].word { + out.push(j); + if s[j].upper { + break; + } + j += 1; + } + out +} + +fn head(s: &[Tok], i: usize) -> Option { + phrase(s, i).last().copied().filter(|&j| s[j].upper) +} + +/// Clause span around `i`: between punctuation tokens. +fn clause(s: &[Tok], i: usize) -> (usize, usize) { + let lo = (0..i).rev().find(|&k| !s[k].word).map_or(0, |k| k + 1); + let hi = (i + 1..s.len()).find(|&k| !s[k].word).unwrap_or(s.len()); + (lo, hi) +} + +struct Tables { + time_nouns: HashSet, + time_adverbs: HashSet, + preps: HashSet, + verb_lemma: HashMap, + pair: HashMap<(String, String), [usize; 2]>, + /// Forms whose train majority UPOS is PROPN. + names: HashSet, + /// Form → [Acc seen, Dat seen]. + acc_dat: HashMap, +} + +impl Tables { + fn mine(train: &[Sent]) -> Self { + let mut t = Tables { + time_nouns: HashSet::new(), + time_adverbs: HashSet::new(), + preps: HashSet::new(), + verb_lemma: HashMap::new(), + pair: HashMap::new(), + acc_dat: HashMap::new(), + names: HashSet::new(), + }; + let mut propn: HashMap = HashMap::new(); + let mut adp: HashMap = HashMap::new(); + let mut verbs: HashMap> = HashMap::new(); + for sent in train { + let s = &sent.toks; + for (i, tok) in s.iter().enumerate() { + adp.entry(tok.form.clone()).or_default()[usize::from(tok.upos != "ADP")] += 1; + propn.entry(tok.form.clone()).or_default()[usize::from(tok.upos != "PROPN")] += 1; + match tok.case { + Some(1) => t.acc_dat.entry(tok.form.clone()).or_default()[0] += 1, + Some(2) => t.acc_dat.entry(tok.form.clone()).or_default()[1] += 1, + _ => {} + } + if tok.upos == "VERB" { + *verbs + .entry(tok.form.clone()) + .or_default() + .entry(tok.lemma.clone()) + .or_default() += 1; + } + if TEMPORAL_ONLY.contains(&tok.form.as_str()) { + if i > 0 && s[i - 1].upos == "ADV" { + t.time_adverbs.insert(s[i - 1].form.clone()); + } + let np = |j: usize| { + s[j].upos == "NOUN" + && s[i + 1..j] + .iter() + .all(|x| x.upos == "DET" || x.upos == "ADJ") + }; + if let Some(j) = head(s, i).filter(|&j| np(j)) { + t.time_nouns.insert(s[j].form.clone()); + } + } + if tok.deprel != "case" { + continue; + } + let Some(n) = tok.head else { continue }; + let rel = s[n].deprel.as_str(); + if rel != "obl" && rel != "obj" { + continue; + } + let Some(v) = s[n].head else { continue }; + if s[v].upos == "VERB" { + t.pair + .entry((s[v].lemma.clone(), tok.form.clone())) + .or_default()[usize::from(rel != "obj")] += 1; + } + } + } + // A noun used as an adverbial with no preposition (`obl` without a + // `case` child: *letzte Woche*, *jeden Tag*) is almost always temporal. + // Exactly `obl`: German `obl:arg` is bare dative objects. No `nummod` + // child: measure phrases (*5 Prozent*, *100 Dollar*) are bare `obl` too. + for sent in train { + let s = &sent.toks; + for (n, tok) in s.iter().enumerate() { + let child = |rel: &str| s.iter().any(|x| x.head == Some(n) && x.deprel == rel); + if tok.upos == "NOUN" && tok.deprel == "obl" && !child("case") && !child("nummod") { + t.time_nouns.insert(tok.form.clone()); + } + } + } + t.names = propn + .into_iter() + .filter(|(_, c)| c[0] > c[1]) + .map(|(f, _)| f) + .collect(); + t.preps = adp + .into_iter() + .filter(|(_, c)| c[0] > c[1]) + .map(|(f, _)| f) + .collect(); + t.verb_lemma = verbs + .into_iter() + .filter_map(|(f, m)| { + m.into_iter() + .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(&a.0))) + .map(|(l, _)| (f, l)) + }) + .collect(); + t + } + + /// A time noun, or a compound whose last part is one (*Geschäftsquartal*, + /// *Wahltag*): German compounds take their meaning from the rightmost part. + fn is_time_noun(&self, form: &str) -> bool { + if self.time_nouns.contains(form) { + return true; + } + let chars: Vec = form.chars().collect(); + (1..chars.len().saturating_sub(2)).any(|k| { + let tail: String = chars[k..].iter().collect(); + tail.chars().count() >= 3 && self.time_nouns.contains(&tail) + }) + } + + fn verb_governed(&self, s: &[Tok], i: usize) -> bool { + let (lo, hi) = clause(s, i); + (lo..hi) + .filter(|&k| k != i) + .filter_map(|k| self.verb_lemma.get(&s[k].form)) + .filter_map(|l| { + let c = self.pair.get(&(l.clone(), s[i].form.clone()))?; + let n = c[0] + c[1]; + (n >= MIN_PAIR).then(|| c[0] as f64 / n as f64) + }) + .any(|r| r >= 0.5) + } + + /// Every voter's output for the preposition at `i`, as named one-hot + /// features grouped by voter. + fn voters(&self, s: &[Tok], i: usize) -> Vec<(&'static str, String)> { + let h = head(s, i); + let noun = match h { + Some(j) if self.is_time_noun(&s[j].form) => "time", + Some(j) if abstract_noun(&s[j].form) => "abstract", + Some(j) if self.names.contains(&s[j].form) => "name", + Some(_) => "other", + None => "none", + }; + let ph = phrase(s, i); + let number = ph.iter().any(|&j| s[j].digit); + let adverb = i > 0 && self.time_adverbs.contains(&s[i - 1].form); + let verb = self.verb_governed(s, i); + let (lo, hi) = clause(s, i); + let vorfeld = i == lo; + let later_prep = (i + 1..hi) + .filter(|&k| !ph.contains(&k)) + .any(|k| self.preps.contains(&s[k].form)); + let article = ph + .first() + .and_then(|&j| self.acc_dat.get(&s[j].form)) + .map_or("unseen", |c| match (c[0] > 0, c[1] > 0) { + (true, false) => "acc", + (false, true) => "dat", + _ => "both", + }); + vec![ + ("noun", noun.to_string()), + ("number", number.to_string()), + ("adverb", adverb.to_string()), + ("verb", verb.to_string()), + ( + "position", + format!("vorfeld={vorfeld},later_prep={later_prep}"), + ), + ("article", format!("{}:{article}", s[i].form)), + ("prep", s[i].form.clone()), + ] + } +} + +/// A labeled item: features and lane. +struct Item { + feats: Vec<(&'static str, String)>, + lane: usize, + abstract_head: bool, + /// `sent_id:token`, for the error dump. + key: String, +} + +/// Multinomial logistic regression over one-hot features. +struct Model { + index: HashMap, + w: Vec<[f64; 3]>, +} + +impl Model { + fn keys(item: &Item, groups: &[&str]) -> Vec { + let mut k: Vec = item + .feats + .iter() + .filter(|(g, _)| groups.contains(g)) + .map(|(g, v)| format!("{g}={v}")) + .collect(); + k.push("bias".into()); + k + } + + fn fit(items: &[Item], groups: &[&str]) -> Self { + let mut index = HashMap::new(); + let rows: Vec<(Vec, usize)> = items + .iter() + .map(|it| { + let ks = Self::keys(it, groups) + .into_iter() + .map(|k| { + let n = index.len(); + *index.entry(k).or_insert(n) + }) + .collect(); + (ks, it.lane) + }) + .collect(); + let mut w = vec![[0.0f64; 3]; index.len()]; + let lambda: f64 = std::env::var("LANE_LAMBDA") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1e-3); + let lr = 0.5; + for _ in 0..3000 { + let mut g = vec![[0.0f64; 3]; w.len()]; + for (ks, y) in &rows { + let p = Self::softmax(&w, ks); + for &k in ks { + for c in 0..3 { + g[k][c] += p[c] - f64::from(u8::from(c == *y)); + } + } + } + let n = rows.len() as f64; + for (wk, gk) in w.iter_mut().zip(&g) { + for c in 0..3 { + wk[c] -= lr * (gk[c] / n + lambda * wk[c]); + } + } + } + Model { index, w } + } + + fn softmax(w: &[[f64; 3]], ks: &[usize]) -> [f64; 3] { + let mut z = [0.0; 3]; + for &k in ks { + for c in 0..3 { + z[c] += w[k][c]; + } + } + let m = z.iter().copied().fold(f64::MIN, f64::max); + let e = z.map(|v| (v - m).exp()); + let sum: f64 = e.iter().sum(); + e.map(|v| v / sum) + } + + fn predict(&self, item: &Item, groups: &[&str]) -> usize { + let ks: Vec = Self::keys(item, groups) + .iter() + .filter_map(|k| self.index.get(k).copied()) + .collect(); + let p = Self::softmax(&self.w, &ks); + (0..3).max_by(|&a, &b| p[a].total_cmp(&p[b])).unwrap_or(0) + } +} + +fn score(model: &Model, test: &[Item], groups: &[&str]) -> (f64, [f64; 3]) { + let mut conf = [[0usize; 3]; 3]; + for it in test { + conf[it.lane][model.predict(it, groups)] += 1; + } + let right: usize = (0..3).map(|c| conf[c][c]).sum(); + let f1 = std::array::from_fn(|c| { + let tp = conf[c][c] as f64; + let pred: usize = (0..3).map(|g| conf[g][c]).sum(); + let gold: usize = conf[c].iter().sum(); + if tp == 0.0 { + 0.0 + } else { + 2.0 * tp / (pred + gold) as f64 + } + }); + (right as f64 / test.len().max(1) as f64, f1) +} + +fn lane_of(s: &str) -> Option { + LANES.iter().position(|l| *l == s) +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let [train_path, test_path, labels_path] = args.as_slice() else { + eprintln!("usage: ud_lane_quorum TRAIN.conllu TEST.conllu LABELS.tsv"); + std::process::exit(2); + }; + let train = read(train_path); + let test = read(test_path); + let tables = Tables::mine(&train); + let by_id = |xs: &[Sent]| -> HashMap { + xs.iter() + .enumerate() + .map(|(k, s)| (s.id.clone(), k)) + .collect() + }; + let (train_ix, test_ix) = (by_id(&train), by_id(&test)); + + let mut fit_items = Vec::new(); + let mut test_items = Vec::new(); + let mut kappa = [[0usize; 3]; 3]; + let mut test_labeled = 0usize; + // Which held-out split to score: `test` (exploratory) or `confirm` (the + // fresh, pre-registered round). + let eval_split = std::env::var("LANE_SPLIT").unwrap_or_else(|_| "test".into()); + let labels = std::fs::read_to_string(labels_path).expect("labels"); + for line in labels.lines().filter(|l| !l.starts_with('#')) { + let c: Vec<&str> = line.split('\t').collect(); + let [split, sid, tok, a, b] = c.as_slice() else { + panic!("labels row must have exactly 5 columns: {line}"); + }; + if *split != "train" && *split != eval_split { + continue; + } + let (sents, ix) = if *split == "train" { + (&train, &train_ix) + } else { + (&test, &test_ix) + }; + let s = &sents[*ix.get(*sid).unwrap_or_else(|| panic!("unknown {sid}"))].toks; + let i: usize = tok.parse::().expect("tok") - 1; + assert!( + WECHSEL.contains(&s[i].form.as_str()), + "{sid}:{tok} is not a Wechsel preposition" + ); + let la = lane_of(a).expect("label a"); + let item = |lane| Item { + feats: tables.voters(s, i), + lane, + abstract_head: head(s, i).is_some_and(|j| abstract_noun(&s[j].form)), + key: format!("{sid}:{tok}"), + }; + if *split == "train" { + fit_items.push(item(la)); + } else { + test_labeled += 1; + let lb = lane_of(b).expect("label b"); + kappa[la][lb] += 1; + if la == lb { + test_items.push(item(la)); + } + } + } + + let n: usize = kappa.iter().flatten().sum(); + let po = (0..3).map(|c| kappa[c][c]).sum::() as f64 / n.max(1) as f64; + let pe: f64 = (0..3) + .map(|c| { + let ra: usize = kappa[c].iter().sum(); + let rb: usize = (0..3).map(|g| kappa[g][c]).sum(); + ra as f64 * rb as f64 / (n * n).max(1) as f64 + }) + .sum(); + let k = (po - pe) / (1.0 - pe); + println!( + "labels: {} train, {test_labeled} test double-labeled, {} agreed ({:.1}%), Cohen kappa {k:.3} {}", + fit_items.len(), + test_items.len(), + 100.0 * po, + if k >= 0.6 { "RELIABLE" } else { "UNRELIABLE (kappa < 0.6)" } + ); + let dist = |xs: &[Item]| -> String { + (0..3) + .map(|c| format!("{} {}", LANES[c], xs.iter().filter(|x| x.lane == c).count())) + .collect::>() + .join(", ") + }; + println!(" train lanes: {}", dist(&fit_items)); + println!(" test lanes (agreed): {}", dist(&test_items)); + println!( + " mined: {} time nouns, {} temporal adverbs, {} (verb, prep) pairs", + tables.time_nouns.len(), + tables.time_adverbs.len(), + tables.pair.len() + ); + + // Operator hypothesis: an abstract head is never PLACE. + let abs: Vec<&Item> = fit_items + .iter() + .chain(&test_items) + .filter(|x| x.abstract_head) + .collect(); + let abs_place = abs.iter().filter(|x| x.lane == 1).count(); + let rate = 100.0 * abs_place as f64 / abs.len().max(1) as f64; + println!( + "\nabstract head -> TIME or FIG, never PLACE: {} abstract items, {abs_place} PLACE ({rate:.1}%) {}", + abs.len(), + if rate <= 5.0 { "PASS" } else { "KILL" } + ); + for (c, name) in LANES.iter().enumerate() { + let m = abs.iter().filter(|x| x.lane == c).count(); + println!(" {name}: {m}"); + } + + let all = [ + "noun", "number", "adverb", "verb", "position", "article", "prep", + ]; + println!("\nsingle voters (each with the preposition prior):"); + let mut best = 0.0f64; + for g in [ + "prep", "noun", "number", "adverb", "verb", "position", "article", + ] { + let groups = if g == "prep" { + vec!["prep"] + } else { + vec![g, "prep"] + }; + let m = Model::fit(&fit_items, &groups); + let (acc, f1) = score(&m, &test_items, &groups); + best = best.max(acc); + println!( + " {g:9} accuracy {:5.1}% F1 TIME {:.3} PLACE {:.3} FIG {:.3}", + 100.0 * acc, + f1[0], + f1[1], + f1[2] + ); + } + let m = Model::fit(&fit_items, &all); + let (acc, f1) = score(&m, &test_items, &all); + if std::env::var("LANE_DUMP").is_ok() { + for it in &test_items { + let p = m.predict(it, &all); + if p != it.lane { + eprintln!( + "{}\t{}\t{}\t{:?}", + it.key, LANES[it.lane], LANES[p], it.feats + ); + } + } + } + println!( + "\nquorum (all voters): accuracy {:5.1}% F1 TIME {:.3} PLACE {:.3} FIG {:.3}", + 100.0 * acc, + f1[0], + f1[1], + f1[2] + ); + for drop in ["noun", "number", "adverb", "verb", "position", "article"] { + let groups: Vec<&str> = all.iter().copied().filter(|g| *g != drop).collect(); + let m = Model::fit(&fit_items, &groups); + let (a, _) = score(&m, &test_items, &groups); + println!(" without {drop:9}: accuracy {:5.1}%", 100.0 * a); + } + let pass = acc >= best + 0.02 && f1[0] >= 0.80 && f1[1] >= 0.60; + println!( + " quorum {} (bars: >= best single + 2 pts ({:.1}%), TIME F1 >= 0.80, PLACE F1 >= 0.60)", + if pass { "PASS" } else { "KILL" }, + 100.0 * (best + 0.02) + ); +} From 1349006fd8d96b02d1c6288f49ec7783aac4db36 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:01:54 +0000 Subject: [PATCH 21/33] deepnsm-v2 lane quorum: confirmation round on fresh labels (D-LXC-25) Frozen design, 400 fresh HDT test phrases labelled after the freeze (kappa 0.969): - noun voter TIME F1 0.831 >= 0.80: PASS; - PLACE F1 0.476 < 0.55: KILL; - quorum 85.3 % vs noun voter 84.8 %: +0.5, below the +2 bar, KILL; - abstract head never PLACE: 3 of 94 (3.2 %), PASS, replicated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 2 +- ...26-10-04-deepnsm-v2-wechsel-lane-quorum.md | 6 - .../examples/data/hdt_wechsel_lanes.tsv | 401 ++++++++++++++++++ 3 files changed, 402 insertions(+), 7 deletions(-) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 554d0d30e..5ba49a96c 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -48,7 +48,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-22** | DECISION: WordNet hypernym metric may feed ndarray CLAM/CHAODA offline as an evaluation instrument (identities unchanged, `48405aa2` stands); grammar is not a thinking style → `tekamolo::read_clause` takes `ReadParams`, style adapter kept for the planner / ogar-loco | In PR | presets pinned to the adapter rows (disable-verified); 195 lib tests. Entry `2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md` | | **D-LXC-23** | first probes from the German grammar map: der/die/das DET vs PRON voters (`ud_pos_eval`), `ud_case_eval` (fixed-case prepositions, article decidability, Wechselpräposition, relative pronoun), `tekamolo_corners` | In PR | DET/PRON German 92.3 → 96.0 % (pronoun P 90.0 / R 54.1) PASS; case on HDT: Acc 90.0 / Dat 92.5 % PASS, Gen KILL, Wechsel article-decided 99.7 %, verb prior KILL (63.0 vs 63.9), relative pronoun 80.1 vs 63.8 % PASS; right-corner-on-Bible prediction KILL (Luther 7.1 % vs GSD 12.3 %). Entry `2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md` | | **D-LXC-24** | prepositional object vs adverbial (`ud_pp_arg_eval`): the (verb, preposition) object rate takes verb-governed phrases out of the TEKAMOLO pool and fixes their case; abstract-noun suffix class | In PR | HDT (gold = `obj` + ADP; German `obl:arg` is bare datives): A1 P 77.2 / R 81.6 %, 896 of 1,098 objects removed, 264 of 19,466 adverbials lost, PASS; A2 case 96.8 vs 81.0 % PASS; abstract heads 10.7 % objects vs 4.6 %; R3a KILL on HDT. Entry `2026-10-04-deepnsm-v2-prepositional-object-detector.md` | -| **D-LXC-25** | TIME / PLACE / FIG lane of a Wechsel phrase (`ud_lane_quorum`): voters for noun class, number, temporal adverb, verb governance, TEKAMOLO position and article case, combined by logistic regression; silver labels (κ 0.958) | In progress | operator hypothesis (abstract → never PLACE) PASS 3.8 %; round 1 KILL (72.7 % < noun 80.4 %); round 2 exploratory: noun 86.5 %, quorum 88.0 %; round 3 (fresh labels) pre-registered, pending. Entry `2026-10-04-deepnsm-v2-wechsel-lane-quorum.md` | +| **D-LXC-25** | TIME / PLACE / FIG lane of a Wechsel phrase (`ud_lane_quorum`): noun, number, adverb, verb governance, TEKAMOLO position and article voters, combined by logistic regression; silver labels | In PR | abstract head never PLACE: PASS, replicated (3.8 %, 3.2 %). Confirmation round (fresh labels, frozen design): noun voter TIME F1 0.831 PASS, PLACE F1 0.476 KILL, quorum +0.5 over noun KILL. Entry `2026-10-04-deepnsm-v2-wechsel-lane-quorum.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md index b83582d42..66bef9846 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md @@ -39,9 +39,3 @@ Operator: - The quorum misses its "+2 points over the best single voter" bar: noun context carries almost all the signal. - TEKAMOLO position, adverb and verb add little on this sample. -**Round 3 (pre-registered before its labels existed): pending.** -- Data: a fresh sample of 400 HDT test phrases, two labelers each. -- Bars: - - κ ≥ 0.6; - - noun voter TIME F1 ≥ 0.80 and PLACE F1 ≥ 0.55; - - quorum ≥ noun + 2 points. diff --git a/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.tsv b/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.tsv index ea3ce7416..9270116ac 100644 --- a/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.tsv +++ b/crates/deepnsm-v2/examples/data/hdt_wechsel_lanes.tsv @@ -1,6 +1,7 @@ # Silver lane labels for German HDT (UD r2.15) Wechsel-preposition phrases: TIME / PLACE / FIG. # Written by independent model labelers from the full sentence (brief: deepnsm-v2 D-LXC-25 entry). # Train: one labeler (label_b = -). Test: two independent labelers. Ids only; no HDT text (CC BY-SA 4.0). +# Confirm: a fresh test sample, labelled after the design was frozen (fresh_0: labelers A+C; B discarded for rule-based labeling). # columns: split, sent_id, token (1-based word index), label_a, label_b train hdt-s31655 5 FIG - train hdt-s23868 8 TIME - @@ -1202,3 +1203,403 @@ test hdt-s17615 1 FIG FIG test hdt-s19181 31 FIG FIG test hdt-s15328 17 FIG FIG test hdt-s121674 1 FIG FIG +confirm hdt-s115992 8 FIG FIG +confirm hdt-s114078 1 TIME TIME +confirm hdt-s18281 8 FIG FIG +confirm hdt-s15871 1 FIG FIG +confirm hdt-s116179 5 FIG FIG +confirm hdt-s112250 13 FIG FIG +confirm hdt-s19997 23 FIG FIG +confirm hdt-s116097 22 FIG FIG +confirm hdt-s10385 5 TIME TIME +confirm hdt-s19009 16 FIG FIG +confirm hdt-s10659 22 PLACE PLACE +confirm hdt-s118251 11 FIG FIG +confirm hdt-s15484 2 TIME TIME +confirm hdt-s15301 5 FIG FIG +confirm hdt-s113477 14 TIME TIME +confirm hdt-s15918 17 FIG FIG +confirm hdt-s19334 21 FIG FIG +confirm hdt-s12194 11 FIG FIG +confirm hdt-s14261 10 TIME TIME +confirm hdt-s11207 11 FIG FIG +confirm hdt-s118001 1 TIME TIME +confirm hdt-s112201 1 FIG FIG +confirm hdt-s17684 5 TIME TIME +confirm hdt-s12676 7 TIME TIME +confirm hdt-s115123 19 TIME TIME +confirm hdt-s119464 12 FIG FIG +confirm hdt-s14435 5 FIG FIG +confirm hdt-s10662 17 FIG FIG +confirm hdt-s14391 2 TIME TIME +confirm hdt-s13621 5 FIG FIG +confirm hdt-s15102 34 FIG FIG +confirm hdt-s16281 1 FIG FIG +confirm hdt-s11988 7 FIG FIG +confirm hdt-s19651 13 FIG FIG +confirm hdt-s113448 1 PLACE PLACE +confirm hdt-s18009 3 FIG FIG +confirm hdt-s10968 4 FIG FIG +confirm hdt-s16402 7 PLACE PLACE +confirm hdt-s118105 1 TIME TIME +confirm hdt-s12125 18 FIG FIG +confirm hdt-s112905 23 FIG FIG +confirm hdt-s116666 5 TIME TIME +confirm hdt-s15177 35 FIG FIG +confirm hdt-s13883 2 PLACE PLACE +confirm hdt-s13446 10 FIG FIG +confirm hdt-s116961 7 FIG FIG +confirm hdt-s10776 1 TIME TIME +confirm hdt-s120062 16 FIG FIG +confirm hdt-s116509 1 TIME TIME +confirm hdt-s18252 8 PLACE PLACE +confirm hdt-s10782 9 PLACE PLACE +confirm hdt-s13160 8 TIME TIME +confirm hdt-s14813 1 FIG FIG +confirm hdt-s12856 1 TIME TIME +confirm hdt-s118902 15 FIG FIG +confirm hdt-s116749 5 FIG FIG +confirm hdt-s117740 13 FIG FIG +confirm hdt-s121464 8 PLACE PLACE +confirm hdt-s112129 1 FIG FIG +confirm hdt-s121740 7 FIG FIG +confirm hdt-s11035 8 TIME TIME +confirm hdt-s10544 3 FIG FIG +confirm hdt-s119075 9 FIG FIG +confirm hdt-s113595 26 TIME TIME +confirm hdt-s16227 9 FIG FIG +confirm hdt-s14940 11 FIG FIG +confirm hdt-s12399 18 PLACE PLACE +confirm hdt-s11458 1 FIG FIG +confirm hdt-s13951 7 FIG FIG +confirm hdt-s14547 5 TIME TIME +confirm hdt-s117224 8 FIG FIG +confirm hdt-s13151 10 FIG FIG +confirm hdt-s121315 15 FIG FIG +confirm hdt-s117405 6 FIG FIG +confirm hdt-s13650 1 FIG FIG +confirm hdt-s18739 4 FIG FIG +confirm hdt-s17874 9 FIG FIG +confirm hdt-s118183 5 FIG FIG +confirm hdt-s115249 9 FIG FIG +confirm hdt-s118939 13 PLACE PLACE +confirm hdt-s117109 7 FIG FIG +confirm hdt-s118674 4 FIG FIG +confirm hdt-s13034 13 FIG FIG +confirm hdt-s19966 11 FIG FIG +confirm hdt-s121831 16 TIME TIME +confirm hdt-s17501 3 PLACE PLACE +confirm hdt-s19003 26 FIG FIG +confirm hdt-s112005 24 FIG FIG +confirm hdt-s115844 10 FIG FIG +confirm hdt-s113781 10 TIME TIME +confirm hdt-s115932 10 FIG FIG +confirm hdt-s112675 5 TIME TIME +confirm hdt-s118321 1 FIG FIG +confirm hdt-s12932 16 FIG FIG +confirm hdt-s17625 9 FIG FIG +confirm hdt-s15803 5 FIG FIG +confirm hdt-s17605 25 FIG FIG +confirm hdt-s11137 15 FIG FIG +confirm hdt-s18018 17 FIG FIG +confirm hdt-s16359 1 FIG FIG +confirm hdt-s119245 9 FIG FIG +confirm hdt-s18699 20 FIG FIG +confirm hdt-s12370 1 FIG FIG +confirm hdt-s18553 1 FIG FIG +confirm hdt-s15492 1 TIME TIME +confirm hdt-s120050 6 FIG FIG +confirm hdt-s113950 5 PLACE PLACE +confirm hdt-s14327 19 FIG FIG +confirm hdt-s18177 1 FIG FIG +confirm hdt-s116122 6 FIG FIG +confirm hdt-s14220 12 PLACE PLACE +confirm hdt-s118573 7 FIG FIG +confirm hdt-s16826 8 TIME TIME +confirm hdt-s120204 31 FIG FIG +confirm hdt-s116449 8 PLACE PLACE +confirm hdt-s114966 1 FIG FIG +confirm hdt-s19790 11 FIG FIG +confirm hdt-s114845 1 FIG FIG +confirm hdt-s11103 8 FIG PLACE +confirm hdt-s16301 5 FIG FIG +confirm hdt-s15462 12 FIG FIG +confirm hdt-s15406 12 FIG FIG +confirm hdt-s120721 9 FIG FIG +confirm hdt-s120884 14 FIG FIG +confirm hdt-s16250 10 FIG FIG +confirm hdt-s117133 2 FIG FIG +confirm hdt-s12640 19 FIG FIG +confirm hdt-s11556 33 FIG FIG +confirm hdt-s120607 22 FIG FIG +confirm hdt-s114784 22 FIG FIG +confirm hdt-s17622 4 FIG FIG +confirm hdt-s116665 14 FIG FIG +confirm hdt-s113134 1 FIG FIG +confirm hdt-s112269 1 FIG FIG +confirm hdt-s15375 17 TIME TIME +confirm hdt-s19368 3 TIME TIME +confirm hdt-s115931 6 TIME TIME +confirm hdt-s18256 2 FIG FIG +confirm hdt-s14358 22 TIME TIME +confirm hdt-s112114 13 PLACE PLACE +confirm hdt-s117335 7 PLACE PLACE +confirm hdt-s112980 12 FIG FIG +confirm hdt-s116797 11 FIG FIG +confirm hdt-s120018 24 FIG FIG +confirm hdt-s18617 2 TIME TIME +confirm hdt-s13585 2 FIG FIG +confirm hdt-s120283 10 FIG FIG +confirm hdt-s119292 16 FIG FIG +confirm hdt-s118327 6 TIME TIME +confirm hdt-s18411 3 FIG PLACE +confirm hdt-s119758 23 FIG FIG +confirm hdt-s16413 1 FIG PLACE +confirm hdt-s119433 14 FIG FIG +confirm hdt-s112566 4 TIME TIME +confirm hdt-s12651 14 FIG FIG +confirm hdt-s18906 1 FIG FIG +confirm hdt-s120098 31 TIME TIME +confirm hdt-s119330 4 PLACE PLACE +confirm hdt-s115730 5 TIME TIME +confirm hdt-s12023 5 FIG FIG +confirm hdt-s121574 21 PLACE FIG +confirm hdt-s10994 7 FIG FIG +confirm hdt-s16908 4 FIG FIG +confirm hdt-s117186 18 PLACE PLACE +confirm hdt-s120217 1 FIG FIG +confirm hdt-s113637 4 PLACE FIG +confirm hdt-s115371 8 FIG FIG +confirm hdt-s16069 4 FIG FIG +confirm hdt-s117820 7 TIME TIME +confirm hdt-s10604 4 FIG FIG +confirm hdt-s17924 31 TIME TIME +confirm hdt-s15857 12 FIG FIG +confirm hdt-s120743 28 FIG FIG +confirm hdt-s116717 5 FIG FIG +confirm hdt-s15247 10 TIME TIME +confirm hdt-s10987 12 FIG FIG +confirm hdt-s121401 25 FIG FIG +confirm hdt-s115725 26 FIG FIG +confirm hdt-s10996 3 FIG FIG +confirm hdt-s18423 8 FIG FIG +confirm hdt-s19788 2 TIME TIME +confirm hdt-s10851 5 FIG FIG +confirm hdt-s121150 20 FIG FIG +confirm hdt-s18884 21 FIG FIG +confirm hdt-s116186 1 TIME TIME +confirm hdt-s16582 1 FIG FIG +confirm hdt-s11657 8 TIME TIME +confirm hdt-s121215 15 FIG FIG +confirm hdt-s113856 13 FIG FIG +confirm hdt-s120623 27 FIG FIG +confirm hdt-s16473 6 FIG FIG +confirm hdt-s119950 3 FIG FIG +confirm hdt-s12354 10 FIG FIG +confirm hdt-s121680 1 FIG FIG +confirm hdt-s11470 21 FIG FIG +confirm hdt-s16703 10 FIG FIG +confirm hdt-s115494 10 FIG FIG +confirm hdt-s16358 1 FIG PLACE +confirm hdt-s19991 16 FIG FIG +confirm hdt-s119784 18 PLACE PLACE +confirm hdt-s13221 12 TIME TIME +confirm hdt-s118230 1 PLACE PLACE +confirm hdt-s10617 29 FIG FIG +confirm hdt-s113219 5 TIME TIME +confirm hdt-s13872 2 TIME TIME +confirm hdt-s113022 5 FIG FIG +confirm hdt-s116290 11 FIG FIG +confirm hdt-s12453 13 TIME TIME +confirm hdt-s121915 9 FIG FIG +confirm hdt-s113472 7 FIG FIG +confirm hdt-s118219 16 FIG FIG +confirm hdt-s115620 3 FIG FIG +confirm hdt-s120697 16 FIG FIG +confirm hdt-s16323 7 FIG FIG +confirm hdt-s13143 19 FIG FIG +confirm hdt-s12641 7 FIG FIG +confirm hdt-s115529 7 FIG FIG +confirm hdt-s116302 8 FIG FIG +confirm hdt-s114733 8 FIG FIG +confirm hdt-s15690 9 PLACE PLACE +confirm hdt-s118874 14 FIG FIG +confirm hdt-s16105 1 FIG FIG +confirm hdt-s19550 12 FIG FIG +confirm hdt-s15649 10 TIME TIME +confirm hdt-s13310 8 TIME TIME +confirm hdt-s12392 6 PLACE PLACE +confirm hdt-s10545 3 FIG FIG +confirm hdt-s113367 14 FIG FIG +confirm hdt-s113143 29 FIG FIG +confirm hdt-s113314 1 TIME TIME +confirm hdt-s113675 1 FIG FIG +confirm hdt-s10587 37 FIG FIG +confirm hdt-s16892 2 TIME TIME +confirm hdt-s11955 7 TIME TIME +confirm hdt-s117650 21 FIG FIG +confirm hdt-s14219 17 FIG FIG +confirm hdt-s15285 15 TIME TIME +confirm hdt-s19944 1 FIG FIG +confirm hdt-s121183 1 FIG FIG +confirm hdt-s19897 8 FIG FIG +confirm hdt-s114559 8 PLACE PLACE +confirm hdt-s15945 1 FIG FIG +confirm hdt-s119370 25 FIG FIG +confirm hdt-s121380 33 FIG FIG +confirm hdt-s116829 11 FIG FIG +confirm hdt-s119760 2 FIG FIG +confirm hdt-s19131 12 TIME TIME +confirm hdt-s19851 5 FIG FIG +confirm hdt-s15722 8 TIME TIME +confirm hdt-s18873 4 TIME TIME +confirm hdt-s10943 11 FIG FIG +confirm hdt-s19965 23 FIG FIG +confirm hdt-s121051 19 TIME TIME +confirm hdt-s11973 8 FIG FIG +confirm hdt-s16835 5 FIG FIG +confirm hdt-s113565 25 FIG FIG +confirm hdt-s112172 27 FIG FIG +confirm hdt-s112257 12 FIG FIG +confirm hdt-s19072 9 FIG FIG +confirm hdt-s112585 1 PLACE PLACE +confirm hdt-s19772 2 TIME TIME +confirm hdt-s112332 9 FIG FIG +confirm hdt-s13416 1 TIME TIME +confirm hdt-s12472 6 FIG FIG +confirm hdt-s19553 1 TIME TIME +confirm hdt-s113053 33 FIG FIG +confirm hdt-s15519 8 FIG FIG +confirm hdt-s116878 5 FIG FIG +confirm hdt-s10479 16 FIG FIG +confirm hdt-s115686 6 FIG FIG +confirm hdt-s18427 3 FIG FIG +confirm hdt-s115680 19 FIG FIG +confirm hdt-s117262 29 PLACE PLACE +confirm hdt-s118675 21 PLACE PLACE +confirm hdt-s117478 20 PLACE PLACE +confirm hdt-s17898 19 FIG FIG +confirm hdt-s11380 1 TIME TIME +confirm hdt-s17165 24 TIME TIME +confirm hdt-s12292 16 FIG FIG +confirm hdt-s112736 1 FIG FIG +confirm hdt-s115487 1 FIG FIG +confirm hdt-s10763 7 FIG FIG +confirm hdt-s16114 4 FIG FIG +confirm hdt-s120098 6 TIME TIME +confirm hdt-s117534 1 FIG FIG +confirm hdt-s13255 1 FIG FIG +confirm hdt-s120619 5 FIG FIG +confirm hdt-s119359 7 FIG FIG +confirm hdt-s11788 11 TIME TIME +confirm hdt-s14258 26 FIG FIG +confirm hdt-s116865 5 FIG FIG +confirm hdt-s118877 1 TIME TIME +confirm hdt-s13018 10 FIG FIG +confirm hdt-s13376 13 FIG FIG +confirm hdt-s10602 16 TIME TIME +confirm hdt-s13198 8 FIG FIG +confirm hdt-s19913 7 FIG FIG +confirm hdt-s118746 19 FIG FIG +confirm hdt-s117932 9 FIG FIG +confirm hdt-s116265 19 TIME TIME +confirm hdt-s19764 17 FIG FIG +confirm hdt-s11250 8 FIG FIG +confirm hdt-s13180 1 FIG FIG +confirm hdt-s121666 1 TIME TIME +confirm hdt-s116873 10 PLACE PLACE +confirm hdt-s116155 12 FIG FIG +confirm hdt-s114884 1 FIG FIG +confirm hdt-s119307 13 FIG FIG +confirm hdt-s17080 10 PLACE PLACE +confirm hdt-s115526 21 TIME TIME +confirm hdt-s13198 48 FIG FIG +confirm hdt-s120944 1 FIG FIG +confirm hdt-s120657 33 PLACE PLACE +confirm hdt-s15647 7 TIME TIME +confirm hdt-s112481 4 FIG FIG +confirm hdt-s17297 7 FIG FIG +confirm hdt-s16468 4 FIG FIG +confirm hdt-s19740 12 FIG FIG +confirm hdt-s19759 18 TIME TIME +confirm hdt-s119819 16 FIG FIG +confirm hdt-s120037 12 TIME TIME +confirm hdt-s13260 9 FIG FIG +confirm hdt-s18821 4 TIME TIME +confirm hdt-s121091 12 FIG FIG +confirm hdt-s17962 1 TIME TIME +confirm hdt-s15683 4 TIME TIME +confirm hdt-s10965 22 FIG FIG +confirm hdt-s12996 6 FIG FIG +confirm hdt-s13226 7 FIG FIG +confirm hdt-s19712 12 FIG FIG +confirm hdt-s113807 7 FIG FIG +confirm hdt-s18893 16 FIG FIG +confirm hdt-s16151 17 PLACE PLACE +confirm hdt-s14586 9 FIG FIG +confirm hdt-s15227 6 FIG FIG +confirm hdt-s16022 5 TIME TIME +confirm hdt-s114752 10 FIG FIG +confirm hdt-s10288 19 PLACE PLACE +confirm hdt-s120072 22 TIME TIME +confirm hdt-s12961 21 PLACE PLACE +confirm hdt-s12999 1 FIG FIG +confirm hdt-s12686 6 FIG FIG +confirm hdt-s14010 11 TIME TIME +confirm hdt-s114799 1 TIME TIME +confirm hdt-s15216 17 TIME TIME +confirm hdt-s13359 29 TIME TIME +confirm hdt-s17101 9 FIG FIG +confirm hdt-s112137 14 TIME TIME +confirm hdt-s115295 16 TIME TIME +confirm hdt-s18771 5 FIG FIG +confirm hdt-s12775 12 FIG FIG +confirm hdt-s15176 9 FIG FIG +confirm hdt-s112833 11 PLACE PLACE +confirm hdt-s13046 9 TIME TIME +confirm hdt-s15715 7 FIG FIG +confirm hdt-s12498 7 TIME TIME +confirm hdt-s16588 22 FIG FIG +confirm hdt-s118887 33 FIG FIG +confirm hdt-s18746 11 FIG FIG +confirm hdt-s121845 1 FIG FIG +confirm hdt-s17540 1 FIG FIG +confirm hdt-s19106 12 FIG FIG +confirm hdt-s119567 14 FIG FIG +confirm hdt-s17799 3 FIG FIG +confirm hdt-s112566 8 FIG FIG +confirm hdt-s13073 1 TIME TIME +confirm hdt-s114634 14 FIG FIG +confirm hdt-s120599 23 FIG FIG +confirm hdt-s117308 11 FIG FIG +confirm hdt-s120181 28 FIG FIG +confirm hdt-s114710 23 FIG FIG +confirm hdt-s118517 17 PLACE PLACE +confirm hdt-s119604 3 PLACE PLACE +confirm hdt-s114586 29 FIG FIG +confirm hdt-s116207 5 FIG FIG +confirm hdt-s113004 21 PLACE PLACE +confirm hdt-s15237 1 TIME TIME +confirm hdt-s13521 9 FIG FIG +confirm hdt-s10821 11 FIG FIG +confirm hdt-s118562 17 FIG FIG +confirm hdt-s15371 10 FIG FIG +confirm hdt-s112496 14 FIG FIG +confirm hdt-s118036 3 FIG FIG +confirm hdt-s119464 10 FIG FIG +confirm hdt-s118889 1 FIG FIG +confirm hdt-s15422 4 TIME TIME +confirm hdt-s119718 2 TIME TIME +confirm hdt-s13280 11 PLACE PLACE +confirm hdt-s114576 6 FIG FIG +confirm hdt-s13018 18 FIG FIG +confirm hdt-s14009 6 FIG FIG +confirm hdt-s121771 14 FIG FIG +confirm hdt-s112499 3 FIG FIG +confirm hdt-s112412 23 FIG FIG +confirm hdt-s19418 32 TIME TIME +confirm hdt-s18655 29 TIME TIME +confirm hdt-s114123 28 FIG FIG +confirm hdt-s17598 1 FIG FIG +confirm hdt-s10226 6 FIG FIG +confirm hdt-s120779 8 FIG FIG From 80760f9b76bb075e2c87ab0a31d4eae57267c547 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:02:31 +0000 Subject: [PATCH 22/33] board: restore the D-LXC-25 confirmation-round section cut by the previous edit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- ...26-10-04-deepnsm-v2-wechsel-lane-quorum.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md index 66bef9846..cad4d9747 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md @@ -39,3 +39,30 @@ Operator: - The quorum misses its "+2 points over the best single voter" bar: noun context carries almost all the signal. - TEKAMOLO position, adverb and verb add little on this sample. +**Round 3 (pre-registered before its labels existed; design frozen).** Data: a fresh sample of 400 HDT test phrases. fresh_0 used labelers A + C, fresh_1 used A + B. κ = 0.969, subject to the same-model caveat above. + +| bar | result | verdict | +|---|---|---| +| κ ≥ 0.6 | 0.969 | — | +| noun voter TIME F1 ≥ 0.80 | **0.831** | PASS | +| noun voter PLACE F1 ≥ 0.55 | 0.476 | **KILL** | +| quorum ≥ noun + 2 points | 85.3 % vs 84.8 % (+0.5) | **KILL** | +| abstract head never PLACE (≤ 5 %) | 3 of 94 (3.2 %) | PASS (replicated) | + +**Verdict.** +- Noun context detects TIME reliably, and the abstract-head rule replicates. +- PLACE is not solved. +- On HDT, TEKAMOLO position, adverb, verb and article add nothing measurable over the noun voter. + +**Exceptions to the abstract rule.** The abstract-head PLACE items across all labelled samples: +- concrete *-ung* artifacts: *Verpackung*, *Packung*; +- a direction: *in Richtung*; +- an institution: *an der Bauhaus-Universität*. + +The suffix class mixes event and abstract nouns with object nouns (*Wohnung*, *Zeitung*, *Leitung*, *Heizung*). + +**Register.** Operator: colloquial *auf der Arbeit* reads as PLACE. HDT is formal IT-news text (heise/c't), so colloquial place readings are under-sampled here. A colloquial frequency source is a prerequisite for that axis. + +**OPEN.** +- A PLACE signal beyond names: object nouns, a physical-artifact lexicon, gender or semantic class. +- The German lexicon (`build_de_codebook.py`, UD-derived, COCA shape) has no gender column, although UD carries `Gender=`. From 60c2ed183e1f859e4efe0c04df9e00a5622d7b30 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:05:20 +0000 Subject: [PATCH 23/33] =?UTF-8?q?deepnsm-v2=20lane=20quorum:=20adverbs=20o?= =?UTF-8?q?f=20time,=20including=20adjectives=20used=20as=20adverbs=20(fr?= =?UTF-8?q?=C3=BCh)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first time-adverb list took the word right before seit/während/bis, which caught focus particles (auch seit, nur bis). The voter now mines by lift: ADV, or ADJ used as advmod (früh, spät), occurring >= 2x more often in clauses with a time-only preposition. It looks two tokens before the phrase and one after. LANE_ADVERBS=codebook uses the codebook builder's fixed TEMPORAL list instead. Exploratory (both samples already seen): neither list helps, and the quorum without the adverb voter scores 86.3 % on confirm. A time adverb in the clause raises the TIME share (32.8 vs 21.0 %): it mostly modifies the time phrase itself, so its signal overlaps the noun voter. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- ...26-10-04-deepnsm-v2-wechsel-lane-quorum.md | 22 +++++ crates/deepnsm-v2/examples/ud_lane_quorum.rs | 98 +++++++++++++++++-- 2 files changed, 114 insertions(+), 6 deletions(-) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md index cad4d9747..1c7d50aa9 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md @@ -66,3 +66,25 @@ The suffix class mixes event and abstract nouns with object nouns (*Wohnung*, *Z **OPEN.** - A PLACE signal beyond names: object nouns, a physical-artifact lexicon, gender or semantic class. - The German lexicon (`build_de_codebook.py`, UD-derived, COCA shape) has no gender column, although UD carries `Gender=`. + +## Addendum: adverbs of time (operator: *früh*) — exploratory, both samples already seen + +**Coverage.** *früh* was **not** covered. +- HDT tags it ADJ + `advmod` (an uninflected adjective used adverbially, the same German adj/adv split as D-LXC-20), and the first miner took UPOS ADV only. +- It occurs about 17 times in HDT train (formal IT news). +- The first "temporal adverb" list was mined as "the word right before *seit/während/bis*". It was mostly focus particles and other lanes: *auch*, *nur*, *sogar*, *aber*, *hier*, *unten*, *daher*. + +| adverb source | test quorum | confirm quorum | confirm, without the adverb voter | +|---|---|---|---| +| word before *seit/während/bis* (first version) | 88.0 % | 85.3 % | 84.8 % | +| lift-mined (ADV, or ADJ-`advmod`, ≥ 2× in clauses with a time-only preposition) | 88.0 % | 85.8 % | 86.3 % | +| codebook `TEMPORAL` list (`build_de_codebook.py`: *früh, spät, heute, damals, …*; `LANE_ADVERBS=codebook`) | 88.3 % | 85.3 % | 86.3 % | + +**Direction (all 1,586 agreed labels).** Testing the operator's Sudoku idea that a filled Te slot pushes the PP to another lane: +- a clause with a time adverb has **more** TIME phrases (32.8 % vs 21.0 %), not fewer; +- time adverbs mostly modify the time PP itself (*erst in 12 Monaten*, *schon vor der Fahrt*) rather than fill a separate Te slot; +- the signal is real but largely redundant with the time-noun voter, so the adverb voter adds nothing. + +**OPEN.** +- A stand-alone temporal adverb (*früh*, *heute*) as a TEKAMOLO Te element is a word-class question, the adj/adv quorum of D-LXC-20, not a PP-lane question. +- The lane reader does not yet place bare adverbs into Te. diff --git a/crates/deepnsm-v2/examples/ud_lane_quorum.rs b/crates/deepnsm-v2/examples/ud_lane_quorum.rs index 75480fff0..35e245b5e 100644 --- a/crates/deepnsm-v2/examples/ud_lane_quorum.rs +++ b/crates/deepnsm-v2/examples/ud_lane_quorum.rs @@ -21,8 +21,10 @@ //! adverbials, or a compound ending in one), an abstract noun (suffix), a //! name (train majority PROPN), or other; //! - **number**: the phrase holds a digit; -//! - **adverb**: the token before the preposition is a temporal adverb -//! (adverbs right before *seit* / *während* / *bis* in train); +//! - **adverb**: a temporal adverb within two tokens before the preposition +//! or right after the phrase (*früh am Morgen*). Temporal adverbs are mined +//! by lift: ADV, or ADJ used as `advmod` (*früh*, *spät*), occurring at +//! least 2x more often in clauses with a time-only preposition; //! - **verb**: the (verb lemma, preposition) object rate from HDT `obj` + ADP, //! checked against every verb in the clause (`ud_pp_arg_eval`); //! - **position** (TEKAMOLO): Vorfeld (clause-initial), and whether another @@ -60,6 +62,37 @@ const ABSTRACT_SUFFIXES: [&str; 18] = [ "ung", "ungen", "heit", "heiten", "keit", "keiten", "schaft", "schaften", "tion", "tionen", "nis", "nisse", "nissen", "ität", "itäten", "ismus", "ismen", "tum", ]; +/// The temporal cue adverbs of the German codebook builder +/// (`lance-graph-planner/examples/data/de/build_de_codebook.py`, `TEMPORAL`), +/// minus its prepositions and conjunctions: a fixed list, not tuned here. +const CODEBOOK_TEMPORAL: [&str; 25] = [ + "anschließend", + "bald", + "damals", + "danach", + "dann", + "früh", + "gestern", + "heute", + "immer", + "jetzt", + "jährlich", + "manchmal", + "monatlich", + "morgen", + "nie", + "noch", + "oft", + "schließlich", + "schon", + "spät", + "täglich", + "wieder", + "wöchentlich", + "zunächst", + "zuvor", +]; + const MIN_PAIR: usize = 3; struct Tok { @@ -192,9 +225,6 @@ impl Tables { .or_default() += 1; } if TEMPORAL_ONLY.contains(&tok.form.as_str()) { - if i > 0 && s[i - 1].upos == "ADV" { - t.time_adverbs.insert(s[i - 1].form.clone()); - } let np = |j: usize| { s[j].upos == "NOUN" && s[i + 1..j] @@ -234,6 +264,45 @@ impl Tables { } } } + // Temporal adverbs by lift: an adverbial word (ADV, or an uninflected + // ADJ used as `advmod` such as *früh*, *spät*) that occurs far more + // often in clauses holding a time-only preposition than in clauses + // overall. Mining "the word right before seit/während" caught focus + // particles (*auch seit*, *nur bis*) instead. + let mut in_time = HashMap::::new(); + let mut overall = HashMap::::new(); + let (mut time_clauses, mut clauses) = (0usize, 0usize); + for sent in train { + let s = &sent.toks; + let mut lo = 0; + while lo < s.len() { + let (_, hi) = clause(s, lo); + let span = &s[lo..hi]; + let timed = span + .iter() + .any(|x| TEMPORAL_ONLY.contains(&x.form.as_str())); + clauses += 1; + time_clauses += usize::from(timed); + for x in span { + let adverbial = x.upos == "ADV" || (x.upos == "ADJ" && x.deprel == "advmod"); + if adverbial { + *overall.entry(x.form.clone()).or_default() += 1; + if timed { + *in_time.entry(x.form.clone()).or_default() += 1; + } + } + } + lo = hi + 1; + } + } + for (form, &k) in &in_time { + let all = overall[form]; + let lift = + (k as f64 / time_clauses.max(1) as f64) / (all as f64 / clauses.max(1) as f64); + if k >= 3 && lift >= 2.0 { + t.time_adverbs.insert(form.clone()); + } + } t.names = propn .into_iter() .filter(|(_, c)| c[0] > c[1]) @@ -294,7 +363,18 @@ impl Tables { }; let ph = phrase(s, i); let number = ph.iter().any(|&j| s[j].digit); - let adverb = i > 0 && self.time_adverbs.contains(&s[i - 1].form); + let after = ph.last().map_or(i + 1, |&j| j + 1); + let adverb = [i.wrapping_sub(2), i.wrapping_sub(1), after] + .into_iter() + .filter(|&k| k < s.len()) + .any(|k| { + let f = s[k].form.as_str(); + if std::env::var("LANE_ADVERBS").is_ok_and(|v| v == "codebook") { + CODEBOOK_TEMPORAL.contains(&f) + } else { + self.time_adverbs.contains(&s[k].form) + } + }); let verb = self.verb_governed(s, i); let (lo, hi) = clause(s, i); let vorfeld = i == lo; @@ -533,6 +613,12 @@ fn main() { tables.pair.len() ); + if std::env::var("LANE_DUMP").is_ok() { + let mut adv: Vec<&String> = tables.time_adverbs.iter().collect(); + adv.sort(); + eprintln!("temporal adverbs: {adv:?}"); + } + // Operator hypothesis: an abstract head is never PLACE. let abs: Vec<&Item> = fit_items .iter() From a737988e9e9befbfb5881278c8620b7764edcdf1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:08:17 +0000 Subject: [PATCH 24/33] deepnsm-v2 ud_case_eval: gender + number decide the article's case (R6, D-LXC-26) R6 reads an article's case from (article form, head-noun gender, head-noun number), using a noun lexicon mined from train. An unseen noun falls back to its German Snowball stem via frostem, the stemmer behind tesseract-paperless search. frostem is a German-only dev-dependency; no AdaWorldAPI fork exists. - Train case purity: article form alone 65 %, with gender + number 79 % (GSD and HDT). - HDT, seen nouns: 76.4 % vs 64.1 % for the article-form majority. - Stem fallback: 63.7 %, KILL. The stem erases number (Firma/Firmen). Exploratory gender-only fallback: 68.4 % vs 66.9 %. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...026-10-04-deepnsm-v2-gender-number-case.md | 40 +++++ .claude/board/entries/README.md | 3 +- crates/deepnsm-v2/Cargo.toml | 3 + crates/deepnsm-v2/examples/ud_case_eval.rs | 169 ++++++++++++++++++ 5 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-gender-number-case.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 5ba49a96c..07b8bcd37 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -49,6 +49,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-23** | first probes from the German grammar map: der/die/das DET vs PRON voters (`ud_pos_eval`), `ud_case_eval` (fixed-case prepositions, article decidability, Wechselpräposition, relative pronoun), `tekamolo_corners` | In PR | DET/PRON German 92.3 → 96.0 % (pronoun P 90.0 / R 54.1) PASS; case on HDT: Acc 90.0 / Dat 92.5 % PASS, Gen KILL, Wechsel article-decided 99.7 %, verb prior KILL (63.0 vs 63.9), relative pronoun 80.1 vs 63.8 % PASS; right-corner-on-Bible prediction KILL (Luther 7.1 % vs GSD 12.3 %). Entry `2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md` | | **D-LXC-24** | prepositional object vs adverbial (`ud_pp_arg_eval`): the (verb, preposition) object rate takes verb-governed phrases out of the TEKAMOLO pool and fixes their case; abstract-noun suffix class | In PR | HDT (gold = `obj` + ADP; German `obl:arg` is bare datives): A1 P 77.2 / R 81.6 %, 896 of 1,098 objects removed, 264 of 19,466 adverbials lost, PASS; A2 case 96.8 vs 81.0 % PASS; abstract heads 10.7 % objects vs 4.6 %; R3a KILL on HDT. Entry `2026-10-04-deepnsm-v2-prepositional-object-detector.md` | | **D-LXC-25** | TIME / PLACE / FIG lane of a Wechsel phrase (`ud_lane_quorum`): noun, number, adverb, verb governance, TEKAMOLO position and article voters, combined by logistic regression; silver labels | In PR | abstract head never PLACE: PASS, replicated (3.8 %, 3.2 %). Confirmation round (fresh labels, frozen design): noun voter TIME F1 0.831 PASS, PLACE F1 0.476 KILL, quorum +0.5 over noun KILL. Entry `2026-10-04-deepnsm-v2-wechsel-lane-quorum.md` | +| **D-LXC-26** | gender + number of the head noun decide the article's case (`ud_case_eval` R6); stemmer fallback via `frostem` German Snowball (the tesseract-paperless search stemmer) | In PR | train case purity 65 → 79 % with gender + number; HDT R6 76.4 vs 64.1 % on seen nouns (+12.3); stem fallback 63.7 % KILL (the stem erases number); exploratory gender-only fallback 68.4 vs 66.9 %. Entry `2026-10-04-deepnsm-v2-gender-number-case.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-gender-number-case.md b/.claude/board/entries/2026-10-04-deepnsm-v2-gender-number-case.md new file mode 100644 index 000000000..dc9b08649 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-gender-number-case.md @@ -0,0 +1,40 @@ +# 2026-10-04 — gender + number of the head noun shrink article case ambiguity (D-LXC-26) + +**Status:** MEASURED (debug-0, release) on UD r2.15 German GSD and HDT. Instrument: `crates/deepnsm-v2/examples/ud_case_eval.rs`, rule R6. The kill bars below were fixed before the run. + +Operator: *"Gender would greatly reduce morphology ambiguity"*. Their suggested stemmer is the one tesseract-paperless search uses: tantivy `Stemmer` with `Language::German`, which is the `frostem` crate (pure-Rust Snowball, zero deps, BSD-3). No AdaWorldAPI fork exists (checked via `list_repos`), so the registry crate is used. It is a **dev-dependency** of deepnsm-v2, German only, floating at major `1`. + +## Method + +- **Noun lexicon (train only):** form → (gender, number) counts. + - Fallback: the noun's German Snowball stem → counts. +- **Paradigm table:** (article form, head-noun gender, head-noun number) → case counts. +- **Head noun:** the first capitalised token within three tokens of the article. +- **Prediction:** the cell's majority case. +- **Baseline:** the majority case of the article form alone, on the same tokens. + +## Results + +| | GSD | HDT | +|---|---|---| +| train case purity, article form alone | 66.4 % | 65.3 % | +| train case purity, form + gender + number | **79.1 %** | **79.1 %** | +| R6, seen noun | 74.7 % vs 62.4 % (1,323) | **76.4 % vs 64.1 % (30,916)** | +| R6, stem fallback | 59.6 % vs 64.4 % (146) | 63.7 % vs 66.9 % (1,606) | +| R6, all | 73.2 % vs 62.6 % | 75.8 % vs 64.2 % (71.6 % of articles) | + +**Verdict.** +- The main bar passes: **+11.6 points** over the form majority on HDT. +- The stem-fallback bar is **KILL**: 63.7 % against 76.4 % for seen nouns, which is worse than the form majority. So R6 as a whole is KILL. +- Gender + number removes about 14 points of article case ambiguity on both treebanks. + +**Diagnosis (exploratory, after the run).** A Snowball stem keeps gender but erases number (*Firma* / *Firmen* share a stem), so it supplies the wrong number for plurals. Reading the (article, gender) table only: +- HDT: **68.4 % vs 66.9 %** (1,640 tokens); +- GSD: 67.5 % vs 63.6 %. + +The gain is small, and unseen nouns are only about 5 % of articles. + +**OPEN.** +- Unseen-noun number from morphology: plural suffix, or the article itself when unambiguous. +- Combining R6 with position. "der + feminine singular" leaves Dat vs Gen; the 2,665 Nom→Gen confusion on HDT is the next target. +- A gender column in the German codebook builder (`build_de_codebook.py`); UD already carries `Gender=`. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 2ffc4d540..61602c045 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -203 entries, 2026-08-06 .. 2026-10-04. +204 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| @@ -40,6 +40,7 @@ exactly one of them, never both. | 2026-10-04 | `D-LXC-16` | | [2026-10-04-deepnsm-v2-learned-position-table.md](2026-10-04-deepnsm-v2-learned-position-table.md) | | 2026-10-04 | `D-LXC-15` | | [2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md](2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md) | | 2026-10-04 | `D-LXC-17` | | [2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md](2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md) | +| 2026-10-04 | `D-LXC-26` | | [2026-10-04-deepnsm-v2-gender-number-case.md](2026-10-04-deepnsm-v2-gender-number-case.md) | | 2026-10-04 | `D-LXC-23` | | [2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md](2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md) | | 2026-10-04 | `D-LXC-21` | | [2026-10-04-deepnsm-deleted-grammar-restored.md](2026-10-04-deepnsm-deleted-grammar-restored.md) | | 2026-10-04 | `coresearch-orkg-architecture` | | [2026-10-04-coresearch-orkg-architecture.md](2026-10-04-coresearch-orkg-architecture.md) | diff --git a/crates/deepnsm-v2/Cargo.toml b/crates/deepnsm-v2/Cargo.toml index 6881391e2..03d58433f 100644 --- a/crates/deepnsm-v2/Cargo.toml +++ b/crates/deepnsm-v2/Cargo.toml @@ -42,3 +42,6 @@ test = true # verse-keyed Bible JSON. The library stays dependency-free. [dev-dependencies] serde_json = "1" +# German Snowball stemmer, as used by tesseract-paperless search (via tantivy). +# Examples only (`ud_case_eval` R6 gender fallback). No AdaWorldAPI fork exists. +frostem = { version = "1", default-features = false, features = ["german"] } diff --git a/crates/deepnsm-v2/examples/ud_case_eval.rs b/crates/deepnsm-v2/examples/ud_case_eval.rs index e4a8a0923..4949b4339 100644 --- a/crates/deepnsm-v2/examples/ud_case_eval.rs +++ b/crates/deepnsm-v2/examples/ud_case_eval.rs @@ -31,6 +31,12 @@ //! the SAME surface condition (a relativizer form right after a comma), plus //! any token marked `PronType=Rel`. Mining on `PronType=Rel` alone fires //! nothing on German GSD, which carries no `PronType=Rel` at all. +//! - **R6** gender and number: an article's case read from (article form, +//! head-noun gender, head-noun number). Gender and number come from a +//! train-mined noun lexicon; an unseen noun falls back to its German +//! Snowball stem (`frostem`, the stemmer tesseract-paperless search uses). +//! Also reported: how much case ambiguity the cell removes compared with the +//! article form alone (train purity). //! - **R5** combined, first rule that fires in the order R3-contraction, R1, R3, //! R4, R2; coverage, precision, baseline and the Nom/Acc/Dat/Gen confusion. //! @@ -50,6 +56,8 @@ //! - R3t, R3a (each): precision <= prep-majority on the same tokens, or fires on < 5 % of //! the article-undecided Wechsel tokens. //! - R4: precision <= the plain form baseline. +//! - R6: precision < the article-form majority + 5 points on the same tokens; +//! stem-fallback precision < seen-noun precision - 10 points. //! //! ```text //! cargo run --release --example ud_case_eval -- de_gsd-ud-train.conllu de_gsd-ud-test.conllu @@ -123,6 +131,16 @@ struct Tok { case: Option, /// `PronType=Rel` in feats. rel: bool, + /// `Gender=` (0 Masc, 1 Fem, 2 Neut) and `Number=` (0 Sing, 1 Plur). + gender: Option, + number: Option, +} + +fn feat(feats: &str, key: &str, values: &[&str]) -> Option { + feats + .split('|') + .find_map(|f| f.strip_prefix(key)) + .and_then(|v| values.iter().position(|x| *x == v)) } fn case_of(feats: &str) -> Option { @@ -155,6 +173,8 @@ fn read(path: &str) -> Vec> { upos: c[3].to_string(), case: case_of(c[5]), rel: c[5].split('|').any(|f| f == "PronType=Rel"), + gender: feat(c[5], "Gender=", &["Masc", "Fem", "Neut"]), + number: feat(c[5], "Number=", &["Sing", "Plur"]), }); } if !cur.is_empty() { @@ -706,4 +726,153 @@ fn main() { CASES[g], row[0], row[1], row[2], row[3] ); } + + r6_report(&train, &test); +} + +/// (gender, number) → index into a 6-cell table. +fn gn(g: usize, n: usize) -> usize { + g * 2 + n +} + +/// The head noun after the article at `i`: the first NOUN-shaped +/// (capitalised) token within three tokens, through words only. +fn article_head(s: &[Tok], i: usize) -> Option { + (i + 1..s.len().min(i + 4)) + .take_while(|&j| s[j].word) + .find(|&j| s[j].upper) +} + +/// R6: case from (article form, head gender, head number). +fn r6_report(train: &[Vec], test: &[Vec]) { + let stemmer = frostem::Stemmer::new(frostem::Algorithm::German); + let majority6 = |c: &[usize; 6]| (0..6).max_by_key(|&k| (c[k], std::cmp::Reverse(k))); + // Noun lexicon: form → (gender, number) counts; stem → counts. + let mut by_form: HashMap = HashMap::new(); + let mut by_stem: HashMap = HashMap::new(); + // (article form, gn cell) → case counts; article form → case counts. + let mut cell: HashMap<(String, usize), [usize; 4]> = HashMap::new(); + let mut form_only: HashMap = HashMap::new(); + // (article form, gender) → case counts: the stem keeps gender but erases + // number (*Firma* / *Firmen*), so the exploratory fallback reads gender only. + let mut cell_g: HashMap<(String, usize), [usize; 4]> = HashMap::new(); + let is_article = |t: &Tok| t.upos == "DET" && t.case.is_some(); + for s in train { + for t in s { + if let (true, Some(g), Some(n)) = (t.upos == "NOUN", t.gender, t.number) { + by_form.entry(t.form.clone()).or_default()[gn(g, n)] += 1; + by_stem + .entry(stemmer.stem(&t.form).into_owned()) + .or_default()[gn(g, n)] += 1; + } + } + for (i, t) in s.iter().enumerate() { + if !is_article(t) { + continue; + } + let c = t.case.expect("article has a case"); + form_only.entry(t.form.clone()).or_default()[c] += 1; + let Some(j) = article_head(s, i) else { + continue; + }; + if let (Some(g), Some(n)) = (s[j].gender, s[j].number) { + cell.entry((t.form.clone(), gn(g, n))).or_default()[c] += 1; + } + if let Some(g) = s[j].gender { + cell_g.entry((t.form.clone(), g)).or_default()[c] += 1; + } + } + } + // Ambiguity: train purity (majority share), weighted by count. + let purity = |m: &mut dyn Iterator| { + let (mut top, mut all) = (0usize, 0usize); + for c in m { + top += c.iter().max().copied().unwrap_or(0); + all += c.iter().sum::(); + } + 100.0 * top as f64 / all.max(1) as f64 + }; + println!("\nR6 gender and number (KILL: precision < form majority + 5 pts; stem fallback < seen - 10 pts)"); + println!( + " noun lexicon: {} forms, {} stems; train case purity: article form alone {:.1}%, form + gender + number {:.1}%", + by_form.len(), + by_stem.len(), + purity(&mut form_only.values()), + purity(&mut cell.values()) + ); + // [seen form, stem fallback] → (fires, R6 right, form-majority right). + let mut tally = [[0usize; 3]; 2]; + let mut no_gn = 0usize; + // Exploratory (after the first run): stem fallback with gender only. + let mut stem_g = [0usize; 3]; + let mut arts = 0usize; + for s in test { + for (i, t) in s.iter().enumerate() { + let (true, Some(gold)) = (t.upos == "DET", t.case) else { + continue; + }; + arts += 1; + let Some(j) = article_head(s, i) else { + no_gn += 1; + continue; + }; + if !by_form.contains_key(&s[j].form) { + let st = stemmer.stem(&s[j].form); + let g = by_stem.get(st.as_ref()).and_then(|c| { + (0..3).max_by_key(|&g| (c[gn(g, 0)] + c[gn(g, 1)], std::cmp::Reverse(g))) + }); + if let Some(pred) = g + .and_then(|g| cell_g.get(&(t.form.clone(), g))) + .and_then(majority) + { + let base = form_only.get(&t.form).and_then(majority); + stem_g[0] += 1; + stem_g[1] += usize::from(pred == gold); + stem_g[2] += usize::from(base == Some(gold)); + } + } + let (src, counts) = match by_form.get(&s[j].form) { + Some(c) => (0, Some(c)), + None => (1, by_stem.get(stemmer.stem(&s[j].form).as_ref())), + }; + let Some(k) = counts.and_then(majority6) else { + no_gn += 1; + continue; + }; + let Some(pred) = cell.get(&(t.form.clone(), k)).and_then(majority) else { + no_gn += 1; + continue; + }; + let base = form_only.get(&t.form).and_then(majority); + tally[src][0] += 1; + tally[src][1] += usize::from(pred == gold); + tally[src][2] += usize::from(base == Some(gold)); + } + } + let p = |a: usize, b: usize| 100.0 * a as f64 / b.max(1) as f64; + let mut pass = true; + for (k, name) in ["seen noun", "stem fallback"].iter().enumerate() { + let [f, r, b] = tally[k]; + println!( + " {name:13}: fires {f:6} R6 {:5.1}% article-form majority {:5.1}%", + p(r, f), + p(b, f) + ); + } + println!( + " stem fallback, gender only (exploratory): fires {:6} R6 {:5.1}% article-form majority {:5.1}%", + stem_g[0], + p(stem_g[1], stem_g[0]), + p(stem_g[2], stem_g[0]) + ); + let [f, r, b] = [0, 1, 2].map(|x| tally[0][x] + tally[1][x]); + pass &= p(r, f) >= p(b, f) + 5.0; + pass &= p(tally[1][1], tally[1][0]) >= p(tally[0][1], tally[0][0]) - 10.0; + println!( + " all : fires {f:6} of {arts} articles ({:.1}%) R6 {:5.1}% article-form majority {:5.1}% no gender/number {no_gn}", + p(f, arts), + p(r, f), + p(b, f) + ); + println!(" R6 {}", verdict(pass)); } From f2ae2d95b442ab6b0c4f0e7ef667bd60cb2d86b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:09:49 +0000 Subject: [PATCH 25/33] deepnsm-v2 data: record the DeReWo / DeReKo-2014 German frequency lists in object storage (licence: non-commercial) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- crates/deepnsm-v2/data/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/deepnsm-v2/data/README.md b/crates/deepnsm-v2/data/README.md index 3e135f849..da7100b58 100644 --- a/crates/deepnsm-v2/data/README.md +++ b/crates/deepnsm-v2/data/README.md @@ -41,3 +41,14 @@ The Tigris bake `lance-graph/codebooks/deepnsm-v2-academic-coca-v1/` carries bake yet; it is the published copy a future loader can verify against. The COCA letter → FSM tag fold lives in one place, `deepnsm_v2::coca`. + +## German frequency lists: DeReWo / DeReKo-2014 (object store, not committed) + +The German counterpart of COCA. Stored unchanged with its documentation under `$AWS_S3_BUCKET_NAME/corpora/de/derewo/` (the deployment's S3 environment; `crates/lance-graph/src/dev_s3_env.rs`). +- Source: https://www.ids-mannheim.de/digspra/kl/projekte/methoden/derewo/ +- `DeReKo-2014-II-MainArchive-STT.100000.freq.zip`: 100k (word form, lemma, STTS POS, frequency) over about 7 billion tokens. + - STTS separates ADJA (attributive) from ADJD (predicative/adverbial), the German adj/adv question. + - No gender column. +- `derewo-v-ww-bll-320000g-2012-12-31-1.0.zip`: 320k lemma list with frequency classes. + +**Licence: non-commercial research only.** DeReWo lists may not be passed on without their documentation. The DeReKo-2014 list is CC BY-NC 3.0, and its attribution is in the bucket README. Never commit these lists, data derived from them, or a product binary that bakes them in. From 0fa066d8dad6c1f5241eb75c4e3aaadc9cd87ff8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:11:31 +0000 Subject: [PATCH 26/33] deepnsm-v2: gender of unseen German nouns via compound head, lemma and stem (D-LXC-27) ud_gender_eval (new) scores test nouns whose form never occurs in train, on HDT: - compound head (the gender of the longest suffix that is a known noun): covers 83.7 % at 98.1 %, PASS; - frostem German stem: 88.0 %, PASS; - DeReKo form -> lemma (CC BY-NC, read from a local path, not committed): 98.4 % on 4.7 %; - nominalised infinitive -> neuter: 91.5 %, below its 95 % bar, KILL; - cascade: 89.9 % coverage at 97.7 %. GSD is weaker (smaller train set, noisier gold). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...026-10-04-deepnsm-v2-unseen-noun-gender.md | 33 +++ .claude/board/entries/README.md | 3 +- crates/deepnsm-v2/examples/ud_gender_eval.rs | 219 ++++++++++++++++++ 4 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-unseen-noun-gender.md create mode 100644 crates/deepnsm-v2/examples/ud_gender_eval.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 07b8bcd37..934812dc6 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -50,6 +50,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-24** | prepositional object vs adverbial (`ud_pp_arg_eval`): the (verb, preposition) object rate takes verb-governed phrases out of the TEKAMOLO pool and fixes their case; abstract-noun suffix class | In PR | HDT (gold = `obj` + ADP; German `obl:arg` is bare datives): A1 P 77.2 / R 81.6 %, 896 of 1,098 objects removed, 264 of 19,466 adverbials lost, PASS; A2 case 96.8 vs 81.0 % PASS; abstract heads 10.7 % objects vs 4.6 %; R3a KILL on HDT. Entry `2026-10-04-deepnsm-v2-prepositional-object-detector.md` | | **D-LXC-25** | TIME / PLACE / FIG lane of a Wechsel phrase (`ud_lane_quorum`): noun, number, adverb, verb governance, TEKAMOLO position and article voters, combined by logistic regression; silver labels | In PR | abstract head never PLACE: PASS, replicated (3.8 %, 3.2 %). Confirmation round (fresh labels, frozen design): noun voter TIME F1 0.831 PASS, PLACE F1 0.476 KILL, quorum +0.5 over noun KILL. Entry `2026-10-04-deepnsm-v2-wechsel-lane-quorum.md` | | **D-LXC-26** | gender + number of the head noun decide the article's case (`ud_case_eval` R6); stemmer fallback via `frostem` German Snowball (the tesseract-paperless search stemmer) | In PR | train case purity 65 → 79 % with gender + number; HDT R6 76.4 vs 64.1 % on seen nouns (+12.3); stem fallback 63.7 % KILL (the stem erases number); exploratory gender-only fallback 68.4 vs 66.9 %. Entry `2026-10-04-deepnsm-v2-gender-number-case.md` | +| **D-LXC-27** | gender of unseen German nouns (`ud_gender_eval`): compound head, DeReKo lemma, `frostem` stem, nominalised infinitive | In PR | HDT unseen nouns: compound head covers 83.7 % at 98.1 % PASS; stem 88.0 % PASS; lemma 98.4 % on 4.7 %; infinitive 91.5 % KILL; cascade 89.9 % coverage at 97.7 %. GSD weaker (compound 88.8 % KILL). Entry `2026-10-04-deepnsm-v2-unseen-noun-gender.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-unseen-noun-gender.md b/.claude/board/entries/2026-10-04-deepnsm-v2-unseen-noun-gender.md new file mode 100644 index 000000000..df0a54af5 --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-unseen-noun-gender.md @@ -0,0 +1,33 @@ +# 2026-10-04 — gender of unseen German nouns from compound head, lemma, stem and nominalisation (D-LXC-27) + +**Status:** MEASURED (debug-0, release) on UD r2.15 German GSD and HDT. Instrument: `crates/deepnsm-v2/examples/ud_gender_eval.rs`. Kill bars were fixed before the run. + +Operator: *"if lemma and word forms have a match, stemmer should mostly help to resolve the gazillions of German prefix and zusammengesetzte Substantive and substantivierte Verben"*. + +## Method + +The unit is a test noun with a single gold `Gender=` whose form is **unseen** in train (HDT: 13,157; GSD: 885). Each method is scored alone, then as a cascade. +- **lemma**: DeReKo-2014 STT `NN` rows (form → lemma; CC BY-NC, read from a local path, never committed) give the lemma, whose gender comes from the train lemmas. +- **infinitive**: a form that is a train VERB lemma ending in *-n* is Neut (*das Essen*). +- **compound**: the gender of the longest proper suffix (≥ 3 letters) that is a train noun form. A compound takes its last part's gender. +- **stem**: the `frostem` German Snowball stem (the tesseract-paperless search stemmer) → majority gender. + +## Results + +| method | HDT coverage | HDT precision | GSD coverage | GSD precision | bar | +|---|---|---|---|---|---| +| lemma (DeReKo) | 4.7 % | 98.4 % | 14.1 % | 87.2 % | — | +| infinitive → Neut | 0.8 % | 91.5 % | 1.4 % | 8.3 % (12 tokens) | 95 %: **KILL** | +| compound head | **83.7 %** | **98.1 %** | 59.7 % | 88.8 % | 90 %: PASS on HDT, KILL on GSD | +| stem | 15.5 % | 88.0 % | 19.8 % | 73.7 % | 85 %: PASS on HDT, KILL on GSD | +| **cascade** | **89.9 %** | **97.7 %** | 71.1 % | 86.8 % | — | + +- The majority-gender baseline is Fem: 39.4 % on HDT, 41.1 % on GSD. +- **The compound head carries the result.** It covers most unseen German nouns at 98 % precision on HDT. +- The lemma route is highly precise but adds little coverage, because train already covers most lemmas. +- **GSD is weaker throughout.** Its train set is about 3× smaller, and its gold gender is noisier (the D-LXC-23 note on GSD Case gold applies too). + +**OPEN.** +- Nominalised infinitives are rare as unseen nouns and miss the 95 % bar on HDT. +- On GSD, the 12 infinitive-shaped "nouns" are mostly not neuter. Plural-noun / verb-lemma collisions such as *Rennen* are suspected, not checked. +- Next: feed this gender into R6 (D-LXC-26) for unseen nouns. Number for unseen nouns is still open. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 61602c045..6f937f8e4 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,12 +25,13 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -204 entries, 2026-08-06 .. 2026-10-04. +205 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| | 2026-10-04 | `D-LXC-22` | | [2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md](2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md) | | 2026-10-04 | `D-LXC-25` | | [2026-10-04-deepnsm-v2-wechsel-lane-quorum.md](2026-10-04-deepnsm-v2-wechsel-lane-quorum.md) | +| 2026-10-04 | `D-LXC-27` | | [2026-10-04-deepnsm-v2-unseen-noun-gender.md](2026-10-04-deepnsm-v2-unseen-noun-gender.md) | | 2026-10-04 | `D-LXC-20` | | [2026-10-04-deepnsm-v2-satzklammer-frageprobe.md](2026-10-04-deepnsm-v2-satzklammer-frageprobe.md) | | 2026-10-04 | `D-LXC-19` | | [2026-10-04-deepnsm-v2-rule-quorum.md](2026-10-04-deepnsm-v2-rule-quorum.md) | | 2026-10-04 | `D-LXC-18` | | [2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md](2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md) | diff --git a/crates/deepnsm-v2/examples/ud_gender_eval.rs b/crates/deepnsm-v2/examples/ud_gender_eval.rs new file mode 100644 index 000000000..1745621f3 --- /dev/null +++ b/crates/deepnsm-v2/examples/ud_gender_eval.rs @@ -0,0 +1,219 @@ +//! `ud_gender_eval` — the gender of a German noun never seen in train. +//! +//! German makes gender recoverable from structure: +//! - a compound takes the gender of its last part +//! (*die Verpackung* → *die Lebensmittelverpackung*); +//! - a nominalised infinitive is always neuter (*das Essen*, *beim Laufen*); +//! - an inflected form shares its lemma's gender (*Firmen* → *Firma*). +//! +//! Each method is scored on its own, then as a cascade. The scored unit is a +//! test NOUN with a single `Gender=` in gold whose form never occurs as a +//! train NOUN. Gender is read from train only. +//! +//! Methods: +//! - **lemma**: an external form → lemma list (DeReKo-2014 STT, `NN` rows) +//! maps the form to a lemma, whose gender comes from the train lemmas. +//! Optional: pass the DeReKo `.freq` path as a third argument. The list is +//! CC BY-NC 3.0 and is never committed (see `data/README.md`). +//! - **infinitive**: the lowercased form is a verb lemma (train VERB lemmas) +//! ending in *-en* / *-n* → Neut. +//! - **compound**: the longest proper suffix of at least 3 letters that is a +//! train noun form → that noun's gender. +//! - **stem**: the German Snowball stem (`frostem`, the stemmer +//! tesseract-paperless search uses) → the majority gender of train nouns +//! with that stem. +//! +//! KILL bars (fixed before the run): compound precision < 0.90; infinitive +//! precision < 0.95; stem precision < 0.85. +//! +//! ```text +//! cargo run --release --example ud_gender_eval -- de_hdt-ud-train-a-1.conllu de_hdt-ud-test.conllu [DeReKo-2014-II-MainArchive-STT.100000.freq] +//! ``` + +use std::collections::{HashMap, HashSet}; + +const GENDERS: [&str; 3] = ["Masc", "Fem", "Neut"]; + +struct Noun { + form: String, + lemma: String, + gender: Option, +} + +/// (NOUN tokens, VERB lemmas) of a CoNLL-U file. +fn read(path: &str) -> (Vec, HashSet) { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut nouns = Vec::new(); + let mut verbs = HashSet::new(); + for line in text.lines() { + let c: Vec<&str> = line.split('\t').collect(); + if line.starts_with('#') || c.len() < 8 || c[0].contains(['-', '.']) { + continue; + } + match c[3] { + "NOUN" => nouns.push(Noun { + form: c[1].to_lowercase(), + lemma: c[2].to_lowercase(), + gender: c[5] + .split('|') + .find_map(|f| f.strip_prefix("Gender=")) + .and_then(|v| GENDERS.iter().position(|g| *g == v)), + }), + "VERB" => { + verbs.insert(c[2].to_lowercase()); + } + _ => {} + } + } + (nouns, verbs) +} + +fn majority(c: &[usize; 3]) -> Option { + (c.iter().sum::() > 0).then(|| (0..3).max_by_key(|&k| (c[k], std::cmp::Reverse(k))))? +} + +#[derive(Default, Clone, Copy)] +struct Tally { + fires: usize, + right: usize, +} + +impl Tally { + fn add(&mut self, ok: bool) { + self.fires += 1; + self.right += usize::from(ok); + } + fn p(&self) -> f64 { + 100.0 * self.right as f64 / self.fires.max(1) as f64 + } +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let (train_path, test_path, dereko) = match args.as_slice() { + [a, b] => (a, b, None), + [a, b, c] => (a, b, Some(c)), + _ => { + eprintln!("usage: ud_gender_eval TRAIN.conllu TEST.conllu [DEREKO.freq]"); + std::process::exit(2); + } + }; + let (train, verbs) = read(train_path); + let (test, _) = read(test_path); + let stemmer = frostem::Stemmer::new(frostem::Algorithm::German); + + let mut by_form: HashMap = HashMap::new(); + let mut by_lemma: HashMap = HashMap::new(); + let mut by_stem: HashMap = HashMap::new(); + let mut seen: HashSet = HashSet::new(); + for n in &train { + seen.insert(n.form.clone()); + if let Some(g) = n.gender { + by_form.entry(n.form.clone()).or_default()[g] += 1; + by_lemma.entry(n.lemma.clone()).or_default()[g] += 1; + by_stem + .entry(stemmer.stem(&n.form).into_owned()) + .or_default()[g] += 1; + } + } + // DeReKo NN rows: form → lemma (highest-frequency row wins). + let mut form_lemma: HashMap = HashMap::new(); + if let Some(p) = dereko { + let text = std::fs::read_to_string(p).unwrap_or_else(|e| panic!("{p}: {e}")); + for line in text.lines() { + let c: Vec<&str> = line.split('\t').collect(); + let [form, lemma, "NN", freq] = c.as_slice() else { + continue; + }; + let f: f64 = freq.parse().unwrap_or(0.0); + let e = form_lemma + .entry(form.to_lowercase()) + .or_insert((lemma.to_lowercase(), f)); + if f > e.1 { + *e = (lemma.to_lowercase(), f); + } + } + } + + let lemma_g = |form: &str| -> Option { + let (l, _) = form_lemma.get(form)?; + by_lemma.get(l).and_then(majority) + }; + let infinitive_g = |form: &str| -> Option { + (form.ends_with('n') && verbs.contains(form)).then_some(2) + }; + let compound_g = |form: &str| -> Option { + let chars: Vec = form.chars().collect(); + (1..chars.len().saturating_sub(2)).find_map(|k| { + let tail: String = chars[k..].iter().collect(); + by_form.get(&tail).and_then(majority) + }) + }; + let stem_g = |form: &str| by_stem.get(stemmer.stem(form).as_ref()).and_then(majority); + + let mut unseen = 0usize; + let mut method = [Tally::default(); 4]; + let mut cascade = Tally::default(); + let mut baseline = [0usize; 3]; + for n in &test { + let Some(g) = n.gender else { continue }; + if seen.contains(&n.form) { + continue; + } + unseen += 1; + baseline[g] += 1; + let preds = [ + lemma_g(&n.form), + infinitive_g(&n.form), + compound_g(&n.form), + stem_g(&n.form), + ]; + for (t, p) in method.iter_mut().zip(preds) { + if let Some(p) = p { + t.add(p == g); + } + } + if let Some(p) = preds.into_iter().flatten().next() { + cascade.add(p == g); + } + } + + println!( + "train: {} nouns, {} forms with gender, {} verb lemmas; DeReKo NN forms: {}", + train.len(), + by_form.len(), + verbs.len(), + form_lemma.len() + ); + let maj = majority(&baseline).unwrap_or(0); + println!( + "test nouns unseen in train (single gold gender): {unseen}; majority gender {} = {:.1}%", + GENDERS[maj], + 100.0 * baseline[maj] as f64 / unseen.max(1) as f64 + ); + let bars = [None, Some(95.0), Some(90.0), Some(85.0)]; + let names = [ + "lemma (DeReKo)", + "infinitive -> Neut", + "compound head", + "stem", + ]; + for ((name, t), bar) in names.iter().zip(method).zip(bars) { + let verdict = match bar { + Some(b) if t.fires == 0 || t.p() < b => format!("KILL (< {b:.0}%)"), + Some(b) => format!("PASS (>= {b:.0}%)"), + None => String::new(), + }; + println!( + " {name:19}: covers {:5} ({:5.1}%) precision {:5.1}% {verdict}", + t.fires, + 100.0 * t.fires as f64 / unseen.max(1) as f64, + t.p() + ); + } + println!( + " cascade (lemma, infinitive, compound, stem): covers {:.1}% precision {:.1}%", + 100.0 * cascade.fires as f64 / unseen.max(1) as f64, + cascade.p() + ); +} From af6a4cdca2d59630f12d6063042d0907484cd672 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:14:33 +0000 Subject: [PATCH 27/33] deepnsm-v2: hydrate joins each marker to its own node; toc_hydrate parses every reading PR #1321 review (Codex): - P2 hydrate: spawn skips a verse past the level capacity, so indexing the dense list of spawned verse nodes by marker position shifted every later triple onto the wrong verse. Each marker now resolves its own (book, chapter, verse) node and is unaddressed when it has none. New test a_skipped_verse_does_not_shift_later_triples, disable-verified against the old dense join. - P1 toc_hydrate: it parsed with the single-reading parse_to_spo while bible_wave uses parse_readings. It now builds the same reading sets and keeps `certain`. KJV: 57,277 triples (= bible_wave's certain pin, was 70,393), 1,187 basins, 0 unaddressed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .../deleted-grammar-heuristics-ledger.md | 2 +- crates/deepnsm-v2/examples/toc_hydrate.rs | 94 +++++++++++++++++-- crates/deepnsm-v2/src/hydrate.rs | 60 ++++++++++-- 3 files changed, 141 insertions(+), 15 deletions(-) diff --git a/.claude/knowledge/deleted-grammar-heuristics-ledger.md b/.claude/knowledge/deleted-grammar-heuristics-ledger.md index 501914927..0bce00924 100644 --- a/.claude/knowledge/deleted-grammar-heuristics-ledger.md +++ b/.claude/knowledge/deleted-grammar-heuristics-ledger.md @@ -28,7 +28,7 @@ for later reference."* | `crates/deepnsm-v2/src/toc.rs` | 426 | `68955ecb` (2026-08-22) | `abd331b9` | **The TOC as an HHTL tree of SoA node addresses.** Book:chapter:verse spawned as the full skeleton before any reasoning (operator: *"bevor wir über das Buch nachdenken wird ein Inhaltsverzeichnis als HHTL-Baum mit SoA-Knoten erstellt"*). Two nibbles per level. | "the hydrated table is rebuilt from scratch" | **RESTORED**. This is the HHTL literal chapter:verse address; the basins below are the cross-chapter nodes. | | `crates/deepnsm-v2/src/hydrate.rs` | 370 | `68955ecb` | `bfc69bf8`, `6e385c88` | Every SPO triple joined to the verse node it was read in. The reading-order chain is kept (the Markov trajectory over the book) along with the scanpath residue. | as `toc.rs` | **RESTORED**. | | `crates/deepnsm-v2/src/promote.rs` | 591 | `68955ecb` | `dd728f17`, `ec50f07b` | Basins written as SoA `NodeRow`s at TOC node keys: keyed by literary unit, not by bare subject (`E-HERMENEUTIK-RUNG-LADDER-1`). Also the TEKAMOLO tenant writer. | "also minted V1 keys" | **RESTORED**. The V1 claim was already false at deletion: `ec50f07b` made the promoter refuse the V1 tail (`mint_for` + read-back check, test-pinned). | -| `crates/deepnsm-v2/examples/toc_hydrate.rs` | 315 | `68955ecb` | `6e385c88` | The end-to-end run of tree → hydrate → promote → loci over the real KJV. | as above | **RESTORED**. Measured 2026-10-04: 32,357 nodes (66 books, 31,102 verses); 70,393 triples, 0 unaddressed; 27,298 cross-verse links (38.8 %); 1,227 basins promoted at TOC keys. | +| `crates/deepnsm-v2/examples/toc_hydrate.rs` | 315 | `68955ecb` | `6e385c88` | The end-to-end run of tree → hydrate → promote → loci over the real KJV. | as above | **RESTORED**. Measured 2026-10-04: 32,357 nodes (66 books, 31,102 verses), 0 unaddressed. Superseded the same day by the PR #1321 review fix, now that toc_hydrate parses with every reading (`parse_readings`, the same stream as `bible_wave`): 57,277 triples (= `bible_wave`'s `certain` pin), 25,459 cross-verse links (44.4 %), 1,187 basins. The first run used the legacy single-reading parse: 70,393 triples, 1,227 basins. | | `crates/deepnsm-v2/src/lexicon.rs` | 307 | `68955ecb` | `ec50f07b` | One COCA lexicon: lemma → forms → archaic → Other, monotone. It replaced three hand-copied taggers. | "taggers inlined into the two remaining examples" | **RESTORED**. The inlined copies in `bible_wave` / `genre_shapes` remain (they are pin-guarded). Merging them back onto this module is a deliberate re-pin. Measured in the file: KJV verses with no triple went from 35.6 % to 12.2 % once the forms table was read. | | `crates/deepnsm-v2/src/loci.rs` | 357 | `68955ecb` | `f47f0528` | Derives `Locus::Antecedent` (a 24×i4 `CausalWitnessFacet` nibble) from the SPO stream. Candidates are non-pronoun subjects within REACH = 8. The ranking is `SelectionalFit` (Cam96 similarity of each candidate to the pronoun clause's predicate) against `Recency`, with a margin gate. | "resolution AND binding already shipped" (`spo_anaphora_nibble`, `l9_loci_real_text`, `probe_antecedent_binder`) | **RESTORED**, with a known defect. It is agreement-blind. On Aesop the shipped agreement resolver scores 0.727 and the agreement-blind rule scores 0.273. **Merge pending:** take the gender/number/case agreement filter (and the German case table, see the research inventory) as the candidate filter, then keep SelectionalFit as the ranker. KJV numbers from `toc_hydrate`: Recency binds 35,025 of 35,613; SelectionalFit binds 27,788 (7,237 thin-margin). | | `crates/deepnsm-v2/src/tekamolo.rs` | 718 | `f3d42b7c` (style half), `c57cc256` (reverted that revert), `68955ecb` (whole file) | `7bae6c84` | **German** right-corner TEKAMOLO lane reading. A left-corner hypothesis is opened per adverbial and committed at the right corner on margin. It reads Luther 1545 and Elberfelder 1905 as a two-translation control, uses `de/tekamolo.tsv`, and hydrates the Kausal/Modal/Lokal lanes. | "insight_coca_read already emits all four lanes" — but that replacement is **English COCA only**, so the German capability was lost (D-LXC-18 blast radius) | **RESTORED**, with a ruling conflict. `V2StyleProvider` (ThinkingStyle → FieldModulation knobs) is the half §3o of `alpha-channel-rung-overlay-v1` refused for deepnsm-v2: reasoning lives in the planner. **Resolved 2026-10-04 (D-LXC-22):** grammar is not a thinking style, so `read_clause` now takes `ReadParams` (`LEFT_CORNER` / `RIGHT_CORNER` presets). The style mapping is kept as the adapter `ReadParams::from_style`; its destination is the planner, or a thinking dialect on `ogar-loco` (the long-term IR substrate). | diff --git a/crates/deepnsm-v2/examples/toc_hydrate.rs b/crates/deepnsm-v2/examples/toc_hydrate.rs index ea3757c73..d7153d108 100644 --- a/crates/deepnsm-v2/examples/toc_hydrate.rs +++ b/crates/deepnsm-v2/examples/toc_hydrate.rs @@ -10,11 +10,13 @@ //! silently disagree with the pipeline that already runs. use deepnsm_v2::basin::basin_self_code; +use deepnsm_v2::coca::{predicate_alternatives, reading_set}; use deepnsm_v2::codebook::{load_cam96_codes, load_cam96_space}; use deepnsm_v2::corpus::split_verses_detailed; -use deepnsm_v2::fsm::{parse_to_spo, Pos, Tagged}; +use deepnsm_v2::fsm::{parse_readings, Pos, PosSet, Reading}; use deepnsm_v2::hydrate::hydrate; -use deepnsm_v2::lexicon::{normalise, Lexicon}; +use deepnsm_v2::lexical::{load_word_forms_csv, LexicalEvidence}; +use deepnsm_v2::lexicon::{archaic_pos, coca_pos, normalise, Lexicon}; use deepnsm_v2::loci::{loci_from_triples, AnteRule}; use deepnsm_v2::promote::{key_at, promote, read_lane, row_of}; use deepnsm_v2::spo::Spo; @@ -99,18 +101,23 @@ fn main() { lexicon.sizes().1 ); - // ── per-verse SPO, same tagging as bible_wave ── + // ── per-verse SPO, parsed exactly as bible_wave does ── + // Every reading the lexicon admits goes to the multi-reading parser + // (D-LXC-2 / D-LXC-13); only its `certain` triples are kept. A + // single-reading `parse_to_spo` here would build the tree and its basins + // from a different triple stream than the main pipeline (PR #1321 review). + let readings = ReadingSource::load(&vocab); let mut per_verse: Vec> = Vec::with_capacity(verses.len()); - let mut tagged: Vec = Vec::new(); + let mut buf: Vec = Vec::new(); for verse in &verses { - tagged.clear(); + buf.clear(); for tok in verse.split_whitespace() { let Some(w) = normalise(tok) else { continue }; let Some(id) = vocab.id(&w) else { continue }; - tagged.push(Tagged::new(id, lexicon.pos(&w))); + buf.push(Reading::new(id, readings.of(&w, id))); } - tagged.push(Tagged::new(0, Pos::Stop)); - per_verse.push(parse_to_spo(&tagged)); + buf.push(Reading::stop()); + per_verse.push(parse_readings(&buf).certain); } let total: usize = per_verse.iter().map(Vec::len).sum(); println!("triples {total} from {} verses", verses.len()); @@ -313,3 +320,74 @@ fn main() { println!("\nEND-TO-END GREEN over the real corpus."); } + +/// The reading set per word, the same as `bible_wave`'s `Tagger::readings`. +/// Order: the lemma table's first row, widened by its noun/verb alternative +/// when any lemma row or the word-forms evidence has one; otherwise every +/// word-forms reading; otherwise the archaic list; otherwise unknown. +/// Merging this with `bible_wave`'s copy onto `deepnsm_v2::lexicon` is a +/// deliberate KJV re-pin (merge queue #3, +/// `.claude/knowledge/deleted-grammar-heuristics-ledger.md`). +struct ReadingSource { + first: HashMap, + all: HashMap, + evidence: LexicalEvidence, +} + +impl ReadingSource { + fn load(vocab: &PaletteVocab) -> Self { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../deepnsm/word_frequency/"); + let lemmas = std::fs::read_to_string(format!("{dir}lemmas_5k.csv")).expect("lemmas_5k.csv"); + let forms = + std::fs::read_to_string(format!("{dir}word_forms.csv")).expect("word_forms.csv"); + let mut first = HashMap::new(); + let mut all: HashMap = HashMap::new(); + for line in lemmas.lines().skip(1) { + let f: Vec<&str> = line.split(',').collect(); + let (Some(lemma), Some(pos)) = (f.get(1), f.get(2)) else { + continue; + }; + first + .entry(lemma.to_lowercase()) + .or_insert_with(|| coca_pos(pos)); + let set = all.entry(lemma.to_lowercase()).or_default(); + *set = set.with(coca_pos(pos)); + } + // Lowercase the `word` column: the corpus tokens are lowercased and + // `load_word_forms_csv` matches surfaces exactly. + let mut lowered = String::with_capacity(forms.len()); + for (i, line) in forms.lines().enumerate() { + match line.rsplit_once(',') { + Some((head, word)) if i > 0 => { + lowered.push_str(head); + lowered.push(','); + lowered.push_str(&word.to_lowercase()); + } + _ => lowered.push_str(line), + } + lowered.push('\n'); + } + let (evidence, _) = load_word_forms_csv(&lowered, vocab).expect("word_forms.csv"); + Self { + first, + all, + evidence, + } + } + + fn of(&self, w: &str, id: deepnsm_v2::vocab::WordId) -> PosSet { + if let Some(&p) = self.first.get(w) { + return predicate_alternatives( + p, + self.all + .get(w) + .copied() + .unwrap_or_default() + .union(reading_set(&self.evidence, id).unwrap_or_default()), + ); + } + reading_set(&self.evidence, id) + .or_else(|| archaic_pos(w).map(PosSet::single)) + .unwrap_or(PosSet::EMPTY) + } +} diff --git a/crates/deepnsm-v2/src/hydrate.rs b/crates/deepnsm-v2/src/hydrate.rs index 21f15da22..645133476 100644 --- a/crates/deepnsm-v2/src/hydrate.rs +++ b/crates/deepnsm-v2/src/hydrate.rs @@ -45,6 +45,7 @@ use crate::spo::Spo; use crate::toc::{spawn, CorpusToc, TocEntry, TocLevel}; use lance_graph_contract::hhtl::NiblePath; +use std::collections::HashMap; /// One triple, addressed at the verse node it was read in. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -133,12 +134,30 @@ pub fn hydrate(markers: &[(u16, u16)], per_verse: &[Vec], basin: u8) -> Hyd let toc = CorpusToc::from_markers(markers); let entries = spawn(&toc, basin); - // Verse nodes in the order the spawn emitted them, which is the order the - // markers were read — so the i-th verse marker is the i-th verse node. - let verse_paths: Vec = entries + // Each marker's OWN node, looked up by its (book, chapter, verse). The + // spawn skips any node past the level capacity, so the dense list of + // spawned verse nodes is NOT aligned with the markers: indexing it by + // marker position would shift every later triple onto the wrong verse + // once one verse is skipped (PR #1321 review). A marker with no node is + // unaddressed. The book counter follows `CorpusToc::from_markers`: a book + // opens at `1:1`, and markers before the first `1:1` belong to book 0. + let node: HashMap<(u16, u16, u16), NiblePath> = entries .iter() .filter(|e| e.level == TocLevel::Verse) - .map(|e| e.path) + .map(|e| ((e.book, e.chapter, e.verse), e.path)) + .collect(); + let mut book: Option = None; + let verse_paths: Vec> = markers + .iter() + .map(|&(ch, vs)| { + let b = match book { + Some(b) if (ch, vs) == (1, 1) => b.saturating_add(1), + Some(b) => b, + None => 0, + }; + book = Some(b); + node.get(&(b, ch, vs)).copied() + }) .collect(); let mut triples = Vec::new(); @@ -151,8 +170,8 @@ pub fn hydrate(markers: &[(u16, u16)], per_verse: &[Vec], basin: u8) -> Hyd barren_verses += 1; continue; } - match verse_paths.get(i) { - Some(&path) => { + match verse_paths.get(i).copied().flatten() { + Some(path) => { for spo in spos { triples.push(AddressedTriple { path, @@ -189,6 +208,35 @@ mod tests { Spo::new(n, n + 100, n + 200) } + /// **A skipped verse does not shift later triples.** Book 0 has a chapter + /// with 256 verses; verse 256 is past the level capacity and gets no node. + /// Book 1's first triple must land on book 1's verse 1:1, not be consumed + /// by book 0's missing verse (PR #1321 review: indexing the dense list of + /// spawned verses by marker position did exactly that). + #[test] + fn a_skipped_verse_does_not_shift_later_triples() { + let mut marks: Vec<(u16, u16)> = (1..=256).map(|v| (1, v)).collect(); + marks.push((1, 1)); + marks.push((1, 2)); + let mut per_verse: Vec> = vec![Vec::new(); marks.len()]; + per_verse[255] = vec![spo(1)]; // book 0, verse 256: no node + per_verse[256] = vec![spo(2)]; // book 1, verse 1:1 + let h = hydrate(&marks, &per_verse, 1); + assert_eq!(h.unaddressed, 1, "verse 256 is past capacity"); + assert_eq!(h.triples.len(), 1); + let landed = h + .entries + .iter() + .find(|e| e.path == h.triples[0].path) + .expect("an addressed triple sits on a spawned node"); + assert_eq!( + (landed.book, landed.chapter, landed.verse), + (1, 1, 1), + "book 1's triple must land on book 1, verse 1:1" + ); + assert_eq!(h.triples[0].spo, spo(2)); + } + /// **Every triple lands on a real minted verse node.** The join is the whole /// deliverable, so it is the test: an address that is not in the tree means /// a triple nothing can ascend from. From b0598a6cc4f4edbbf72ca6b8cf7a8d2a0f78a5b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:22:25 +0000 Subject: [PATCH 28/33] deepnsm-v2: DeReKo profile and TEKAMOLO order across genres (D-LXC-28) - dereko_profile (new): coverage, Spearman rank agreement, POS ambiguity and ADJD share against the DeReKo-2014 list (read from a local path, CC BY-NC, not committed). Adverbial adjectives are about 3.4 % of tokens in novels and about 1 % in every Bible. - tekamolo_order (new): Te < Ka < Mo < Lo pair order by position, using unambiguous cue words. Pre-registered rounds: - round 1: KILL, because news sits at chance like the Bibles; - round 2, fresh novels: 58.9 % vs 47.1 % for the Bibles, KILL on interval overlap (z ~ 2.1). Temporal and manner adverbs are 3-5x rarer per token in the Bibles. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...dereko-and-tekamolo-order-across-genres.md | 70 ++++++ .claude/board/entries/README.md | 3 +- crates/deepnsm-v2/examples/dereko_profile.rs | 184 ++++++++++++++ crates/deepnsm-v2/examples/tekamolo_order.rs | 238 ++++++++++++++++++ 5 files changed, 495 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-dereko-and-tekamolo-order-across-genres.md create mode 100644 crates/deepnsm-v2/examples/dereko_profile.rs create mode 100644 crates/deepnsm-v2/examples/tekamolo_order.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 934812dc6..e9b6dca7b 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -51,6 +51,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-25** | TIME / PLACE / FIG lane of a Wechsel phrase (`ud_lane_quorum`): noun, number, adverb, verb governance, TEKAMOLO position and article voters, combined by logistic regression; silver labels | In PR | abstract head never PLACE: PASS, replicated (3.8 %, 3.2 %). Confirmation round (fresh labels, frozen design): noun voter TIME F1 0.831 PASS, PLACE F1 0.476 KILL, quorum +0.5 over noun KILL. Entry `2026-10-04-deepnsm-v2-wechsel-lane-quorum.md` | | **D-LXC-26** | gender + number of the head noun decide the article's case (`ud_case_eval` R6); stemmer fallback via `frostem` German Snowball (the tesseract-paperless search stemmer) | In PR | train case purity 65 → 79 % with gender + number; HDT R6 76.4 vs 64.1 % on seen nouns (+12.3); stem fallback 63.7 % KILL (the stem erases number); exploratory gender-only fallback 68.4 vs 66.9 %. Entry `2026-10-04-deepnsm-v2-gender-number-case.md` | | **D-LXC-27** | gender of unseen German nouns (`ud_gender_eval`): compound head, DeReKo lemma, `frostem` stem, nominalised infinitive | In PR | HDT unseen nouns: compound head covers 83.7 % at 98.1 % PASS; stem 88.0 % PASS; lemma 98.4 % on 4.7 %; infinitive 91.5 % KILL; cascade 89.9 % coverage at 97.7 %. GSD weaker (compound 88.8 % KILL). Entry `2026-10-04-deepnsm-v2-unseen-noun-gender.md` | +| **D-LXC-28** | DeReKo-2014 profile across genres (`dereko_profile`); TEKAMOLO order by position in edited novels vs Bible translations (`tekamolo_order`) | In PR | ADJD about 3.4 % of tokens in novels vs about 1 % in every Bible; order: fresh novels 58.9 % vs Bibles 47.1 %, intervals overlap, KILL (z ≈ 2.1); density Te/Mo 3–5× lower in Bibles (robust). Entry `2026-10-04-deepnsm-v2-dereko-and-tekamolo-order-across-genres.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-dereko-and-tekamolo-order-across-genres.md b/.claude/board/entries/2026-10-04-deepnsm-v2-dereko-and-tekamolo-order-across-genres.md new file mode 100644 index 000000000..11126437e --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-dereko-and-tekamolo-order-across-genres.md @@ -0,0 +1,70 @@ +# 2026-10-04 — DeReKo profile and TEKAMOLO order: edited German books vs Bible translations (D-LXC-28) + +**Status:** MEASURED (debug-0, release). +- Instruments: `crates/deepnsm-v2/examples/{dereko_profile,tekamolo_order}.rs`. +- Texts, none committed: + - Project Gutenberg public-domain novels; + - the PD Bible bundle (Luther 1545 in modernised spelling, Elberfelder 1905); + - getbible Elberfelder 1871 and Schlachter 1951; + - UD GSD and HDT. +- Schlachter 1951 was used only as a local measurement input: its copyright status is unclear. + +Operator: test DeReWo against other literature, and compare TEKAMOLO by position between edited books (*lektorierte Bücher*) and the grammar-distorted Bible. + +## 1. DeReKo-2014 profile (top-100k word / lemma / STTS / frequency list) + +| text | tokens | covered | Spearman | POS-ambiguous | ADJD | +|---|---|---|---|---|---| +| UD GSD (news/web) | 222,938 | 90.2 % | 0.670 | 13.4 % | 2.42 % | +| UD HDT (IT news) | 630,538 | 91.7 % | 0.509 | 15.2 % | 2.39 % | +| Kafka, *Verwandlung* 1915 | 19,230 | 94.9 % | 0.595 | 18.1 % | 3.69 % | +| Mann, *Buddenbrooks* 1901 | 230,254 | 91.2 % | 0.570 | 16.2 % | 3.44 % | +| Fontane, *Effi Briest* 1895 | 95,624 | 93.6 % | 0.620 | 18.7 % | 3.36 % | +| Storm, *Schimmelreiter* 1888 | 38,473 | 91.9 % | 0.523 | 17.3 % | 2.28 % | +| Goethe, *Faust I* (verse) | 30,959 | 88.6 % | 0.547 | 18.9 % | 3.47 % | +| Schlachter 1951 | 708,524 | 91.9 % | 0.475 | 18.7 % | 1.12 % | +| Elberfelder 1905 | 722,778 | 91.2 % | 0.450 | 18.0 % | 0.91 % | +| Elberfelder 1871 | 718,761 | 91.3 % | 0.452 | 18.0 % | 0.92 % | +| Luther 1545 (modernised spelling) | 696,534 | 89.0 % | 0.468 | 18.9 % | 1.20 % | + +- **Token coverage barely separates genres** (89–95 %). +- **Rank agreement with contemporary usage is lower for every Bible** (0.45–0.48) than for the novels (0.52–0.62). +- **ADJD** (adjective used predicatively or adverbially, *früh*, *schnell*) is about 3.4 % of tokens in the novels and about 1 % in every Bible, including the 20th-century Schlachter. +- Type-OOV rates are not comparable across texts: they grow with text length and proper names. + +## 2. TEKAMOLO order by position + +**Method.** +- A clause is the span between punctuation marks. +- Lanes are read from **unambiguous** cue words only: no prepositions, no modal particles, no *da* / *so* / *noch*. +- A pair of two different lanes in one clause counts as in order when Te < Ka < Mo < Lo holds. + +**Round 1 (pre-registered): KILL.** +- Prediction: every Bible lies below the pooled edited texts, with non-overlapping 95 % intervals. +- The pool included the two news treebanks, which sit at chance (GSD 50.0 %, HDT 46.5 %), as do the Bibles (45.5–50.0 %). +- The novels alone are higher: Storm 74.3 %, Fontane 67.5 %, Mann 60.3 %. Pooled, 101/157 = 64.3 % vs the Bibles 73/155 = 47.1 %, non-overlapping. That split was made after seeing the data. + +**Round 2 (pre-registered before download; novels fixed by Gutenberg id).** +- Novels: *Dr. Mabuse*, *Taugenichts*, *Liebe ist ewig*, *Zauberberg I*, *Beate Hoyermann*, *Elixiere des Teufels*. +- *Winnetou I* was excluded before any run: no plain-text file on Gutenberg. + +| | in order | 95 % Wilson | pairs | +|---|---|---|---| +| fresh novels, pooled | **58.9 %** | [51.1, 66.2] | 158 | +| four Bibles, pooled | **47.1 %** | [39.4, 54.9] | 155 | + +**Verdict: KILL on the pre-registered non-overlap criterion.** The direction replicates (+11.8 points; two-proportion z ≈ 2.1, p ≈ 0.04), but the intervals overlap by 3.8 points. The bottleneck is that only about 150 clauses per group contain two unambiguous cues. + +**The robust difference is density, not order (both rounds).** + +| | Te per 10k tokens | Mo per 10k tokens | Ka per 10k tokens | +|---|---|---|---| +| novels | 47–93 | 16–55 | 7–11 | +| news | 51–53 | 13–17 | — | +| Bibles | 14–18 | 4–5 | 16–20 (Luther's *darum*, *deshalb*) | + +Biblical German rarely stacks two adverbials in one clause; it chains short clauses with *und* (parataxis). This agrees with the ADJD gap in §1. + +**OPEN.** +- Order needs more two-lane clauses: a lane reader with ambiguous cues resolved (D-LXC-25 lanes for PPs, D-LXC-20 adj/adv for bare adverbs), not a larger cue list. +- No public-domain contemporary German Bible exists. Luther 2017, Einheitsübersetzung 2016, Elberfelder 2006, Schlachter 2000 and the BasisBibel are copyrighted. The *Offene Bibel* is an openly licensed, partial contemporary translation (licence to verify). diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 6f937f8e4..8a70312d5 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -205 entries, 2026-08-06 .. 2026-10-04. +206 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| @@ -43,6 +43,7 @@ exactly one of them, never both. | 2026-10-04 | `D-LXC-17` | | [2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md](2026-10-04-deepnsm-v2-german-tekamolo-verb-position.md) | | 2026-10-04 | `D-LXC-26` | | [2026-10-04-deepnsm-v2-gender-number-case.md](2026-10-04-deepnsm-v2-gender-number-case.md) | | 2026-10-04 | `D-LXC-23` | | [2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md](2026-10-04-deepnsm-v2-first-probes-from-the-grammar-map.md) | +| 2026-10-04 | `D-LXC-28` | | [2026-10-04-deepnsm-v2-dereko-and-tekamolo-order-across-genres.md](2026-10-04-deepnsm-v2-dereko-and-tekamolo-order-across-genres.md) | | 2026-10-04 | `D-LXC-21` | | [2026-10-04-deepnsm-deleted-grammar-restored.md](2026-10-04-deepnsm-deleted-grammar-restored.md) | | 2026-10-04 | `coresearch-orkg-architecture` | | [2026-10-04-coresearch-orkg-architecture.md](2026-10-04-coresearch-orkg-architecture.md) | | 2026-10-04 | `coresearch-german-grammar-evidence` | | [2026-10-04-coresearch-german-grammar-evidence.md](2026-10-04-coresearch-german-grammar-evidence.md) | diff --git a/crates/deepnsm-v2/examples/dereko_profile.rs b/crates/deepnsm-v2/examples/dereko_profile.rs new file mode 100644 index 000000000..ee6ebb18f --- /dev/null +++ b/crates/deepnsm-v2/examples/dereko_profile.rs @@ -0,0 +1,184 @@ +//! `dereko_profile` — how well does the contemporary German frequency list +//! (DeReKo-2014, about 7 billion tokens) describe a text? +//! +//! For each text it reports: +//! - **coverage**: the share of tokens whose surface form is in the list, and +//! the share of word types missing from it; +//! - **rank agreement**: the Spearman correlation between the text's +//! frequency ranks and DeReKo's, over the text's 1,000 most frequent types +//! that the list knows; +//! - **POS ambiguity**: the share of tokens whose form has two or more STTS +//! tags, each carrying at least 10 % of its DeReKo frequency; +//! - **ADJD share**: tokens whose dominant tag is ADJD, the German adjective +//! used predicatively or adverbially (*früh*, *schnell*). +//! +//! Inputs: Project Gutenberg `.txt` (boilerplate stripped), getbible-style +//! Bible `.json`, or UD `.conllu`. DeReKo is CC BY-NC 3.0 and is never +//! committed; it is passed as a path (see `data/README.md`). +//! +//! ```text +//! cargo run --release --example dereko_profile -- DeReKo-2014-II-MainArchive-STT.100000.freq TEXT... +//! ``` + +use std::collections::HashMap; + +/// Per form: total frequency and per-STTS-tag frequency. +struct Entry { + total: f64, + tags: HashMap, +} + +fn load_dereko(path: &str) -> (HashMap, HashMap) { + let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut forms: HashMap = HashMap::new(); + for line in text.lines() { + let c: Vec<&str> = line.split('\t').collect(); + let [form, _lemma, tag, freq] = c.as_slice() else { + continue; + }; + let f: f64 = freq.parse().unwrap_or(0.0); + let e = forms.entry((*form).to_string()).or_insert(Entry { + total: 0.0, + tags: HashMap::new(), + }); + e.total += f; + *e.tags.entry((*tag).to_string()).or_default() += f; + } + let mut by_freq: Vec<(&String, f64)> = forms.iter().map(|(k, e)| (k, e.total)).collect(); + by_freq.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(b.0))); + let rank = by_freq + .into_iter() + .enumerate() + .map(|(i, (k, _))| (k.clone(), i)) + .collect(); + (forms, rank) +} + +/// The running text of one input file. +fn text_of(path: &str) -> String { + let raw = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + if path.ends_with(".json") { + let v: serde_json::Value = serde_json::from_str(&raw).expect("json"); + let mut out = String::new(); + for b in v["books"].as_array().into_iter().flatten() { + for c in b["chapters"].as_array().into_iter().flatten() { + for vs in c["verses"].as_array().into_iter().flatten() { + out.push_str(vs["text"].as_str().unwrap_or("")); + out.push('\n'); + } + } + } + out + } else if path.ends_with(".conllu") { + raw.lines() + .filter_map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + (!l.starts_with('#') && c.len() > 1 && !c[0].contains(['-', '.'])).then(|| c[1]) + }) + .collect::>() + .join(" ") + } else { + // Project Gutenberg: keep the body between the START and END markers. + let start = raw + .find("*** START") + .and_then(|i| raw[i..].find('\n').map(|j| i + j)) + .unwrap_or(0); + let end = raw.find("*** END").unwrap_or(raw.len()); + raw[start..end.max(start)].to_string() + } +} + +/// Word tokens: maximal alphabetic runs, case kept (German nouns). +fn tokens(text: &str) -> Vec<&str> { + text.split(|c: char| !c.is_alphabetic()) + .filter(|w| !w.is_empty()) + .collect() +} + +/// Spearman correlation of two rank lists of equal length. +fn spearman(a: &[f64], b: &[f64]) -> f64 { + let n = a.len() as f64; + let rank = |x: &[f64]| { + let mut idx: Vec = (0..x.len()).collect(); + idx.sort_by(|&i, &j| x[i].total_cmp(&x[j])); + let mut r = vec![0.0; x.len()]; + for (k, &i) in idx.iter().enumerate() { + r[i] = k as f64; + } + r + }; + let (ra, rb) = (rank(a), rank(b)); + let d2: f64 = ra.iter().zip(&rb).map(|(x, y)| (x - y).powi(2)).sum(); + 1.0 - 6.0 * d2 / (n * (n * n - 1.0)) +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let Some((dereko, texts)) = args.split_first() else { + eprintln!("usage: dereko_profile DEREKO.freq TEXT..."); + std::process::exit(2); + }; + let (forms, rank) = load_dereko(dereko); + // A form found as given, or with its first letter lowercased (a + // sentence-initial capital). + let find = |w: &str| -> Option<&String> { + if let Some((k, _)) = forms.get_key_value(w) { + return Some(k); + } + let mut cs = w.chars(); + let first = cs.next()?; + let lowered: String = first.to_lowercase().chain(cs).collect(); + forms.get_key_value(&lowered).map(|(k, _)| k) + }; + println!("DeReKo forms: {}", forms.len()); + println!( + "{:<34} {:>9} {:>8} {:>9} {:>9} {:>8} {:>7}", + "text", "tokens", "covered", "type OOV", "spearman", "POS amb", "ADJD" + ); + for path in texts { + let text = text_of(path); + let toks = tokens(&text); + let mut counts: HashMap<&str, usize> = HashMap::new(); + for t in &toks { + *counts.entry(t).or_default() += 1; + } + let (mut covered, mut ambiguous, mut adjd) = (0usize, 0usize, 0usize); + for t in &toks { + let Some(k) = find(t) else { continue }; + covered += 1; + let e = &forms[k]; + let strong = e.tags.values().filter(|&&f| f >= 0.1 * e.total).count(); + ambiguous += usize::from(strong >= 2); + let top = e.tags.iter().max_by(|a, b| a.1.total_cmp(b.1)); + adjd += usize::from(top.is_some_and(|(t, _)| t == "ADJD")); + } + let oov_types = counts.keys().filter(|w| find(w).is_none()).count(); + let mut by_count: Vec<(&&str, &usize)> = counts.iter().collect(); + by_count.sort_by(|a, b| b.1.cmp(a.1).then_with(|| a.0.cmp(b.0))); + let (mut text_rank, mut ref_rank) = (Vec::new(), Vec::new()); + for (i, (w, _)) in by_count.iter().enumerate() { + if text_rank.len() >= 1000 { + break; + } + if let Some(k) = find(w) { + text_rank.push(i as f64); + ref_rank.push(rank[k] as f64); + } + } + let name = std::path::Path::new(path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(path); + let n = toks.len().max(1) as f64; + println!( + "{:<34} {:>9} {:>7.1}% {:>8.1}% {:>9.3} {:>7.1}% {:>6.2}%", + name, + toks.len(), + 100.0 * covered as f64 / n, + 100.0 * oov_types as f64 / counts.len().max(1) as f64, + spearman(&text_rank, &ref_rank), + 100.0 * ambiguous as f64 / n, + 100.0 * adjd as f64 / n + ); + } +} diff --git a/crates/deepnsm-v2/examples/tekamolo_order.rs b/crates/deepnsm-v2/examples/tekamolo_order.rs new file mode 100644 index 000000000..98eb1e084 --- /dev/null +++ b/crates/deepnsm-v2/examples/tekamolo_order.rs @@ -0,0 +1,238 @@ +//! `tekamolo_order` — do edited German books keep the TEKAMOLO order +//! (Temporal < Kausal < Modal < Lokal) more often than the Bible translations? +//! +//! The reading is by **position only**, on raw text: +//! - a clause is the span between punctuation marks; +//! - in each clause, the first occurrence of each lane's cue word gives that +//! lane's position; +//! - every pair of two different lanes counts as **in order** when the +//! earlier lane comes first in Te < Ka < Mo < Lo. +//! +//! Cue words are UNAMBIGUOUS lane markers only: no prepositions (*in*, *an* +//! read both time and place), no modal particles (*wohl*), no words with a +//! second common reading (*da*, *so*, *noch*). +//! +//! Pre-registered prediction (fixed before the run): every Bible translation +//! has a lower in-order rate than the pooled edited literature, with 95 % +//! Wilson intervals that do not overlap. +//! +//! Inputs: Project Gutenberg `.txt`, getbible-style `.json`, UD `.conllu`. +//! +//! ```text +//! cargo run --release --example tekamolo_order -- --lit A.txt B.txt -- --bible X.json Y.json +//! ``` + +const TE: [&str; 17] = [ + "heute", + "gestern", + "morgen", + "damals", + "jetzt", + "bald", + "später", + "früher", + "immer", + "nie", + "niemals", + "oft", + "manchmal", + "bereits", + "neulich", + "zuvor", + "anschließend", +]; +const KA: [&str; 10] = [ + "deshalb", "deswegen", "darum", "daher", "wegen", "aufgrund", "trotz", "infolge", "dank", + "folglich", +]; +const MO: [&str; 16] = [ + "gern", + "gerne", + "schnell", + "langsam", + "plötzlich", + "allmählich", + "sorgfältig", + "gemeinsam", + "leise", + "laut", + "ruhig", + "heimlich", + "mühsam", + "eilig", + "vorsichtig", + "zusammen", +]; +const LO: [&str; 17] = [ + "hier", "dort", "drinnen", "draußen", "oben", "unten", "vorn", "vorne", "hinten", "links", + "rechts", "überall", "nirgends", "daheim", "dorthin", "hierher", "dahin", +]; + +fn lane(w: &str) -> Option { + [&TE[..], &KA[..], &MO[..], &LO[..]] + .iter() + .position(|set| set.contains(&w)) +} + +fn text_of(path: &str) -> String { + let raw = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}")); + if path.ends_with(".json") { + let v: serde_json::Value = serde_json::from_str(&raw).expect("json"); + let mut out = String::new(); + for b in v["books"].as_array().into_iter().flatten() { + for c in b["chapters"].as_array().into_iter().flatten() { + for vs in c["verses"].as_array().into_iter().flatten() { + out.push_str(vs["text"].as_str().unwrap_or("")); + out.push('\n'); + } + } + } + out + } else if path.ends_with(".conllu") { + raw.lines() + .filter_map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + (!l.starts_with('#') && c.len() > 1 && !c[0].contains(['-', '.'])).then(|| c[1]) + }) + .collect::>() + .join(" ") + } else { + let start = raw + .find("*** START") + .and_then(|i| raw[i..].find('\n').map(|j| i + j)) + .unwrap_or(0); + let end = raw.find("*** END").unwrap_or(raw.len()); + raw[start..end.max(start)].to_string() + } +} + +/// (pairs in order, pairs total, clauses with >= 2 lanes, [lane counts], +/// lane-in-Vorfeld counts). +#[derive(Default)] +struct Count { + in_order: usize, + pairs: usize, + clauses: usize, + lanes: [usize; 4], + vorfeld: [usize; 4], + tokens: usize, +} + +fn count(text: &str) -> Count { + let mut c = Count::default(); + for clause in text.split([ + '.', ',', ';', ':', '!', '?', '(', ')', '"', '«', '»', '„', '“', + ]) { + let words: Vec = clause + .split(|ch: char| !ch.is_alphabetic()) + .filter(|w| !w.is_empty()) + .map(str::to_lowercase) + .collect(); + c.tokens += words.len(); + let mut first: [Option; 4] = [None; 4]; + for (i, w) in words.iter().enumerate() { + if let Some(l) = lane(w) { + c.lanes[l] += 1; + if i == 0 { + c.vorfeld[l] += 1; + } + first[l].get_or_insert(i); + } + } + let present: Vec<(usize, usize)> = + (0..4).filter_map(|l| first[l].map(|p| (l, p))).collect(); + if present.len() < 2 { + continue; + } + c.clauses += 1; + for a in 0..present.len() { + for b in a + 1..present.len() { + // `present` is in lane order, so lane a < lane b. + c.pairs += 1; + c.in_order += usize::from(present[a].1 < present[b].1); + } + } + } + c +} + +/// Wilson 95 % interval of k / n. +fn wilson(k: usize, n: usize) -> (f64, f64, f64) { + if n == 0 { + return (0.0, 0.0, 0.0); + } + let (k, n, z) = (k as f64, n as f64, 1.96); + let p = k / n; + let d = 1.0 + z * z / n; + let c = (p + z * z / (2.0 * n)) / d; + let h = z * ((p * (1.0 - p) / n + z * z / (4.0 * n * n)).sqrt()) / d; + (p, c - h, c + h) +} + +fn line(name: &str, c: &Count) { + let (p, lo, hi) = wilson(c.in_order, c.pairs); + let per_10k = |x: usize| 1e4 * x as f64 / c.tokens.max(1) as f64; + println!( + " {name:<32} in order {:5.1}% [{:4.1}, {:4.1}] pairs {:6} clauses {:6} per 10k tokens Te {:5.1} Ka {:4.1} Mo {:5.1} Lo {:5.1} Te in Vorfeld {:4.1}%", + 100.0 * p, + 100.0 * lo, + 100.0 * hi, + c.pairs, + c.clauses, + per_10k(c.lanes[0]), + per_10k(c.lanes[1]), + per_10k(c.lanes[2]), + per_10k(c.lanes[3]), + 100.0 * c.vorfeld[0] as f64 / c.lanes[0].max(1) as f64 + ); +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let split = args.iter().position(|a| a == "--").unwrap_or(args.len()); + let (lit, bible) = args.split_at(split); + let lit: Vec<&String> = lit.iter().filter(|a| *a != "--lit").collect(); + let bible: Vec<&String> = bible + .iter() + .filter(|a| *a != "--" && *a != "--bible") + .collect(); + if lit.is_empty() || bible.is_empty() { + eprintln!("usage: tekamolo_order --lit TEXT... -- --bible TEXT..."); + std::process::exit(2); + } + let name = |p: &str| { + std::path::Path::new(p) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(p) + .to_string() + }; + println!("edited literature / contemporary text:"); + let mut pooled = Count::default(); + for p in &lit { + let c = count(&text_of(p)); + line(&name(p), &c); + pooled.in_order += c.in_order; + pooled.pairs += c.pairs; + pooled.clauses += c.clauses; + pooled.tokens += c.tokens; + for l in 0..4 { + pooled.lanes[l] += c.lanes[l]; + pooled.vorfeld[l] += c.vorfeld[l]; + } + } + line("POOLED", &pooled); + let (_, lit_lo, _) = wilson(pooled.in_order, pooled.pairs); + println!("Bible translations:"); + let mut all_below = true; + for p in &bible { + let c = count(&text_of(p)); + line(&name(p), &c); + let (_, _, hi) = wilson(c.in_order, c.pairs); + all_below &= hi < lit_lo; + } + println!( + "prediction (every Bible below the pooled literature, non-overlapping 95% intervals): {}", + if all_below { "PASS" } else { "KILL" } + ); +} From efb0fe394288c41b35474cc99147cd9b49afb96b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:24:47 +0000 Subject: [PATCH 29/33] deepnsm-v2 toc_hydrate: measure 6 vs 8 rails per 16-byte tenant on KJV basins (D-LXC-29) Basins per verse: median 3, p95 6, max 12. One tenant fits 95.66 % of verses with 6 rails (facet classid 4 + 12 bytes) and 99.41 % with 8 rails (16 bytes, classid in the row key); overflow drops 1,106 -> 149. Exact-set many-to-many group nodes do not deduplicate (1,099 of 1,106 sets unique). The reading-order chain gives a degenerate edge measure (one link per verse). No layout changed: an 8-rail tenant is a contract decision. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 1 + ...26-10-04-deepnsm-v2-tenant-rails-6-vs-8.md | 38 ++++++++++ .claude/board/entries/README.md | 3 +- crates/deepnsm-v2/examples/toc_hydrate.rs | 72 +++++++++++++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index e9b6dca7b..e71fb5244 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -52,6 +52,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-26** | gender + number of the head noun decide the article's case (`ud_case_eval` R6); stemmer fallback via `frostem` German Snowball (the tesseract-paperless search stemmer) | In PR | train case purity 65 → 79 % with gender + number; HDT R6 76.4 vs 64.1 % on seen nouns (+12.3); stem fallback 63.7 % KILL (the stem erases number); exploratory gender-only fallback 68.4 vs 66.9 %. Entry `2026-10-04-deepnsm-v2-gender-number-case.md` | | **D-LXC-27** | gender of unseen German nouns (`ud_gender_eval`): compound head, DeReKo lemma, `frostem` stem, nominalised infinitive | In PR | HDT unseen nouns: compound head covers 83.7 % at 98.1 % PASS; stem 88.0 % PASS; lemma 98.4 % on 4.7 %; infinitive 91.5 % KILL; cascade 89.9 % coverage at 97.7 %. GSD weaker (compound 88.8 % KILL). Entry `2026-10-04-deepnsm-v2-unseen-noun-gender.md` | | **D-LXC-28** | DeReKo-2014 profile across genres (`dereko_profile`); TEKAMOLO order by position in edited novels vs Bible translations (`tekamolo_order`) | In PR | ADJD about 3.4 % of tokens in novels vs about 1 % in every Bible; order: fresh novels 58.9 % vs Bibles 47.1 %, intervals overlap, KILL (z ≈ 2.1); density Te/Mo 3–5× lower in Bibles (robust). Entry `2026-10-04-deepnsm-v2-dereko-and-tekamolo-order-across-genres.md` | +| **D-LXC-29** | 6 vs 8 rails of u8:u8 per 16-byte tenant (classid already in the row key), measured on KJV basin membership (`toc_hydrate` TENANT CAPACITY) | In PR | basins per verse median 3 / p95 6 / max 12; one tenant fits 95.66 % (6 rails) vs 99.41 % (8 rails); overflow 1,106 → 149; exact-set group nodes do not deduplicate (1,099 of 1,106 unique). Layout change OPEN: needs an envelope audit and an operator ruling. Entry `2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md` | ## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md new file mode 100644 index 000000000..eea9447ef --- /dev/null +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md @@ -0,0 +1,38 @@ +# 2026-10-04 — 6 vs 8 rails per 16-byte tenant, measured on KJV basin membership (D-LXC-29) + +**Status:** MEASURED on the full KJV with `crates/deepnsm-v2/examples/toc_hydrate.rs` (debug-0, release), section "TENANT CAPACITY". No layout is changed. + +## The proposal + +The operator proposed using 8 × (u8:u8) per 16-byte tenant instead of 6 × (u8:u8): +- the V3 facet is classid(4) + 12 bytes = 6 rails; +- a tenant **inside a row whose key already carries the classid** repeats that classid; +- so the full 16 bytes could be rails, giving 8. + +Suggested tenants: +- tenant 1: the verse address; +- tenant 2: basins (`part_of:is_a` style) or edges; +- many-to-many tenants as group nodes. + +## Measurement + +- One rail holds one 16-bit reference (a basin id; 1,187 basins > 255, so one `u8:u8` pair per basin). +- Per verse: the distinct promoted basins whose subject or object occurs in that verse's triples. 25,460 verses have triples. +- "Group nodes" = distinct overflowing reference sets, i.e. many-to-many group nodes with exact-set sharing. + +| | median | p95 | max | fit in one tenant | overflow verses | group nodes | +|---|---|---|---|---|---|---| +| 6 rails (facet 4 + 12) | 3 | 6 | 12 | 95.66 % | 1,106 | 1,099 | +| **8 rails (16 B, classid in the slab)** | 3 | 6 | 12 | **99.41 %** | **149** | **146** | + +**Findings.** +1. **8 rails cut the overflow about 7.4×.** The p95 sits exactly at 6, so 6 rails cut into the tail and 8 rails clear almost all of it. +2. **Exact-set group nodes do not deduplicate**: 1,099 of 1,106 overflowing sets are unique. A useful many-to-many group would be a frequent SUBSET (itemset) shared across verses, not an exact set. Not measured. +3. **The edge measure here is degenerate.** Links come from the reading-order chain, so every verse has exactly one next link (max 1). An edge tenant needs coreference or shared-basin links, not measured. +4. The verse address (book, chapter, verse, each ≤ 255) needs 2 rails under either carving. + +**OPEN, a contract decision rather than a measurement.** +- The 4 + 12 facet is canon (`E-V3-FACET-4-PLUS-12`, `.claude/v3/soa_layout/le-contract.md` §3). +- A classid-free 8-rail register as a value-slab tenant is a new reading. +- It needs the envelope auditor (`v3-envelope-auditor`: field-isolation matrix, read-mode alias) and an operator ruling before any lane changes. +- BASIS: this table. REVISIT WHEN: the ruling lands, or edge/coreference degree is measured. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 8a70312d5..7e4550194 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,13 +25,14 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -206 entries, 2026-08-06 .. 2026-10-04. +207 entries, 2026-08-06 .. 2026-10-04. | date | entry id | finding | file | |---|---|---|---| | 2026-10-04 | `D-LXC-22` | | [2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md](2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md) | | 2026-10-04 | `D-LXC-25` | | [2026-10-04-deepnsm-v2-wechsel-lane-quorum.md](2026-10-04-deepnsm-v2-wechsel-lane-quorum.md) | | 2026-10-04 | `D-LXC-27` | | [2026-10-04-deepnsm-v2-unseen-noun-gender.md](2026-10-04-deepnsm-v2-unseen-noun-gender.md) | +| 2026-10-04 | `D-LXC-29` | | [2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md](2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md) | | 2026-10-04 | `D-LXC-20` | | [2026-10-04-deepnsm-v2-satzklammer-frageprobe.md](2026-10-04-deepnsm-v2-satzklammer-frageprobe.md) | | 2026-10-04 | `D-LXC-19` | | [2026-10-04-deepnsm-v2-rule-quorum.md](2026-10-04-deepnsm-v2-rule-quorum.md) | | 2026-10-04 | `D-LXC-18` | | [2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md](2026-10-04-deepnsm-v2-question-test-2cube-and-blast-radius.md) | diff --git a/crates/deepnsm-v2/examples/toc_hydrate.rs b/crates/deepnsm-v2/examples/toc_hydrate.rs index d7153d108..3d2071b68 100644 --- a/crates/deepnsm-v2/examples/toc_hydrate.rs +++ b/crates/deepnsm-v2/examples/toc_hydrate.rs @@ -255,6 +255,78 @@ fn main() { promoted += 1; } + // ── TENANT CAPACITY: 6 vs 8 rails of u8:u8 per 16-byte tenant ── + // A 16-byte tenant is either the V3 facet (classid 4 + 12 bytes = 6 rails) + // or, when the row's own key already carries the classid, the full 16 + // bytes (8 rails). One rail holds one 16-bit reference (a basin id or a + // target verse). Measured here: how many verses fit their references in + // one tenant, and how many many-to-many group nodes the overflow needs + // when identical reference sets share one group node. + let basin_ids: HashSet = rows.iter().map(|r| r.key.identity_v2()).collect(); + let mut basins_at: HashMap> = HashMap::new(); + for t in &h.triples { + let set = basins_at.entry(t.path).or_default(); + for x in [t.spo.subject, t.spo.object] { + if basin_ids.contains(&x) { + set.insert(x); + } + } + } + let mut links_at: HashMap> = HashMap::new(); + for (a, b) in &chain { + if a.path != b.path { + links_at.entry(a.path).or_default().insert(b.path); + } + } + let report = |name: &str, sets: Vec>| { + let n = sets.len().max(1); + let mut sizes: Vec = sets.iter().map(Vec::len).collect(); + sizes.sort_unstable(); + let pct = |q: usize| { + sizes + .get((sizes.len() * q / 100).min(sizes.len().saturating_sub(1))) + .copied() + .unwrap_or(0) + }; + print!( + "tenant {name:<17} {} verses median {} p95 {} max {}", + sets.len(), + pct(50), + pct(95), + sizes.last().copied().unwrap_or(0) + ); + for k in [6usize, 8] { + let over: Vec<&Vec> = sets.iter().filter(|s| s.len() > k).collect(); + let groups: HashSet<&Vec> = over.iter().copied().collect(); + print!( + " | {k} rails: fit {:.2}% overflow {} group nodes {}", + 100.0 * (sets.len() - over.len()) as f64 / n as f64, + over.len(), + groups.len() + ); + } + println!(); + }; + let sorted = |it: &mut dyn Iterator| { + let mut v: Vec = it.collect(); + v.sort_unstable(); + v + }; + report( + "basins per verse", + basins_at + .values() + .map(|s| sorted(&mut s.iter().map(|&x| u64::from(x)))) + .collect(), + ); + report( + "verse links out", + links_at + .values() + .map(|s| sorted(&mut s.iter().map(|p| p.packed().0))) + .collect(), + ); + // Every promoted basin must occupy its OWN address. Basins that first // appear in the same verse share a path, so the identity half is what // separates them — this is the assertion that proves it does. From e4c4783993c9d3ee40f95c3ee7812fe65a98712f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:28:06 +0000 Subject: [PATCH 30/33] board D-LXC-29: group nodes are explicit (AD-like) membership, not storage sharing; exploratory KJV group scan Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- ...26-10-04-deepnsm-v2-tenant-rails-6-vs-8.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md index eea9447ef..3d277cda1 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md @@ -36,3 +36,31 @@ Suggested tenants: - A classid-free 8-rail register as a value-slab tenant is a new reading. - It needs the envelope auditor (`v3-envelope-auditor`: field-isolation matrix, read-mode alias) and an operator ruling before any lane changes. - BASIS: this table. REVISIT WHEN: the ruling lands, or edge/coreference degree is measured. + +## Correction: group nodes are explicit membership, not storage sharing + +Operator: *"Group nodes wasn't about saving, it was about active directory like group memberships for explicit members as opposed to 24xi4 abstract qualities."* + +Finding 2 above ("exact-set group nodes do not deduplicate") measured storage sharing, which is the wrong question. + +- **A group node is an enumerated, crisp membership**, like an AD group: *the sons of Levi* = {Gershon, Kohath, Merari}. +- Groups nest: a member is itself a group. +- A verse refers to the group with one rail; members resolve through the group node's own member tenant, transitively through nested groups. +- Contrast: the 24×i4 tenant flavours hold graded, abstract qualities that are inferred, not enumerated. + +**Exploratory scan.** A regex over the KJV for "the sons/children/… of X; A, B, and C" (uncommitted lab script, noisy): +- **74 explicit groups**, 380 members, median 5 members, maximum 14. +- **29 nested groups**: a member heads its own group. Example chain: sons of Israel → Levi → sons of Levi → Kohath → sons of Kohath → Amram, four levels. +- **Groups are named collectively far more often than they are enumerated.** Collective mentions: *sons of Aaron* 28, *sons of Levi* 21, *children of Israel* 647, *the twelve* 38, *twelve tribes* 10. + +**Known noise:** +- Dinah (a daughter) is captured under the sons of Jacob. +- The variant spellings Gershom/Gershon are not merged. +- *Children of Israel* is never enumerated by this pattern. + +**OPEN.** +- A group-node reader: + - enumerations → member tenant (8 × u8:u8 per 16 B, chained for larger groups); + - nested membership as group → group edges; + - a collective mention → one group rail in the verse tenant. +- Then measure how many verse rails collective mentions replace. From 0374c450a5756d053103d08ff20c475ddac8120e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 10:29:36 +0000 Subject: [PATCH 31/33] board D-LXC-29: 16-byte register as 16 x u8 / 32 x u4 tags; clause-length fit (98 % <= 16 tokens) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- ...26-10-04-deepnsm-v2-tenant-rails-6-vs-8.md | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md index 3d277cda1..75dc2d0dd 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md @@ -64,3 +64,34 @@ Finding 2 above ("exact-set group nodes do not deduplicate") measured storage sh - nested membership as group → group edges; - a collective mention → one group rail in the verse tenant. - Then measure how many verse rails collective mentions replace. + +## Addendum: the same 16 classid-free bytes, read as 16 × u8 or 32 × u4 + +The operator points out that a classid-free 16-byte register can be read three ways (the content-blind register idea, one size up from the 12-byte facet): +- 8 × (u8:u8) rails; +- 16 × u8 tags; +- 32 × u4 sparse tags. + +**Current contract.** +- `CascadeShape` defines only 12-byte carvings (G6D2 / G4D3 / G3D4, G·D = 12). +- `CausalWitnessFacet` (the 24 × i4 tenant flavours) is exactly 12 bytes. +- A 16-byte register as 32 × u4 holds the 24 witness loci plus 8 spare nibbles. +- None of the 16-byte readings exist yet; adding them is a contract change. + +**Natural fits:** +- 16 × u8: verb atom (144), STTS (54) or UPOS (17), DeReKo frequency class, per token; +- 32 × u4: case + number, lane, or sparse flags, per token. Zero means unbound. + +**Measured: clause length (tokens between punctuation; lab scan, uncommitted).** + +| corpus | median | p95 | ≤ 16 tokens | ≤ 32 tokens | +|---|---|---|---|---| +| KJV | 6 | 13 | 98.5 % | 100.0 % | +| Luther 1545 | 5 | 13 | 98.1 % | 100.0 % | +| Buddenbrooks | 5 | 13 | 98.2 % | 100.0 % | +| Effi Briest | 5 | 12 | 98.4 % | 99.9 % | +| UD HDT (news) | 7 | 19 | 91.4 % | 99.9 % | + +One 16-byte tenant per clause therefore carries a per-token u8 tag for about 98 % of literary and biblical clauses (91 % of news), and a per-token nibble for essentially all clauses. + +**OPEN.** This is the same contract decision as above. It needs `CascadeShape` variants for 16 bytes (G8D2, G16D1, nibble×32), the envelope auditor's field-isolation matrix, and an operator ruling. From 9062218b4214037c92f08f08d01cf833e110a2c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 11:26:15 +0000 Subject: [PATCH 32/33] board D-LXC-29: 128-bit classid-free tenants via hot-plug ReadMode / metadata envelope; precedents verified in code Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- ...26-10-04-deepnsm-v2-tenant-rails-6-vs-8.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md index 75dc2d0dd..9b919afef 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md @@ -95,3 +95,33 @@ The operator points out that a classid-free 16-byte register can be read three w One 16-byte tenant per clause therefore carries a per-token u8 tag for about 98 % of literary and biblical clauses (91 % of news), and a per-token nibble for essentially all clauses. **OPEN.** This is the same contract decision as above. It needs `CascadeShape` variants for 16 bytes (G8D2, G16D1, nibble×32), the envelope auditor's field-isolation matrix, and an operator ruling. + +## Proposed design: 128-bit classid-free tenants, classid inherited (operator direction, precedents verified) + +Operator: *"for uniform shaped substrate we can define 128 bits instead of 96 by inheritance of the classid of the consumer via hotplug.rs in plug and play and its adjacent ogar-vocab plug and play registry, and/or via metadata envelope which already carries the 6 or 8 byte gamma metadata of palette256 … AD IAM already uses it."* + +Each precedent was read in code (VERIFIED-IN-CODE): + +| precedent | location | what it shows | +|---|---|---| +| hot-plug inheritance | `lance-graph-contract/src/hotplug.rs` (`Activation`), `canonical_node.rs:1454` (`ReadMode`) | The authority hands each hot-plugged classid a `ReadMode {tail_variant, value_schema, edge_codec}`: *"which value tenants materialise"*. The reading is resolved from the plug, never stored in the bytes. | +| metadata envelope | `bgz-tensor/src/shared_palette.rs:73` (`FisherZTable`: k×k i8 + **8 bytes family gamma**, once per palette group); `hhtl_cache.rs:251` (16 B); `gamma_phi.rs:67` (36 B) | Cells hold pure payload; calibration lives once in the group header. No 6-byte variant was found. | +| AD / IAM | `lance-graph-contract/src/rbac.rs:262` (`ClassGrant {target_classid: u16, op_mask}`), membership → member_role → role folding | Each grant stores only the 16-bit concept half (a `u8:u8` rail); the app prefix is inherited. Group membership folding is the group-node shape. | + +**DECISION (proposed, not yet landed).** +- A value-slab tenant may be a 128-bit classid-free register. +- Its classid comes from the row key / the consumer's hot-plug `ReadMode` (`value_schema`), or from a group-level metadata envelope. +- Readings: 8 × (u8:u8), 16 × u8, 32 × u4. + +**SCOPE.** +- Value-slab tenants only. The 16-byte KEY stays the V3 4 + 12 facet: classid canon-high plus payload; the classid must live somewhere. + +**BASIS.** Measured: 8 rails fit 99.41 % vs 6 rails 95.66 % of KJV verses' basin sets; one 16 × u8 tenant per clause fits 98 % of literary/biblical clauses; 32 × u4 fits about 100 %. Three precedents above. + +**Remaining work before it lands:** +1. 16-byte `CascadeShape` readings: G8D2, G16D1, nibble×32. +2. A `ValueSchema` entry that the hot-plug `ReadMode` resolves to. +3. The `v3-envelope-auditor` field-isolation matrix. +4. `le-contract.md` §3 extended beside the 12-byte carvings, without replacing them. + +**REVISIT WHEN:** the envelope audit, or a consumer whose rows cannot resolve a `ReadMode` through hot-plug (it would have nowhere to inherit a classid from). From d8edb747afdb137f7815ac916a823fb9cf07e194 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 11:39:28 +0000 Subject: [PATCH 33/33] deepnsm: CodeRabbit round on #1321 - fsm: a determiner that follows a carried subject is not a floating quantifier ("he gave him the charge" -> Noun); KJV re-pinned - lexicon: coca_pos delegates to coca::fsm_pos_tag - ud_lane_quorum: abstract-head counts on held-out items only; the confirm split prints its registered Round 3 bars - ontology_vocab: synset identity is (pos, offset) - persona_chain_replay: noun-only candidates; G1 fails on zero replays, G2 on any committed Ambiguous pronoun - board: stale numbers updated (KJV pin, toc_hydrate, persona ratio, held-out abstract counts, tenant fit) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SNGJgggrebw2JzdajXm59 --- .claude/board/STATUS_BOARD.md | 2 +- ...-10-04-deepnsm-deleted-grammar-restored.md | 8 +++ ...-10-04-deepnsm-v2-predicate-by-position.md | 9 ++++ ...26-10-04-deepnsm-v2-tenant-rails-6-vs-8.md | 5 ++ ...26-10-04-deepnsm-v2-wechsel-lane-quorum.md | 14 ++++++ .../deleted-grammar-heuristics-ledger.md | 4 +- ...deepnsm-v2-lexical-evidence-consumer-v1.md | 6 +++ crates/deepnsm-v2/Cargo.toml | 2 +- crates/deepnsm-v2/examples/bible_wave.rs | 9 +++- crates/deepnsm-v2/examples/ud_lane_quorum.rs | 44 +++++++++++----- crates/deepnsm-v2/src/fsm.rs | 39 ++++++++++++--- crates/deepnsm-v2/src/lexicon.rs | 27 ++++------ .../deepnsm/examples/persona_chain_replay.rs | 50 ++++++++++++++++--- crates/deepnsm/src/ontology_vocab.rs | 49 ++++++++++++++++-- 14 files changed, 216 insertions(+), 52 deletions(-) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index e71fb5244..44ad6e802 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -36,7 +36,7 @@ Plan: `.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md`. Convergence br | **D-LXC-8** | scrub the model-naming `Co-Authored-By` trailer from history (120 commits on `main`) | Deferred | operator: no force push | | **D-LXC-9** | `archaic_pos` cannot override COCA-known KJV words (`art` is Noun, `word_forms.csv:1047`) | Queued | pre-existing | | **D-LXC-10** | `Pos::Rel` produced by no in-crate tagger in deepnsm-v2 (only FSM match arms + one unit test) | Queued | pre-existing; external callers can still pass `Pos::Rel` through the public `Tagged`/`parse_to_spo` | -| **D-LXC-13** | noun/verb homograph decided by position: lemma N/V tag widened to its other reading (`coca::predicate_alternatives`), slot rule after a fresh subject | In PR | slot rule + widening disable-verified; KJV re-pinned (34,824 ambiguous / 451 words, certain 57,350 + alternative 29,001, 0 unexplained). Entry `2026-10-04-deepnsm-v2-predicate-by-position.md` | +| **D-LXC-13** | noun/verb homograph decided by position: lemma N/V tag widened to its other reading (`coca::predicate_alternatives`), slot rule after a fresh subject | In PR | slot rule + widening disable-verified; KJV re-pinned (34,824 ambiguous / 451 words, 0 unexplained). Current pin after the floating-quantifier and carried-subject fixes: certain 57,325 + alternative 29,287, reading accounting 14,111 / 20,713 / 1,898 / 155→161. Entry `2026-10-04-deepnsm-v2-predicate-by-position.md` | | **D-LXC-14** | `Pos::Adv`; `Typology` (measured, gates adjective licensing); `AttributeRule` adjective/adverb clauses (off: below frequency); `ud_pos_eval` gold harness (en/de/fr) | In PR | noun/verb with COCA: position 96.7 % vs frequency 85.1 % on 457 decided UD-EWT tokens; adjective/adverb: no clause beats frequency in every language. Entry `2026-10-04-deepnsm-v2-position-rules-on-ud-gold.md` | | **D-LXC-15** | KJV silver noun/verb labels from Luther 1545 casing (release `v0.1.0-codebooks-2026-07-26`); clause rule as an off-by-default switch; WordNet as a parallel coordinate | In PR | KJV: slot + licensing 85.8 % vs COCA lemma tag 74.9 % on 756 decided; clause rule below slot rule and frequency in en/de/fr/KJV; WordNet chooser 59.3 %, existence filter changes 1 form. Entry `2026-10-04-deepnsm-v2-kjv-silver-labels-from-luther.md` | | **D-LXC-16** | learned position table (neighbours' readings + copula → P(reading)), German case kept, Animal Farm silver reference | In PR | position × frequency beats frequency on adjective/adverb in 5/5 settings (Animal Farm 78.1 → 88.4 %), noun/verb in 4/5. Harness only. Entry `2026-10-04-deepnsm-v2-learned-position-table.md` | diff --git a/.claude/board/entries/2026-10-04-deepnsm-deleted-grammar-restored.md b/.claude/board/entries/2026-10-04-deepnsm-deleted-grammar-restored.md index d811ba91f..bd305684b 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-deleted-grammar-restored.md +++ b/.claude/board/entries/2026-10-04-deepnsm-deleted-grammar-restored.md @@ -43,3 +43,11 @@ Operator (2026-10-04): restore the deleted grammar heuristics, repurpose or merg - `tekamolo.rs` still carries `V2StyleProvider`. §3o refused thinking styles in deepnsm-v2; the operator decides whether to keep it here or move it to the planner. - `loci.rs` is agreement-blind. The shipped agreement resolver scores 0.727 against 0.273 for the agreement-blind rule. Merge queue in the ledger. + +> **⊘ Updated 2026-10-04 (PR #1321 review, same day).** +> - `toc_hydrate` now parses with every reading (`parse_readings(...).certain`, the same stream as `bible_wave`). +> - The FSM no longer treats an object carried into the subject slot as a subject head. +> - Current KJV numbers: **57,325 triples** (= `bible_wave`'s `certain` pin), 0 unaddressed, 1,187 basins. +> - `persona_chain_replay` candidates are now nouns only (the resolver's own lexicon): 2 replays run, both escalate, and the margin gate is **617×** too coarse (spread 0.000162 vs 0.1). +> - deepnsm-v2 lib tests: 196. +> - The figures above are the first run's. diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-predicate-by-position.md b/.claude/board/entries/2026-10-04-deepnsm-v2-predicate-by-position.md index 031436255..624f7dc69 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-predicate-by-position.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-predicate-by-position.md @@ -67,3 +67,12 @@ in the reading set. tag ("They record the deeds" → `record:Noun`); it adopts `coca::predicate_alternatives` after this merges. - Downstream `--export` still carries certain triples only. + +> **⊘ Re-pinned 2026-10-04 (PR #1321 reviews). The table above is the D-LXC-13 run.** +> +> | | narrowed | still ambiguous | eliminated | words | certain | alternative | +> |---|---|---|---|---|---|---| +> | floating quantifier (Bugbot) | 13,732 | 21,092 | 1,805 | 155 | 57,277 | 29,684 | +> | carried subject (CodeRabbit: "gave him **the** charge") | **14,111** | **20,713** | **1,898** | **161** | **57,325** | **29,287** | +> +> Both are pinned in `bible_wave::print_and_gate`, with the previous values kept in its comment. diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md index 9b919afef..bd7ab2825 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-tenant-rails-6-vs-8.md @@ -125,3 +125,8 @@ Each precedent was read in code (VERIFIED-IN-CODE): 4. `le-contract.md` §3 extended beside the 12-byte carvings, without replacing them. **REVISIT WHEN:** the envelope audit, or a consumer whose rows cannot resolve a `ReadMode` through hot-plug (it would have nowhere to inherit a classid from). + +> **⊘ Re-measured 2026-10-04 after the carried-subject FSM fix (KJV certain 57,325).** +> - 6 rails fit 95.62 %, 8 rails fit 99.40 %. +> - Overflow: 1,114 vs 152 verses; group nodes 1,107 vs 149. +> - The conclusions are unchanged. diff --git a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md index 1c7d50aa9..23c2d6194 100644 --- a/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md +++ b/.claude/board/entries/2026-10-04-deepnsm-v2-wechsel-lane-quorum.md @@ -88,3 +88,17 @@ The suffix class mixes event and abstract nouns with object nouns (*Wohnung*, *Z **OPEN.** - A stand-alone temporal adverb (*früh*, *heute*) as a TEKAMOLO Te element is a word-class question, the adj/adv quorum of D-LXC-20, not a PP-lane question. - The lane reader does not yet place bare adverbs into Te. + +> **⊘ Corrected 2026-10-04 (PR #1321 review): the abstract-head counts included the shared train labels.** +> Both rounds counted the 800 train items, so the two rounds were not independent. On held-out items only: +> - exploratory test: 1 of 35 abstract heads is PLACE (2.9 %), PASS; +> - fresh confirmation: 0 of 24 (0 %), PASS. +> +> Both are small samples. +> +> `LANE_SPLIT=confirm` now prints the Round 3 bars it registered: +> - noun-voter TIME F1 0.831: PASS; +> - PLACE F1 0.476: KILL; +> - quorum ≥ noun + 2 points: KILL. +> +> The verdicts are unchanged. diff --git a/.claude/knowledge/deleted-grammar-heuristics-ledger.md b/.claude/knowledge/deleted-grammar-heuristics-ledger.md index 0bce00924..e5f5dd038 100644 --- a/.claude/knowledge/deleted-grammar-heuristics-ledger.md +++ b/.claude/knowledge/deleted-grammar-heuristics-ledger.md @@ -28,12 +28,12 @@ for later reference."* | `crates/deepnsm-v2/src/toc.rs` | 426 | `68955ecb` (2026-08-22) | `abd331b9` | **The TOC as an HHTL tree of SoA node addresses.** Book:chapter:verse spawned as the full skeleton before any reasoning (operator: *"bevor wir über das Buch nachdenken wird ein Inhaltsverzeichnis als HHTL-Baum mit SoA-Knoten erstellt"*). Two nibbles per level. | "the hydrated table is rebuilt from scratch" | **RESTORED**. This is the HHTL literal chapter:verse address; the basins below are the cross-chapter nodes. | | `crates/deepnsm-v2/src/hydrate.rs` | 370 | `68955ecb` | `bfc69bf8`, `6e385c88` | Every SPO triple joined to the verse node it was read in. The reading-order chain is kept (the Markov trajectory over the book) along with the scanpath residue. | as `toc.rs` | **RESTORED**. | | `crates/deepnsm-v2/src/promote.rs` | 591 | `68955ecb` | `dd728f17`, `ec50f07b` | Basins written as SoA `NodeRow`s at TOC node keys: keyed by literary unit, not by bare subject (`E-HERMENEUTIK-RUNG-LADDER-1`). Also the TEKAMOLO tenant writer. | "also minted V1 keys" | **RESTORED**. The V1 claim was already false at deletion: `ec50f07b` made the promoter refuse the V1 tail (`mint_for` + read-back check, test-pinned). | -| `crates/deepnsm-v2/examples/toc_hydrate.rs` | 315 | `68955ecb` | `6e385c88` | The end-to-end run of tree → hydrate → promote → loci over the real KJV. | as above | **RESTORED**. Measured 2026-10-04: 32,357 nodes (66 books, 31,102 verses), 0 unaddressed. Superseded the same day by the PR #1321 review fix, now that toc_hydrate parses with every reading (`parse_readings`, the same stream as `bible_wave`): 57,277 triples (= `bible_wave`'s `certain` pin), 25,459 cross-verse links (44.4 %), 1,187 basins. The first run used the legacy single-reading parse: 70,393 triples, 1,227 basins. | +| `crates/deepnsm-v2/examples/toc_hydrate.rs` | 315 | `68955ecb` | `6e385c88` | The end-to-end run of tree → hydrate → promote → loci over the real KJV. | as above | **RESTORED**. Measured 2026-10-04: 32,357 nodes (66 books, 31,102 verses), 0 unaddressed. Superseded the same day by the PR #1321 review fix, now that toc_hydrate parses with every reading (`parse_readings`, the same stream as `bible_wave`): 57,277 triples (= `bible_wave`'s `certain` pin), 25,459 cross-verse links (44.4 %), 1,187 basins; after the carried-subject FSM fix, 57,325 (= the re-pinned `certain`). The first run used the legacy single-reading parse: 70,393 triples, 1,227 basins. | | `crates/deepnsm-v2/src/lexicon.rs` | 307 | `68955ecb` | `ec50f07b` | One COCA lexicon: lemma → forms → archaic → Other, monotone. It replaced three hand-copied taggers. | "taggers inlined into the two remaining examples" | **RESTORED**. The inlined copies in `bible_wave` / `genre_shapes` remain (they are pin-guarded). Merging them back onto this module is a deliberate re-pin. Measured in the file: KJV verses with no triple went from 35.6 % to 12.2 % once the forms table was read. | | `crates/deepnsm-v2/src/loci.rs` | 357 | `68955ecb` | `f47f0528` | Derives `Locus::Antecedent` (a 24×i4 `CausalWitnessFacet` nibble) from the SPO stream. Candidates are non-pronoun subjects within REACH = 8. The ranking is `SelectionalFit` (Cam96 similarity of each candidate to the pronoun clause's predicate) against `Recency`, with a margin gate. | "resolution AND binding already shipped" (`spo_anaphora_nibble`, `l9_loci_real_text`, `probe_antecedent_binder`) | **RESTORED**, with a known defect. It is agreement-blind. On Aesop the shipped agreement resolver scores 0.727 and the agreement-blind rule scores 0.273. **Merge pending:** take the gender/number/case agreement filter (and the German case table, see the research inventory) as the candidate filter, then keep SelectionalFit as the ranker. KJV numbers from `toc_hydrate`: Recency binds 35,025 of 35,613; SelectionalFit binds 27,788 (7,237 thin-margin). | | `crates/deepnsm-v2/src/tekamolo.rs` | 718 | `f3d42b7c` (style half), `c57cc256` (reverted that revert), `68955ecb` (whole file) | `7bae6c84` | **German** right-corner TEKAMOLO lane reading. A left-corner hypothesis is opened per adverbial and committed at the right corner on margin. It reads Luther 1545 and Elberfelder 1905 as a two-translation control, uses `de/tekamolo.tsv`, and hydrates the Kausal/Modal/Lokal lanes. | "insight_coca_read already emits all four lanes" — but that replacement is **English COCA only**, so the German capability was lost (D-LXC-18 blast radius) | **RESTORED**, with a ruling conflict. `V2StyleProvider` (ThinkingStyle → FieldModulation knobs) is the half §3o of `alpha-channel-rung-overlay-v1` refused for deepnsm-v2: reasoning lives in the planner. **Resolved 2026-10-04 (D-LXC-22):** grammar is not a thinking style, so `read_clause` now takes `ReadParams` (`LEFT_CORNER` / `RIGHT_CORNER` presets). The style mapping is kept as the adapter `ReadParams::from_style`; its destination is the planner, or a thinking dialect on `ogar-loco` (the long-term IR substrate). | | `crates/deepnsm/src/ontology_vocab.rs` | 141 | `48405aa2` (2026-08-22) | `ae8e762e` (`5ff56bdf` is cited by `48405aa2` but not in this clone) | An OBO ontology concept register (`0x03XX`) over `ogar_codebook::concepts_in_domain`: name ↔ id, with the COCA rank space kept apart. Lookups only; it never used CAM-PQ itself. | "CAM-PQ is prohibited for ontologies", and it sat in v1 when the instruction named v2 | **RESTORED+REPURPOSED**. (1) The OBO half returns empty: `98de36eb` retracted the 14 0x03XX mirror rows as never minted, so the domain is reserved with zero rows, and a test pins that. (2) **New WordNet half:** `WordNetRail` (all senses, all hypernym edges from `wordnet31_isa_v2.tsv`, exact 7-column arity), `ancestors`, `is_a`, and `ConceptMask`, which matches by membership of the synset or any ancestor. Identity only, no distance, so the A1 ruling holds. Tested on real WordNet rows: whale/dolphin under `aquatic_mammal`, shark/manta under `cartilaginous_fish`, and a three-root water-animal mask. | -| `crates/deepnsm/examples/persona_chain_replay.rs` | 376 | `8c03aaa9` (revert of `a74c3bbd`, no reason given) | `a74c3bbd` (not in this clone; its file content survives at `8c03aaa9^`) | The first caller of `ContextChain::disambiguate_with` with REAL candidates (window NP heads), scored against the agreement resolver's gold. Pronoun case gate: Ambiguous never decides. | none recorded | **RESTORED**. Measured 2026-10-04: the margin gate and the rank metric are on incompatible scales. The widest coherence spread is 0.0003 against a threshold of 0.1, 335× too small, so every replay escalates. This is the baseline for the counterfactual "does this make sense" test. | +| `crates/deepnsm/examples/persona_chain_replay.rs` | 376 | `8c03aaa9` (revert of `a74c3bbd`, no reason given) | `a74c3bbd` (not in this clone; its file content survives at `8c03aaa9^`) | The first caller of `ContextChain::disambiguate_with` with REAL candidates (window NP heads), scored against the agreement resolver's gold. Pronoun case gate: Ambiguous never decides. | none recorded | **RESTORED**. Measured 2026-10-04: the margin gate and the rank metric are on incompatible scales. The widest coherence spread is 0.0003 against a threshold of 0.1, 335× too small, so every replay escalates. With the candidates restricted to the resolver's nouns (PR #1321 review), 2 replays run and both escalate; the spread is 0.000162, 617× too small. This is the baseline for the counterfactual "does this make sense" test. | | `crates/deepnsm/src/markov_soa.rs` | 360 | `9a5f54c1` (2026-05-31) | — | The Markov wave over SoA SPO rows. | layer inversion: a core concern sat in a linguistics sensor | **SUPERSEDED-IN-PLACE** at `crates/lance-graph/src/graph/arigraph/markov_soa.rs` (vocabulary-agnostic, with an injected distance). | | `crates/deepnsm/src/{content_fp,markov_bundle,trajectory}.rs` | 98 / 250 / 298 | `0ae9f906` (2026-04-24) | D5 | 10K-bit XOR content fingerprints, bit-rotation braiding, Hamming recovery margin. | wrong substrate: GF(2)/XOR where the stack uses ℝ multiply+add (I-VSA-IDENTITIES, CHANGELOG 2026-04-21) | `markov_bundle.rs` and `trajectory.rs` exist again on the ℝ substrate. **`content_fp.rs` is HISTORY-ONLY**, because its `Vsa10k` type was removed from the contract. Its idea (a deterministic per-rank content vector) lives in the ℝ switchboard. | diff --git a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md index 360773027..28cfe2f90 100644 --- a/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md +++ b/.claude/plans/deepnsm-v2-lexical-evidence-consumer-v1.md @@ -80,6 +80,12 @@ Read in full before this rewrite: - F1 #1299 boundary: evidence, not transition; no dependency on causal-edge, cognitive-shader-driver, arm-discovery or the deprecated `deepnsm` crate. + - ⊘ **Narrowed 2026-10-04 (D-LXC-20).** + - DECISION: `deepnsm-v2` depends on `causal-edge` for ONE type, `pearl::CausalMask`. `fsm::answered_questions` returns the Pearl 2³ mask (operator: *"2³ ladder is in causaledge64"*). + - SCOPE: the mask type only. No transition, no `CausalEdge64` packing, no inference code is used; "evidence, not transition" stands. + - BASIS: `causal-edge` is zero-dependency, and one 2³ ladder beats a second copy. + - REVISIT WHEN: anything beyond the mask type is imported. + - The other three exclusions are unchanged. - F2 Routing `WordId`s and Cam96 `codes[word_id]` do not move. - F3 Integer counts; unknown = `None`; aggregates checked; unknown beats overflow (`lexical.rs:293-306`). diff --git a/crates/deepnsm-v2/Cargo.toml b/crates/deepnsm-v2/Cargo.toml index 03d58433f..fed14a490 100644 --- a/crates/deepnsm-v2/Cargo.toml +++ b/crates/deepnsm-v2/Cargo.toml @@ -39,7 +39,7 @@ name = "bible_wave" test = true # Examples only: `bible_wave`'s optional Rosetta arm reads the release's -# verse-keyed Bible JSON. The library stays dependency-free. +# verse-keyed Bible JSON. The library's dependencies are the path crates above. [dev-dependencies] serde_json = "1" # German Snowball stemmer, as used by tesseract-paperless search (via tantivy). diff --git a/crates/deepnsm-v2/examples/bible_wave.rs b/crates/deepnsm-v2/examples/bible_wave.rs index 6fe82ac22..2751c5470 100644 --- a/crates/deepnsm-v2/examples/bible_wave.rs +++ b/crates/deepnsm-v2/examples/bible_wave.rs @@ -269,6 +269,11 @@ impl LexicalDecodeReport { // the subject's head no longer licenses away the next verb reading. // Before it: reading (…, 14_383, 20_441, 1_943, 162), triples // (70_393, 57_350, 29_001, 12_984, 1_231, 1_190, 12_437, 256). + // Carried subject (CodeRabbit on #1321): after "gave him", the object + // carried into the subject slot is not a subject head, so "the" + // opens the next object group again. Before it: reading (…, 13_732, + // 21_092, 1_805, 155), triples (70_393, 57_277, 29_684, 13_151, + // 1_130, 1_183, 12_581, 256). // A deliberate decoder change re-pins these with the difference // reported. assert_eq!( @@ -283,7 +288,7 @@ impl LexicalDecodeReport { self.legacy_eliminated, self.eliminated_words.len(), ), - (771_176, 652_344, 34_824, 451, 84_008, 13_732, 21_092, 1_805, 155), + (771_176, 652_344, 34_824, 451, 84_008, 14_111, 20_713, 1_898, 161), "KILL D-LXC-2: reading accounting moved from the pinned KJV layout" ); assert_eq!( @@ -297,7 +302,7 @@ impl LexicalDecodeReport { self.verses_changed, self.peak_configs, ), - (70_393, 57_277, 29_684, 13_151, 1_130, 1_183, 12_581, 256), + (70_393, 57_325, 29_287, 13_040, 1_202, 1_192, 12_496, 256), "KILL D-LXC-2: triple accounting moved from the pinned KJV layout" ); println!("D-LXC-2 PASS accounting matches the pinned KJV layout"); diff --git a/crates/deepnsm-v2/examples/ud_lane_quorum.rs b/crates/deepnsm-v2/examples/ud_lane_quorum.rs index 35e245b5e..c3c212fae 100644 --- a/crates/deepnsm-v2/examples/ud_lane_quorum.rs +++ b/crates/deepnsm-v2/examples/ud_lane_quorum.rs @@ -41,7 +41,12 @@ //! such; //! - quorum accuracy < best single voter + 2 points; //! - TIME F1 < 0.80, or PLACE F1 < 0.60; -//! - hypothesis: more than 5 % of abstract-head items are labelled PLACE. +//! - hypothesis: more than 5 % of the held-out abstract-head items are +//! labelled PLACE (train labels are not counted). +//! +//! Confirmation round (`LANE_SPLIT=confirm`, bars registered before its +//! labels existed): noun-voter TIME F1 < 0.80, noun-voter PLACE F1 < 0.55, +//! or quorum accuracy < noun-voter accuracy + 2 points. //! //! `LANE_SPLIT=confirm` scores the fresh confirmation sample instead of the //! exploratory `test` sample; `LANE_LAMBDA` overrides the L2 strength. @@ -620,11 +625,10 @@ fn main() { } // Operator hypothesis: an abstract head is never PLACE. - let abs: Vec<&Item> = fit_items - .iter() - .chain(&test_items) - .filter(|x| x.abstract_head) - .collect(); + // Held-out split only: the train labels are shared by every round, so + // counting them would make two rounds look independent when they are not + // (PR #1321 review). + let abs: Vec<&Item> = test_items.iter().filter(|x| x.abstract_head).collect(); let abs_place = abs.iter().filter(|x| x.lane == 1).count(); let rate = 100.0 * abs_place as f64 / abs.len().max(1) as f64; println!( @@ -642,6 +646,7 @@ fn main() { ]; println!("\nsingle voters (each with the preposition prior):"); let mut best = 0.0f64; + let mut noun = (0.0f64, [0.0f64; 3]); for g in [ "prep", "noun", "number", "adverb", "verb", "position", "article", ] { @@ -653,6 +658,9 @@ fn main() { let m = Model::fit(&fit_items, &groups); let (acc, f1) = score(&m, &test_items, &groups); best = best.max(acc); + if g == "noun" { + noun = (acc, f1); + } println!( " {g:9} accuracy {:5.1}% F1 TIME {:.3} PLACE {:.3} FIG {:.3}", 100.0 * acc, @@ -687,10 +695,22 @@ fn main() { let (a, _) = score(&m, &test_items, &groups); println!(" without {drop:9}: accuracy {:5.1}%", 100.0 * a); } - let pass = acc >= best + 0.02 && f1[0] >= 0.80 && f1[1] >= 0.60; - println!( - " quorum {} (bars: >= best single + 2 pts ({:.1}%), TIME F1 >= 0.80, PLACE F1 >= 0.60)", - if pass { "PASS" } else { "KILL" }, - 100.0 * (best + 0.02) - ); + if eval_split == "confirm" { + // Round 3 bars, registered before the confirmation labels existed. + let checks = [ + ("noun voter TIME F1 >= 0.80", noun.1[0] >= 0.80), + ("noun voter PLACE F1 >= 0.55", noun.1[1] >= 0.55), + ("quorum >= noun voter + 2 pts", acc >= noun.0 + 0.02), + ]; + for (bar, ok) in checks { + println!(" confirm {}: {bar}", if ok { "PASS" } else { "KILL" }); + } + } else { + let pass = acc >= best + 0.02 && f1[0] >= 0.80 && f1[1] >= 0.60; + println!( + " quorum {} (bars: >= best single + 2 pts ({:.1}%), TIME F1 >= 0.80, PLACE F1 >= 0.60)", + if pass { "PASS" } else { "KILL" }, + 100.0 * (best + 0.02) + ); + } } diff --git a/crates/deepnsm-v2/src/fsm.rs b/crates/deepnsm-v2/src/fsm.rs index 1abc4285d..3eff8e4ee 100644 --- a/crates/deepnsm-v2/src/fsm.rs +++ b/crates/deepnsm-v2/src/fsm.rs @@ -160,6 +160,10 @@ struct Core { /// The previous token was a noun. Read only to recognise a floating /// quantifier, which follows the subject's head directly. after_noun: bool, + /// The subject slot holds an object carried over from the previous + /// triple ("gave **him** the charge"). A determiner after it opens the + /// next object's group, so it is never a floating quantifier. + subject_carried: bool, /// The previous token opened the clause as its subject — a noun taken in /// [`State::Start`] with no determiner or adjective in front ("they", /// "men", "God"). An object carried into the subject slot ("gave him …") @@ -184,6 +188,7 @@ impl Core { adjective_open: false, floating_quantifier: false, after_noun: false, + subject_carried: false, fresh_subject: false, predicated: false, }; @@ -196,8 +201,10 @@ impl Core { self.fresh_subject = false; self.nominal_open = matches!(t.pos, Pos::Det | Pos::Adj); self.adjective_open = t.pos == Pos::Adj; - self.floating_quantifier = - t.pos == Pos::Det && self.after_noun && matches!(self.state, State::HaveSubject); + self.floating_quantifier = t.pos == Pos::Det + && self.after_noun + && !self.subject_carried + && matches!(self.state, State::HaveSubject); self.after_noun = t.pos == Pos::Noun; // While a relative clause is open, its own tiny machine consumes the @@ -276,6 +283,7 @@ impl Core { self.subject = t.id; self.state = State::HaveSubject; self.fresh_subject = !nominal_was_open; + self.subject_carried = false; } (State::HaveSubject, Pos::Verb) => { self.predicated = true; @@ -287,12 +295,17 @@ impl Core { // Serial-verb chain: the object seeds the next subject. self.subject = t.id; self.state = State::HaveSubject; + self.subject_carried = true; return Some(out); } // A verb before a subject, or a second verb, restarts cleanly. (State::Start, Pos::Verb) => {} - (State::HaveSubject, Pos::Noun) => self.subject = t.id, // re-anchor subject - (State::HaveVerb, Pos::Verb) => self.predicate = t.id, // last verb wins + (State::HaveSubject, Pos::Noun) => { + // Re-anchor the subject; it is no longer a carried object. + self.subject = t.id; + self.subject_carried = false; + } + (State::HaveVerb, Pos::Verb) => self.predicate = t.id, // last verb wins } None } @@ -734,9 +747,10 @@ fn slot_allows(core: &Core, pos: Pos, set: PosSet) -> bool { /// subject and object is the predicate, however often COCA counts the word /// as a noun. Relative like licensing, so it never empties a token. /// -/// There is deliberately no "a sentence needs a predicate" rule: KJV verses -/// are often verbless fragments ("the goats for sin offering"), and on the -/// KJV such a rule turned nouns into verbs about three times in four. +/// The "a sentence needs a predicate" clause rule +/// ([`Typology::predicate_required`]) exists but is off here: KJV verses are +/// often verbless fragments ("the goats for sin offering"), and on the KJV +/// that rule turned nouns into verbs about three times in four. /// /// **Output.** At each `Stop` (and at the end of input) the sentence's /// configurations are compared: triples found on all of them are @@ -1154,6 +1168,17 @@ mod tests { assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Noun)); let p = parse_readings(&[one(9, Pos::Det), noun_or_verb(2)]); assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Noun)); + // CodeRabbit (#1321): "he gave him the charge". "him" is an object + // carried into the subject slot, so "the" opens the next object's + // group and "charge" loses its verb reading. + let p = parse_readings(&[ + one(1, Pos::Noun), + one(4, Pos::Verb), + one(3, Pos::Noun), + one(9, Pos::Det), + noun_or_verb(2), + ]); + assert_eq!(p.ambiguous[0].survived, PosSet::single(Pos::Noun)); // "man when the land": a determiner after a non-noun opens a group. let p = parse_readings(&[ one(1, Pos::Noun), diff --git a/crates/deepnsm-v2/src/lexicon.rs b/crates/deepnsm-v2/src/lexicon.rs index 0d0a239e0..aad60203b 100644 --- a/crates/deepnsm-v2/src/lexicon.rs +++ b/crates/deepnsm-v2/src/lexicon.rs @@ -36,32 +36,27 @@ use crate::fsm::Pos; use std::collections::HashMap; -/// Map a COCA part-of-speech letter to a [`Pos`]. -/// -/// `n`/`p` are noun-ish, `v` verbal, `j` adjectival, `a`/`d` determiners. -/// Everything else is [`Pos::Other`] — the FSM skips it. +/// Map a COCA part-of-speech letter to a [`Pos`], through the crate's one +/// fold ([`crate::coca::fsm_pos_tag`]): `n`/`p` noun, `v` verb, `j` +/// adjective, `a`/`d` determiner, `r` adverb, everything else +/// [`Pos::Other`]. A separate mapping here had drifted (it read `r` as +/// `Other`; PR #1321 review). #[must_use] pub fn coca_pos(letter: &str) -> Pos { - match letter { - "n" | "p" => Pos::Noun, - "v" => Pos::Verb, - "j" => Pos::Adj, - "a" | "d" => Pos::Det, - _ => Pos::Other, - } + crate::coca::fsm_pos_tag(letter) } -/// Early-modern English forms COCA does not carry. -/// -/// The explicit word list is load-bearing and must not be paraphrased down to -/// the `-eth`/`-est` rule: `thou`/`hath`/`shall`/`saith` are among the most -/// frequent tokens in the corpus and none of them matches that suffix. /// Early-modern personal pronouns. COCA tags pronouns `p` but carries none of /// these, so they are named ONCE here and read by both [`archaic_pos`] (which /// gives them a noun slot) and [`Lexicon::is_pronoun`] (which remembers the /// slot holds a POINTER, not a referent). pub const ARCHAIC_PRONOUNS: [&str; 3] = ["thou", "thee", "ye"]; +/// Early-modern English forms COCA does not carry. +/// +/// The explicit word list is load-bearing and must not be paraphrased down to +/// the `-eth`/`-est` rule: `thou`/`hath`/`shall`/`saith` are among the most +/// frequent tokens in the corpus and none of them matches that suffix. #[must_use] pub fn archaic_pos(w: &str) -> Option { if ARCHAIC_PRONOUNS.contains(&w) { diff --git a/crates/deepnsm/examples/persona_chain_replay.rs b/crates/deepnsm/examples/persona_chain_replay.rs index 4a10a04c2..b777c3d9d 100644 --- a/crates/deepnsm/examples/persona_chain_replay.rs +++ b/crates/deepnsm/examples/persona_chain_replay.rs @@ -140,11 +140,23 @@ fn candidates_at(stream: &[&str], pos: usize, ranks: &HashMap) -> V let lo = pos.saturating_sub(WINDOW); (lo..pos) .rev() - .filter(|&k| !is_pronoun(stream[k])) + .filter(|&k| !is_pronoun(stream[k]) && is_noun(stream[k])) .filter_map(|k| ranks.get(stream[k]).map(|&r| (k, r))) .collect() } +/// The shipped resolver's noun lexicon: exactly the forms +/// `spo_anaphora_nibble::noun_features` recognises (examples cannot import +/// each other, so the list is mirrored here). Both doors then rank the same +/// candidates, and a determiner or a verb can never be picked as an +/// antecedent (PR #1321 review). +fn is_noun(tok: &str) -> bool { + matches!( + tok, + "man" | "dog" | "girl" | "bird" | "girls" | "men" | "dogs" | "book" | "car" | "house" + ) +} + /// Fill an 11-slot `ContextChain` centred on `pos`: slot `focal + delta` carries /// the fingerprint of the token at `pos + delta`, or `None` where the token has /// no rank (out of the COCA table) or falls outside the stream. @@ -201,6 +213,9 @@ fn main() { let mut scored = 0usize; let mut out_of_catalogue: Vec = Vec::new(); let mut min_candidates = usize::MAX; + // Replays that actually ran, and how many of them committed. + let mut replays = 0usize; + let mut commits = 0usize; let mut widest_spread = 0.0f32; for &(pos, want) in gold { @@ -222,6 +237,7 @@ fn main() { } min_candidates = min_candidates.min(cands.len()); + replays += 1; let chain = chain_around(&stream, pos, &ranks); let res = chain.disambiguate_with( ContextChain::focal_index(), @@ -268,6 +284,14 @@ fn main() { None => CausalWitnessFacet::ZERO, }; + // G2, checked rather than asserted in prose: an Ambiguous pronoun + // never commits. + if case_blocked && committed.is_some() { + fail.push(format!("@{pos} G2: an Ambiguous pronoun committed")); + } + if committed.is_some() { + commits += 1; + } let picked = committed.map(|(k, _)| stream[k]); let reason = if case_blocked { "case=Ambiguous -> withheld" @@ -337,9 +361,18 @@ fn main() { )); } + // G1 needs at least one real replay: with none, `min_candidates` stays at + // its sentinel and the gate would pass on nothing. + if replays == 0 { + fail.push("G1: no pronoun had >= 2 candidates; no replay ran".to_string()); + } + // ── report ── println!(); - println!("G1 REAL-CANDIDATES min candidates per replay = {min_candidates} (sentinel path never taken)"); + println!( + "G1 REAL-CANDIDATES {replays} replay(s) ran, min candidates per replay = {}", + if replays == 0 { 0 } else { min_candidates } + ); println!("G2 CASE-GATE Ambiguous pronouns withheld, never committed on case"); println!("G3 METRIC-EXACT {g3_pairs} pairs, {g3_nonzero} with non-zero distance, hamming == |dRank|/{RANK_SCALE}"); println!( @@ -353,13 +386,16 @@ fn main() { } println!(" escalation threshold = {DISAMBIGUATION_MARGIN_THRESHOLD} (contract constant, untouched)"); println!( - "G5 SCALE widest coherence spread across candidates = {widest_spread:.6}; \n\ - \x20 threshold = {DISAMBIGUATION_MARGIN_THRESHOLD} -> ratio {:.0}x too small. The rank metric\n\ - \x20 and the margin gate are on INCOMPATIBLE SCALES: common-word\n\ - \x20 ranks differ by ~15 bits out of 16,384, so every replay\n\ - \x20 escalates regardless of input. Reported, not tuned.", + "G5 SCALE widest coherence spread across candidates = {widest_spread:.6}; \ + threshold = {DISAMBIGUATION_MARGIN_THRESHOLD} -> ratio {:.0}x too small; {commits} of {replays} replay(s) committed.", f64::from(DISAMBIGUATION_MARGIN_THRESHOLD) / f64::from(widest_spread.max(1e-9)) ); + if replays > 0 && commits == 0 { + println!( + " The rank metric and the margin gate are on INCOMPATIBLE SCALES:\n\ + \x20 every replay escalated. Reported, not tuned." + ); + } if fail.is_empty() { println!("\nGATES PASS — the real-candidate replay path runs end to end: candidates from"); diff --git a/crates/deepnsm/src/ontology_vocab.rs b/crates/deepnsm/src/ontology_vocab.rs index c48fd303e..ffc4b88f1 100644 --- a/crates/deepnsm/src/ontology_vocab.rs +++ b/crates/deepnsm/src/ontology_vocab.rs @@ -98,8 +98,17 @@ pub fn concept_at(concept_id: u16) -> Option<&'static str> { // WordNet — an identity register over synset offsets // ═══════════════════════════════════════════════════════════════════════════ -/// A WordNet synset offset: an identity, never a coordinate in a metric space. -pub type Synset = u32; +/// A WordNet synset: its offset AND its part of speech. An offset is unique +/// only inside one part-of-speech data file, so a noun and a verb can share +/// one (PR #1321 review); the pair is the identity. Never a coordinate in a +/// metric space. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Synset { + /// The part of speech (`'n'`, `'v'`, `'a'`, `'r'`, …). + pub pos: char, + /// The offset inside that part of speech's data file. + pub offset: u32, +} /// One sense of a word: its synset and WordNet's own sense rank (1-based). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -159,7 +168,10 @@ impl WordNetRail { .ok_or(RailError::Number { line: line_no })?; let sense_num = u16::try_from(num(c[2])?).map_err(|_| RailError::Number { line: line_no })?; - let synset = num(c[3])?; + let synset = Synset { + pos, + offset: num(c[3])?, + }; let senses = rail.senses.entry((c[0].to_string(), pos)).or_default(); if !senses.iter().any(|s| s.synset == synset) { senses.push(Sense { synset, sense_num }); @@ -167,7 +179,12 @@ impl WordNetRail { } rail.names.entry(synset).or_insert_with(|| c[0].to_string()); if !c[6].is_empty() { - let parent = num(c[6])?; + // Hypernym edges never cross parts of speech, so the parent + // shares the row's part of speech. + let parent = Synset { + pos, + offset: num(c[6])?, + }; // The hypernym column carries the parent's canonical name. rail.names.insert(parent, c[5].to_string()); let ps = rail.parents.entry(synset).or_default(); @@ -370,6 +387,30 @@ mod tests { } /// Exact arity: a 6-column line is refused, never skipped. + #[test] + fn a_noun_and_a_verb_with_the_same_offset_stay_distinct() { + // Same offset 00000123 in the noun file and the verb file: the verb + // must not inherit the noun's ancestors. + let text = "dog\tn\t1\t00000123\tisa\tanimal\t00000999\n\ + run\tv\t1\t00000123\tisa\tmove\t00000555\n"; + let r = WordNetRail::parse(text).expect("parses"); + let dog = r.synset("dog", 'n', 1).expect("dog"); + let run = r.synset("run", 'v', 1).expect("run"); + assert_ne!(dog, run); + assert_eq!(dog.offset, run.offset); + let animal = Synset { + pos: 'n', + offset: 999, + }; + assert!(r.is_a(dog, animal)); + assert!( + !r.is_a(run, animal), + "the verb inherited the noun's ancestor" + ); + assert!(!ConceptMask::new([animal]).covers(&r, run)); + assert_eq!(r.ancestors(run).len(), 1); + } + #[test] fn a_line_with_the_wrong_arity_is_refused() { let bad = "whale\tn\t2\t02065397\tisa\tcetacean";