From 5e866784620b211d44606d7a900249810f2a7be0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 30 Aug 2026 15:05:11 -0700 Subject: [PATCH 1/3] docs(rules,lexicon): the never-given set supplies a default, not a prohibition Three sites said a never-given particle "cannot be a name at all" or "can never be a given name", and used that to justify the readings built on it. It is false: NON_GIVEN_NAME_PARTICLES supplies the reading position leaves open and forbids nothing. Measured, all 33 members report given == themselves when parsed bare. The conclusions those sites draw are right; the reason given for them was not, and it is load-bearing. A particle OPENING the name has the rest of the name to join forward to -- that is evidence about the writing rather than about the word, which is why no declared name_order contradicts it. Stating it as a property of the vocabulary invites the opposite inference for a particle ENDING the name, where there is nothing to join forward to and the writing is all there is. This is the premise #466 was argued from, and the second correction of this same over-claim: decisions.md#P6 already records dropping "no particle is a name by itself" for the same reason. Behavior descriptions that the rest of #467 will change are left alone here; this commit moves no parse. --- docs/design/rules.md | 19 +++++++++++-------- nameparser/_lexicon.py | 7 +++++-- nameparser/config/particles.py | 11 +++++++---- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index 02347166..95140998 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -358,14 +358,17 @@ P5. Rationale: some given-name words are incomplete alone — "abdul" history: decisions.md#P5 · interacts: S2, M2, H1, P2, P4, P6 · implemented: nameparser/_pipeline/_group.py, nameparser/_pipeline/_post_rules.py P6. Rationale: a particle ending the name has nothing to link - forward to, so it is not doing a particle's work there. A - never-given particle in that position cannot be a name at all - and must belong to the family written beside it; an ambiguous - particle could genuinely be the name (Vietnamese "Van"), and - after a comma there is no signal that separates the two - readings. Dutch and Flemish names are listed exactly - this way ("Beethoven, Ludwig van"), the tussenvoegsel trailing - the given name but belonging to the surname. + forward to, so it is not doing a particle's work there. What it + is doing instead is decided by what the WRITING says, not by the + word: where something has already named the family — a comma, or + a declared family-first order — the particle belongs to that + family, which is how Dutch and Flemish names are listed + ("Beethoven, Ludwig van", the tussenvoegsel trailing the given + name but belonging to the surname). Where nothing has, it is + read where it stands, and a particle CAN be a given name there: + the never-given vocabulary supplies the reading position leaves + open, and forbids nothing (parse("de") reports given "de", and + so does every other word in that set). Where a family comma has already named the family, a particle ending the name attaches to that family name and is written before it — provided at least one given word remains, so that a diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 74d5ab5d..32baae46 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -374,8 +374,11 @@ class Lexicon: #: folded back into the family name once roles exist, so the whole #: name is the surname ("de Mesnil" -- a bare "de", with nothing to #: fold into, is left alone). That fold is order-independent too - #: (#359): a word that can never be a given name leaves - #: ``name_order`` nothing to decide. Which field a MEMBER's piece + #: (#359) -- not because the word could not be a given name, which + #: it can where position forces it (``parse("de")`` reports given + #: "de"), but because a particle OPENING the name has the rest of + #: the name to join forward to, and that evidence is positional + #: rather than vocabulary, so no declared order contradicts it. Which field a MEMBER's piece #: lands in is ``name_order``'s question, not this set's. #: No constant of its own -- the default derives #: as particles minus diff --git a/nameparser/config/particles.py b/nameparser/config/particles.py index 78127869..a2fe6b32 100644 --- a/nameparser/config/particles.py +++ b/nameparser/config/particles.py @@ -6,10 +6,13 @@ #: name has no given name -- the whole thing is a surname (e.g. "de Mesnil" #: -> family name "de Mesnil") -- and that reading holds under EVERY #: ``name_order`` (#359). It is not scoped to the default order the way the -#: rest of the positional read is: ``name_order`` says which side of the -#: name the family sits on, and a word that can never be a given name -#: leaves it nothing to decide, so ``Policy(name_order=FAMILY_FIRST)`` -#: reads "de Mesnil" as the family name too. What is asked about is the +#: rest of the positional read is: what settles it is that a particle +#: opening the name has the rest of the name to JOIN FORWARD TO, which +#: is evidence about the writing rather than about the word, so +#: ``Policy(name_order=FAMILY_FIRST)`` reads "de Mesnil" as the family +#: name too. Membership here does NOT mean the word can never be a +#: given name -- where position forces the reading it is one, and every +#: member of this set reports given "de"-style for a bare parse. What is asked about is the #: opening *piece*, not the first word of the string: a particle that has #: already chained onto the word behind it is part of that piece rather #: than standing alone. From 417c35af0d8700d9221a55f6ac9b27df9a348325 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 30 Aug 2026 15:15:55 -0700 Subject: [PATCH 2/3] fix(post_rules): the trailing particle is decided by its slot, not its spelling (#467) A declared family-first order names the family as a comma does, so P6's attachment fires without a comma too -- keyed on the SLOT the trailing run landed in. MIDDLE is the one position that means nothing for a particle, middles being further given names, and FAMILY_FIRST is the only order that puts a trailing piece there. Jong, Anke de family 'de Jong' (unchanged) Jong Anke de FF family 'Jong' middle 'de' -> family 'de Jong' P1 loses its given-position site in the same change, and that half is a correction rather than a narrowing: that slot holds what the caller DECLARED to be the given name. `Menil de` now reports given 'de' under both family-first orders. No vocabulary test at the new site, and that is the design. A comma cannot separate the Dutch reading from the Vietnamese; the declared order can, and does both at once -- 'Beethoven Ludwig van' gives family 'van Beethoven' under FAMILY_FIRST while 'Nguyen Thi Van' keeps given 'Van' under FAMILY_FIRST_GIVEN_LAST, the same word in the same set. A never-given test would have excluded 37 ambiguous particles including von, di, da, del, le and `van` itself. No re-layout either, and that is a property of the order: the roles run family, given, middle, middle..., so dropping a trailing middle leaves every other piece where it was. One guard remains, the base -- attaching to an all-particle family reorders words R2 reads as ordinary names. The invariant this rests on: `Family, Given particle` under the default order and `Family Given particle` under FAMILY_FIRST parse the same. 0 of 216 constructed names agreed before, 216 of 216 after, including all 72 with an ambiguous particle. One example line was deleted rather than updated -- "Nguyen Thi Van" under FAMILY_FIRST -- because the name is not written in that format and its reading there pins nothing (#470 is the general question). 30 of 6594 parses move, twelve names, none under the default order. Gate green at all three baselines and blind to all of it. The leading fold's reach under the default order is #471, split out: that one is a 1.4.0 parity break where this is new-feature territory. --- docs/design/decisions.md | 13 ++++ docs/design/rules.md | 92 +++++++++++++++++++-------- docs/release_log.rst | 5 +- nameparser/_pipeline/_post_rules.py | 86 +++++++++++++++++++++---- tests/v2/pipeline/test_post_rules.py | 88 +++++++++++++++++++++---- tools/differential/corpus_rules.jsonl | 5 ++ 6 files changed, 235 insertions(+), 54 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index c794ae8a..1433f140 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -118,6 +118,19 @@ Open: [#380](https://github.com/derek73/python-nameparser/issues/380) covers "Be Measured over the 1094-name differential corpus: P6's attachment fires on 9 names, of which 5 take the ambiguous arm (`Beethoven, Ludwig van`, `Berg, Jan van der`, `Nguyen, Thi Van`, `Vega, Juan de la`, `der, y van`), 2 the suffix arm (`Berg, Jan vd`, `Berg, abdul vd`) and 2 stay silent (`Jong, Anke de`, `Jong, Piet de`). Seven names gain an ambiguity kind and no role field moves that was not already moving. Read that seven as evidence about the corpus rather than about the blast radius, the caution the 2026-08-16 entry above already gives for this rule: 304 of the 1094 names carry a comma and only 13 end in a particle, 11 of those being rows the #379/#380 arc added itself (this change added four corpus names and not one of them carries a comma, so the 11 is unchanged by it). The reach is a SHAPE — every family-comma listing ending in an ambiguous particle, which is to say ordinary Dutch, Flemish and unaccented Vietnamese listings — and a caller holding such records sees a report on all of them, not on seven. The LEDGER shape that follows from that is not the one the plan predicted, and the difference is worth recording because it will recur for any report added to a rule whose fields already move. A new `fields = ["_ambiguities"]` rule explains none of the seven: `classify` matches on a rule's `fields` being a SUPERSET of the whole diff, and against 2.0.0 and 2.1.0 these names diff in `middle`/`family` (or `family`/`suffix`) from #379 and #380 as well. So the repair is to WIDEN those two existing rules to `{middle, family, _ambiguities}` and `{family, suffix, _ambiguities}` in both ledgers, not to add a rule. Each widened rule still passes the #452 equality check, since the union over the names it explains is exactly what it declares — the two silent `Jong` names contribute `{middle, family}` and the five ambiguous ones contribute `_ambiguities` on top. 1.4.0 needs nothing at all, having no ambiguity surface to diff. Gate before and after: byte-identical output at all three baselines apart from the provenance header — 1094 names, 229/194/102 intentional, 0 unexplained, and no per-heading count moved. The seven names were already in the diff; only their field sets grew. The most visible consequence, and it is not the Vietnamese one: `Beethoven, Ludwig van` — the textbook-correct Dutch listing, read exactly right — now carries a report. That is honest rather than regrettable. The parser cannot separate it from `Nguyen, Thi Van`; the two are the same string shape over the same vocabulary, and a report on one is a report on the other. A caller who wants only the doubtful cases has nothing here to filter on, because there is nothing to filter on in the input either. +- 2026-08-30 #467 (the no-comma site, and the given position given back) — a declared family-first order names the family as a comma does, so P6's attachment fires without a comma too. It is keyed on the SLOT the trailing run landed in: MIDDLE, the one position that means nothing for a particle, since middles are further given names and a particle is not one. Only FAMILY_FIRST puts a trailing piece there. This DECIDES the parenthesis the 2026-08-16 Scope bullet left open ("arguably a missing comma under a declared family-first order") and narrows that bullet to the default order. + P1 LOSES ITS GIVEN-POSITION SITE in the same change, and that half is a correction rather than a narrowing. That slot holds what the caller DECLARED to be the given name; the never-given vocabulary supplies a reading where position leaves the question open and vetoes none that position has already given. `Ménil de` under either family-first order now reports given `de`. + THE PREMISE THIS REPLACES WAS FALSE, and it had been stated in three places (rules.md#P6's rationale, `_lexicon.py`, `config/particles.py`): that a never-given particle "cannot be a name at all". Measured, all 33 members report given == themselves when parsed bare. This is the SECOND correction of the same over-claim in this entry — the 2026-08-16 note above records dropping "no particle is a name by itself" for the identical reason. The conclusions drawn from it were mostly right; the reason was not, and it is load-bearing, because stating it as a property of the VOCABULARY invites the same inference for a particle ending the name, where there is nothing to join forward to and the writing is all there is. + NO VOCABULARY TEST AT THE NEW SITE, and that is the substance of the design. The comma path needs one because a comma cannot separate the Dutch reading from the Vietnamese; the declared order can, and does both at once — `Beethoven Ludwig van` under FAMILY_FIRST gives family 'van Beethoven' though `van` is ambiguous vocabulary, and `Nguyen Thi Van` under FAMILY_FIRST_GIVEN_LAST keeps given 'Van' though it is the same word in the same set. A never-given test here would have excluded 37 ambiguous particles — von, di, da, del, le, and `van` itself, the flagship word of the listing the rule is named for. An earlier attempt (#466) carried exactly that test and failed on the Dutch case because of it. + NO RE-LAYOUT EITHER, and this is a property of the order rather than a simplification: under these orders the roles run family, given, middle, middle…, so dropping a trailing MIDDLE leaves every other piece's role untouched. #466 removed a piece and re-laid the leftover out, which lost a given name outright on `van Berg Jan de` and promoted a post-nominal into the given slot on `Berg Jan Jr. de`. Both defects are structurally unreachable here. The one guard that remains is the base: attaching to an all-particle family renders one particle in front of another, and R2 reads those words as ordinary name words, which no rule reorders. + THE INVARIANT THIS RESTS ON, and the reason to trust it over the argument it replaced: for a name of the form `Family Given [Particle]`, the family-comma writing under the default order and the comma-less writing under FAMILY_FIRST parse identically. Measured over 216 constructed names — 0 of 216 agreed before this change, 216 of 216 after, including all 72 with an ambiguous particle. The invariant draws the never-given/ambiguous line by itself instead of having it asserted. Scope it to that SHAPE: 181 of 247 corpus comma-names parse differently with the comma removed, because a comma also delimits titles and suffixes. + An asymmetry falls out of it rather than being carved: shape 2 places the given name BEFORE the middles, so it corresponds to FAMILY_FIRST only. There is no comma format that puts the given name last, so FAMILY_FIRST_GIVEN_LAST has no twin to inherit an equivalence from. That is why the trailing particle is one order's business (#469). + ONE EXAMPLE LINE WAS DELETED RATHER THAN UPDATED: `"Nguyen Thi Van" family-first → middle="Van"`. The name is written in FAMILY_FIRST_GIVEN_LAST; its FAMILY_FIRST reading is wrong by construction, so the line pinned a value nobody wants and nothing should depend on. Recorded here because the general question — whether a rules.md example should declare its format, and whether the doc test should refuse to run one outside it — is #470 and is NOT decided by this change. + REACH: 30 of 6594 parses move — this tree's 1099-name corpus × three `name_order` values × `middle_as_family` off and on — over twelve names, NONE under the default order. Recompute by parsing the four `tools/differential/corpus*.jsonl` files of this tree, deduped, under each of the six policies, and comparing the seven role fields against a checkout of the parent commit reading the SAME corpus files; the comparator is the pre-change code, never a construction of this rule. Dated snapshot, measured 2026-08-30. Note the corpus grew from master's 1094 by this change's own example lines, which is why the count must name the tree it was taken on. + The differential gate cannot see any of it: `compare.py` parses every name as `HumanName(name)` and sweeps no policy, so no non-default `name_order` behavior has ever been compared across versions. Output is byte-identical at all three baselines apart from the added corpus names, none of which diffs. + Open: [#471](https://github.com/derek73/python-nameparser/issues/471) is the leading fold's REACH under the default order (`de Mesnil Juan` reports the whole string as the family). Split out of this change deliberately: it is a 1.4.0 parity break on the order every existing caller is on — prototyped, 5 unexplained diffs at that baseline and a failing v1 parity test — where nothing under a family-first order has a v1 answer at all. + + ### M2 — the maiden-marker rule - 2026-07-03 (maiden-bucket design; #274 filed 2026-07-07, landed in the v2 core, PR #288) — the marker takes everything after it up to a trailing suffix, greedily: "née Jones Smith" is a two-word maiden name, matching how the marker is actually used in running text. The marker itself is dropped as structural, like a delimiter character. diff --git a/docs/design/rules.md b/docs/design/rules.md index 95140998..c386ff06 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -102,28 +102,37 @@ H3. Rationale: compound titles are written as a run of title words, Background: particles ("de", "la", "van", "von", "bin") link forward to a surname and are written as part of it. Some are never anyone's given name; others ("Van", "Bin") are ordinary given names in some cultures, so the vocabulary distinguishes never-given particles from ambiguous ones, and only the never-given ones license special treatment. A separate small vocabulary binds forward to a GIVEN name instead: words like "abdul" that are not complete given names alone (P5). Which particles fall on which side of the never-given line is its own open question (#360). -P1. Rationale: a never-given particle standing alone cannot be - someone's given name; a name that opens with one, or offers only - one as the given name, is a surname written out in full. - A never-given particle standing alone where the given name would - go — or opening the name — marks the name as surname-only: the - particle run and the name words it attaches to are the family. It - needs another name word to attach to. The run is every particle - in sequence, never-given and ambiguous alike ("de la Vega" is - one group, not "de" plus a separate "la Vega"). An ambiguous particle keeps - whatever reading its position gives it. That the particle claims - the FAMILY rather than a given name holds under every order: a - never-given particle is evidence about how the name is written, - and a declared order governs only what no vocabulary has claimed - (O4) — the same precedence the script license takes in W4. - - How MANY name words it attaches to depends on the order, and on - which of the two positions above the particle stands in. Opening - the name, under a family-first order, it takes exactly ONE — - declaring that order asserts that what follows the family is not - more surname. Opening the name under the default order, or - standing in the given position under any order, it takes the rest - of the name: nothing there marks where the surname ends. One name +P1. Rationale: a particle OPENING a name has the whole rest of the + name to link forward to, so it is doing a particle's work and the + piece it heads is a surname written out in full. That is evidence + about the writing, not about the word, which is why no declared + order contradicts it. + A never-given particle opening the name marks the name as + surname-only: the particle run and the name words it attaches to + are the family. It needs another name word to attach to. The run + is every particle in sequence, never-given and ambiguous alike + ("de la Vega" is one group, not "de" plus a separate "la Vega"). + An ambiguous particle keeps whatever reading its position gives + it. That the particle claims the FAMILY rather than a given name + holds under every order: this is evidence about how the name is + written, and a declared order governs only what no vocabulary has + claimed (O4) — the same precedence the script license takes in W4. + The OPENING position is the whole of this rule's subject. A + particle standing where the given name would go was a second site + until #467, and dropping it is a correction rather than a + narrowing: that slot holds what the caller DECLARED to be the + given name, and the never-given vocabulary supplies a reading + where position leaves the question open rather than vetoing one + position has already given. A particle ending the name is P6's, + and only where it landed in a MIDDLE. + + How MANY name words it attaches to depends on the order. Under a + family-first order it takes exactly ONE — declaring that order + asserts that what follows the family is not more surname. Under + the default order it takes the rest of the name: nothing there + marks where the surname ends. Whether the default order should + narrow the same way is #471 — today `de Mesnil Juan` reports the + whole string as the family, and that reach is 1.4.0's. One name word means one UNIT — a particle chain (P2), a conjunction join (P3) or a bound given-name pair (P5) is taken whole or not at all. A title does not move the opening position (P4), but a @@ -136,7 +145,7 @@ P1. Rationale: a never-given particle standing alone cannot be the family slot being already filled. "de la Vega" → family="de la Vega" "Sir de Mesnil" → family="de Mesnil" - "Mesnil de" family-first → family="Mesnil de" + "Mesnil de" family-first → given="de" "de Mesnil Juan" → family="de Mesnil Juan" "de Mesnil Juan" family-first → family="de Mesnil" "de Mesnil Juan" family-first → given="Juan" @@ -391,6 +400,33 @@ P6. Rationale: a particle ending the name has nothing to link "Beethoven, Ludwig van" → family_base="Beethoven" "Beethoven, Ludwig van" → family_particles="van" "Nguyen, Van" → given="Van" · boundary + Without a comma, a declared family-first order has named the + family in the same way and the attachment fires there too — but + only where the run landed in a MIDDLE, which is the one position + that means nothing for it. Middles are further given names and a + particle is not one. FAMILY_FIRST is the only order that puts a + trailing piece there: FAMILY_FIRST_GIVEN_LAST puts it in GIVEN, + where the caller's declaration says it IS the given name, and the + default order puts it in FAMILY, where it already is one. + Nothing is asked about the WORD, and that is what lets one rule + read both traditions: the same vocabulary that spells the Dutch + tussenvoegsel spells the Vietnamese given name, and the declared + order separates them where a comma cannot. + Nothing is re-laid-out either — under that order the roles run + family, given, middle, middle…, so dropping a trailing middle + leaves every other piece where it was. The family must hold a + base of its own, since a family that is all particles is not a + family written beside anything (R2). + "Jong Anke de" family-first → family="de Jong" + "Jong Anke de" family-first → given="Anke" + "Ménil Christophe de" family-first → family="de Ménil" + "Ménil Christophe de" family-first → given="Christophe" + "Beethoven Ludwig van" family-first → family="van Beethoven" + "Ménil Christophe de" family-first-given-last → given="de" + "Berg Jan de Jr." family-first → family="de Berg" + "Berg Jan de Jr." family-first → suffix="Jr." + "van Berg Jan de" family-first → given="Berg" · boundary + "Ménil de" family-first → given="de" · boundary Accepted: an ambiguous particle attaches on the same terms as a never-given one, so a Vietnamese name written in this listing loses its given name — but only in the UNACCENTED @@ -414,12 +450,14 @@ P6. Rationale: a particle ending the name has nothing to link "Berg, Jan vd" → ambiguities=("suffix-or-name",) "Jong, Piet de" → ambiguities=() · boundary Accepted: the colliding spelling has a format that reads - correctly, and it is ONE order, not both: FAMILY_FIRST still - sends the given name to the middle, and only - FAMILY_FIRST_GIVEN_LAST recovers it. + correctly, and it is ONE order, not both — FAMILY_FIRST_GIVEN_LAST, + which is the order this name is written in. A line asserting what + it does under FAMILY_FIRST stood here until #467 and was removed + rather than updated: the name is not written in that format, so + its reading there is wrong by construction and pins nothing worth + keeping (#470). "Nguyen Thi Van" family-first-given-last → family="Nguyen" "Nguyen Thi Van" family-first-given-last → given="Van" - "Nguyen Thi Van" family-first → middle="Van" Accepted: without a family comma the name's written shape is not settled — "Jong Anke de" may be a misformatted listing, and a bare "Jong de" may be a given name beside a particle — so the diff --git a/docs/release_log.rst b/docs/release_log.rst index 7a22ec8d..c5efa437 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -16,6 +16,8 @@ Release Log the shapes and the corpus names it moves, and the gate output is the source of record. The unusual part is the family-first orders, which change too: seven names from the family-first fix, + twelve more from the trailing-particle change below (#467), which + is family-first only and moves nothing under the default order, plus the names #367's title fix moves, which it moves in every order. The other title fix below, #410, is default-order only -- under either family-first order the word behind the title is @@ -61,7 +63,7 @@ Release Log - Fix a space-separated credential run after a family comma rendering with a comma the name never had: ``"Smith, MD PhD"`` gives suffix ``MD PhD`` where it gave ``MD, PhD``, and ``"Smith, CBE MC"``, ``"Smith, BSc MBA"``, ``"Smith, PhD Jr."`` and ``"Smith, Dr. MD PhD"`` the same. The roles are unchanged; only the rendered string carried the extra comma. One comma segment is one suffix entry, and grouping decided which segments those were by their position while assignment decided the same segment by its content, so the two disagreed about the segment a family comma leaves after it. The reading follows assignment now, which is where the segment's roles were already decided. This reaches any family comma whose following segment holds no name word, not only a one-word family: ``"John Smith, Jr. III"`` gives suffix ``Jr. III``, which is also what 1.4.0 gave. The full-name comma form was never affected -- ``"John Smith, MD PhD"`` has given suffix ``MD PhD`` since 1.4.0 -- and this brings the rest into line with it. A run written with commas keeps them (``"Smith, MD, PhD"`` is still suffix ``MD, PhD``), a comma between two entries survives (``"Smith Jr., Mr. Jr."`` is still suffix ``Jr., Jr.``), and a name word in the segment still makes it the given-and-suffix reading (``"Smith, John Jr."``). A name written without a comma is unaffected and still renders its run comma-joined, so re-parsing ``str()`` output does not reproduce the run (closes #429) - Fix a one-character suffix word after a comma being read by the wrong neighbour: ``"Smith, PSM I"`` gives suffix ``PSM I`` where it gave given ``PSM`` and suffix ``I``, and ``"Smith, John V."`` gives middle ``V.`` where it gave suffix ``V.``. Inside a comma part a suffix word short enough to be mistaken for an initial -- ``I``, ``V`` and ``2`` in the shipped vocabulary -- is read by what stands before it. Behind a credential it is describing that credential -- ``PSM I`` is Professional Scrum Master level I -- so the run continues through it whether or not a period is written, since an initial in that position is not a name shape anyone writes; ``"Smith, MD I"`` and ``"Smith, Jr. I"`` reached the same wrong answer by a different route and move with it. Behind a name the period decides, marking an abbreviation and so a middle initial, exactly as ``"Smith, John B."`` always read. A numeral written bare after a name is still the generation it looks like (``"Smith, John V"`` is suffix ``V``), and a name with no comma is untouched (closes #430, closes #432) - - Fix a name opening with a particle that is *never* a given name being split at the particle under a family-first name order -- ``Policy(name_order=FAMILY_FIRST)`` and ``Policy(name_order=FAMILY_FIRST_GIVEN_LAST)`` alike, and identically: ``"de Mesnil"`` read as family ``de``, given ``Mesnil``, and ``"de la Vega"`` as family ``de``, given ``la Vega``. Each is now the whole surname, as it has always been in the default order. The rule enforcing it asked for the particle by the ``GIVEN`` role, which under a family-first order belongs to the token *after* the particle, so the test read the wrong word and declined. It now also asks by position -- the piece that opens the name -- so both shapes of the same rule are caught: where such a particle stands alone as a piece, either opening the name or in the given position, the name is left with no given name at all, the given and the middles folding into the family. Standing *alone* is the whole of it, and the rule claims nothing wider: ``"Juan de la Vega"`` under ``FAMILY_FIRST`` still reports given ``de la Vega``, because there the particle chained onto the words after it rather than standing alone, and a bare ``"de"`` with nothing to fold into is still reported as the given name. The decision behind the fix: a word that can never be a given name leaves ``name_order`` nothing to decide, so declaring family-first is not a reason to make ``de`` a surname on its own. A leading particle that *may* be a given name is genuinely order-dependent and is untouched -- ``"van Gogh"`` still reads as family ``van``, given ``Gogh`` under both family-first orders. This is also what gives ``Lexicon.particles_ambiguous`` an effect outside the default order: taking a word out of it now changes the parsed fields under a family-first order, where before it moved only the ambiguity report. Seven of the 751 differential corpus names move, the same seven under each family-first order; default-order output is byte-identical over all 751, at the 1.4.0, 2.0.0 and 2.1.0 differential baselines alike (closes #359) + - Fix a name opening with a particle that is *never* a given name being split at the particle under a family-first name order -- ``Policy(name_order=FAMILY_FIRST)`` and ``Policy(name_order=FAMILY_FIRST_GIVEN_LAST)`` alike, and identically: ``"de Mesnil"`` read as family ``de``, given ``Mesnil``, and ``"de la Vega"`` as family ``de``, given ``la Vega``. Each is now the whole surname, as it has always been in the default order. The rule enforcing it asked for the particle by the ``GIVEN`` role, which under a family-first order belongs to the token *after* the particle, so the test read the wrong word and declined. It now also asks by position -- the piece that opens the name -- so both shapes of the same rule are caught: where such a particle stands alone as a piece OPENING the name, the given and the middles fold into the family. (The rule reached a particle standing in the GIVEN position too until #467 below, which gives that slot back to the declared name order.) Standing *alone* is the whole of it, and the rule claims nothing wider: ``"Juan de la Vega"`` under ``FAMILY_FIRST`` still reports given ``de la Vega``, because there the particle chained onto the words after it rather than standing alone, and a bare ``"de"`` with nothing to fold into is still reported as the given name. The decision behind the fix: a word that can never be a given name leaves ``name_order`` nothing to decide, so declaring family-first is not a reason to make ``de`` a surname on its own. A leading particle that *may* be a given name is genuinely order-dependent and is untouched -- ``"van Gogh"`` still reads as family ``van``, given ``Gogh`` under both family-first orders. This is also what gives ``Lexicon.particles_ambiguous`` an effect outside the default order: taking a word out of it now changes the parsed fields under a family-first order, where before it moved only the ambiguity report. Seven of the 751 differential corpus names move, the same seven under each family-first order; default-order output is byte-identical over all 751, at the 1.4.0, 2.0.0 and 2.1.0 differential baselines alike (closes #359) - Fix a family name made only of particle words reporting no base on the 2.0 API, so the surname vanished from ``family_base`` and, on both APIs, from the initials: ``parse("Anh Do")`` gave family ``Do`` with ``family_base`` ``''`` and initials ``A.``, and under ``Policy(name_order=FAMILY_FIRST)`` ``"Del Toro"`` gave family ``Del`` the same way. The empty base was a 2.0 regression rather than a longstanding bug -- 1.4.0's own guard kept ``HumanName("Anh Do").last_base`` at ``Do``, and the facade has kept it right throughout; what 1.4.0 and 2.1 shared was the missing initial. A particle earns its name by joining forward to the word it modifies, so a particle standing alone in a name part is not doing a particle's work there and reads as an ordinary name word: it anchors the base, leaves the particles view, and contributes an initial. ``"Anh Do"`` is now base ``Do``, initials ``A. D.``; ``"Juan van der"`` is base ``van der``, initials ``J. v. d.``; ``"Nguyen, Van Le"`` initials ``V. L. N.`` where the middle name used to be dropped. Position decides this, not vocabulary -- whether the word is borne as a surname somewhere does not enter into it, which reverses the reasoning ``rules.md#R2`` carried before. The invariant it exists to hold: a non-empty family always has a non-empty base, because a particle needs a base to attach to. Where the particles DO join a name word nothing changes -- ``"Juan de la Vega"`` keeps base ``Vega``, particles ``de la`` and initials ``J. V.``. The parse fields themselves do not move: ``title``, ``given``, ``middle``, ``family``, ``suffix``, ``nickname`` and ``maiden`` are byte-identical over all 751 differential corpus names in all three name orders, and only the derived views and the initials change -- which is why the differential harness, which compares those seven roles, reports no diffs for this at any baseline. One consequence to know about: where the whole family is particles AND a tussenvoegsel was attached to it (#379), this change exposed a pre-existing ordering gap between ``initials()`` and the family field of the same name -- fixed in the #408 entry below, so the two agree as shipped (closes #385, closes #402) @@ -69,6 +71,7 @@ Release Log - Change case repair to read the parser's own ``conjunction`` tag instead of re-deciding, from the word's spelling, whether a word is a conjunction or an initial. The parse answers that question already -- ``"Scott E. Werner"`` reads ``E.`` as an initial rather than the Italian conjunction -- and the other views honor the answer; case repair asked again, with a shape test applied to each word of a token's text rather than to the token. Two spellings of one name disagreed because of it: ``"juan e-f smith"`` capitalized to ``Juan e-F Smith`` while ``"JUAN E-F SMITH"`` gave ``Juan E-F Smith``; both give ``Juan E-F Smith`` now, ``e-f`` being a middle name and no conjunction of the parse's reading. A conjunction written as a word of its own is untouched, and so is the one-letter carve-out where it applies -- ``"juan y garcia"`` still repairs to ``Juan y Garcia``, ``"JUAN Y GARCIA"`` still to ``Juan Y Garcia``. A field assigned after the parse is unaffected: its text was never classified, so there is no reading to honor and repair asks the vocabulary, applying v1's own predicate the way every earlier version applied it everywhere -- ``h.last = "velasquez y garcia"`` still repairs to ``Velasquez y Garcia`` and ``h.middle = "e."`` to ``E.``. That is the predicate over TODAY's vocabulary, which is narrower than parity with 1.4.0 and the difference is real: ``h.last = "хосе и мария сантос"`` gives ``Хосе И Мария Сантос`` on 1.4.0 and ``Хосе и Мария Сантос`` here, because the Cyrillic ``и`` is a 2.x conjunction and was not a 1.4.0 one. What decides which path a token takes is a mark the assignment leaves, not the absence of a span: a value revised through ``Parser.revise()`` is classified by a sub-parse and keeps its tags, so it repairs as the parse does. One reading does change for hand-built ``Token``\ s in the 2.0 API: an untagged token whose text is conjunction vocabulary is now an ordinary name word and capitalizes, where 2.1 lowercased it -- tags are what the views read, and a hand-built token that carries none is a token with nothing to declare. Case repair is not one of the seven role fields the differential harness compares, so no gate run can see this change either way and none of its counts move; measured directly instead, no name of the 1094-name differential corpus moves under ``capitalized()`` or ``capitalized(force=True)``, its uppercased and lowercased spellings included -- 6564 name/spelling/lexicon rows and 13128 calls (closes #458) + - Fix a trailing surname particle being stranded as a standalone middle name under ``Policy(name_order=FAMILY_FIRST)``, where the same listing written with a comma reads it as part of the surname. A particle ending the name has nothing to link forward to, so what it is doing there is decided by what the writing says: after a family comma it joins the family the comma named and is written before it, and a declared family-first order names the family the same way. ``Parser(policy=Policy(name_order=FAMILY_FIRST)).parse("Jong Anke de")`` gave family ``Jong`` with ``de`` left as a middle name, and now gives family ``de Jong``, given ``Anke`` -- the same answer ``parse("Jong, Anke de")`` has always given. The test is the SLOT the particle landed in, not the word: a middle name is a further given name, which a particle is not, and ``FAMILY_FIRST`` is the only order that puts a trailing piece there. ``FAMILY_FIRST_GIVEN_LAST`` puts it in the given slot, where the caller's own declaration says it is the given name, so ``"Nguyen Thi Van"`` under that order still reads given ``Van``. That one test reads both traditions without asking about the vocabulary at all: ``"Beethoven Ludwig van"`` under ``FAMILY_FIRST`` now gives family ``van Beethoven`` even though ``van`` is one of the 37 particles that are ordinary given names elsewhere. In the same change, a particle standing alone where a family-first order puts the GIVEN name is no longer folded into the family: ``"Ménil de"`` reports given ``de`` under both family-first orders, because that slot holds what the caller declared, and the never-given word list supplies a reading where position leaves the question open rather than overriding one position has already given. Nothing moves under the DEFAULT name order. Measured, 30 of 6594 parses move -- this release's 1099-name corpus under three ``name_order`` values with ``middle_as_family`` off and on -- over twelve names. Those counts cannot come from the differential gate: it parses every corpus name with the default policy and sweeps no policy at all, so no non-default ``name_order`` behavior has ever been compared across versions, and the gate output here is unchanged at all three baselines apart from the corpus names this change's own rules.md examples add. What moves is recomputed by the recipe in the ``P6`` entry of ``docs/design/decisions.md``, which compares the seven role fields against a checkout of the parent commit reading the same corpus files; the count is a dated snapshot, measured 2026-08-30. The ``rules.md#P1`` and ``rules.md#P6`` example lines and ``tests/v2/pipeline/test_post_rules.py`` are what pin the behavior (closes #467) - Fix ``initials()`` reading a name in a different order than the fields of the same name. Two rules fold words into the family name and render them before the rest of it -- ``Policy(middle_as_family=True)``, which sends every middle word to the family, and the tussenvoegsel attachment after a family comma -- and both do it by marking the words rather than moving them, since a parsed word keeps the position it was written at. The ``family`` field reads that mark and ``initials()`` did not, so one parse gave two orders: ``parse("der, y van")`` gave family ``van der`` and initials ``y. d. v.``, and now gives ``y. v. d.``. This RESTORES v1: ``middle_name_as_last`` is v1's spelling of the same option, so most of what moves has a 1.4.0 answer to be measured against, and measured over the 1094-name differential corpus at the default name order, 71 names move under that option, 54 of them back to exactly what 1.4.0 returns and not one of them away from it -- ``"Doe, Dr. John A."`` gives ``J. A. D.`` again where 2.0 through 2.2 gave ``J. D. A.``, and ``"Brundridge, Contessa A"`` gives ``C. A. B.`` where they gave ``C. B. A.``. Of the 17 that match 1.4.0 neither before nor after, 14 now agree with it on the ORDER and differ only in how v1 grouped initials -- 1.4.0 gives one initial per element of its own ``last_list``, so a conjunction-joined surname yields ``V G.`` where the 2.x view, one initial per word, yields ``V. G.``; one more is ``"der, y van"``, whose family is nothing but particles, where 1.4.0 contributes no initial at all and 2.x contributes its words, a difference this release does not touch; and the remaining two parse differently from 1.4.0 for reasons that predate this fix. Without the option, one corpus name moves, the ``"der, y van"`` above -- and it is one of one, since it is the only name in the corpus whose family holds two contributing words with a folded one behind the other, the shape an order change can be seen in at all. ``HumanName.initials()`` was already right and is unchanged, measured: no name of the 1094-name corpus moves through the facade, with the option or without it. It reads ``first_list``/``middle_list``/``last_list``, which prepend the folded words as v1 did, so it is the 2.0 API's ``ParsedName.initials()`` that was out of step -- with the field beside it, with the facade, and with 1.4.0 at once. ``initials()`` is not one of the seven role fields the differential harness compares, so no gate run can see this change: run at all three baselines before and after, the output is identical to the byte -- 1094 corpus names, 229 / 194 / 102 intentional diffs and ``unexplained: 0`` at 1.4.0 / 2.0.0 / 2.1.0, with every per-heading count unchanged. So where the counts in this bullet come from has to be said, the gate's classified summary not being able to supply them and the ``rules.md#R3`` example line witnessing the order without counting anything: what MOVES is recomputed by the recipe in the ``R3`` entry of ``docs/design/decisions.md``, which compares this view against the pre-change rendering over these same four corpora and reproduces the 71 and the one-of-one; the 1.4.0 comparisons and the facade sweep are dated snapshots rather than re-derivable ones, measured 2026-08-30 against the released 1.4.0 wheel and against the pre-change tree, which nothing in the repository re-runs. The ``rules.md#R3`` example line and ``tests/v2/test_render.py`` are what pin the behavior (closes #408) - Fix a tussenvoegsel attached to the family name after a comma deciding a genuinely uncertain reading and reporting nothing. ``"Van Johnson"`` reports a ``PARTICLE_OR_GIVEN`` ambiguity -- ``Van`` is a Dutch particle and a Vietnamese given name, and the parser has to pick one -- while ``"Nguyen, Thi Van"`` picked the same word the same way, silently, and lost the given name doing it. The attachment now reports the fork it decides, in the kind that names the reading it declined. A particle that could be an ordinary name reports ``PARTICLE_OR_GIVEN``: ``"Nguyen, Thi Van"``, ``"Berg, Jan van der"`` and ``"Vega, Juan de la"`` each gain one, the ``detail`` naming the ambiguous word. A particle the parser had already read as a post-nominal reports ``SUFFIX_OR_NAME`` instead, because the credential reading is what the attachment overrode: ``"Berg, Jan vd"`` gains one, ``vd`` being read as *van der* rather than as the Volunteer Decoration. Which kind you get follows the reading that was overridden rather than the word's vocabulary, so ``"Berg, Jan do"`` reports ``PARTICLE_OR_GIVEN`` even though ``do`` is a postnominal too -- it was already being read as a name word, so no credential reading was overridden. A particle where nothing was overridden reports nothing at all: ``"Jong, Piet de"`` and ``"Jong, Anke de"`` are unchanged, ``de`` being no name in any reading and no postnominal either, and so is ``"Berg, Jan de vd"``, whose run was read as name words whole. Worth knowing before you filter on this: ``"Beethoven, Ludwig van"`` -- the textbook Dutch listing, read exactly right -- now carries a report too. It is the same string shape over the same vocabulary as ``"Nguyen, Thi Van"``, and nothing in the input separates them, so a report on one is a report on both. This adds the report and nothing else: every field these names parse to is exactly what the tussenvoegsel fix below already gave them, and ``ambiguities`` is the only value that grows. Seven differential corpus names gain a kind against the 2.0.0 and 2.1.0 baselines, seven of the nine the tussenvoegsel fix below already moved -- five of ``fix(#379)``'s seven and both of ``fix(#380)``'s two, now carrying ``_ambiguities`` in their diff as well; the two left out are the pair named unchanged above, ``Jong, Piet de`` and ``Jong, Anke de`` -- and none against 1.4.0, which had no ambiguity reporting at all (closes #405) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 1869dae8..19306eff 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -245,11 +245,10 @@ def post_rules(state: ParseState) -> ParseState: middles = _idx(tokens, Role.MIDDLE) families = _idx(tokens, Role.FAMILY) - # rules.md#P1: "a never-given particle standing alone where the - # given name would go — or opening the name — marks the name as - # surname-only: the particle run and the name words it attaches - # to are the family." (v1 handle_non_first_name_prefix; history: - # decisions.md#P1) + # rules.md#P1: "A never-given particle opening the name marks the + # name as surname-only: the particle run and the name words it + # attaches to are the family." (v1 handle_non_first_name_prefix; + # history: decisions.md#P1) # How far the fold reaches depends on the order the name was READ # under (#395; decisions.md#P1, 2026-08-17): declaring a # family-first order asserts that what follows the family is not @@ -266,15 +265,22 @@ def post_rules(state: ParseState) -> ParseState: # having already fixed the surname, so `order is None` here. # Anything that later gives that path an order turns the # narrowing on for it. - # Code-local: a lone PIECE is the test at both sites, so a - # particle group already chained forward is not a lone particle, - # and rule H1 above cannot be what produces the fold's family - # reading -- H1 is gated on `not families`. + # ONE site, the opening piece. A particle standing in the GIVEN + # position was a second site until #467, and it was the parser + # overriding a declaration rather than reading one: under a + # family-first order that slot holds what the caller SAID is the + # given name, and the never-given vocabulary supplies a default + # where position leaves the question open, not a veto over an + # answer position already gave. So "Menil de" under either + # family-first order reports given "de" -- P6 takes the trailing + # particle only where it landed in a MIDDLE, which means nothing. + # Code-local: a lone PIECE is the test, so a particle group + # already chained forward is not a lone particle, and rule H1 + # above cannot be what produces the fold's family reading -- H1 is + # gated on `not families`. lead = _leading_name_piece(state, tokens) lead_fires = _is_lone_never_given_particle(lead, tokens) - sites_fire = lead_fires or _is_lone_never_given_particle( - tuple(givens), tokens) - if len(givens) + len(middles) + len(families) > 1 and sites_fire: + if len(givens) + len(middles) + len(families) > 1 and lead_fires: order = state.order if lead_fires and order is not None and order[0] is Role.FAMILY: # `state.order`, not policy.name_order: a script_orders @@ -335,6 +341,62 @@ def post_rules(state: ParseState) -> ParseState: _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) # rules.md#P6: "a particle ending the name attaches to that family + # name and is written before it" -- the no-comma site (#467). + # A comma is not the only thing that names the family; a declared + # family-first order does too, and the same listing is written + # both ways ("Jong, Anke de" and "Jong Anke de" under + # FAMILY_FIRST both give family "de Jong"). + # + # Keyed on the SLOT the run landed in, not on its vocabulary. + # MIDDLE is the one position that means nothing here: middles are + # further given names, and a particle is not one. Only + # FAMILY_FIRST puts a trailing piece there -- + # FAMILY_FIRST_GIVEN_LAST puts it in GIVEN, where the caller's own + # declaration says it IS the given name, and P1's site above no + # longer overrides that either. + # + # Which is why no vocabulary test appears here and none is wanted. + # The comma path needs one because a comma cannot separate the + # Dutch reading from the Vietnamese; the declared ORDER can, and + # does it for both at once: "Beethoven Ludwig van" under + # FAMILY_FIRST gives family "van Beethoven" though `van` is + # ambiguous vocabulary, and "Nguyen Thi Van" under + # FAMILY_FIRST_GIVEN_LAST keeps given "Van" though the same word + # is in the same set. A never-given test here would have excluded + # 37 ambiguous particles -- von, di, da, del, le and `van` itself, + # the flagship word of the listing this rule is named for. + # + # NO RE-LAYOUT, and that is a property of the order rather than a + # simplification: under FAMILY_FIRST the roles run family, given, + # middle, middle..., so dropping a trailing MIDDLE leaves every + # other piece's role untouched. An earlier draft (#466) removed a + # piece and re-laid the leftover out, which lost a given name + # outright on "van Berg Jan de" and promoted a post-nominal into + # the given slot on "Berg Jan Jr. de". + # + # The family must still hold a base of its own: attaching to an + # all-particle family renders one particle in front of another + # ("van Berg Jan de" -> family 'de van'), and R2 reads those words + # as ordinary name words, which no rule reorders. + mids = _idx(tokens, Role.MIDDLE) + if mids and families and state.order is not None: + run: list[int] = [] + i = len(mids) + while i and "particle" in tokens[mids[i - 1]].tags: + i -= 1 + run.append(mids[i]) + if run and any("particle" not in tokens[j].tags for j in families): + for j in run: + tokens[j] = dataclasses.replace( + tokens[j], role=Role.FAMILY, + tags=tokens[j].tags | {FOLDED_TAG}) + # recomputed for H1's reason, stated at H1: a stale index + # list is the bug shape #359 fixed + givens = _idx(tokens, Role.GIVEN) + middles = _idx(tokens, Role.MIDDLE) + families = _idx(tokens, Role.FAMILY) + + # rules.md#P6: "a particle ending the name attaches to that family # name and is written before it" -- where a family comma has # already named the family, and provided at least one given word # remains (history: decisions.md#P6). The Dutch alphabetized diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index 6f835a99..337b6edf 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -9,7 +9,8 @@ from nameparser._policy import (FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, PatronymicRule, Policy, Script) -from nameparser._types import STABLE_TAGS, AmbiguityKind, Role +from nameparser._types import (FOLDED_TAG, STABLE_TAGS, AmbiguityKind, + Role) # A reduced lexicon, the convention in every pipeline stage module: a # stage test should not move when shipped vocabulary does. What it must @@ -306,18 +307,80 @@ def test_family_comma_fold_is_order_independent(policy: Policy) -> None: @pytest.mark.parametrize("policy", _FAMILY_FIRST) -def test_lone_never_given_particle_in_given_position_folds( +def test_lone_never_given_particle_in_given_position_is_the_given_name( policy: Policy) -> None: - # The opening-position test alone does not carry the rule: under a - # family-first order the given position is the TRAILING piece, and - # a lone 'de' landing there has to fold into the family beside it - # or the parse leaves the whole given name as a word the vocabulary - # says is never a given name. Guarded here because a refactor that - # reads the rule as leading-particle-only drops exactly this shape, - # silently and under a non-default order (#359 review). + # The reverse of what this file asserted until #467. A particle + # standing where the declared order puts the GIVEN name is the + # given name: the never-given vocabulary supplies a reading where + # position leaves the question open, and vetoes none that position + # has already given. `Mesnil de` has two pieces under either + # family-first order, so `de` lands in GIVEN and stays there. + # + # P6's no-comma site does not reach it either, and the negative + # assertion says so: that site takes a trailing particle only out + # of a MIDDLE, which two pieces never produce. out = _parsed("Mesnil de", policy) - assert _by_role(out, Role.FAMILY) == "Mesnil de" - assert not _by_role(out, Role.GIVEN) + assert _by_role(out, Role.FAMILY) == "Mesnil" + assert _by_role(out, Role.GIVEN) == "de" + assert not _folded(out) + + +def _folded(state: ParseState) -> str: + return " ".join(t.text for t in state.tokens if FOLDED_TAG in t.tags) + + +@pytest.mark.parametrize("policy", _FAMILY_FIRST) +def test_trailing_particle_out_of_a_middle_joins_the_family( + policy: Policy) -> None: + # #467: three pieces put the trailing particle in MIDDLE under + # FAMILY_FIRST and in GIVEN under FAMILY_FIRST_GIVEN_LAST, and the + # slot is the whole criterion -- a middle is a further given name, + # which a particle is not, so only that one is reinterpreted. + out = _parsed("Mesnil Garcia de", policy) + if policy.name_order[1] is Role.GIVEN: # FAMILY_FIRST + assert _by_role(out, Role.FAMILY) == "Mesnil de" + assert _by_role(out, Role.GIVEN) == "Garcia" + assert _folded(out) == "de" + else: # FAMILY_FIRST_GIVEN_LAST + assert _by_role(out, Role.FAMILY) == "Mesnil" + assert _by_role(out, Role.GIVEN) == "de" + assert not _folded(out) + + +def test_the_slot_test_reads_both_traditions_without_asking_the_word() -> None: + # The same vocabulary spells the Dutch tussenvoegsel and the + # Vietnamese given name, and `van` is in it. A never-given test + # here would have to lose one of these two; the declared order + # loses neither, which is why no vocabulary test appears at that + # site. (`van` is one of 37 ambiguous particles, so such a test + # would also have excluded von, di, da, del and le.) + dutch = _parsed("Beethoven Ludwig van", Policy(name_order=FAMILY_FIRST)) + assert _by_role(dutch, Role.FAMILY) == "Beethoven van" + assert _folded(dutch) == "van" + viet = _parsed("Nguyen Thi Van", + Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) + assert _by_role(viet, Role.GIVEN) == "Van" + assert not _folded(viet) + + +@pytest.mark.parametrize("policy", _FAMILY_FIRST) +def test_the_family_the_run_joins_must_have_a_base(policy: Policy) -> None: + # Attaching to an all-particle family renders one particle in + # front of another ('de van'), and R2 reads those words as + # ordinary name words, which no rule reorders. + out = _parsed("van Berg Jan de", policy) + assert not _folded(out) + + +@pytest.mark.parametrize("policy", _FAMILY_FIRST) +def test_no_leftover_is_re_laid_out(policy: Policy) -> None: + # Under these orders the roles run family, given, middle, middle..., + # so dropping a trailing middle leaves every other piece where it + # was. An earlier draft removed the piece and re-laid the leftover + # out, which lost the given name here and promoted a post-nominal + # into the given slot on "Berg Jan Jr. de". + out = _parsed("Berg Jan Jr. de", policy) + assert _by_role(out, Role.GIVEN) != "Jr." def test_lone_never_given_particle_needs_no_repair_by_default() -> None: @@ -488,9 +551,6 @@ def test_the_two_family_first_orders_differ_at_two_leftovers() -> None: # positional read for a declared order to narrow (state.order is # None on this path) ("Smith, de Mesnil", "Smith de Mesnil"), - # the particle stands in the GIVEN slot rather than opening the - # name -- the second fold site, and not a leading run - ("Juan de", "Juan de"), ]) def test_shapes_the_stop_does_not_reach( policy: Policy, text: str, family: str) -> None: diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index ebad1264..ae3dd4fc 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -20,7 +20,9 @@ "Anna z (domu) Nowak" "Anna z Nowak" "Asst. Vice Chancellor John Smith" +"Beethoven Ludwig van" "Beethoven, Ludwig van" +"Berg Jan de Jr." "Berg, Jan van der" "Berg, Jan vd" "Berg, abd née Jones" @@ -125,6 +127,8 @@ "Mr. Jack and Jill" "Mr. Johnson" "Mrs. Garcia" +"Ménil Christophe de" +"Ménil de" "Nguyen Thi Van" "Nguyen, Thi Van" "Nguyen, Van" @@ -205,6 +209,7 @@ "juan mcdonald" "mohamad ali smith" "née Jones" +"van Berg Jan de" "van Gogh" "van der Berg, abdul née Jones" "Иван Петрович Абрамович" From 5769e98ff727d01dbeb0d781c4894776f3a9aa5e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 30 Aug 2026 16:39:35 -0700 Subject: [PATCH 3/3] fix(post_rules,docs): the four-agent review findings on #467, measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The slot criterion was right and the code did not implement it. Two defects, found independently by two reviewers each, and both invisible to the corpus. (i) THE SITE FIRED UNDER THE DEFAULT ORDER. The gate was `state.order is not None`, which every no-comma order satisfies, while the reasoning above it argued from the FAMILY_FIRST layout. Where a conjunction stops a particle's forward chain the particle stands in a middle under any order, so `Maria Luisa y de la Cruz` gave family 'de la Cruz' where master gives 'la Cruz' -- 52 such names, on plain HumanName() with no policy. Three shipped documents asserted the opposite in my words. (ii) IT FOLDED A PARTICLE THAT DOES NOT END THE NAME. Under FAMILY_FIRST_GIVEN_LAST the trailing MIDDLE is not the trailing name word -- the given name stands behind the middles -- so `de Anke van y` gave family 'van de Anke', hoisting a particle in front of a base it was written after. 366 folds. The repair is P6's own word: the run must END the name. That single test subsumes the order test a first repair added -- measured over 542,592 generated parses, an explicit order test never decides anything the ends-the-name test has not, so it is not in the code. The default order ends with the FAMILY and FFGL with the GIVEN name, so only FAMILY_FIRST can put a name's last word in a middle at all. Also repaired: a stale `families` list (O1/O2 retag without recomputing, 33,840 stale observations; no shipped input reaches a wrong answer through it, so it is recorded rather than tested), and the site now REPORTS the fork it decides -- decisions.md#P6 settled that in #405, and the state that decision fixed is the one this site recreated, the comma writing reporting while the comma-less writing decided the identical fork in silence. MY INVARIANT WAS MEASURED ON A RIGGED SAMPLE. It is 603 of 630 over the whole particle vocabulary, not 216 of 216: the script sliced `sorted(particles)[:14]` and excluded `vd`, `mc` and `do`, the three words that are also suffix vocabulary and the only ones where the two writings genuinely differ. AGENTS.md axis 2 exactly -- the detector agreed with the design because it inherited the design's blind spot. The rule now states that limit; the entry records the failure. Docs corrected with it: two paragraphs still described the removed given-position site as current, one in rendered API documentation; P6's Accepted block still scoped the rule to the comma form; P6's rationale predicted the opposite of its own example; "181 of 247" reproduces under no definition (219 of 304); and a boundary example asserted the one field its guard cannot move. Both regressions are now `· boundary` examples so the doc runner holds them. Six mutations verified. Reach unchanged at 30 of 6606, 12 names, none under the default order -- the corpus never saw either defect. --- docs/design/decisions.md | 14 ++-- docs/design/rules.md | 49 ++++++++----- docs/release_log.rst | 2 +- nameparser/_lexicon.py | 4 +- nameparser/_pipeline/_post_rules.py | 95 ++++++++++++++++++++----- nameparser/config/particles.py | 17 ++--- tests/v2/cases.py | 8 ++- tests/v2/pipeline/test_post_rules.py | 99 +++++++++++++++++++++++---- tools/differential/corpus_rules.jsonl | 2 + 9 files changed, 223 insertions(+), 67 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 1433f140..86dcbcef 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -118,16 +118,22 @@ Open: [#380](https://github.com/derek73/python-nameparser/issues/380) covers "Be Measured over the 1094-name differential corpus: P6's attachment fires on 9 names, of which 5 take the ambiguous arm (`Beethoven, Ludwig van`, `Berg, Jan van der`, `Nguyen, Thi Van`, `Vega, Juan de la`, `der, y van`), 2 the suffix arm (`Berg, Jan vd`, `Berg, abdul vd`) and 2 stay silent (`Jong, Anke de`, `Jong, Piet de`). Seven names gain an ambiguity kind and no role field moves that was not already moving. Read that seven as evidence about the corpus rather than about the blast radius, the caution the 2026-08-16 entry above already gives for this rule: 304 of the 1094 names carry a comma and only 13 end in a particle, 11 of those being rows the #379/#380 arc added itself (this change added four corpus names and not one of them carries a comma, so the 11 is unchanged by it). The reach is a SHAPE — every family-comma listing ending in an ambiguous particle, which is to say ordinary Dutch, Flemish and unaccented Vietnamese listings — and a caller holding such records sees a report on all of them, not on seven. The LEDGER shape that follows from that is not the one the plan predicted, and the difference is worth recording because it will recur for any report added to a rule whose fields already move. A new `fields = ["_ambiguities"]` rule explains none of the seven: `classify` matches on a rule's `fields` being a SUPERSET of the whole diff, and against 2.0.0 and 2.1.0 these names diff in `middle`/`family` (or `family`/`suffix`) from #379 and #380 as well. So the repair is to WIDEN those two existing rules to `{middle, family, _ambiguities}` and `{family, suffix, _ambiguities}` in both ledgers, not to add a rule. Each widened rule still passes the #452 equality check, since the union over the names it explains is exactly what it declares — the two silent `Jong` names contribute `{middle, family}` and the five ambiguous ones contribute `_ambiguities` on top. 1.4.0 needs nothing at all, having no ambiguity surface to diff. Gate before and after: byte-identical output at all three baselines apart from the provenance header — 1094 names, 229/194/102 intentional, 0 unexplained, and no per-heading count moved. The seven names were already in the diff; only their field sets grew. The most visible consequence, and it is not the Vietnamese one: `Beethoven, Ludwig van` — the textbook-correct Dutch listing, read exactly right — now carries a report. That is honest rather than regrettable. The parser cannot separate it from `Nguyen, Thi Van`; the two are the same string shape over the same vocabulary, and a report on one is a report on the other. A caller who wants only the doubtful cases has nothing here to filter on, because there is nothing to filter on in the input either. -- 2026-08-30 #467 (the no-comma site, and the given position given back) — a declared family-first order names the family as a comma does, so P6's attachment fires without a comma too. It is keyed on the SLOT the trailing run landed in: MIDDLE, the one position that means nothing for a particle, since middles are further given names and a particle is not one. Only FAMILY_FIRST puts a trailing piece there. This DECIDES the parenthesis the 2026-08-16 Scope bullet left open ("arguably a missing comma under a declared family-first order") and narrows that bullet to the default order. +- 2026-08-30 #467 (the no-comma site, and the given position given back) — a declared family-first order names the family as a comma does, so P6's attachment fires without a comma too. It is keyed on TWO things, and review showed the second is not implied by the first: the run must stand in a MIDDLE — the one position that means nothing for a particle, middles being further given names — and it must END the name. Together they name the order without asking it: the default order ends with the FAMILY and FAMILY_FIRST_GIVEN_LAST with the GIVEN name, so only FAMILY_FIRST can put a name's last word in a middle. This DECIDES the parenthesis the 2026-08-16 Scope bullet left open ("arguably a missing comma under a declared family-first order") and narrows that bullet to the default order. P1 LOSES ITS GIVEN-POSITION SITE in the same change, and that half is a correction rather than a narrowing. That slot holds what the caller DECLARED to be the given name; the never-given vocabulary supplies a reading where position leaves the question open and vetoes none that position has already given. `Ménil de` under either family-first order now reports given `de`. THE PREMISE THIS REPLACES WAS FALSE, and it had been stated in three places (rules.md#P6's rationale, `_lexicon.py`, `config/particles.py`): that a never-given particle "cannot be a name at all". Measured, all 33 members report given == themselves when parsed bare. This is the SECOND correction of the same over-claim in this entry — the 2026-08-16 note above records dropping "no particle is a name by itself" for the identical reason. The conclusions drawn from it were mostly right; the reason was not, and it is load-bearing, because stating it as a property of the VOCABULARY invites the same inference for a particle ending the name, where there is nothing to join forward to and the writing is all there is. NO VOCABULARY TEST AT THE NEW SITE, and that is the substance of the design. The comma path needs one because a comma cannot separate the Dutch reading from the Vietnamese; the declared order can, and does both at once — `Beethoven Ludwig van` under FAMILY_FIRST gives family 'van Beethoven' though `van` is ambiguous vocabulary, and `Nguyen Thi Van` under FAMILY_FIRST_GIVEN_LAST keeps given 'Van' though it is the same word in the same set. A never-given test here would have excluded 37 ambiguous particles — von, di, da, del, le, and `van` itself, the flagship word of the listing the rule is named for. An earlier attempt (#466) carried exactly that test and failed on the Dutch case because of it. - NO RE-LAYOUT EITHER, and this is a property of the order rather than a simplification: under these orders the roles run family, given, middle, middle…, so dropping a trailing MIDDLE leaves every other piece's role untouched. #466 removed a piece and re-laid the leftover out, which lost a given name outright on `van Berg Jan de` and promoted a post-nominal into the given slot on `Berg Jan Jr. de`. Both defects are structurally unreachable here. The one guard that remains is the base: attaching to an all-particle family renders one particle in front of another, and R2 reads those words as ordinary name words, which no rule reorders. - THE INVARIANT THIS RESTS ON, and the reason to trust it over the argument it replaced: for a name of the form `Family Given [Particle]`, the family-comma writing under the default order and the comma-less writing under FAMILY_FIRST parse identically. Measured over 216 constructed names — 0 of 216 agreed before this change, 216 of 216 after, including all 72 with an ambiguous particle. The invariant draws the never-given/ambiguous line by itself instead of having it asserted. Scope it to that SHAPE: 181 of 247 corpus comma-names parse differently with the comma removed, because a comma also delimits titles and suffixes. + NO RE-LAYOUT EITHER, and this is a property of ONE order rather than a simplification: under FAMILY_FIRST the roles run family, given, middle, middle…, so dropping a trailing MIDDLE leaves every other piece's role untouched. It is false of FAMILY_FIRST_GIVEN_LAST (family, middle…, given) and of the default order, which is the second thing the ends-the-name test secures. #466 removed a piece and re-laid the leftover out, which lost a given name outright on `van Berg Jan de` and promoted a post-nominal into the given slot on `Berg Jan Jr. de`. Both defects are structurally unreachable here. The one guard that remains is the base: attaching to an all-particle family renders one particle in front of another, and R2 reads those words as ordinary name words, which no rule reorders. + THE INVARIANT THIS RESTS ON, and the reason to trust it over the argument it replaced: for a name of the form `Family Given [Particle]`, the family-comma writing under the default order and the comma-less writing under FAMILY_FIRST parse identically. Measured over ALL 70 particles × 3 families × 3 givens = 630 names: 0 of 630 agreed before this change, **603 of 630** after. The invariant draws the never-given/ambiguous line by itself instead of having it asserted, which is why no vocabulary test appears at the site. + The 27 failures are the three words that are BOTH particle and suffix vocabulary — `vd`, `mc`, `do` — and they are a real limit, not noise: P6's precedence over S2 is stated inside the comma paragraph and justified by the comma ("a trailing abbreviation after a family comma is the tussenvoegsel far more often"), so the comma form gives family 'vd Berg' where the comma-less family-first form gives suffix 'vd'. The rule now says so where it states the correspondence. + A FIRST DRAFT OF THIS ENTRY CLAIMED 216 of 216, and the number came from a script that sliced the particle vocabulary — `sorted(particles)[:14]` — which excluded exactly those three words. That is AGENTS.md's axis 2 ("your detector is a second, unreviewed implementation") in its purest form: the detector agreed with the design because it inherited the design's blind spot. Recompute over the WHOLE vocabulary, never a sample. + Scope the invariant to that SHAPE as well: 219 of the 304 comma-bearing corpus names parse differently with the comma simply removed, because a comma also delimits titles and suffixes. (An earlier draft said "181 of 247", which reproduces under no definition of "comma name" — the 304 matches the count the 2026-08-29 entry above already records.) An asymmetry falls out of it rather than being carved: shape 2 places the given name BEFORE the middles, so it corresponds to FAMILY_FIRST only. There is no comma format that puts the given name last, so FAMILY_FIRST_GIVEN_LAST has no twin to inherit an equivalence from. That is why the trailing particle is one order's business (#469). ONE EXAMPLE LINE WAS DELETED RATHER THAN UPDATED: `"Nguyen Thi Van" family-first → middle="Van"`. The name is written in FAMILY_FIRST_GIVEN_LAST; its FAMILY_FIRST reading is wrong by construction, so the line pinned a value nobody wants and nothing should depend on. Recorded here because the general question — whether a rules.md example should declare its format, and whether the doc test should refuse to run one outside it — is #470 and is NOT decided by this change. - REACH: 30 of 6594 parses move — this tree's 1099-name corpus × three `name_order` values × `middle_as_family` off and on — over twelve names, NONE under the default order. Recompute by parsing the four `tools/differential/corpus*.jsonl` files of this tree, deduped, under each of the six policies, and comparing the seven role fields against a checkout of the parent commit reading the SAME corpus files; the comparator is the pre-change code, never a construction of this rule. Dated snapshot, measured 2026-08-30. Note the corpus grew from master's 1094 by this change's own example lines, which is why the count must name the tree it was taken on. + REACH: 30 of 6606 parses move — this tree's 1101-name corpus × three `name_order` values × `middle_as_family` off and on — over twelve names, NONE under the default order. Recompute by parsing the four `tools/differential/corpus*.jsonl` files of this tree, deduped, under each of the six policies, and comparing the seven role fields against a checkout of the parent commit reading the SAME corpus files; the comparator is the pre-change code, never a construction of this rule. Dated snapshot, measured 2026-08-30. Note the corpus grew from master's 1094 by this change's own example lines, which is why the count must name the tree it was taken on. The differential gate cannot see any of it: `compare.py` parses every name as `HumanName(name)` and sweeps no policy, so no non-default `name_order` behavior has ever been compared across versions. Output is byte-identical at all three baselines apart from the added corpus names, none of which diffs. + WHAT REVIEW FOUND, because the corpus could not. Four agents on PR #472 found the site firing outside the order it was reasoned from, in two directions, and the differential corpus reported neither: 52 default-order names (a conjunction stops a particle's forward chain and leaves it standing in a middle — `Maria Luisa y de la Cruz` gave family 'de la Cruz' where master gives 'la Cruz') and 366 FAMILY_FIRST_GIVEN_LAST folds of a particle that does not end the name (`de Anke van y` gave family 'van de Anke' with the given name still behind it). Of 1099 corpus names only 6 hold a conjunction-then-particle adjacency and none in this shape, so the REACH recipe honestly reported 0 default-order movers throughout. The count was right and the population was blind; "none under the default order" was a property of the corpus, not of the code. Both are now `· boundary` examples in rules.md so the doc runner holds them. + The repair is one clause — the run must END the name — and it subsumes the order test a first repair added: measured over 542,592 generated parses, an explicit `state.order[0] is Role.FAMILY` never decides anything the ends-the-name test has not already decided, so it is not in the code. What IS kept is `state.order is not None`, which is what keeps this site off the family-comma path (0 co-firings with P6's comma site over ~892,000 parses). + A stale index list was repaired with it: O1 and O2 retag between roles without recomputing, and this site was the first consumer of `families` after them (33,840 stale observations instrumented). No shipped input reaches a wrong answer through it — no `_TURKIC`/`_EAST_SLAVIC`-matching word is particle vocabulary — so the fix is unwitnessed by any test and is recorded here instead. Open: [#471](https://github.com/derek73/python-nameparser/issues/471) is the leading fold's REACH under the default order (`de Mesnil Juan` reports the whole string as the family). Split out of this change deliberately: it is a 1.4.0 parity break on the order every existing caller is on — prototyped, 5 unexplained diffs at that baseline and a failing v1 parity test — where nothing under a family-first order has a v1 answer at all. diff --git a/docs/design/rules.md b/docs/design/rules.md index c386ff06..c4a67f1c 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -369,8 +369,9 @@ P5. Rationale: some given-name words are incomplete alone — "abdul" P6. Rationale: a particle ending the name has nothing to link forward to, so it is not doing a particle's work there. What it is doing instead is decided by what the WRITING says, not by the - word: where something has already named the family — a comma, or - a declared family-first order — the particle belongs to that + word: where something has already named the family AND left the + particle in a position that means nothing — after a family comma, + or in the middle of a FAMILY_FIRST reading — it belongs to that family, which is how Dutch and Flemish names are listed ("Beethoven, Ludwig van", the tussenvoegsel trailing the given name but belonging to the surname). Where nothing has, it is @@ -402,19 +403,27 @@ P6. Rationale: a particle ending the name has nothing to link "Nguyen, Van" → given="Van" · boundary Without a comma, a declared family-first order has named the family in the same way and the attachment fires there too — but - only where the run landed in a MIDDLE, which is the one position - that means nothing for it. Middles are further given names and a - particle is not one. FAMILY_FIRST is the only order that puts a - trailing piece there: FAMILY_FIRST_GIVEN_LAST puts it in GIVEN, - where the caller's declaration says it IS the given name, and the - default order puts it in FAMILY, where it already is one. + only where the run ENDS the name and stands in a MIDDLE — the one + position that means nothing for it, middles being further given + names, which a particle is not. Those two together name the + order without asking it: the default order ends with the FAMILY + and FAMILY_FIRST_GIVEN_LAST with the GIVEN name, so only + FAMILY_FIRST can put a name's last word in a middle at all. + Ending the name is not implied by the slot and has to be said. A + draft that tested the slot alone fired on 52 default-order names, + where a conjunction stops a particle's forward chain and leaves it + standing in a middle, and on 366 FAMILY_FIRST_GIVEN_LAST middles + with the given name still behind them. Nothing is asked about the WORD, and that is what lets one rule read both traditions: the same vocabulary that spells the Dutch tussenvoegsel spells the Vietnamese given name, and the declared - order separates them where a comma cannot. - Nothing is re-laid-out either — under that order the roles run - family, given, middle, middle…, so dropping a trailing middle - leaves every other piece where it was. The family must hold a + order separates them where a comma cannot — one order, not both, + as the Accepted clause below records. + Nothing is re-laid-out either — under the order this can reach, + the roles run family, given, middle, middle…, so dropping a + trailing middle leaves every other piece where it was. That is a + property of FAMILY_FIRST alone, and it is the second thing the + two tests above secure. The family must hold a base of its own, since a family that is all particles is not a family written beside anything (R2). "Jong Anke de" family-first → family="de Jong" @@ -425,8 +434,12 @@ P6. Rationale: a particle ending the name has nothing to link "Ménil Christophe de" family-first-given-last → given="de" "Berg Jan de Jr." family-first → family="de Berg" "Berg Jan de Jr." family-first → suffix="Jr." - "van Berg Jan de" family-first → given="Berg" · boundary + "van Berg Jan de" family-first → family="van" · boundary + "Maria Luisa y de la Cruz" → family="la Cruz" · boundary + "de Anke van y" family-first-given-last → middle="van" · boundary "Ménil de" family-first → given="de" · boundary + "Beethoven Ludwig van" family-first → ambiguities=("particle-or-given",) + "Jong Anke de" family-first → ambiguities=() · boundary Accepted: an ambiguous particle attaches on the same terms as a never-given one, so a Vietnamese name written in this listing loses its given name — but only in the UNACCENTED @@ -458,11 +471,11 @@ P6. Rationale: a particle ending the name has nothing to link keeping (#470). "Nguyen Thi Van" family-first-given-last → family="Nguyen" "Nguyen Thi Van" family-first-given-last → given="Van" - Accepted: without a family comma the name's written shape is not - settled — "Jong Anke de" may be a misformatted listing, and a - bare "Jong de" may be a given name beside a particle — so the - attachment is scoped to the comma form, and the comma-less - shapes keep their positional reading. + Accepted: under the DEFAULT order a comma-less name's written + shape is not settled — "Jong Anke de" may be a misformatted + listing, and a bare "Jong de" may be a given name beside a + particle — so nothing there names the family, the positional + reading stands, and the same input reports family "de" here. "Jong Anke de" → family="de" Accepted: the precedence over S2 is stated for the shape, so it sweeps in every word that is both particle and suffix diff --git a/docs/release_log.rst b/docs/release_log.rst index c5efa437..f6222fdb 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -71,7 +71,7 @@ Release Log - Change case repair to read the parser's own ``conjunction`` tag instead of re-deciding, from the word's spelling, whether a word is a conjunction or an initial. The parse answers that question already -- ``"Scott E. Werner"`` reads ``E.`` as an initial rather than the Italian conjunction -- and the other views honor the answer; case repair asked again, with a shape test applied to each word of a token's text rather than to the token. Two spellings of one name disagreed because of it: ``"juan e-f smith"`` capitalized to ``Juan e-F Smith`` while ``"JUAN E-F SMITH"`` gave ``Juan E-F Smith``; both give ``Juan E-F Smith`` now, ``e-f`` being a middle name and no conjunction of the parse's reading. A conjunction written as a word of its own is untouched, and so is the one-letter carve-out where it applies -- ``"juan y garcia"`` still repairs to ``Juan y Garcia``, ``"JUAN Y GARCIA"`` still to ``Juan Y Garcia``. A field assigned after the parse is unaffected: its text was never classified, so there is no reading to honor and repair asks the vocabulary, applying v1's own predicate the way every earlier version applied it everywhere -- ``h.last = "velasquez y garcia"`` still repairs to ``Velasquez y Garcia`` and ``h.middle = "e."`` to ``E.``. That is the predicate over TODAY's vocabulary, which is narrower than parity with 1.4.0 and the difference is real: ``h.last = "хосе и мария сантос"`` gives ``Хосе И Мария Сантос`` on 1.4.0 and ``Хосе и Мария Сантос`` here, because the Cyrillic ``и`` is a 2.x conjunction and was not a 1.4.0 one. What decides which path a token takes is a mark the assignment leaves, not the absence of a span: a value revised through ``Parser.revise()`` is classified by a sub-parse and keeps its tags, so it repairs as the parse does. One reading does change for hand-built ``Token``\ s in the 2.0 API: an untagged token whose text is conjunction vocabulary is now an ordinary name word and capitalizes, where 2.1 lowercased it -- tags are what the views read, and a hand-built token that carries none is a token with nothing to declare. Case repair is not one of the seven role fields the differential harness compares, so no gate run can see this change either way and none of its counts move; measured directly instead, no name of the 1094-name differential corpus moves under ``capitalized()`` or ``capitalized(force=True)``, its uppercased and lowercased spellings included -- 6564 name/spelling/lexicon rows and 13128 calls (closes #458) - - Fix a trailing surname particle being stranded as a standalone middle name under ``Policy(name_order=FAMILY_FIRST)``, where the same listing written with a comma reads it as part of the surname. A particle ending the name has nothing to link forward to, so what it is doing there is decided by what the writing says: after a family comma it joins the family the comma named and is written before it, and a declared family-first order names the family the same way. ``Parser(policy=Policy(name_order=FAMILY_FIRST)).parse("Jong Anke de")`` gave family ``Jong`` with ``de`` left as a middle name, and now gives family ``de Jong``, given ``Anke`` -- the same answer ``parse("Jong, Anke de")`` has always given. The test is the SLOT the particle landed in, not the word: a middle name is a further given name, which a particle is not, and ``FAMILY_FIRST`` is the only order that puts a trailing piece there. ``FAMILY_FIRST_GIVEN_LAST`` puts it in the given slot, where the caller's own declaration says it is the given name, so ``"Nguyen Thi Van"`` under that order still reads given ``Van``. That one test reads both traditions without asking about the vocabulary at all: ``"Beethoven Ludwig van"`` under ``FAMILY_FIRST`` now gives family ``van Beethoven`` even though ``van`` is one of the 37 particles that are ordinary given names elsewhere. In the same change, a particle standing alone where a family-first order puts the GIVEN name is no longer folded into the family: ``"Ménil de"`` reports given ``de`` under both family-first orders, because that slot holds what the caller declared, and the never-given word list supplies a reading where position leaves the question open rather than overriding one position has already given. Nothing moves under the DEFAULT name order. Measured, 30 of 6594 parses move -- this release's 1099-name corpus under three ``name_order`` values with ``middle_as_family`` off and on -- over twelve names. Those counts cannot come from the differential gate: it parses every corpus name with the default policy and sweeps no policy at all, so no non-default ``name_order`` behavior has ever been compared across versions, and the gate output here is unchanged at all three baselines apart from the corpus names this change's own rules.md examples add. What moves is recomputed by the recipe in the ``P6`` entry of ``docs/design/decisions.md``, which compares the seven role fields against a checkout of the parent commit reading the same corpus files; the count is a dated snapshot, measured 2026-08-30. The ``rules.md#P1`` and ``rules.md#P6`` example lines and ``tests/v2/pipeline/test_post_rules.py`` are what pin the behavior (closes #467) + - Fix a trailing surname particle being stranded as a standalone middle name under ``Policy(name_order=FAMILY_FIRST)``, where the same listing written with a comma reads it as part of the surname. A particle ending the name has nothing to link forward to, so what it is doing there is decided by what the writing says: after a family comma it joins the family the comma named and is written before it, and a declared family-first order names the family the same way. ``Parser(policy=Policy(name_order=FAMILY_FIRST)).parse("Jong Anke de")`` gave family ``Jong`` with ``de`` left as a middle name, and now gives family ``de Jong``, given ``Anke`` -- the same answer ``parse("Jong, Anke de")`` has always given. The test is the SLOT the particle landed in, not the word: a middle name is a further given name, which a particle is not, and ``FAMILY_FIRST`` is the only order that puts a trailing piece there. ``FAMILY_FIRST_GIVEN_LAST`` puts it in the given slot, where the caller's own declaration says it is the given name, so ``"Nguyen Thi Van"`` under that order still reads given ``Van``. That one test reads both traditions without asking about the vocabulary at all: ``"Beethoven Ludwig van"`` under ``FAMILY_FIRST`` now gives family ``van Beethoven`` even though ``van`` is one of the 37 particles that are ordinary given names elsewhere. In the same change, a particle standing alone where a family-first order puts the GIVEN name is no longer folded into the family: ``"Ménil de"`` reports given ``de`` under both family-first orders, because that slot holds what the caller declared, and the never-given word list supplies a reading where position leaves the question open rather than overriding one position has already given. Nothing moves under the DEFAULT name order. Measured, 30 of 6606 parses move -- this release's 1101-name corpus under three ``name_order`` values with ``middle_as_family`` off and on -- over twelve names. Those counts cannot come from the differential gate: it parses every corpus name with the default policy and sweeps no policy at all, so no non-default ``name_order`` behavior has ever been compared across versions, and the gate output here is unchanged at all three baselines apart from the corpus names this change's own rules.md examples add. What moves is recomputed by the recipe in the ``P6`` entry of ``docs/design/decisions.md``, which compares the seven role fields against a checkout of the parent commit reading the same corpus files; the count is a dated snapshot, measured 2026-08-30. The ``rules.md#P1`` and ``rules.md#P6`` example lines and ``tests/v2/pipeline/test_post_rules.py`` are what pin the behavior (closes #467) - Fix ``initials()`` reading a name in a different order than the fields of the same name. Two rules fold words into the family name and render them before the rest of it -- ``Policy(middle_as_family=True)``, which sends every middle word to the family, and the tussenvoegsel attachment after a family comma -- and both do it by marking the words rather than moving them, since a parsed word keeps the position it was written at. The ``family`` field reads that mark and ``initials()`` did not, so one parse gave two orders: ``parse("der, y van")`` gave family ``van der`` and initials ``y. d. v.``, and now gives ``y. v. d.``. This RESTORES v1: ``middle_name_as_last`` is v1's spelling of the same option, so most of what moves has a 1.4.0 answer to be measured against, and measured over the 1094-name differential corpus at the default name order, 71 names move under that option, 54 of them back to exactly what 1.4.0 returns and not one of them away from it -- ``"Doe, Dr. John A."`` gives ``J. A. D.`` again where 2.0 through 2.2 gave ``J. D. A.``, and ``"Brundridge, Contessa A"`` gives ``C. A. B.`` where they gave ``C. B. A.``. Of the 17 that match 1.4.0 neither before nor after, 14 now agree with it on the ORDER and differ only in how v1 grouped initials -- 1.4.0 gives one initial per element of its own ``last_list``, so a conjunction-joined surname yields ``V G.`` where the 2.x view, one initial per word, yields ``V. G.``; one more is ``"der, y van"``, whose family is nothing but particles, where 1.4.0 contributes no initial at all and 2.x contributes its words, a difference this release does not touch; and the remaining two parse differently from 1.4.0 for reasons that predate this fix. Without the option, one corpus name moves, the ``"der, y van"`` above -- and it is one of one, since it is the only name in the corpus whose family holds two contributing words with a folded one behind the other, the shape an order change can be seen in at all. ``HumanName.initials()`` was already right and is unchanged, measured: no name of the 1094-name corpus moves through the facade, with the option or without it. It reads ``first_list``/``middle_list``/``last_list``, which prepend the folded words as v1 did, so it is the 2.0 API's ``ParsedName.initials()`` that was out of step -- with the field beside it, with the facade, and with 1.4.0 at once. ``initials()`` is not one of the seven role fields the differential harness compares, so no gate run can see this change: run at all three baselines before and after, the output is identical to the byte -- 1094 corpus names, 229 / 194 / 102 intentional diffs and ``unexplained: 0`` at 1.4.0 / 2.0.0 / 2.1.0, with every per-heading count unchanged. So where the counts in this bullet come from has to be said, the gate's classified summary not being able to supply them and the ``rules.md#R3`` example line witnessing the order without counting anything: what MOVES is recomputed by the recipe in the ``R3`` entry of ``docs/design/decisions.md``, which compares this view against the pre-change rendering over these same four corpora and reproduces the 71 and the one-of-one; the 1.4.0 comparisons and the facade sweep are dated snapshots rather than re-derivable ones, measured 2026-08-30 against the released 1.4.0 wheel and against the pre-change tree, which nothing in the repository re-runs. The ``rules.md#R3`` example line and ``tests/v2/test_render.py`` are what pin the behavior (closes #408) - Fix a tussenvoegsel attached to the family name after a comma deciding a genuinely uncertain reading and reporting nothing. ``"Van Johnson"`` reports a ``PARTICLE_OR_GIVEN`` ambiguity -- ``Van`` is a Dutch particle and a Vietnamese given name, and the parser has to pick one -- while ``"Nguyen, Thi Van"`` picked the same word the same way, silently, and lost the given name doing it. The attachment now reports the fork it decides, in the kind that names the reading it declined. A particle that could be an ordinary name reports ``PARTICLE_OR_GIVEN``: ``"Nguyen, Thi Van"``, ``"Berg, Jan van der"`` and ``"Vega, Juan de la"`` each gain one, the ``detail`` naming the ambiguous word. A particle the parser had already read as a post-nominal reports ``SUFFIX_OR_NAME`` instead, because the credential reading is what the attachment overrode: ``"Berg, Jan vd"`` gains one, ``vd`` being read as *van der* rather than as the Volunteer Decoration. Which kind you get follows the reading that was overridden rather than the word's vocabulary, so ``"Berg, Jan do"`` reports ``PARTICLE_OR_GIVEN`` even though ``do`` is a postnominal too -- it was already being read as a name word, so no credential reading was overridden. A particle where nothing was overridden reports nothing at all: ``"Jong, Piet de"`` and ``"Jong, Anke de"`` are unchanged, ``de`` being no name in any reading and no postnominal either, and so is ``"Berg, Jan de vd"``, whose run was read as name words whole. Worth knowing before you filter on this: ``"Beethoven, Ludwig van"`` -- the textbook Dutch listing, read exactly right -- now carries a report too. It is the same string shape over the same vocabulary as ``"Nguyen, Thi Van"``, and nothing in the input separates them, so a report on one is a report on both. This adds the report and nothing else: every field these names parse to is exactly what the tussenvoegsel fix below already gave them, and ``ambiguities`` is the only value that grows. Seven differential corpus names gain a kind against the 2.0.0 and 2.1.0 baselines, seven of the nine the tussenvoegsel fix below already moved -- five of ``fix(#379)``'s seven and both of ``fix(#380)``'s two, now carrying ``_ambiguities`` in their diff as well; the two left out are the pair named unchanged above, ``Jong, Piet de`` and ``Jong, Anke de`` -- and none against 1.4.0, which had no ambiguity reporting at all (closes #405) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 32baae46..a95a873a 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -371,8 +371,8 @@ class Lexicon: #: decides is what becomes of that piece afterwards. Under ANY #: ``name_order`` a member records a particle-or-given ambiguity #: and a non-member records none, and a non-member is additionally - #: folded back into the family name once roles exist, so the whole - #: name is the surname ("de Mesnil" -- a bare "de", with nothing to + #: folded back into the family name once roles exist WHERE IT OPENS + #: THE NAME, so the whole name is the surname ("de Mesnil" -- a bare "de", with nothing to #: fold into, is left alone). That fold is order-independent too #: (#359) -- not because the word could not be a given name, which #: it can where position forces it (``parse("de")`` reports given diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 19306eff..2eb7286b 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -255,9 +255,7 @@ def post_rules(state: ParseState) -> ParseState: # more surname, which is the very question of where the run stops. # Under that declaration the run takes its own particles and ONE # name word; under the default order it keeps taking the rest of - # the name, nothing having marked where the surname ends. The - # narrowing is the LEADING site's alone: a particle - # standing in the given slot keeps the old reach. Note what + # the name, nothing having marked where the surname ends. Note what # actually holds the family-comma shape back, since it is NOT # that test -- "Smith, de Mesnil Juan" DOES fire the leading site, # segment 1 opening with the particle. It keeps the old reach @@ -357,35 +355,94 @@ def post_rules(state: ParseState) -> ParseState: # # Which is why no vocabulary test appears here and none is wanted. # The comma path needs one because a comma cannot separate the - # Dutch reading from the Vietnamese; the declared ORDER can, and - # does it for both at once: "Beethoven Ludwig van" under - # FAMILY_FIRST gives family "van Beethoven" though `van` is - # ambiguous vocabulary, and "Nguyen Thi Van" under + # Dutch reading from the Vietnamese; the declared ORDER can, when + # the caller declares the right one for the name: "Beethoven + # Ludwig van" under FAMILY_FIRST gives family "van Beethoven" + # though `van` is ambiguous vocabulary, and "Nguyen Thi Van" under # FAMILY_FIRST_GIVEN_LAST keeps given "Van" though the same word - # is in the same set. A never-given test here would have excluded - # 37 ambiguous particles -- von, di, da, del, le and `van` itself, + # is in the same set. (Under the WRONG order each loses: the + # Vietnamese name read as FAMILY_FIRST gives family "Van Nguyen". + # rules.md#P6 records that as accepted -- it is one order, not + # both.) A never-given test here would have excluded every + # ambiguous particle -- von, di, da, del, le and `van` itself, # the flagship word of the listing this rule is named for. # - # NO RE-LAYOUT, and that is a property of the order rather than a + # NO RE-LAYOUT, and that is a property of ONE order rather than a # simplification: under FAMILY_FIRST the roles run family, given, # middle, middle..., so dropping a trailing MIDDLE leaves every - # other piece's role untouched. An earlier draft (#466) removed a - # piece and re-laid the leftover out, which lost a given name - # outright on "van Berg Jan de" and promoted a post-nominal into - # the given slot on "Berg Jan Jr. de". + # other piece's role untouched. It is not true of + # FAMILY_FIRST_GIVEN_LAST (family, middle..., given) and not true + # of the default order, which is why this site tests the order + # rather than trusting the slot to imply it -- review found the + # first draft firing on 52 default-order names, where a + # conjunction stops a particle's forward chain and leaves it + # standing in a middle ("Maria Luisa y de la Cruz"). An earlier + # draft still (#466) removed a piece and re-laid the leftover out, + # which lost a given name outright on "van Berg Jan de" and + # promoted a post-nominal into the given slot on + # "Berg Jan Jr. de"; neither is reachable without a re-layout. # - # The family must still hold a base of its own: attaching to an - # all-particle family renders one particle in front of another - # ("van Berg Jan de" -> family 'de van'), and R2 reads those words - # as ordinary name words, which no rule reorders. + # The family must still hold a base of its own -- R2's invariant, + # that a non-empty family has a non-empty base, not a ban on + # particle-before-particle. Without it "van Berg Jan de" reports + # family 'de van', whose words R2 reads as ordinary name words, + # and no rule reorders those. It does NOT stop a second particle + # joining a family that merely OPENS with one: "de Mesnil Jan de" + # gives 'de de Mesnil', which keeps its base and is accepted. mids = _idx(tokens, Role.MIDDLE) + # O1 and O2 above retag between roles WITHOUT recomputing, so this + # is the first consumer of a stale list -- the bug shape #359 fixed, + # and every other retagging block in this stage recomputes for it. + families = _idx(tokens, Role.FAMILY) + names = _idx(tokens, Role.GIVEN) + mids + families if mids and families and state.order is not None: run: list[int] = [] i = len(mids) while i and "particle" in tokens[mids[i - 1]].tags: i -= 1 run.append(mids[i]) - if run and any("particle" not in tokens[j].tags for j in families): + # ENDING the name is the rule's own word, and it is the whole + # test: the trailing MIDDLE is not always the trailing NAME + # word. Under FAMILY_FIRST_GIVEN_LAST the given name stands + # behind the middles, and under the default order the family + # does, so in both a middle that ends the name is impossible + # and this declines without asking which order was declared. + # Only FAMILY_FIRST can put a name's last word in a middle. + # + # A draft tested the SLOT alone and fired on 52 default-order + # names, where a conjunction stops a particle's forward chain + # and leaves it standing in a middle ("Maria Luisa y de la + # Cruz"), and on 366 FAMILY_FIRST_GIVEN_LAST middles with the + # given name still behind them. A later draft tested the order + # as well; measured over 542,592 generated parses, that second + # test never decides anything this one has not already decided, + # so it is not here. + trailing = bool(run) and max(run) == max(names) + if trailing and any("particle" not in tokens[j].tags + for j in families): + # mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE: "Emit at + # the site that takes the branch, not where an ambiguous + # tag sits". decisions.md#P6 (#405) settled that this + # attachment reports the fork it decides, and the state it + # fixed is the one this site would otherwise recreate -- + # `Beethoven, Ludwig van` reporting while `Beethoven Ludwig + # van` under FAMILY_FIRST decides the identical fork in + # silence. ONE arm here where the comma path has two: the + # suffix reading stands without a comma, so no post-nominal + # reading is ever overridden for this site to report. + ambiguous = [j for j in run + if "vocab:particle-ambiguous" in tokens[j].tags] + if ambiguous: + word = tokens[ambiguous[0]].text + text = " ".join(tokens[j].text for j in sorted(run)) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.PARTICLE_OR_GIVEN, + f"{word!r} is both a family-name particle and an " + f"ordinary given name; ending a name read " + f"family-first, {text!r} joins the family that " + f"order named rather than standing as a name word " + f"of its own", + tuple(sorted(run)))) for j in run: tokens[j] = dataclasses.replace( tokens[j], role=Role.FAMILY, diff --git a/nameparser/config/particles.py b/nameparser/config/particles.py index a2fe6b32..79658fda 100644 --- a/nameparser/config/particles.py +++ b/nameparser/config/particles.py @@ -1,8 +1,8 @@ from nameparser.config._invariants import assert_normalized from nameparser.config.bound_given_names import BOUND_GIVEN_NAMES -#: The sub-set of :py:data:`PARTICLES` that are *never* a standalone given -#: name. Where one of these stands ALONE as the piece opening a name, that +#: The sub-set of :py:data:`PARTICLES` that no culture uses as a standalone +#: given name. Where one of these stands ALONE as the piece opening a name, that #: name has no given name -- the whole thing is a surname (e.g. "de Mesnil" #: -> family name "de Mesnil") -- and that reading holds under EVERY #: ``name_order`` (#359). It is not scoped to the default order the way the @@ -16,12 +16,13 @@ #: opening *piece*, not the first word of the string: a particle that has #: already chained onto the word behind it is part of that piece rather #: than standing alone. -#: Opening the name is only the commonest shape. The rule enforcing it -#: (rules.md#P1; the pre-2.2 docstrings called it rule 1b) reaches a -#: member standing alone as a piece in -#: the given position too, folding it into the family beside it, so that -#: neither shape leaves a given name behind -- as long as there is another -#: name token to fold into. A bare "de" stays as it is. Where a chain is +#: OPENING the name is the whole of that rule's subject (rules.md#P1; +#: the pre-2.2 docstrings called it rule 1b). A member standing alone +#: where a declared order puts the GIVEN name was a second site until +#: #467, and is not one now: that slot holds what the caller declared, +#: so ``Policy(name_order=FAMILY_FIRST)`` reads "Mesnil de" as family +#: "Mesnil", given "de". A member ENDING the name under a family-first +#: order is rules.md#P6's, which joins it to the family. A bare "de" stays as it is. Where a chain is #: reported as the given name anyway it is because the member is no #: longer standing alone: under ``Policy(name_order=FAMILY_FIRST)`` the #: given position of "Juan de la Vega" holds the whole three-token diff --git a/tests/v2/cases.py b/tests/v2/cases.py index f36bd668..3caf5425 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -406,8 +406,12 @@ def __post_init__(self) -> None: "someone deciding to move it: Nguyen Thi Van is " "family-middle-given, so the given name Van is lost " "here. The listing is identical to the Dutch one and " - "nothing separates them. The comma-LESS family-first " - "spelling reads it correctly, which is what makes the " + "nothing separates them. The comma-less " + "FAMILY_FIRST_GIVEN_LAST spelling reads it correctly -- " + "that ONE order, not family-first generally, which #467 " + "made load-bearing by giving comma-less FAMILY_FIRST the " + "same attachment (there it reads family Van Nguyen too). " + "That surviving format is what makes the " "loss acceptable -- see rules.md#P6. Since #405 the " "loss is at least REPORTED: 'Van' is ambiguous " "vocabulary, so the attachment declines a live reading " diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index 337b6edf..db3609cc 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -325,6 +325,9 @@ def test_lone_never_given_particle_in_given_position_is_the_given_name( assert not _folded(out) +# --- P6's no-comma site (#467) ------------------------------------ + + def _folded(state: ParseState) -> str: return " ".join(t.text for t in state.tokens if FOLDED_TAG in t.tags) @@ -352,8 +355,8 @@ def test_the_slot_test_reads_both_traditions_without_asking_the_word() -> None: # Vietnamese given name, and `van` is in it. A never-given test # here would have to lose one of these two; the declared order # loses neither, which is why no vocabulary test appears at that - # site. (`van` is one of 37 ambiguous particles, so such a test - # would also have excluded von, di, da, del and le.) + # site. Such a test would also have excluded every other ambiguous + # particle -- von, di, da, del, le. dutch = _parsed("Beethoven Ludwig van", Policy(name_order=FAMILY_FIRST)) assert _by_role(dutch, Role.FAMILY) == "Beethoven van" assert _folded(dutch) == "van" @@ -363,24 +366,94 @@ def test_the_slot_test_reads_both_traditions_without_asking_the_word() -> None: assert not _folded(viet) -@pytest.mark.parametrize("policy", _FAMILY_FIRST) -def test_the_family_the_run_joins_must_have_a_base(policy: Policy) -> None: +def test_the_family_the_run_joins_must_have_a_base() -> None: # Attaching to an all-particle family renders one particle in # front of another ('de van'), and R2 reads those words as # ordinary name words, which no rule reorders. - out = _parsed("van Berg Jan de", policy) + # + # FAMILY_FIRST only: under FAMILY_FIRST_GIVEN_LAST this name puts + # `de` in GIVEN, so the run is empty for an unrelated reason and + # the guard is never consulted. `family` is asserted as well as the + # tag, because inverting the guard retags the run to FAMILY without + # the tag -- which `_folded` alone cannot see. + out = _parsed("van Berg Jan de", Policy(name_order=FAMILY_FIRST)) + assert _by_role(out, Role.FAMILY) == "van" + assert _by_role(out, Role.MIDDLE) == "Jan de" assert not _folded(out) -@pytest.mark.parametrize("policy", _FAMILY_FIRST) -def test_no_leftover_is_re_laid_out(policy: Policy) -> None: - # Under these orders the roles run family, given, middle, middle..., +def test_no_leftover_is_re_laid_out() -> None: + # Under FAMILY_FIRST the roles run family, given, middle, middle..., # so dropping a trailing middle leaves every other piece where it - # was. An earlier draft removed the piece and re-laid the leftover - # out, which lost the given name here and promoted a post-nominal - # into the given slot on "Berg Jan Jr. de". - out = _parsed("Berg Jan Jr. de", policy) - assert _by_role(out, Role.GIVEN) != "Jr." + # was. NOT under FAMILY_FIRST_GIVEN_LAST (family, middle..., given), + # which is one of the two reasons the site tests the order. + # + # An earlier draft (#466) removed the piece and re-laid the leftover + # out. The whole role map is asserted because that draft failed in + # two directions: it lost a given name outright on "van Berg Jan de" + # and promoted a post-nominal into the given slot here. A test on + # `GIVEN != "Jr."` alone passes on an EMPTY given, which is the + # first of those. + out = _parsed("Berg Jan Jr. de", Policy(name_order=FAMILY_FIRST)) + assert _by_role(out, Role.GIVEN) == "Jan" + assert _by_role(out, Role.MIDDLE) == "Jr." + assert _by_role(out, Role.FAMILY) == "Berg de" + assert _folded(out) == "de" + + +@pytest.mark.parametrize("policy", _FAMILY_FIRST) +def test_a_multi_token_run_is_taken_whole(policy: Policy) -> None: + # `van de` and `van der` are the commonest real tussenvoegsels and + # the shape this rule exists for, and every other fixture here runs + # the walk for a single token -- so a walk that took only the last + # particle passed the entire suite when review mutated it in. + out = _parsed("Jong Anke van de", policy) + if policy.name_order[1] is Role.GIVEN: # FAMILY_FIRST + assert _by_role(out, Role.FAMILY) == "Jong van de" + assert _folded(out) == "van de" + assert not _by_role(out, Role.MIDDLE) + else: + assert not _folded(out) + + +def test_the_run_must_end_the_name() -> None: + # The trailing MIDDLE is not the trailing NAME word under + # FAMILY_FIRST_GIVEN_LAST, where the given name stands behind the + # middles. Without this test the rule hoists a particle in front of + # a base it was written after, with a given name still following it + # -- 366 such folds over a generated sweep, all of them shapes where + # a conjunction stopped the particle's forward chain. + out = _parsed("de Anke van y", Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) + assert _by_role(out, Role.MIDDLE) == "van" + assert _by_role(out, Role.FAMILY) == "de Anke" + assert not _folded(out) + + +def test_the_site_reports_the_fork_it_decides() -> None: + # decisions.md#P6 (#405): the attachment reports the fork it + # decides. The state that decision fixed is the one this site + # would otherwise recreate -- the comma writing reporting while + # the comma-less writing decides the identical fork in silence. + # ONE arm, since the suffix reading stands without a comma. + ff = _parsed("Beethoven Ludwig van", Policy(name_order=FAMILY_FIRST)) + kinds = [a.kind for a in ff.ambiguities] + assert kinds == [AmbiguityKind.PARTICLE_OR_GIVEN] + # a never-given run overrides no live reading, so it stays silent + quiet = _parsed("Jong Anke de", Policy(name_order=FAMILY_FIRST)) + assert not quiet.ambiguities + + +def test_the_default_order_is_not_reached() -> None: + # The site reads `state.order`, which is recorded under EVERY + # no-comma order, so the order itself must be tested -- the slot + # does not imply it. A conjunction stops a particle's forward chain + # and leaves it standing in a middle under the default order too: + # measured, 52 names moved before this test was added, and the + # differential corpus holds no instance of the shape, so the gate + # reported none of them. + out = _parsed("Maria Luisa y de la Cruz") + assert _by_role(out, Role.FAMILY) == "la Cruz" + assert not _folded(out) def test_lone_never_given_particle_needs_no_repair_by_default() -> None: diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index ae3dd4fc..e2de31d0 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -118,6 +118,7 @@ "Maria Kowalska (z domu Nowak)" "Maria Kowalska (z domu)" "Maria Kowalska z domu Nowak" +"Maria Luisa y de la Cruz" "Marquess of Bath" "Mary Beth Smith" "Mc Donald" @@ -198,6 +199,7 @@ "abdul salam ahmed salem" "anh van do" "de" +"de Anke van y" "de Mesnil Juan" "de la Cruz Juan Carlos" "de la Vega"