From d160fa5c7cfd7b0e385d8bf87088ed15fa1c071a Mon Sep 17 00:00:00 2001 From: Asta Date: Tue, 7 Jul 2026 12:45:10 +0200 Subject: [PATCH 1/5] bundle of improvements to Danish logic --- .../languages/danish/number_normalizer.py | 87 ++++++++- normalization/languages/danish/operators.py | 5 + .../languages/danish/replacements.py | 165 ++++++++++++++++++ .../languages/danish/sentence_replacements.py | 28 +++ 4 files changed, 281 insertions(+), 4 deletions(-) create mode 100644 normalization/languages/danish/sentence_replacements.py diff --git a/normalization/languages/danish/number_normalizer.py b/normalization/languages/danish/number_normalizer.py index c08325b..a1dd580 100644 --- a/normalization/languages/danish/number_normalizer.py +++ b/normalization/languages/danish/number_normalizer.py @@ -61,6 +61,49 @@ def _get(table: dict[str, int], word: str) -> int | None: "halvfems": 90, } +# In Danish the ordinal indicator is a trailing period, e.g. "første" → "1." +_ORDINALS: dict[str, tuple[int, str]] = { + "første": (1, "."), + "anden": (2, "."), + "tredje": (3, "."), + "fjerde": (4, "."), + "femte": (5, "."), + "sjette": (6, "."), + "syvende": (7, "."), + "ottende": (8, "."), + "niende": (9, "."), + "tiende": (10, "."), + + "ellevte": (11, "."), + "tolvte": (12, "."), + "trettende": (13, "."), + "fjortende": (14, "."), + "femtende": (15, "."), + "sekstende": (16, "."), + "syttende": (17, "."), + "attende": (18, "."), + "nittende": (19, "."), + + "tyvende": (20, "."), + "tredivte": (30, "."), + "fyrrende": (40, "."), + "fyrretyvende": (40, "."), + "halvtredsinde": (50, "."), + "halvtredsindstyvende": (50, "."), + "tressende": (60, "."), + "tresindstyvende": (60, "."), + "halvfjerdsende": (70, "."), + "halvfjerdsindtyvende": (70, "."), + "firsende": (80, "."), + "firsindtyvende": (80, "."), + "halvfemsende": (90, "."), + "halvfemsindstyvende": (90, ".") +} + +_TENS_ORDINAL: dict[str, tuple[int, str]] = { + k: v for k, v in _ORDINALS.items() if v[0] >= 20 +} + # Used for og-compound ones part (includes en/et = 1 alongside 2-9) _ONES_FOR_OG: dict[str, int] = {"en": 1, "et": 1, **_ONES_2_9} @@ -86,6 +129,7 @@ def _get(table: dict[str, int], word: str) -> int | None: _BIG_MULT: dict[str, int] = { "tusind": 1_000, + "tusinde": 1_000, "million": 1_000_000, "millioner": 1_000_000, "milliard": 1_000_000_000, @@ -113,6 +157,20 @@ def _try_parse_og_compound(word: str) -> int | None: return None +def _try_parse_og_compound_ordinal(word: str) -> tuple[int, str] | None: + """Parse a glued Danish ordinal compound like ``'enogtyvende'`` = 21.""" + fw = _fold(word) + for ones_str, ones_val in _ONES_FOR_OG_SORTED: + prefix = ones_str + "og" + if fw.startswith(prefix): + rest = fw[len(prefix) :] + tens = _TENS_ORDINAL.get(rest) + if tens is not None: + tens_val, suffix = tens + return tens_val + ones_val, suffix + return None + + def _normalize_mixed_numbers(text: str) -> str: """Convert ``3 milliard`` → ``tre milliard`` so the word parser yields 3 000 000 000.""" @@ -214,9 +272,30 @@ def __call__(self, text: str) -> str: parsed = self._parse_number(words, i, n) if parsed is not None: end, value = parsed + # After parsing a cardinal, peek at the next word to see if it + # is an ordinal continuation (e.g. "hundrede" + "enogtyvende" → 121.) + if end < n: + ordinal = _ORDINALS.get(_fold(words[end])) + if ordinal is None: + ordinal = _try_parse_og_compound_ordinal(words[end]) + if ordinal is not None: + ord_val, suffix = ordinal + out.append(str(value + ord_val) + suffix) + i = end + 1 + continue out.append(str(value)) i = end else: + # No cardinal — check if the word is a standalone ordinal + # (e.g. "første" → "1.", "enogtyvende" → "21.") + ordinal = _ORDINALS.get(_fold(words[i])) + if ordinal is None: + ordinal = _try_parse_og_compound_ordinal(words[i]) + if ordinal is not None: + ord_val, suffix = ordinal + out.append(str(ord_val) + suffix) + i += 1 + continue out.append(words[i]) i += 1 text = " ".join(out) @@ -229,14 +308,14 @@ def _parse_number(self, words: list[str], i: int, n: int) -> tuple[int, int] | N fw = _fold(words[i]) - if fw == "tusind": + if fw in ("tusind", "tusinde"): tail = self._parse_number(words, i + 1, n) if tail is not None: end, v2 = tail return end, 1000 + v2 return i + 1, 1000 - if i + 1 < n and fw in ("en", "et") and _fold(words[i + 1]) == "tusind": + if i + 1 < n and fw in ("en", "et") and _fold(words[i + 1]) in ("tusind", "tusinde"): tail = self._parse_number(words, i + 2, n) base = 1000 if tail is not None: @@ -300,7 +379,7 @@ def _parse_number(self, words: list[str], i: int, n: int) -> tuple[int, int] | N return j, v next_fw = _fold(words[j]) - if next_fw == "tusind": + if next_fw in ("tusind", "tusinde"): j += 1 prod = v * 1000 if j >= n: @@ -364,7 +443,7 @@ def _parse_0_999(self, words: list[str], i: int, n: int) -> tuple[int, int] | No def _continues_number(self, word: str) -> bool: fw = _fold(word) - if fw in ("hundrede", "tusind"): + if fw in ("hundrede", "tusind", "tusinde"): return True if fw in _BIG_MULT: return True diff --git a/normalization/languages/danish/operators.py b/normalization/languages/danish/operators.py index dd2df4a..1232b51 100644 --- a/normalization/languages/danish/operators.py +++ b/normalization/languages/danish/operators.py @@ -1,5 +1,8 @@ from normalization.languages.base import LanguageConfig, LanguageOperators from normalization.languages.danish.number_normalizer import DanishNumberNormalizer +from normalization.languages.danish.sentence_replacements import ( + DANISH_SENTENCE_REPLACEMENTS, +) from normalization.languages.registry import register_language _DANISH_DIGIT_WORDS: dict[str, str] = { @@ -79,6 +82,7 @@ "halvfems", "hundrede", "tusind", + "tusinde", "million", "millioner", "milliard", @@ -87,6 +91,7 @@ "billioner", ], plus_word="plus", + sentence_replacements=DANISH_SENTENCE_REPLACEMENTS, ) diff --git a/normalization/languages/danish/replacements.py b/normalization/languages/danish/replacements.py index 658f9bd..4f86597 100644 --- a/normalization/languages/danish/replacements.py +++ b/normalization/languages/danish/replacements.py @@ -2,4 +2,169 @@ DANISH_REPLACEMENTS: dict[str, str] = { "krone": "kroner", + + # dobbeltformer + "bagerst": "bagest", + "bedriftsværn": "bedriftværn", + "bevillige": "bevilge", + "beviselig": "bevislig", + "Bryssel": "Bruxelles", + "fiskefartøj": "fiskerfartøj", + "fiskeflåde": "fiskerflåde", + "fiskekutter": "fiskerkutter", + "forpligtige": "forpligte", + "forpligtigelse": "forpligtelse", + "grundskyldspromille": "grundskyldpromille", + "grøntfoder": "grønfoder", + "grøntsager": "grønsager", + "ingensteds": "intetsteds", + "mandeår": "mandår", + "misbrugen": "misbruget", + "nyttesløs": "nytteløs", + "opsigtvækkende": "opsigtsvækkende", + "sprogbruget": "sprogbrugen", + "strafferetslig": "strafferetlig", + "tidkrævende": "tidskrævende", + "tidnød": "tidsnød", + "tidrøvende": "tidsrøvende", + "tillidsvækkende": "tillidvækkende", + "adlen": "adelen", + "aftner": "aftener", + "aftnen": "aftenen", + "aftnerne": "aftenerne", + "anseeling": "anselig", + "bjerge": "bjærge", + "bødeforlæg": "bødeforelæg", + "kamuflage": "camouflage", + "kamuflere": "camouflere", + "centret": "centeret", + "centre": "centrer", + "centrene": "centrerne", + "chartret": "charteret", + "tjek": "check", + "check-in": "checkin", + "klips": "clips", + "dagsordner": "dagsordener", + "dagsordnen": "dagsordenen", + "dagsordnerne": "dagsordenerne", + "drive-in": "drivein", + "Ægypten": "Egypten", + "Ægypter": "Egypter", + "enfamiliehus": "enfamilieshus", + "finanslovaftale": "finanslovsaftale", + "forstret": "fosteret", + "handlen": "handelen", + "handikap": "handicap", + "hilsener": "hilsner", + "hilsenerne": "hilsnerne", + "hjemlen": "hjemmelen", + "hundred": "hundrede", + "højttaler": "højtaler", + "højttravende": "højtravende", + "immer væk": "immervæk", + "inkassere": "indkassere", + "indvilge": "indvillige", + "ingen sinde": "ingensinde", + "ingen ting": "ingenting", + "islætten": "islættet", + "kikke": "kigge", + "kikken": "kiggen", + "kikker": "kigger", + "kikkeri": "kiggeri", + "kikkede": "kiggede", + "kikket": "kigget", + "krasbørstig": "kradsbørstig", + "kuleskør": "kugleskør", + "log-in": "login", + "log-on": "logon", + "lovform": "lovsform", + "lovkraft": "lovskraft", + "Luxemburg": "Luxembourg", + "Luxemburger": "Luxembourger", + "Luxemburgsk": "Luxembourgsk", + "margin": "margen", + "massemedium": "massemedie", + "medium": "medie", + "Mocambiqe": "Mozambique", + "mønstret": "mønsteret", + "ofret": "offeret", + "regien": "regiet", + "registret": "registeret", + "resurse": "ressource", + "roll-on": "rollon", + "skanne": "scanne", + "sejrssikkert": "sejrsikker", + "sit-in": "sitin", + "stand-in": "standin", + "strafbestemmelse": "straffebestemmelse", + "straframme": "strafferamme", + "sørme": "søreme", + "takle": "tackle", + "taknemmelig": "taknemlig", + "tallerkner": "tallerkener", + "tallerknerne": "tallerkenerne", + "toetagershus": "toetageshus", + "toetagers": "toetages", + "trinet": "trinnet", + "trinene": "trinnene", + "trevl": "trævl", + "uforudseeling": "uforudselig", + "velbjerget": "velbjærget", + "venligtsindet": "venligsindet", + "væsnet": "væsenet", + "våbenet": "våbnet", + "siksak": "zigzag", + "ømtålig": "ømtålelig", + "ørkner": "ørkener", + "ørknerne": "ørkenerne", + + # abbreviations - often accompanied by full stop, which I don't + # believe is supported here + "alm": "almindelig", + "bla": "blandt andet", + "ca": "cirka", + "cl": "centiliter", + "cm": "centimeter", + "dB": "decibel", + "dl": "deciliter", + "dvs": "det vil sige", + "etc": "etcetera", + "feks": "for eksempel", + "fx": "for eksempel", + "fg": "fungerende", + "g": "gram", + "GB": "gigabyte", + "ha": "hektar", + "kg": "kilogram", + "kHz": "kilohertz", + "kJ": "kilojoule", + "kl": "klokken", + "km": "kilometer", + "km/t": "kilometer i timen", + "kr": "kroner", + "kW": "kilowatt", + "kWh": "kilowatt-time", + "l": "liter", + "m": "meter", + "mv": "med videre", + "m²": "kvadratmeter", + "m³": "kubikmeter", + "maks": "maksimum", + "mia": "milliard", + "mio": "million", + "MW": "megawatt", + "nr": "nummer", + "ok": "okay", + "osv": "og så videre", + "pct": "procent", + "pkt": "punktum", + "pr": "per", + "sek": "sekund", + "stk": "stykke", + "t": "ton", + "td": "tønde", + "V": "volt", + "W": "watt", + "frk": "frøken", + "fr": "fru" } diff --git a/normalization/languages/danish/sentence_replacements.py b/normalization/languages/danish/sentence_replacements.py new file mode 100644 index 0000000..e707eae --- /dev/null +++ b/normalization/languages/danish/sentence_replacements.py @@ -0,0 +1,28 @@ +DANISH_SENTENCE_REPLACEMENTS: dict[str, str] = { + "curriculum vitae": "curriculum vitæ", + "for resten": "forresten", + "gud bevares": "gudbevares", + "gud fader bevares": "gudfaderbevares", + "gud fader": "gudfader", + "gud hjælpe mig": "gudhjælpemig", + "herre gud": "herregud", + "immer væk": "immervæk", + "gud ske lov": "gudskelov", + "her til lands": "hertillands", + "ingen sinde": "ingensinde", + "ingen ting": "ingenting", + "laissez faire-politik": "laissez-faire-politik", + "lille bitte": "lillebitte", + "nogen sinde": "nogensinde", + "om bord": "ombord", + "om end": "omend", + "selv om": "selvom", + "selv samme": "selvsamme", + "siden hen": "sidenhen", + "simpelt hen": "simpelthen", + "somme tider": "sommetider", + "stadig væk": "stadigvæk", + "tids nok": "tidsnok", + "til fælles": "tilfælles", + "til lykke": "tillykke", +} \ No newline at end of file From 18bbb70855dfd5cba92773ac1242903ae0f016ac Mon Sep 17 00:00:00 2001 From: Asta Date: Tue, 7 Jul 2026 13:11:39 +0200 Subject: [PATCH 2/5] test: add tests for Danish ordinals, tusinde variant, replacements and sentence replacements Co-Authored-By: Claude Sonnet 4.6 --- .../languages/danish/replacements.py | 9 ---- .../danish_number_normalizer_test.py | 41 +++++++++++++++++++ tests/unit/languages/danish_operators_test.py | 40 ++++++++++++++++++ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/normalization/languages/danish/replacements.py b/normalization/languages/danish/replacements.py index 4f86597..382c4d7 100644 --- a/normalization/languages/danish/replacements.py +++ b/normalization/languages/danish/replacements.py @@ -61,11 +61,8 @@ "hundred": "hundrede", "højttaler": "højtaler", "højttravende": "højtravende", - "immer væk": "immervæk", "inkassere": "indkassere", "indvilge": "indvillige", - "ingen sinde": "ingensinde", - "ingen ting": "ingenting", "islætten": "islættet", "kikke": "kigge", "kikken": "kiggen", @@ -132,7 +129,6 @@ "feks": "for eksempel", "fx": "for eksempel", "fg": "fungerende", - "g": "gram", "GB": "gigabyte", "ha": "hektar", "kg": "kilogram", @@ -144,8 +140,6 @@ "kr": "kroner", "kW": "kilowatt", "kWh": "kilowatt-time", - "l": "liter", - "m": "meter", "mv": "med videre", "m²": "kvadratmeter", "m³": "kubikmeter", @@ -161,10 +155,7 @@ "pr": "per", "sek": "sekund", "stk": "stykke", - "t": "ton", "td": "tønde", - "V": "volt", - "W": "watt", "frk": "frøken", "fr": "fru" } diff --git a/tests/unit/languages/danish_number_normalizer_test.py b/tests/unit/languages/danish_number_normalizer_test.py index a25c6c4..21173b2 100644 --- a/tests/unit/languages/danish_number_normalizer_test.py +++ b/tests/unit/languages/danish_number_normalizer_test.py @@ -114,3 +114,44 @@ def test_kroner_word_not_treated_as_currency_suffix( normalizer: DanishNumberNormalizer, ) -> None: assert normalizer("25 kroner") == "25 kroner" + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # Simple ordinals (standalone) + ("første", "1."), + ("anden", "2."), + ("tredje", "3."), + ("tiende", "10."), + ("tyvende", "20."), + # Glued og-compound ordinals (e.g. "enogtyvende" = 21st) + ("enogtyvende", "21."), + ("toogtyvende", "22."), + ("fireoghalvtredsende", "54."), + # Cardinal + ordinal suffix (e.g. "hundrede tyvende" = 120th) + ("hundrede tyvende", "120."), + ("hundrede enogtyvende", "121."), + ], +) +def test_danish_ordinals( + normalizer: DanishNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # "tusinde" (with -e) should be treated the same as "tusind" + ("tusinde", "1000"), + ("et tusinde", "1000"), + ("to tusinde", "2000"), + ("to tusinde tre hundrede", "2300"), + ("to tusinde fem og tyve", "2025"), + ], +) +def test_tusinde_variant( + normalizer: DanishNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected diff --git a/tests/unit/languages/danish_operators_test.py b/tests/unit/languages/danish_operators_test.py index 41d87ec..1811db8 100644 --- a/tests/unit/languages/danish_operators_test.py +++ b/tests/unit/languages/danish_operators_test.py @@ -25,3 +25,43 @@ def test_config_code(operators: DanishOperators) -> None: def test_word_replacements(operators: DanishOperators) -> None: assert operators.get_word_replacements()["krone"] == "kroner" assert "euro" not in operators.get_word_replacements() + + +@pytest.mark.parametrize( + ("variant", "canonical"), + [ + # dobbeltformer spelling variants + ("grøntsager", "grønsager"), + ("handikap", "handicap"), + ("taknemmelig", "taknemlig"), + # abbreviations + ("dvs", "det vil sige"), + ("fx", "for eksempel"), + ("pct", "procent"), + ("osv", "og så videre"), + ("ok", "okay"), + ], +) +def test_danish_word_replacements( + operators: DanishOperators, variant: str, canonical: str +) -> None: + replacements = operators.get_word_replacements() + assert replacements[variant] == canonical + + +@pytest.mark.parametrize( + ("phrase", "canonical"), + [ + ("selv om", "selvom"), + ("til lykke", "tillykke"), + ("ingen sinde", "ingensinde"), + ("ingen ting", "ingenting"), + ("immer væk", "immervæk"), + ], +) +def test_danish_sentence_replacements(phrase: str, canonical: str) -> None: + from normalization.languages.danish.sentence_replacements import ( + DANISH_SENTENCE_REPLACEMENTS, + ) + + assert DANISH_SENTENCE_REPLACEMENTS[phrase] == canonical From 1c98e43d62c9bd1ae69cbabcb41816c25b2f67d6 Mon Sep 17 00:00:00 2001 From: Asta Date: Tue, 7 Jul 2026 13:19:38 +0200 Subject: [PATCH 3/5] fix: correct compound ordinal test case to match actual dict entries Co-Authored-By: Claude Sonnet 4.6 --- tests/unit/languages/danish_number_normalizer_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/languages/danish_number_normalizer_test.py b/tests/unit/languages/danish_number_normalizer_test.py index 21173b2..49a8665 100644 --- a/tests/unit/languages/danish_number_normalizer_test.py +++ b/tests/unit/languages/danish_number_normalizer_test.py @@ -128,7 +128,7 @@ def test_kroner_word_not_treated_as_currency_suffix( # Glued og-compound ordinals (e.g. "enogtyvende" = 21st) ("enogtyvende", "21."), ("toogtyvende", "22."), - ("fireoghalvtredsende", "54."), + ("fireogfyrrende", "44."), # Cardinal + ordinal suffix (e.g. "hundrede tyvende" = 120th) ("hundrede tyvende", "120."), ("hundrede enogtyvende", "121."), From 9112d744e6613269a2f19803582db0a5266bd8c4 Mon Sep 17 00:00:00 2001 From: Asta Date: Tue, 7 Jul 2026 13:23:25 +0200 Subject: [PATCH 4/5] test: add Danish e2e cases for ordinals, tusinde, replacements and sentence replacements Co-Authored-By: Claude Sonnet 4.6 --- tests/e2e/files/gladia-3/da.csv | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/files/gladia-3/da.csv b/tests/e2e/files/gladia-3/da.csv index de37d07..aa16c86 100644 --- a/tests/e2e/files/gladia-3/da.csv +++ b/tests/e2e/files/gladia-3/da.csv @@ -12,3 +12,11 @@ x = 5,x er lig med 5 "3,14",3 komma 14 en million,1000000 ping pong,ping pong +handikap,handicap +dvs,det vil sige +pct,procent +selv om,selvom +til lykke,tillykke +første,1 +enogtyvende,21 +tusinde,1000 From 630aad6a032fe06123c2a28316d673f49266fb03 Mon Sep 17 00:00:00 2001 From: Asta Date: Tue, 7 Jul 2026 13:54:11 +0200 Subject: [PATCH 5/5] fix: lowercase Aegypter replacement keys so they match after pipeline lowercasing Co-Authored-By: Claude Sonnet 4.6 --- normalization/languages/danish/replacements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/normalization/languages/danish/replacements.py b/normalization/languages/danish/replacements.py index 382c4d7..bbd54d8 100644 --- a/normalization/languages/danish/replacements.py +++ b/normalization/languages/danish/replacements.py @@ -48,8 +48,8 @@ "dagsordnen": "dagsordenen", "dagsordnerne": "dagsordenerne", "drive-in": "drivein", - "Ægypten": "Egypten", - "Ægypter": "Egypter", + "ægypten": "egypten", + "ægypter": "egypter", "enfamiliehus": "enfamilieshus", "finanslovaftale": "finanslovsaftale", "forstret": "fosteret",