Feat/danish normalization replacement bundle

normalization/languages/danish/number_normalizer.py

@@ -61,6 +61,49 @@ def _get(table: dict[str, int], word: str) -> int | None:
     "halvfems": 90,
 }
 
+# In Danish the ordinal indicator is a trailing period, e.g. "første" → "1."
+_ORDINALS: dict[str, tuple[int, str]] = {
+    "første": (1, "."),
+    "anden": (2, "."),
+    "tredje": (3, "."),
+    "fjerde": (4, "."),
+    "femte": (5, "."),
+    "sjette": (6, "."),
+    "syvende": (7, "."),
+    "ottende": (8, "."),
+    "niende": (9, "."),
+    "tiende": (10, "."),
+
+    "ellevte": (11, "."),
+    "tolvte": (12, "."),
+    "trettende": (13, "."),
+    "fjortende": (14, "."),
+    "femtende": (15, "."),
+    "sekstende": (16, "."),
+    "syttende": (17, "."),
+    "attende": (18, "."),
+    "nittende": (19, "."),
+
+    "tyvende": (20, "."),
+    "tredivte": (30, "."),
+    "fyrrende": (40, "."),
+    "fyrretyvende": (40, "."),
+    "halvtredsinde": (50, "."),
+    "halvtredsindstyvende": (50, "."),
+    "tressende": (60, "."),
+    "tresindstyvende": (60, "."),
+    "halvfjerdsende": (70, "."),
+    "halvfjerdsindtyvende": (70, "."),
+    "firsende": (80, "."),
+    "firsindtyvende": (80, "."),
+    "halvfemsende": (90, "."),
+    "halvfemsindstyvende": (90, ".")
+}
+
+_TENS_ORDINAL: dict[str, tuple[int, str]] = {
+    k: v for k, v in _ORDINALS.items() if v[0] >= 20
+}
+
 # Used for og-compound ones part (includes en/et = 1 alongside 2-9)
 _ONES_FOR_OG: dict[str, int] = {"en": 1, "et": 1, **_ONES_2_9}
 
@@ -86,6 +129,7 @@ def _get(table: dict[str, int], word: str) -> int | None:
 
 _BIG_MULT: dict[str, int] = {
     "tusind": 1_000,
+    "tusinde": 1_000,
     "million": 1_000_000,
     "millioner": 1_000_000,
     "milliard": 1_000_000_000,
@@ -113,6 +157,20 @@ def _try_parse_og_compound(word: str) -> int | None:
     return None
 
 
+def _try_parse_og_compound_ordinal(word: str) -> tuple[int, str] | None:
+    """Parse a glued Danish ordinal compound like ``'enogtyvende'`` = 21."""
+    fw = _fold(word)
+    for ones_str, ones_val in _ONES_FOR_OG_SORTED:
+        prefix = ones_str + "og"
+        if fw.startswith(prefix):
+            rest = fw[len(prefix) :]
+            tens = _TENS_ORDINAL.get(rest)
+            if tens is not None:
+                tens_val, suffix = tens
+                return tens_val + ones_val, suffix
+    return None
+
+
 def _normalize_mixed_numbers(text: str) -> str:
     """Convert ``3 milliard`` → ``tre milliard`` so the word parser yields 3 000 000 000."""
 
@@ -214,9 +272,30 @@ def __call__(self, text: str) -> str:
             parsed = self._parse_number(words, i, n)
             if parsed is not None:
                 end, value = parsed
+                # After parsing a cardinal, peek at the next word to see if it
+                # is an ordinal continuation (e.g. "hundrede" + "enogtyvende" → 121.)
+                if end < n:
+                    ordinal = _ORDINALS.get(_fold(words[end]))
+                    if ordinal is None:
+                        ordinal = _try_parse_og_compound_ordinal(words[end])
+                    if ordinal is not None:
+                        ord_val, suffix = ordinal
+                        out.append(str(value + ord_val) + suffix)
+                        i = end + 1
+                        continue
                 out.append(str(value))
                 i = end
             else:
+                # No cardinal — check if the word is a standalone ordinal
+                # (e.g. "første" → "1.", "enogtyvende" → "21.")
+                ordinal = _ORDINALS.get(_fold(words[i]))
+                if ordinal is None:
+                    ordinal = _try_parse_og_compound_ordinal(words[i])
+                if ordinal is not None:
+                    ord_val, suffix = ordinal
+                    out.append(str(ord_val) + suffix)
+                    i += 1
+                    continue
                 out.append(words[i])
                 i += 1
         text = " ".join(out)
@@ -229,14 +308,14 @@ def _parse_number(self, words: list[str], i: int, n: int) -> tuple[int, int] | N
 
         fw = _fold(words[i])
 
-        if fw == "tusind":
+        if fw in ("tusind", "tusinde"):
             tail = self._parse_number(words, i + 1, n)
             if tail is not None:
                 end, v2 = tail
                 return end, 1000 + v2
             return i + 1, 1000
 
-        if i + 1 < n and fw in ("en", "et") and _fold(words[i + 1]) == "tusind":
+        if i + 1 < n and fw in ("en", "et") and _fold(words[i + 1]) in ("tusind", "tusinde"):
             tail = self._parse_number(words, i + 2, n)
             base = 1000
             if tail is not None:
@@ -300,7 +379,7 @@ def _parse_number(self, words: list[str], i: int, n: int) -> tuple[int, int] | N
             return j, v
 
         next_fw = _fold(words[j])
-        if next_fw == "tusind":
+        if next_fw in ("tusind", "tusinde"):
             j += 1
             prod = v * 1000
             if j >= n:
@@ -364,7 +443,7 @@ def _parse_0_999(self, words: list[str], i: int, n: int) -> tuple[int, int] | No
 
     def _continues_number(self, word: str) -> bool:
         fw = _fold(word)
-        if fw in ("hundrede", "tusind"):
+        if fw in ("hundrede", "tusind", "tusinde"):
             return True
         if fw in _BIG_MULT:
             return True

normalization/languages/danish/operators.py

@@ -1,5 +1,8 @@
 from normalization.languages.base import LanguageConfig, LanguageOperators
 from normalization.languages.danish.number_normalizer import DanishNumberNormalizer
+from normalization.languages.danish.sentence_replacements import (
+    DANISH_SENTENCE_REPLACEMENTS,
+)
 from normalization.languages.registry import register_language
 
 _DANISH_DIGIT_WORDS: dict[str, str] = {
@@ -79,6 +82,7 @@
         "halvfems",
         "hundrede",
         "tusind",
+        "tusinde",
         "million",
         "millioner",
         "milliard",
@@ -87,6 +91,7 @@
         "billioner",
     ],
     plus_word="plus",
+    sentence_replacements=DANISH_SENTENCE_REPLACEMENTS,
 )
 
 

normalization/languages/danish/replacements.py

@@ -2,4 +2,160 @@
 
 DANISH_REPLACEMENTS: dict[str, str] = {
     "krone": "kroner",
+
+    # dobbeltformer
+    "bagerst": "bagest",
+    "bedriftsværn": "bedriftværn",
+    "bevillige": "bevilge",
+    "beviselig": "bevislig",
+    "Bryssel": "Bruxelles",
+    "fiskefartøj": "fiskerfartøj",
+    "fiskeflåde": "fiskerflåde",
+    "fiskekutter": "fiskerkutter",
+    "forpligtige": "forpligte",
+    "forpligtigelse": "forpligtelse",
+    "grundskyldspromille": "grundskyldpromille",
+    "grøntfoder": "grønfoder",
+    "grøntsager": "grønsager",
+    "ingensteds": "intetsteds",
+    "mandeår": "mandår",
+    "misbrugen": "misbruget",
+    "nyttesløs": "nytteløs",
+    "opsigtvækkende": "opsigtsvækkende",
+    "sprogbruget": "sprogbrugen",
+    "strafferetslig": "strafferetlig",
+    "tidkrævende": "tidskrævende",
+    "tidnød": "tidsnød",
+    "tidrøvende": "tidsrøvende",
+    "tillidsvækkende": "tillidvækkende",
+    "adlen": "adelen",
+    "aftner": "aftener",
+    "aftnen": "aftenen",
+    "aftnerne": "aftenerne",
+    "anseeling": "anselig",
+    "bjerge": "bjærge",
+    "bødeforlæg": "bødeforelæg",
+    "kamuflage": "camouflage",
+    "kamuflere": "camouflere",
+    "centret": "centeret",
+    "centre": "centrer",
+    "centrene": "centrerne",
+    "chartret": "charteret",
+    "tjek": "check",
+    "check-in": "checkin",
+    "klips": "clips",
+    "dagsordner": "dagsordener",
+    "dagsordnen": "dagsordenen",
+    "dagsordnerne": "dagsordenerne",
+    "drive-in": "drivein",
+    "ægypten": "egypten",
+    "ægypter": "egypter",
+    "enfamiliehus": "enfamilieshus",
+    "finanslovaftale": "finanslovsaftale",
+    "forstret": "fosteret",
+    "handlen": "handelen",
+    "handikap": "handicap",
+    "hilsener": "hilsner",
+    "hilsenerne": "hilsnerne",
+    "hjemlen": "hjemmelen",
+    "hundred": "hundrede",
+    "højttaler": "højtaler",
+    "højttravende": "højtravende",
+    "inkassere": "indkassere",
+    "indvilge": "indvillige",
+    "islætten": "islættet",
+    "kikke": "kigge",
+    "kikken": "kiggen",
+    "kikker": "kigger",
+    "kikkeri": "kiggeri",
+    "kikkede": "kiggede",
+    "kikket": "kigget",
+    "krasbørstig": "kradsbørstig",
+    "kuleskør": "kugleskør",
+    "log-in": "login",
+    "log-on": "logon",
+    "lovform": "lovsform",
+    "lovkraft": "lovskraft",
+    "Luxemburg": "Luxembourg",
+    "Luxemburger": "Luxembourger",
+    "Luxemburgsk": "Luxembourgsk",
+    "margin": "margen",
+    "massemedium": "massemedie",
+    "medium": "medie",
+    "Mocambiqe": "Mozambique",
+    "mønstret": "mønsteret",
+    "ofret": "offeret",
+    "regien": "regiet",
+    "registret": "registeret",
+    "resurse": "ressource",
+    "roll-on": "rollon",
+    "skanne": "scanne",
+    "sejrssikkert": "sejrsikker",
+    "sit-in": "sitin",
+    "stand-in": "standin",
+    "strafbestemmelse": "straffebestemmelse",
+    "straframme": "strafferamme",
+    "sørme": "søreme",
+    "takle": "tackle",
+    "taknemmelig": "taknemlig",
+    "tallerkner": "tallerkener",
+    "tallerknerne": "tallerkenerne",
+    "toetagershus": "toetageshus",
+    "toetagers": "toetages",
+    "trinet": "trinnet",
+    "trinene": "trinnene",
+    "trevl": "trævl",
+    "uforudseeling": "uforudselig",
+    "velbjerget": "velbjærget",
+    "venligtsindet": "venligsindet",
+    "væsnet": "væsenet",
+    "våbenet": "våbnet",
+    "siksak": "zigzag",
+    "ømtålig": "ømtålelig",
+    "ørkner": "ørkener",
+    "ørknerne": "ørkenerne",
+
+    # abbreviations - often accompanied by full stop, which I don't 
+    # believe is supported here
+    "alm": "almindelig",
+    "bla": "blandt andet",
+    "ca": "cirka",
+    "cl": "centiliter",
+    "cm": "centimeter",
+    "dB": "decibel",
+    "dl": "deciliter",
+    "dvs": "det vil sige",
+    "etc": "etcetera",
+    "feks": "for eksempel",
+    "fx": "for eksempel",
+    "fg": "fungerende",
+    "GB": "gigabyte",
+    "ha": "hektar",
+    "kg": "kilogram",
+    "kHz": "kilohertz",
+    "kJ": "kilojoule",
+    "kl": "klokken",
+    "km": "kilometer",
+    "km/t": "kilometer i timen",
+    "kr": "kroner",
+    "kW": "kilowatt",
+    "kWh": "kilowatt-time",
+    "mv": "med videre",
+    "m²": "kvadratmeter",
+    "m³": "kubikmeter",
+    "maks": "maksimum",
+    "mia": "milliard",
+    "mio": "million",
+    "MW": "megawatt",
+    "nr": "nummer",
+    "ok": "okay",
+    "osv": "og så videre",
+    "pct": "procent",
+    "pkt": "punktum",
+    "pr": "per",
+    "sek": "sekund",
+    "stk": "stykke",
+    "td": "tønde",
+    "frk": "frøken",
+    "fr": "fru"
 }

normalization/languages/danish/sentence_replacements.py

@@ -0,0 +1,28 @@
+DANISH_SENTENCE_REPLACEMENTS: dict[str, str] = {
+    "curriculum vitae": "curriculum vitæ",
+    "for resten": "forresten",
+    "gud bevares": "gudbevares",
+    "gud fader bevares": "gudfaderbevares",
+    "gud fader": "gudfader",
+    "gud hjælpe mig": "gudhjælpemig",
+    "herre gud": "herregud",
+    "immer væk": "immervæk",
+    "gud ske lov": "gudskelov",
+    "her til lands": "hertillands",
+    "ingen sinde": "ingensinde",
+    "ingen ting": "ingenting",
+    "laissez faire-politik": "laissez-faire-politik",
+    "lille bitte": "lillebitte",
+    "nogen sinde": "nogensinde",
+    "om bord": "ombord",
+    "om end": "omend",
+    "selv om": "selvom",
+    "selv samme": "selvsamme",
+    "siden hen": "sidenhen",
+    "simpelt hen": "simpelthen",
+    "somme tider": "sommetider",
+    "stadig væk": "stadigvæk",
+    "tids nok": "tidsnok",
+    "til fælles": "tilfælles",
+    "til lykke": "tillykke",
+}

tests/e2e/files/gladia-3/da.csv

@@ -12,3 +12,11 @@ x = 5,x er lig med 5
 "3,14",3 komma 14
 en million,1000000
 ping pong,ping pong
+handikap,handicap
+dvs,det vil sige
+pct,procent
+selv om,selvom
+til lykke,tillykke
+første,1
+enogtyvende,21
+tusinde,1000