Add Portuguese normalization for gladia-3 WER scoring.

normalization/languages/__init__.py

@@ -7,6 +7,7 @@
     german,
     italian,
     norwegian,
+    portuguese,
     spanish,
     swedish,
 )
@@ -24,6 +25,7 @@
     "german",
     "italian",
     "norwegian",
+    "portuguese",
     "spanish",
     "swedish",
     "get_language_registry",

normalization/languages/portuguese/__init__.py

@@ -0,0 +1,7 @@
+from .operators import PortugueseOperators
+from .replacements import PORTUGUESE_REPLACEMENTS
+
+__all__ = [
+    "PortugueseOperators",
+    "PORTUGUESE_REPLACEMENTS",
+]

normalization/languages/portuguese/clitics.py

@@ -0,0 +1,38 @@
+"""Merge hyphenated Portuguese clitic pronouns into single tokens for WER scoring.
+
+Oral Portuguese often attaches clitics to imperatives (``envie-me`` / ``enviame``).
+ASR may hyphenate or space them; this step canonicalizes to the glued form.
+"""
+
+import re
+
+_CLITIC_SUFFIXES = (
+    "me",
+    "te",
+    "se",
+    "lo",
+    "la",
+    "lhe",
+    "lhes",
+    "nos",
+    "vos",
+    "los",
+    "las",
+    "no",
+    "na",
+    "lho",
+    "lha",
+    "lhes",
+)
+
+_RE_HYPHEN_CLITIC = re.compile(
+    rf"\b(\w+)-({'|'.join(_CLITIC_SUFFIXES)})\b",
+    re.IGNORECASE,
+)
+
+
+def merge_hyphenated_clitics(text: str) -> str:
+    """``digame-me`` / ``diga-me`` → ``digame``."""
+    return _RE_HYPHEN_CLITIC.sub(
+        lambda m: m.group(1).lower() + m.group(2).lower(), text
+    )

normalization/languages/portuguese/number_normalizer.py

@@ -0,0 +1,66 @@
+"""Portuguese number normalizer using text2num's alpha2digit.
+
+Converts spelled-out numbers to digits (e.g. vinte e tres → 23) and handles
+mixed digit+word forms (e.g. 3 milhoes → tres milhoes) before conversion.
+"""
+
+import re
+
+from text_to_num import alpha2digit
+
+_RE_MIXED_NUMBER = re.compile(
+    r"\b(\d+)\s+(mil|milhao|milhoes|milhão|milhões|bilhao|bilhoes|bilhão|bilhões)\b",
+    re.IGNORECASE,
+)
+
+_RE_UM = re.compile(r"\bum\b", re.IGNORECASE)
+_RE_DOIS = re.compile(r"\bdois\b", re.IGNORECASE)
+
+# text2num expects accented Portuguese forms; STT often drops accents.
+_UNACCENTED_NUMBER_FORMS: dict[str, str] = {
+    "milhoes": "milhões",
+    "milhao": "milhão",
+    "bilhoes": "bilhões",
+    "bilhao": "bilhão",
+    "tres": "três",
+    "seis": "seis",
+}
+
+
+def _restore_number_accents(text: str) -> str:
+    for unaccented, canonical in _UNACCENTED_NUMBER_FORMS.items():
+        text = re.sub(rf"\b{unaccented}\b", canonical, text, flags=re.IGNORECASE)
+    return text
+
+
+def _fix_remaining_words(text: str) -> str:
+    """Replace number words alpha2digit did not convert."""
+    text = _RE_UM.sub("1", text)
+    text = _RE_DOIS.sub("2", text)
+    return text
+
+
+class PortugueseNumberNormalizer:
+    """Convert Portuguese spelled-out numbers to digits via text2num.alpha2digit."""
+
+    def __init__(self, digit_words: dict[str, str]) -> None:
+        self._digit_to_word = {v: k for k, v in digit_words.items()}
+
+    def _normalize_mixed_numbers(self, text: str) -> str:
+        """Convert ``3 milhoes`` → ``tres milhoes`` so alpha2digit yields 3000000."""
+
+        def replace(match: re.Match[str]) -> str:
+            number = match.group(1)
+            multiplier = match.group(2)
+            if len(number) == 1 and number in self._digit_to_word:
+                return f"{self._digit_to_word[number]} {multiplier}"
+            return match.group(0)
+
+        return _RE_MIXED_NUMBER.sub(replace, text)
+
+    def __call__(self, text: str) -> str:
+        text = _restore_number_accents(text)
+        text = self._normalize_mixed_numbers(text)
+        text = alpha2digit(text, "pt")
+        text = _fix_remaining_words(text)
+        return text

normalization/languages/portuguese/operators.py

@@ -0,0 +1,133 @@
+import re
+
+from normalization.languages.base import LanguageConfig, LanguageOperators
+from normalization.languages.portuguese.clitics import merge_hyphenated_clitics
+from normalization.languages.portuguese.number_normalizer import (
+    PortugueseNumberNormalizer,
+)
+from normalization.languages.portuguese.replacements import PORTUGUESE_REPLACEMENTS
+from normalization.languages.portuguese.sentence_replacements import (
+    PORTUGUESE_SENTENCE_REPLACEMENTS,
+)
+from normalization.languages.registry import register_language
+
+_PORTUGUESE_DIGIT_WORDS: dict[str, str] = {
+    "zero": "0",
+    "um": "1",
+    "dois": "2",
+    "tres": "3",
+    "quatro": "4",
+    "cinco": "5",
+    "seis": "6",
+    "sete": "7",
+    "oito": "8",
+    "nove": "9",
+}
+
+PORTUGUESE_CONFIG = LanguageConfig(
+    code="pt",
+    decimal_separator=",",
+    decimal_word="virgula",
+    thousand_separator=" ",
+    symbols_to_words={
+        "@": "arroba",
+        ".": "ponto",
+        "+": "mais",
+        "=": "igual a",
+        ">": "maior que",
+        "<": "menor que",
+        "°": "grau",
+        "°C": "graus celsius",
+        "°F": "graus fahrenheit",
+        "%": "por cento",
+    },
+    currency_symbol_to_word={
+        "€": "euros",
+        "$": "dolares",
+        "£": "libras",
+        "¢": "centavos",
+        "¥": "ienes",
+    },
+    filler_words=[
+        # Hesitations / backchannels only — avoid stripping meaningful words
+        # (e.g. "claro", "bem" in "tudo bem", "pronto" in "e pronto, foi assim").
+        "ah",
+        "eh",
+        "ehm",
+        "entao",
+        "portanto",
+        "mm",
+        "mmm",
+        "mhm",
+        "hmm",
+        "hm",
+        "hum",
+        "humm",
+    ],
+    sentence_replacements=PORTUGUESE_SENTENCE_REPLACEMENTS,
+    digit_words=_PORTUGUESE_DIGIT_WORDS,
+    number_words=[
+        *_PORTUGUESE_DIGIT_WORDS,
+        "dez",
+        "onze",
+        "doze",
+        "treze",
+        "catorze",
+        "quinze",
+        "dezasseis",
+        "dezesseis",
+        "dezassete",
+        "dezessete",
+        "dezoito",
+        "dezenove",
+        "vinte",
+        "trinta",
+        "quarenta",
+        "cinquenta",
+        "sessenta",
+        "setenta",
+        "oitenta",
+        "noventa",
+        "cem",
+        "cento",
+        "mil",
+        "milhao",
+        "milhoes",
+        "milhão",
+        "milhões",
+        "bilhao",
+        "bilhoes",
+        "bilhão",
+        "bilhões",
+    ],
+    plus_word="mais",
+)
+
+
+@register_language
+class PortugueseOperators(LanguageOperators):
+    """Portuguese language operators: clitics, written numbers, word replacements."""
+
+    def __init__(self) -> None:
+        super().__init__(PORTUGUESE_CONFIG)
+        self._number_normalizer = PortugueseNumberNormalizer(
+            PORTUGUESE_CONFIG.digit_words or {}
+        )
+
+    def expand_contractions(self, text: str) -> str:
+        """Merge hyphenated clitic pronouns (envie-me → enviame)."""
+        return merge_hyphenated_clitics(text)
+
+    def expand_written_numbers(self, text: str) -> str:
+        """Convert Portuguese spelled-out numbers to digits (vinte e tres → 23)."""
+        return self._number_normalizer(text)
+
+    def fix_one_word_in_numeric_contexts(self, text: str) -> str:
+        text = re.sub(r"(\d+)\s+um\b", r"\1 1", text, flags=re.IGNORECASE)
+        text = re.sub(r"\bum\s+(\d)", r"1 \1", text, flags=re.IGNORECASE)
+        text = re.sub(r"(\d+)um\b", r"\1 1", text, flags=re.IGNORECASE)
+        text = re.sub(r"\bum(\d)", r"1 \1", text, flags=re.IGNORECASE)
+        return text
+
+    def get_word_replacements(self) -> dict[str, str]:
+        return PORTUGUESE_REPLACEMENTS

normalization/languages/portuguese/replacements.py

@@ -0,0 +1,42 @@
+"""Single-token Portuguese colloquial / ASR variants → canonical form for WER."""
+
+PORTUGUESE_REPLACEMENTS: dict[str, str] = {
+    # Oral preposition / contraction variants
+    "pra": "para",
+    # Gender agreement in short confirmations (telephony slips)
+    "propria": "proprio",
+    "proprias": "proprios",
+    # Adjective / adverb spelling variants
+    "maioritariamente": "maioritario",
+    "media": "medio",
+    # Email / compound spelling
+    "emails": "email",
+    "e-mail": "email",
+    "e-mails": "email",
+    # Morphological number (singular/plural oral agreement)
+    "rotativos": "rotativo",
+    "turnos": "turno",
+    "pesos": "peso",
+    "quantos": "quanto",
+    # PT-PT / PT-BR orthography (canonical: PT-PT)
+    "controles": "controlos",
+    "equipe": "equipa",
+    "confirmar-se": "confirmares",
+    # Clitic / verb form variants (ASR subjunctive vs imperative)
+    "envieme": "enviame",
+    "adeco": "adecco",
+    "adec": "adecco",
+    # Abbreviations / titles
+    "dr": "doutor",
+    "dra": "doutora",
+    "sr": "senhor",
+    "sra": "senhora",
+    "exmo": "excelentissimo",
+    "exma": "excelentissima",
+    "etc": "etcetera",
+    "tel": "telefone",
+    "vs": "versus",
+    "versus": "versus",
+    # Acknowledgments / spelling variants
+    "okay": "ok",
+}

normalization/languages/portuguese/sentence_replacements.py

@@ -0,0 +1,17 @@
+"""Multi-word Portuguese phrase normalization (oral variants, compounds, ASR repairs)."""
+
+PORTUGUESE_SENTENCE_REPLACEMENTS: dict[str, str] = {
+    # Email compounds (hyphenated forms run in text_pre; spaced forms after cleanup)
+    "e-mails": "email",
+    "e-mail": "email",
+    "e mail": "email",
+    "e mails": "email",
+    # Morphological number / construction variants
+    "turnos rotativos": "turno rotativo",
+    "em quantos pesos": "de quanto peso",
+    # PT-PT / PT-BR orthography
+    "confirmar se": "confirmares",
+    # Clitic spacing (ASR splits glued imperatives)
+    "envie me": "enviame",
+    "diga me": "digame",
+}

tests/e2e/files/gladia-3/pt.csv

@@ -0,0 +1,30 @@
+input,expected
+eh entao sim,sim
+ah mm sim,sim
+claro estou,claro estou
+esta tudo bem,esta tudo bem
+e pronto foi assim,e pronto foi assim
+pra trabalhar,para trabalhar
+Se vou ser chamada pra trabalhar,se vou ser chamada para trabalhar
+Sim e propria,sim e proprio
+maioritariamente,maioritario
+media,medio
+emails,email
+e-mails,email
+turnos rotativos,turno rotativo
+em quantos pesos,de quanto peso
+envie-me,enviame
+enviame,enviame
+diga-me,digame
+diga me,digame
+controles,controlos
+equipe,equipa
+adeco,adecco
+adec,adecco
+vinte e tres,23
+cem euros,100 euros
+Maria,maria
+María,maria
+test@example.com,test arroba example ponto com
+"12,5 euros",12 virgula 5 euros
+posso ajudar,posso ajudar

tests/unit/languages/portuguese_number_normalizer_test.py

@@ -0,0 +1,37 @@
+import pytest
+
+from normalization.languages.portuguese.number_normalizer import (
+    PortugueseNumberNormalizer,
+)
+
+_DIGIT_WORDS = {
+    "zero": "0",
+    "um": "1",
+    "dois": "2",
+    "tres": "3",
+    "quatro": "4",
+    "cinco": "5",
+    "seis": "6",
+    "sete": "7",
+    "oito": "8",
+    "nove": "9",
+}
+
+
+@pytest.fixture
+def normalizer():
+    return PortugueseNumberNormalizer(_DIGIT_WORDS)
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("vinte e tres", "23"),
+        ("cem", "100"),
+        ("duzentos", "200"),
+        ("3 milhoes", "3000000"),
+        ("um", "1"),
+    ],
+)
+def test_portuguese_number_normalizer(normalizer, text: str, expected: str):
+    assert normalizer(text) == expected